Skip to content

Commit 08b5fd6

Browse files
author
Sarah Wang
committed
webusb: add zstd decompress support for wic image
1 parent 787880f commit 08b5fd6

7 files changed

Lines changed: 318 additions & 10 deletions

File tree

.gitignore

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,4 +14,5 @@ CMakeCache.txt
1414
*.clst
1515
*.snap
1616
node_modules
17-
build
17+
build
18+
*-lock.json

webusb/index.js

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
window.ZstdCodec = require('zstd-codec').ZstdCodec;
2+
require("zstd-codec/lib/module.js").run((binding) => {console.log("running", binding); window.binding = binding} )

webusb/package.json

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -3,26 +3,24 @@
33
"version": "0.1.0",
44
"private": true,
55
"dependencies": {
6+
"-": "^0.0.1",
67
"@testing-library/jest-dom": "^5.16.5",
78
"@testing-library/react": "^13.4.0",
89
"@testing-library/user-event": "^13.5.0",
10+
"fs": "^0.0.1-security",
11+
"g": "^2.0.1",
912
"react": "^18.2.0",
1013
"react-dom": "^18.2.0",
1114
"react-scripts": "^5.0.1",
12-
"web-vitals": "^2.1.4"
15+
"web-vitals": "^2.1.4",
16+
"zstd-codec": "^0.1.4"
1317
},
1418
"scripts": {
1519
"start": "react-scripts start",
16-
"build": "react-scripts build",
20+
"build": "react-scripts build; browserify index.js -o build/bundle.js -t babelify --presets es2015; ",
1721
"test": "react-scripts test",
1822
"eject": "react-scripts eject"
1923
},
20-
"eslintConfig": {
21-
"extends": [
22-
"react-app",
23-
"react-app/jest"
24-
]
25-
},
2624
"browserslist": {
2725
"production": [
2826
">0.2%",
@@ -35,5 +33,8 @@
3533
"last 1 safari version"
3634
]
3735
},
38-
"proxy": "http://localhost:5000"
36+
"devDependencies": {
37+
"@babel/core": "^7.22.9",
38+
"babelify": "^10.0.0"
39+
}
3940
}

webusb/public/index.html

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,14 @@
2525
Learn how to configure a non-root public URL by running `npm run build`.
2626
-->
2727
<title>React App</title>
28+
<script src="bundle.js"></script>
2829
</head>
30+
2931
<body>
32+
<h1>Sample Applications</h1>
33+
<div id="container"></div>
34+
35+
3036
<noscript>You need to enable JavaScript to run this app.</noscript>
3137
<div id="root"></div>
3238
<!--

webusb/src/App.js

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import React, { useState } from 'react';
22
import Combined from "./components/Combined";
3+
import Decompress2 from "./components/Decompress";
34
import usePopup from "./logic/usePopup";
45

56
import './App.css'
@@ -12,6 +13,9 @@ const App = () => {
1213
return (
1314
<div className="App">
1415
<div className="u-flex u-column">
16+
<div>
17+
<Decompress2 />
18+
</div>
1519
<div className="u-row">
1620
<span className="link-container">bootloader [optional]: </span>
1721
<span className="file-name">{bootFile? bootFile.name:""}</span>

webusb/src/index.js

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ import ReactDOM from 'react-dom/client';
33
import './index.css';
44
import App from './App';
55
import reportWebVitals from './reportWebVitals';
6+
// import * as Z from 'zstd-codec'
67

78
const root = ReactDOM.createRoot(document.getElementById('root'));
89
root.render(

webusb/src/logic/useDecompress.js

Lines changed: 293 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,293 @@
1+
2+
import {useEffect, useState} from 'react';
3+
import {str_to_arr, ab_to_str} from '../helper/functions.js'
4+
import {CHUNK_SZ, BLK_SZ, CHUNK_TYPE_RAW, CHUNK_TYPE_DONT_CARE, build_sparse_header, build_chunk_header} from '../helper/sparse.js'
5+
6+
const USBrequestParams = { filters: [{ vendorId: 8137, productId: 0x0152 }] };
7+
8+
const PACKET_SZ = 0x10000;
9+
const DATA_SZ = CHUNK_SZ * BLK_SZ; // in bytes
10+
11+
const useDecompress = (flashFile) => {
12+
const [USBDevice, setUSBDevice] = useState();
13+
const [flashProgress, setFlashProgress] = useState();
14+
const [flashTotal, setFlashTotal] = useState();
15+
16+
useEffect (()=> {
17+
if (USBDevice) {
18+
USBDevice.open().then(()=>{
19+
console.log(`Opened USB Device: ${USBDevice.productName}`);
20+
}, (err)=> {
21+
console.log(err)
22+
})
23+
.then(()=> {
24+
if (flashFile)
25+
decompressFileToTransform();
26+
})
27+
}
28+
29+
}, [USBDevice])
30+
31+
const requestUSBDevice = () => { // first thing called with button
32+
navigator.usb
33+
.requestDevice(USBrequestParams)
34+
.then((device) => {
35+
setUSBDevice(device);
36+
})
37+
.catch((e) => {
38+
console.error(`There is no device. ${e}`);
39+
});
40+
}
41+
42+
async function preBoot () {
43+
await USBDevice.claimInterface(0);
44+
45+
await send_data(await str_to_arr("UCmd:setenv fastboot_dev mmc"), "OKAY");
46+
await send_data(await str_to_arr("UCmd:setenv mmcdev ${emmc_dev}"), "OKAY");
47+
await send_data(await str_to_arr("UCmd:mmc dev ${emmc_dev}"), "OKAY");
48+
49+
console.log("preboot complete")
50+
}
51+
52+
const processChunk = async(data, curr_len, i) => {
53+
console.log(data, curr_len, i);
54+
55+
// ignore, fill with zeroes
56+
if (curr_len != data.length) {
57+
let fill_len = (Math.ceil(curr_len/BLK_SZ)*BLK_SZ - curr_len);
58+
// let fill = new Uint8Array(fill_len);
59+
data.set(curr_len, new Uint8Array(fill_len));
60+
data = data.slice(0, curr_len+fill_len);
61+
}
62+
63+
let hex_len = (data.length + 52).toString(16); // 52 comes from headers
64+
await send_data(await str_to_arr(`download:${hex_len}`), `DATA${hex_len}`);
65+
await send_headers(data.length, i);
66+
67+
let offset=0;
68+
while (offset < data.length) {
69+
let packet_len = Math.min(PACKET_SZ, DATA_SZ - offset);
70+
await USBDevice.transferOut(1, data.slice(offset, offset + packet_len));
71+
console.log("transferout", data.slice(offset, offset + packet_len));
72+
offset += packet_len;
73+
}
74+
75+
let result = await USBDevice.transferIn(1, 1048);
76+
if ("OKAY" !== await ab_to_str(result.data.buffer)) {
77+
throw new Error ("failed to send data:", await ab_to_str(result.data.buffer))
78+
}
79+
80+
await flash_all();
81+
console.log("FLASH SUCCESS")
82+
}
83+
84+
async function send_packet(raw_data) {
85+
await USBDevice.transferOut(1, raw_data);
86+
console.log("transferout", raw_data);
87+
}
88+
89+
async function send_chunk_headers (chunk_len, i) {
90+
let hex_len = (chunk_len + 52).toString(16); // 52 comes from headers
91+
await send_data(await str_to_arr(`download:${hex_len}`), `DATA${hex_len}`);
92+
await send_headers(chunk_len, i);
93+
}
94+
95+
async function send_headers(raw_data_bytelength, i) {
96+
let sparse = await build_sparse_header(raw_data_bytelength, i);
97+
let dont_care = await build_chunk_header(CHUNK_TYPE_DONT_CARE, raw_data_bytelength, i);
98+
let raw = await build_chunk_header(CHUNK_TYPE_RAW, raw_data_bytelength, i);
99+
100+
let headers = new Uint8Array(52);
101+
headers.set(sparse, 0);
102+
headers.set(dont_care, 28);
103+
headers.set(raw, 40);
104+
105+
await USBDevice.transferOut(1, headers);
106+
}
107+
108+
async function send_flash () {
109+
let result = await USBDevice.transferIn(1, 1048);
110+
if ("OKAY" !== await ab_to_str(result.data.buffer)) {
111+
throw new Error ("failed to send data:", await ab_to_str(result.data.buffer))
112+
}
113+
114+
await flash_all();
115+
}
116+
117+
/*
118+
* data: string or arraybuffer
119+
* success_str: checks response of usb
120+
* Throws error if USB input does not match success_str
121+
*/
122+
async function send_data(data, success_str) {
123+
await USBDevice.transferOut(1, data);
124+
let result = await USBDevice.transferIn(1, 1048);
125+
126+
if (success_str !== await ab_to_str(result.data.buffer)) {
127+
throw new Error ("failed to send data:",await ab_to_str(result.data.buffer))
128+
}
129+
}
130+
131+
async function flash_all () {
132+
await send_data(await str_to_arr("flash:all"), "OKAY");
133+
console.log("flash");
134+
}
135+
136+
const decompressFileToTransform = async(e) => {
137+
console.log(USBDevice);
138+
139+
if (!flashFile) return;
140+
141+
const file = flashFile;
142+
let buff = await file.arrayBuffer()
143+
let data = new Uint8Array(buff);
144+
let totalBytes = 0;
145+
146+
console.log(data)
147+
148+
if (!window.binding) {
149+
console.log("no binding");
150+
return;
151+
}
152+
153+
const stream = new binding.ZstdDecompressStreamBinding();
154+
const transform = new TransformStream();
155+
const writer = transform.writable.getWriter();
156+
157+
const callback = (decompressed) => {
158+
totalBytes += decompressed.length;
159+
writer.write(decompressed);
160+
}
161+
162+
if (!stream.begin()) {
163+
console.log("stream.begin() error");
164+
return null;
165+
}
166+
167+
console.log("start unzipping");
168+
169+
let i = 0;
170+
const size = 1024*1024*2;
171+
while (size*i < data.length) {
172+
let end = Math.min(size*(i+1), data.length);
173+
let slice = data.slice(size*i, end);
174+
175+
if (!stream.transform(slice, callback)) {
176+
console.log(`stream.transform() error on slice ${size*i} to ${end}`);
177+
return null;
178+
}
179+
180+
i++;
181+
}
182+
183+
if (!stream.end(callback)) {
184+
console.log("stream.end() error");
185+
return null;
186+
}
187+
188+
console.log("finishing unzipping");
189+
190+
const readable = transform.readable;
191+
doFlash(readable, PACKET_SZ, totalBytes);
192+
}
193+
194+
const doFlash = async(readable, size, totalSize) => {
195+
await preBoot();
196+
197+
const reader = readable.getReader();
198+
199+
let offset = 0;
200+
let sofar = new Uint8Array(size);
201+
let i=0;
202+
let totalBytes = 0;
203+
204+
let i_last = Math.floor(totalSize/DATA_SZ);
205+
let last_len = totalSize - Math.floor(totalSize/DATA_SZ)*DATA_SZ;
206+
207+
console.log(i_last, last_len);
208+
209+
let isFirst = true;
210+
let bytesChunk = 0;
211+
212+
reader.read().then(async function processText ({ done, value }) {
213+
// Result objects contain two properties:
214+
// done - true if the stream has already given you all its data.
215+
// value - some data. Always undefined when done is true.
216+
217+
totalBytes += value.length;
218+
console.log(totalBytes);
219+
220+
// Send chunk headers
221+
if (isFirst) {
222+
let send_len;
223+
if (i==i_last) {
224+
send_len = Math.ceil(last_len/BLK_SZ)*BLK_SZ
225+
}
226+
else {
227+
send_len = DATA_SZ;
228+
}
229+
await send_chunk_headers(send_len, i);
230+
console.log("Sent headers", send_len, i)
231+
232+
isFirst = false;
233+
}
234+
235+
while (offset + value.length >= size) {
236+
sofar.set(value.slice(0, size-offset), offset); // Whole packet is ready to send
237+
bytesChunk += size-offset;
238+
await send_packet(sofar);
239+
240+
console.log("bytes", bytesChunk, i);
241+
242+
if (bytesChunk == DATA_SZ) {
243+
await send_flash();
244+
bytesChunk = 0;
245+
i++;
246+
isFirst = true;
247+
}
248+
value = value.slice(size-offset, value.length);
249+
offset = 0;
250+
}
251+
252+
sofar.set(value, offset);
253+
offset += value.length;
254+
bytesChunk += value.length
255+
256+
if (i==i_last && bytesChunk==last_len) {
257+
console.log("reached last send", i, bytesChunk);
258+
259+
// Pad last packet with zeros
260+
let fill_len = Math.ceil(last_len/BLK_SZ)*BLK_SZ - last_len;
261+
sofar.set(new Uint8Array(fill_len), offset);
262+
sofar = sofar.slice(0, offset+fill_len);
263+
264+
console.log(sofar);
265+
266+
await send_packet(sofar);
267+
await send_flash();
268+
}
269+
270+
if (done) {
271+
console.log("stream done")
272+
return;
273+
}
274+
// Read some more, and call this function again
275+
return reader.read().then(processText);
276+
});
277+
}
278+
279+
return [{
280+
requestUSBDevice,
281+
USBDevice,
282+
flashProgress,
283+
flashTotal,
284+
preBoot,
285+
processChunk,
286+
send_chunk_headers,
287+
send_packet,
288+
send_flash,
289+
flash_all,
290+
}]
291+
}
292+
293+
export default useDecompress;

0 commit comments

Comments
 (0)