From a5cfd219f646f9c6eed5c6b287ffb0ff70a76f27 Mon Sep 17 00:00:00 2001 From: Lucas Winther Date: Wed, 15 Jul 2026 14:45:40 +0200 Subject: [PATCH] Implement speedups for transcription. --- docs/Decisions.md | 40 ++++++++++++++++++++ src/index.ts | 68 ++++++++++++++++++++++++++-------- src/worker/inference.worker.ts | 26 +++++++++++++ 3 files changed, 118 insertions(+), 16 deletions(-) diff --git a/docs/Decisions.md b/docs/Decisions.md index 6d61c24..21100e2 100644 --- a/docs/Decisions.md +++ b/docs/Decisions.md @@ -356,3 +356,43 @@ configuration). If a one-word/garbage transcript ever shows up on those, suspect their fp16 encoders and extend this decision. **Revisit:** if the onnx-community medium conversions are re-exported with working fp16 files, or once WebNN/WebGPU runs can be exercised in CI. + +## D-023 — Cross-origin isolation (COOP/COEP) so WASM inference is multithreaded +**Status:** Accepted (2026-07-15) +Reported symptom: after D-022 moved medium to q4, a 14 s clip transcribed +correctly in-browser but "took ages" while native whisper needs seconds. +Cause: without `crossOriginIsolated` there is no SharedArrayBuffer, and +onnxruntime-web silently runs its (SIMD-threaded build!) WASM backend on +**one** thread. Measured on the exact browser runtime (ort-web wasm EP, +medium q4, the 14.4 s en_aiproblem clip): 52.1 s at 1 thread, 15.3 s at 4 +(ort's default cap once isolation exists), 9.5 s at 12 — a 5.5× speedup. +**Fix:** `src/index.ts` sends `Cross-Origin-Opener-Policy: same-origin` + +`Cross-Origin-Embedder-Policy: require-corp` on every response in both serve +modes (`withIsolationHeaders`). Bun cannot attach headers to an HTMLBundle +route, so in development the shell moved to an internal `/__shell` route and +`fetch()` re-serves `/` (and SPA fallbacks) with the headers stamped on. The +inference worker raises `env.backends.onnx.wasm.numThreads` to +`min(hardwareConcurrency − 1, 12)` when `crossOriginIsolated` is true (ort +would otherwise cap at 4; without isolation the default is left alone since +ort clamps to 1 anyway). +**Consequences:** COEP `require-corp` blocks cross-origin subresources that +are neither CORS- nor CORP-approved. Everything the app loads today survives: +Hub model downloads use `fetch()` in CORS mode, all other assets are +same-origin, and the service worker preserves the headers it cached, so the +installed/offline app stays isolated too. Any future cross-origin ``, +font, or iframe must ship CORS/CORP headers or be proxied same-origin. +**Verified:** headers present on `/`, worker, sw.js, `/ort/*` in both dev and +production serve modes; dev HMR asset routes unaffected by the `/__shell` +indirection. Real-browser thread scaling still deserves a spot check (the +5.5× number is ort-web under Bun, same runtime but not a browser tab). +**Amendment (2026-07-15, same day):** threads are per-device now. With +SharedArrayBuffer newly available, onnxruntime-web's *own* default thread +count becomes ~4 — and the JSEP accelerator EPs (WebGPU, WebNN) are not +thread-safe: real-browser WebGPU runs failed at OrtRun with "WebGPU +validation failed. Buffer with '' label is invalid". The worker now sets +`numThreads` right before each device attempt (`configureWasmThreads`): +`min(cores − 1, 12)` for the pure `wasm` device, **explicitly 1** for +`webgpu`/`webnn` (relying on the default is what broke). onnxruntime-web +fixes its thread pool at first-session init, so the first attempt in a +worker decides; a wasm fallback after a failed accelerator attempt may run +single-threaded until the next worker respawn — correct over fast. diff --git a/src/index.ts b/src/index.ts index 27d5484..da09cf2 100644 --- a/src/index.ts +++ b/src/index.ts @@ -16,6 +16,36 @@ const IS_PRODUCTION = process.env.NODE_ENV === "production"; const NOT_FOUND = () => new Response("Not Found", { status: 404 }); +/** + * Cross-origin isolation (D-023): COOP + COEP make `crossOriginIsolated` + * true in the browser, which unlocks SharedArrayBuffer — without it, + * onnxruntime-web silently runs its WASM backend on a SINGLE thread and + * browser inference is several times slower than it should be. Applied to + * every response: only documents/workers consume them, and they are harmless + * on assets and API responses. Cross-origin model downloads keep working — + * the Hugging Face Hub is fetched with CORS, which `require-corp` allows. + */ +const ISOLATION_HEADERS: Record = { + "Cross-Origin-Opener-Policy": "same-origin", + "Cross-Origin-Embedder-Policy": "require-corp", +}; + +function withIsolationHeaders(response: Response): Response { + try { + for (const [name, value] of Object.entries(ISOLATION_HEADERS)) { + response.headers.set(name, value); + } + return response; + } catch { + // Responses produced by fetch() have immutable headers — rebuild. + const copy = new Response(response.body, response); + for (const [name, value] of Object.entries(ISOLATION_HEADERS)) { + copy.headers.set(name, value); + } + return copy; + } +} + // onnxruntime-web fetches its .wasm binaries at runtime — from the jsdelivr // CDN by default, which would break offline. The worker points // `env.backends.onnx.wasm.wasmPaths` at /ort/ instead (see @@ -138,15 +168,15 @@ async function productionServeOptions(): Promise { // API routes are dynamic (M4/M5) and network-only in the SW (D-007). if (url.pathname.includes("/api/")) { const response = await handleApiRequest(request, server); - if (response) return response; + if (response) return withIsolationHeaders(response); return new Response("Upgrading...", { status: 101 }); // actually Bun handles the real response, but TypeScript requires a return } let relative = url.pathname === "/" ? "index.html" : decodeURIComponent(url.pathname.slice(1)); if (relative.endsWith("/")) relative += "index.html"; - + let path = normalize(join(dist, relative)); - + // Auto-strip proxy subpaths for static assets while (relative.includes("/") && !(await Bun.file(path).exists())) { relative = relative.substring(relative.indexOf("/") + 1); @@ -163,9 +193,9 @@ async function productionServeOptions(): Promise { ? "public, max-age=31536000, immutable" : "no-cache", ); - if (response.status !== 404) return response; + if (response.status !== 404) return withIsolationHeaders(response); // SPA fallback: serve the app shell for unknown navigations. - return serveFile(join(dist, "index.html"), "no-cache"); + return withIsolationHeaders(await serveFile(join(dist, "index.html"), "no-cache")); }, }; } @@ -215,7 +245,10 @@ async function developmentServeOptions(): Promise { routes: { // The HTML import is an HTMLBundle, which Bun only serves as a route // value — returning it from fetch() throws "Expected a Response object". - "/": index, + // It is mounted on an internal path so "/" can flow through fetch() + // below, which re-serves it with the COOP/COEP isolation headers + // (D-023) that Bun cannot attach to an HTMLBundle route directly. + "/__shell": index, }, // Fallback for everything the route table doesn't match, including @@ -226,34 +259,37 @@ async function developmentServeOptions(): Promise { if (pathname.includes("/api/")) { const response = await handleApiRequest(request, server); - if (response) return response; + if (response) return withIsolationHeaders(response); return new Response("Upgrading...", { status: 101 }); } if (pathname.endsWith("/inference.worker.js")) { - return new Response(workerScript, { headers: scriptHeaders }); + return withIsolationHeaders(new Response(workerScript, { headers: scriptHeaders })); } if (pathname.endsWith("/sw.js")) { - return new Response(swScript, { headers: { ...scriptHeaders, "Cache-Control": "no-cache" } }); + return withIsolationHeaders( + new Response(swScript, { headers: { ...scriptHeaders, "Cache-Control": "no-cache" } }), + ); } if (pathname.endsWith("/manifest.webmanifest")) { - return serveFile(join(import.meta.dir, "pwa/manifest.webmanifest")); + return withIsolationHeaders(await serveFile(join(import.meta.dir, "pwa/manifest.webmanifest"))); } const iconsMatch = pathname.match(/\/icons\/(icon-\d+\.png)$/); if (iconsMatch?.[1]) { - return serveFile(join(ICONS_DIR, iconsMatch[1])); + return withIsolationHeaders(await serveFile(join(ICONS_DIR, iconsMatch[1]))); } const ortMatch = pathname.match(/\/ort\/(.*(?:wasm|mjs))$/); if (ortMatch?.[1]) { - return serveOrtFile(ORT_DIST, ortMatch[1]); + return withIsolationHeaders(await serveOrtFile(ORT_DIST, ortMatch[1])); } - // SPA fallback: unknown navigations (including subpath ones) get the - // app shell. The HTMLBundle itself can only live in `routes`, so - // re-enter the server at "/" to let it produce the dev-bundled HTML. - return fetch(new URL("/", server.url)); + // The app shell ("/" and SPA fallback for unknown navigations, + // including subpath ones). The HTMLBundle itself can only live in + // `routes`, so re-enter the server at "/__shell" to let it produce the + // dev-bundled HTML, then stamp the isolation headers on the document. + return withIsolationHeaders(await fetch(new URL("/__shell", server.url))); }, development: { diff --git a/src/worker/inference.worker.ts b/src/worker/inference.worker.ts index b38baf5..9b95916 100644 --- a/src/worker/inference.worker.ts +++ b/src/worker/inference.worker.ts @@ -38,6 +38,31 @@ if (env.backends.onnx.wasm) { env.backends.onnx.wasm.wasmPaths = "./ort/"; } +/** + * Set the WASM thread count for the device we are about to create a pipeline + * on (D-023). The server sends COOP/COEP, so SharedArrayBuffer exists and + * threads genuinely run — which cuts both ways: + * - Pure WASM inference scales well past onnxruntime-web's 4-thread default, + * so use most of the machine (measured 52 s → 9.5 s on a 14 s clip). + * - The JSEP-based accelerator EPs (WebGPU, WebNN) are NOT thread-safe: + * with numThreads > 1 (even ort's own default of ~4 once SAB exists), + * inference fails with "WebGPU validation failed. Buffer with '' label is + * invalid". They must run the wasm side single-threaded. + * Caveat: onnxruntime-web initializes its runtime on the first session + * creation and ignores later numThreads changes, so the first attempt in + * this worker decides. A wasm *fallback* after a failed accelerator attempt + * therefore may run single-threaded — correct, just slower; the next worker + * respawn (cancel does one, D-018) picks the full count up again. + */ +function configureWasmThreads(device: InferenceDevice): void { + const wasm = env.backends.onnx.wasm; + if (!wasm) return; + wasm.numThreads = + device === "wasm" && globalThis.crossOriginIsolated + ? Math.max(1, Math.min((navigator.hardwareConcurrency || 4) - 1, 12)) + : 1; +} + // `lib: ["DOM"]` types `self` as Window; give it the worker-scope shape we use. interface DedicatedWorkerScope { postMessage(message: WorkerToUiMessage): void; @@ -153,6 +178,7 @@ async function createPipeline( let lastError: unknown; for (const device of attempts) { + configureWasmThreads(device); const dtype = message.dtype ?? resolveDefaultDtype(message.modelId, device); try { const asr = await pipeline("automatic-speech-recognition", message.modelId, {