Implement speedups for transcription.

docs/Decisions.md

@@ -356,3 +356,43 @@ configuration). If a one-word/garbage transcript ever shows up on those, suspect
 their fp16 encoders and extend this decision.
 **Revisit:** if the onnx-community medium conversions are re-exported with
 working fp16 files, or once WebNN/WebGPU runs can be exercised in CI.
+
+## D-023 — Cross-origin isolation (COOP/COEP) so WASM inference is multithreaded
+**Status:** Accepted (2026-07-15)
+Reported symptom: after D-022 moved medium to q4, a 14 s clip transcribed
+correctly in-browser but "took ages" while native whisper needs seconds.
+Cause: without `crossOriginIsolated` there is no SharedArrayBuffer, and
+onnxruntime-web silently runs its (SIMD-threaded build!) WASM backend on
+**one** thread. Measured on the exact browser runtime (ort-web wasm EP,
+medium q4, the 14.4 s en_aiproblem clip): 52.1 s at 1 thread, 15.3 s at 4
+(ort's default cap once isolation exists), 9.5 s at 12 — a 5.5× speedup.
+**Fix:** `src/index.ts` sends `Cross-Origin-Opener-Policy: same-origin` +
+`Cross-Origin-Embedder-Policy: require-corp` on every response in both serve
+modes (`withIsolationHeaders`). Bun cannot attach headers to an HTMLBundle
+route, so in development the shell moved to an internal `/__shell` route and
+`fetch()` re-serves `/` (and SPA fallbacks) with the headers stamped on. The
+inference worker raises `env.backends.onnx.wasm.numThreads` to
+`min(hardwareConcurrency − 1, 12)` when `crossOriginIsolated` is true (ort
+would otherwise cap at 4; without isolation the default is left alone since
+ort clamps to 1 anyway).
+**Consequences:** COEP `require-corp` blocks cross-origin subresources that
+are neither CORS- nor CORP-approved. Everything the app loads today survives:
+Hub model downloads use `fetch()` in CORS mode, all other assets are
+same-origin, and the service worker preserves the headers it cached, so the
+installed/offline app stays isolated too. Any future cross-origin `<img>`,
+font, or iframe must ship CORS/CORP headers or be proxied same-origin.
+**Verified:** headers present on `/`, worker, sw.js, `/ort/*` in both dev and
+production serve modes; dev HMR asset routes unaffected by the `/__shell`
+indirection. Real-browser thread scaling still deserves a spot check (the
+5.5× number is ort-web under Bun, same runtime but not a browser tab).
+**Amendment (2026-07-15, same day):** threads are per-device now. With
+SharedArrayBuffer newly available, onnxruntime-web's *own* default thread
+count becomes ~4 — and the JSEP accelerator EPs (WebGPU, WebNN) are not
+thread-safe: real-browser WebGPU runs failed at OrtRun with "WebGPU
+validation failed. Buffer with '' label is invalid". The worker now sets
+`numThreads` right before each device attempt (`configureWasmThreads`):
+`min(cores − 1, 12)` for the pure `wasm` device, **explicitly 1** for
+`webgpu`/`webnn` (relying on the default is what broke). onnxruntime-web
+fixes its thread pool at first-session init, so the first attempt in a
+worker decides; a wasm fallback after a failed accelerator attempt may run
+single-threaded until the next worker respawn — correct over fast.

src/index.ts

@@ -16,6 +16,36 @@ const IS_PRODUCTION = process.env.NODE_ENV === "production";
 
 const NOT_FOUND = () => new Response("Not Found", { status: 404 });
 
+/**
+ * Cross-origin isolation (D-023): COOP + COEP make `crossOriginIsolated`
+ * true in the browser, which unlocks SharedArrayBuffer — without it,
+ * onnxruntime-web silently runs its WASM backend on a SINGLE thread and
+ * browser inference is several times slower than it should be. Applied to
+ * every response: only documents/workers consume them, and they are harmless
+ * on assets and API responses. Cross-origin model downloads keep working —
+ * the Hugging Face Hub is fetched with CORS, which `require-corp` allows.
+ */
+const ISOLATION_HEADERS: Record<string, string> = {
+  "Cross-Origin-Opener-Policy": "same-origin",
+  "Cross-Origin-Embedder-Policy": "require-corp",
+};
+
+function withIsolationHeaders(response: Response): Response {
+  try {
+    for (const [name, value] of Object.entries(ISOLATION_HEADERS)) {
+      response.headers.set(name, value);
+    }
+    return response;
+  } catch {
+    // Responses produced by fetch() have immutable headers — rebuild.
+    const copy = new Response(response.body, response);
+    for (const [name, value] of Object.entries(ISOLATION_HEADERS)) {
+      copy.headers.set(name, value);
+    }
+    return copy;
+  }
+}
+
 // onnxruntime-web fetches its .wasm binaries at runtime — from the jsdelivr
 // CDN by default, which would break offline. The worker points
 // `env.backends.onnx.wasm.wasmPaths` at /ort/ instead (see
@@ -138,15 +168,15 @@ async function productionServeOptions(): Promise<ServeOptions> {
       // API routes are dynamic (M4/M5) and network-only in the SW (D-007).
       if (url.pathname.includes("/api/")) {
         const response = await handleApiRequest(request, server);
-        if (response) return response;
+        if (response) return withIsolationHeaders(response);
         return new Response("Upgrading...", { status: 101 }); // actually Bun handles the real response, but TypeScript requires a return
       }
 
       let relative = url.pathname === "/" ? "index.html" : decodeURIComponent(url.pathname.slice(1));
       if (relative.endsWith("/")) relative += "index.html";
-      
+
       let path = normalize(join(dist, relative));
-      
+
       // Auto-strip proxy subpaths for static assets
       while (relative.includes("/") && !(await Bun.file(path).exists())) {
         relative = relative.substring(relative.indexOf("/") + 1);
@@ -163,9 +193,9 @@ async function productionServeOptions(): Promise<ServeOptions> {
           ? "public, max-age=31536000, immutable"
           : "no-cache",
       );
-      if (response.status !== 404) return response;
+      if (response.status !== 404) return withIsolationHeaders(response);
       // SPA fallback: serve the app shell for unknown navigations.
-      return serveFile(join(dist, "index.html"), "no-cache");
+      return withIsolationHeaders(await serveFile(join(dist, "index.html"), "no-cache"));
     },
   };
 }
@@ -215,7 +245,10 @@ async function developmentServeOptions(): Promise<ServeOptions> {
     routes: {
       // The HTML import is an HTMLBundle, which Bun only serves as a route
       // value — returning it from fetch() throws "Expected a Response object".
-      "/": index,
+      // It is mounted on an internal path so "/" can flow through fetch()
+      // below, which re-serves it with the COOP/COEP isolation headers
+      // (D-023) that Bun cannot attach to an HTMLBundle route directly.
+      "/__shell": index,
     },
 
     // Fallback for everything the route table doesn't match, including
@@ -226,34 +259,37 @@ async function developmentServeOptions(): Promise<ServeOptions> {
 
       if (pathname.includes("/api/")) {
         const response = await handleApiRequest(request, server);
-        if (response) return response;
+        if (response) return withIsolationHeaders(response);
         return new Response("Upgrading...", { status: 101 });
       }
 
       if (pathname.endsWith("/inference.worker.js")) {
-        return new Response(workerScript, { headers: scriptHeaders });
+        return withIsolationHeaders(new Response(workerScript, { headers: scriptHeaders }));
       }
       if (pathname.endsWith("/sw.js")) {
-        return new Response(swScript, { headers: { ...scriptHeaders, "Cache-Control": "no-cache" } });
+        return withIsolationHeaders(
+          new Response(swScript, { headers: { ...scriptHeaders, "Cache-Control": "no-cache" } }),
+        );
       }
       if (pathname.endsWith("/manifest.webmanifest")) {
-        return serveFile(join(import.meta.dir, "pwa/manifest.webmanifest"));
+        return withIsolationHeaders(await serveFile(join(import.meta.dir, "pwa/manifest.webmanifest")));
       }
 
       const iconsMatch = pathname.match(/\/icons\/(icon-\d+\.png)$/);
       if (iconsMatch?.[1]) {
-        return serveFile(join(ICONS_DIR, iconsMatch[1]));
+        return withIsolationHeaders(await serveFile(join(ICONS_DIR, iconsMatch[1])));
       }
 
       const ortMatch = pathname.match(/\/ort\/(.*(?:wasm|mjs))$/);
       if (ortMatch?.[1]) {
-        return serveOrtFile(ORT_DIST, ortMatch[1]);
+        return withIsolationHeaders(await serveOrtFile(ORT_DIST, ortMatch[1]));
       }
 
-      // SPA fallback: unknown navigations (including subpath ones) get the
+      // The app shell ("/" and SPA fallback for unknown navigations,
-      // app shell. The HTMLBundle itself can only live in `routes`, so
+      // including subpath ones). The HTMLBundle itself can only live in
-      // re-enter the server at "/" to let it produce the dev-bundled HTML.
+      // `routes`, so re-enter the server at "/__shell" to let it produce the
-      return fetch(new URL("/", server.url));
+      // dev-bundled HTML, then stamp the isolation headers on the document.
+      return withIsolationHeaders(await fetch(new URL("/__shell", server.url)));
     },
 
     development: {

src/worker/inference.worker.ts

@@ -38,6 +38,31 @@ if (env.backends.onnx.wasm) {
   env.backends.onnx.wasm.wasmPaths = "./ort/";
 }
 
+/**
+ * Set the WASM thread count for the device we are about to create a pipeline
+ * on (D-023). The server sends COOP/COEP, so SharedArrayBuffer exists and
+ * threads genuinely run — which cuts both ways:
+ * - Pure WASM inference scales well past onnxruntime-web's 4-thread default,
+ *   so use most of the machine (measured 52 s → 9.5 s on a 14 s clip).
+ * - The JSEP-based accelerator EPs (WebGPU, WebNN) are NOT thread-safe:
+ *   with numThreads > 1 (even ort's own default of ~4 once SAB exists),
+ *   inference fails with "WebGPU validation failed. Buffer with '' label is
+ *   invalid". They must run the wasm side single-threaded.
+ * Caveat: onnxruntime-web initializes its runtime on the first session
+ * creation and ignores later numThreads changes, so the first attempt in
+ * this worker decides. A wasm *fallback* after a failed accelerator attempt
+ * therefore may run single-threaded — correct, just slower; the next worker
+ * respawn (cancel does one, D-018) picks the full count up again.
+ */
+function configureWasmThreads(device: InferenceDevice): void {
+  const wasm = env.backends.onnx.wasm;
+  if (!wasm) return;
+  wasm.numThreads =
+    device === "wasm" && globalThis.crossOriginIsolated
+      ? Math.max(1, Math.min((navigator.hardwareConcurrency || 4) - 1, 12))
+      : 1;
+}
+
 // `lib: ["DOM"]` types `self` as Window; give it the worker-scope shape we use.
 interface DedicatedWorkerScope {
   postMessage(message: WorkerToUiMessage): void;
@@ -153,6 +178,7 @@ async function createPipeline(
 
   let lastError: unknown;
   for (const device of attempts) {
+    configureWasmThreads(device);
     const dtype = message.dtype ?? resolveDefaultDtype(message.modelId, device);
     try {
       const asr = await pipeline("automatic-speech-recognition", message.modelId, {