From cc8e031d6b859994b2996d4d59379d9fa1fa33a3 Mon Sep 17 00:00:00 2001 From: Lucas Winther Date: Fri, 30 Oct 2026 00:45:46 +0100 Subject: [PATCH] Run WASM sessions at the default graph optimization level now that ort-web 1.31 loads the 8-bit decoders Co-Authored-By: Claude Opus 5.5 --- docs/Decisions.md | 43 +++++++++++++++++++++++- docs/WorkBreakdown.md | 2 ++ docs/plans/001-1-transformers-upgrade.md | 4 +-- src/worker/inference.worker.ts | 14 ++++---- tests/offline.md | 5 +++ tests/transformers-runtime.test.ts | 13 +++++++ 6 files changed, 70 insertions(+), 11 deletions(-) diff --git a/docs/Decisions.md b/docs/Decisions.md index eab0ebb..a124be5 100644 --- a/docs/Decisions.md +++ b/docs/Decisions.md @@ -125,7 +125,8 @@ than ONNX Runtime under Bun; the server engine becomes genuinely useful rather t slow convenience. ## D-014 — WASM sessions run at graphOptimizationLevel "basic" -**Status:** Accepted (bug workaround, 2026-07-10) +**Status:** **Superseded by D-WS1c** (bug workaround, 2026-07-10; the ort-web +build in Transformers.js 4.3.2 no longer has the bug, so the cap is gone) On the WASM backend, the inference worker passes `session_options: { graphOptimizationLevel: "basic" }` to the Transformers.js pipeline. The onnxruntime-web build bundled with Transformers.js 4.2 @@ -1319,3 +1320,43 @@ share the CPU), so read it only as "the seek loop is slower", not as a version-to-version speed comparison. **Decision:** keep `max_new_tokens: 440` in `src/worker/inference.worker.ts` and in `tests/live-real-model.test.ts`. + +## D-WS1c — WASM sessions back at the default graph optimization level (D-014's "basic" cap dropped) +**Status:** Accepted (2026-10-30) — supersedes D-014 +**Re-test** of D-014's revisit condition after the Transformers.js 4.3.2 bump +(D-WS1a), which brings onnxruntime-web `1.31.0-dev.20260914-8d85527a0`. +Measured under Bun with onnxruntime-web's own wasm EP (not onnxruntime-node), +1 thread, AMD Ryzen 9 9950X shared with other workstreams. +1. **Session creation**, `onnx-community/whisper-tiny` + `decoder_model_merged_quantized.onnx` (D-014's repro file), every level: + + | onnxruntime-web | disabled | basic | extended | all | + |---|---|---|---|---| + | 1.26.0-dev.20260416 (control) | OK | OK | FAIL | FAIL | + | 1.31.0-dev.20260914 | OK | OK | OK | OK | + + The control still fails with D-014's exact error (`qdq_actions.cc:137 + TransposeDQWeightsForMatMulNBits Missing required scale: + model.decoder.embed_tokens.weight_merged_0_scale`); 1.31 loads at all four. +2. **Transcription**, Transformers.js 4.3.2 pipeline on that ort-web build, + `whisper-tiny.en` q8 (catalog WASM default), `en_aiproblem` (14.4 s), + worker options (30 s chunks, 5 s stride, timestamps, `max_new_tokens: + 440`). Both pipelines in one process, 1 warm-up each, then 8 interleaved + runs (A/B order alternating) so machine-load drift hits both equally: + + | level | warm runs (s) | median | WER | text | + |---|---|---|---|---| + | `basic` (D-014) | 1.49 1.44 1.36 1.39 1.41 1.47 1.48 1.60 | 1.47 s | 0.000 | identical | + | default (`all`) | 1.38 1.47 1.34 1.39 1.37 1.42 1.43 1.43 | 1.42 s | 0.000 | identical | + + Same transcript byte-for-byte at both levels and across every run; the + default level is ~3 % faster here (within noise; certainly not slower). +**Decision:** `src/worker/inference.worker.ts` no longer passes +`session_options` — WASM and WebGPU both use onnxruntime's default level. +`tests/transformers-runtime.test.ts` pins the onnxruntime-web version this +was measured on and that the worker sets no `graphOptimizationLevel`; a +different ort-web build fails the fast suite until this session-creation +check is re-run. +**Not covered here:** a real browser's ort-web wasm build (the same +`ort-wasm-simd-threaded.*` files, but multithreaded under COOP/COEP) — +`tests/offline.md` §2's WASM step is the owner's browser check. diff --git a/docs/WorkBreakdown.md b/docs/WorkBreakdown.md index d69541b..cb1963f 100644 --- a/docs/WorkBreakdown.md +++ b/docs/WorkBreakdown.md @@ -342,6 +342,8 @@ cached. ## M8 — Transcription quality and speed (spec 001) - [x] Transformers.js 4.2.0 → 4.3.2, sequential-session patch regenerated for 4.3.2 and guarded by `tests/transformers-runtime.test.ts` (D-WS1a). +- [x] WASM sessions back at the default graph optimization level: ort-web + 1.31.0-dev loads the q8 decoders at every level (D-WS1c supersedes D-014). --- diff --git a/docs/plans/001-1-transformers-upgrade.md b/docs/plans/001-1-transformers-upgrade.md index 0c623bf..998bbf9 100644 --- a/docs/plans/001-1-transformers-upgrade.md +++ b/docs/plans/001-1-transformers-upgrade.md @@ -98,12 +98,12 @@ onnxruntime-node 1.30.0), bun:test. **Files:** `docs/Decisions.md`; `src/worker/inference.worker.ts` only if the outcome is "drop". -- [ ] **Step 1: Measure** (`/tmp/ws1/d014.ts`): `globalThis[Symbol.for("onnxruntime")] = +- [x] **Step 1: Measure** (`/tmp/ws1/d014.ts`): `globalThis[Symbol.for("onnxruntime")] = await import("onnxruntime-web")` before importing transformers, wasm EP, 1 thread; for `onnx-community/whisper-tiny` `decoder_model_merged_quantized.onnx` create a session at `disabled/basic/extended/all`; record success/error. If all load: transcribe `en_aiproblem` with tiny.en q8 at `basic` vs default (`all`), 3 warm runs each, wall-clock + text identity. -- [ ] **Step 2:** If default fails → keep, `D-WS1c` "still needed on ort-web 1.31.0-dev". If it +- [x] **Step 2:** If default fails → keep, `D-WS1c` "still needed on ort-web 1.31.0-dev". If it loads and is not slower → remove the override (worker comment cites D-WS1c), run a q8 transcribe again through the harness with the worker's new options. Commit. diff --git a/src/worker/inference.worker.ts b/src/worker/inference.worker.ts index bd3e656..79c2492 100644 --- a/src/worker/inference.worker.ts +++ b/src/worker/inference.worker.ts @@ -260,14 +260,12 @@ async function createPipeline( const pipelineOptions: PretrainedModelOptions & { sequentialSessionLoad?: boolean } = { device, dtype, - // WASM sessions cap graph optimization at "basic" (D-014): the level-2 - // QDQ transform in the bundled onnxruntime-web build rejects the 8-bit - // decoders' tied embed_tokens weight ("Missing required scale … - // DequantizeLinear"). q4 models use native MatMulNBits ops, so the - // lower level costs them nothing. WebGPU keeps the default level. - ...(device === "wasm" && { - session_options: { graphOptimizationLevel: "basic" as const }, - }), + // No session_options: every backend runs at onnxruntime's default + // graph optimization level. D-014's WASM "basic" cap worked around a + // level-2 QDQ bug in ort-web 1.26.0-dev that rejected the 8-bit + // decoders; the 1.31.0-dev build bundled with Transformers.js 4.3.2 + // loads them at every level (D-WS1c, pinned in + // tests/transformers-runtime.test.ts). progress_callback: forwardProgress, // D-027/D-029: sequential sub-model session loading trades load speed // for a lower peak-memory footprint — only worth it where the UI diff --git a/tests/offline.md b/tests/offline.md index c1f0912..1206587 100644 --- a/tests/offline.md +++ b/tests/offline.md @@ -69,6 +69,11 @@ The worker now caps WASM sessions at `graphOptimizationLevel: "basic"` outside the browser. **Re-run this step with Backend = WASM to confirm in a real browser.** +**Update (2026-10-30, D-WS1c):** the Transformers.js 4.3.2 bump brings an +onnxruntime-web build without that bug, so the cap is gone and WASM sessions +run at the default level again. Re-run this step with Backend = WASM (Whisper +Tiny and Tiny.en) to confirm the session still loads in a real browser. + ## 3. Go offline and transcribe 1. DevTools → Network → throttling **Offline** (or OS airplane mode — stronger, diff --git a/tests/transformers-runtime.test.ts b/tests/transformers-runtime.test.ts index b790df7..06fac6c 100644 --- a/tests/transformers-runtime.test.ts +++ b/tests/transformers-runtime.test.ts @@ -54,4 +54,17 @@ describe("@huggingface/transformers runtime", () => { const source = read(join(PKG_DIR, "src/models/whisper/modeling_whisper.js")); expect(source).toContain("if (generation_config.return_timestamps && !kwargs.max_new_tokens)"); }); + + test("onnxruntime-web is the build D-WS1c re-tested 8-bit decoders on at the default optimization level", () => { + // D-014's "basic" cap was dropped because this build loads the q8 merged + // decoders at every graphOptimizationLevel. A different build needs the + // D-WS1c session-creation check re-run before it ships. + const ortWeb = JSON.parse(read(join(ROOT, "node_modules/onnxruntime-web/package.json"))).version; + expect(ortWeb).toBe("1.31.0-dev.20260914-8d85527a0"); + }); + + test("worker leaves WASM sessions at the default graph optimization level (D-WS1c)", () => { + const worker = read(join(ROOT, "src/worker/inference.worker.ts")); + expect(worker).not.toContain("graphOptimizationLevel"); + }); });