From 9f1d58c0b7da036bcc3606a5183bbaaac42e4707 Mon Sep 17 00:00:00 2001 From: Lucas Winther Date: Thu, 29 Oct 2026 22:59:35 +0100 Subject: [PATCH] Keep max_new_tokens: 440 on Transformers.js 4.3.2 and pin the seek-loop condition it avoids Co-Authored-By: Claude Opus 5.5 --- docs/Decisions.md | 27 ++++++++++++++++++++++++ docs/plans/001-1-transformers-upgrade.md | 6 +++--- src/worker/inference.worker.ts | 7 +++--- tests/transformers-runtime.test.ts | 7 ++++++ 4 files changed, 41 insertions(+), 6 deletions(-) diff --git a/docs/Decisions.md b/docs/Decisions.md index 5aa8acb..eab0ebb 100644 --- a/docs/Decisions.md +++ b/docs/Decisions.md @@ -1292,3 +1292,30 @@ cannot be re-measured here. What the upgrade changes on that side, from the "WASM (on-device, N threads)" with N > 1). 5. **Safari 26 (macOS/iOS):** what the footer reports for "auto"; if WebGPU, check speed and output. + +## D-WS1b — D-021's `max_new_tokens: 440` is still needed on Transformers.js 4.3.2 +**Status:** Accepted (2026-10-29) — re-test of D-021 part 3, which stands unchanged +**Source:** 4.3.2's `WhisperForConditionalGeneration.generate()` still takes +the seek loop on exactly D-021's condition, +`generation_config.return_timestamps && !kwargs.max_new_tokens` +(`src/models/whisper/modeling_whisper.js`); `_generate_with_seek` is +unchanged. `tests/transformers-runtime.test.ts` now pins that line, so a +future bump that changes it fails the fast suite and prompts a re-measure. +**Measured** (tiny.en q8 — the catalog WASM default — on onnxruntime-node, +CPU, 16-core shared machine, same harness for both versions; pipeline options +as the worker: 30 s chunks, 5 s stride, `return_timestamps`; 888 s +`en_alien.opus`, 44 windows): + +| Transformers.js | `max_new_tokens` | generate passes | wall-clock | WER | +|---|---|---|---|---| +| 4.2.0 | 440 | 44 | 54.7 s | 0.315 | +| 4.2.0 | unset (seek loop) | 83 | 66.6 s | 0.447 | +| 4.3.2 | 440 | 44 | 55.5 s | 0.315 | +| 4.3.2 | unset (seek loop) | 84 | 119.7 s | 0.449 | + +Pass counts and WER are deterministic and reproduce D-021 (80 passes, +0.447 vs 0.315); wall-clock is noisy here (other workstreams' light jobs +share the CPU), so read it only as "the seek loop is slower", not as a +version-to-version speed comparison. +**Decision:** keep `max_new_tokens: 440` in `src/worker/inference.worker.ts` +and in `tests/live-real-model.test.ts`. diff --git a/docs/plans/001-1-transformers-upgrade.md b/docs/plans/001-1-transformers-upgrade.md index 25d5424..0c623bf 100644 --- a/docs/plans/001-1-transformers-upgrade.md +++ b/docs/plans/001-1-transformers-upgrade.md @@ -82,16 +82,16 @@ onnxruntime-node 1.30.0), bun:test. **Files:** Modify `tests/transformers-runtime.test.ts`, `docs/Decisions.md`, `src/worker/inference.worker.ts` (comment only: "4.2" → "4.2–4.3", cite D-WS1b). -- [ ] **Step 1: Test** `"Whisper generate still takes the seek loop when max_new_tokens is unset (D-021)"` +- [x] **Step 1: Test** `"Whisper generate still takes the seek loop when max_new_tokens is unset (D-021)"` — `src/models/whisper/modeling_whisper.js` contains `if (generation_config.return_timestamps && !kwargs.max_new_tokens)`. Run → PASS expected (tripwire; it encodes the revisit condition, verified against the 4.3.2 source). -- [ ] **Step 2: Measure** (`/tmp/ws1/d021.ts`, onnxruntime-node, under the lock): tiny.en q8 on +- [x] **Step 2: Measure** (`/tmp/ws1/d021.ts`, onnxruntime-node, under the lock): tiny.en q8 on `en_alien.opus` (ffmpeg-static → 16 kHz f32), pipeline options as the worker (`chunk_length_s 30, stride_length_s 5, return_timestamps true`), with and without `max_new_tokens: 440`; record wall-clock, generate-pass count (wrap `model.generate`), window count, WER vs `en_alien.txt` (`tests/dsp.ts` `wordErrorRate`). -- [ ] **Step 3:** `D-WS1b` with the numbers; keep `max_new_tokens: 440` unless the seek path is now +- [x] **Step 3:** `D-WS1b` with the numbers; keep `max_new_tokens: 440` unless the seek path is now both faster and not worse on WER. Commit. ### Task 3: D-014 re-test (WASM graphOptimizationLevel) diff --git a/src/worker/inference.worker.ts b/src/worker/inference.worker.ts index 2d5b4b2..bd3e656 100644 --- a/src/worker/inference.worker.ts +++ b/src/worker/inference.worker.ts @@ -348,9 +348,10 @@ async function transcribe(message: TranscribeMessage): Promise { stride_length_s: STRIDE_LENGTH_S, return_timestamps: true, // With return_timestamps and no explicit max_new_tokens, Transformers.js - // 4.2's Whisper generate() takes a seek-loop path that re-decodes each - // window's tail (1–3 generate passes per window). Under this pipeline's - // own 30 s/5 s chunking that is pure overhead: ~1.4× slower on long + // 4.2–4.3 Whisper generate() takes a seek-loop path (D-021; re-measured + // on 4.3.2, D-WS1b) that re-decodes each window's tail (1–3 generate + // passes per window). Under this pipeline's own 30 s/5 s chunking that + // is pure overhead: ~1.4× slower on long // files, duplicated boundary text, on_finalize firing once per pass // (breaking TranscribeProgressTracker's one-finalize-per-window // assumption — the progress bar saturated early, then looked frozen), diff --git a/tests/transformers-runtime.test.ts b/tests/transformers-runtime.test.ts index 73babd5..b790df7 100644 --- a/tests/transformers-runtime.test.ts +++ b/tests/transformers-runtime.test.ts @@ -47,4 +47,11 @@ describe("@huggingface/transformers runtime", () => { expect(body).toContain("Promise.all("); expect(body).toContain("for (const name of Object.keys(names))"); }); + + test("Whisper generate still takes the seek loop when max_new_tokens is unset (D-021, D-WS1b)", () => { + // The worker's `max_new_tokens: 440` exists only to dodge this branch. + // If upstream changes the condition, re-measure D-021 before touching it. + const source = read(join(PKG_DIR, "src/models/whisper/modeling_whisper.js")); + expect(source).toContain("if (generation_config.return_timestamps && !kwargs.max_new_tokens)"); + }); });