diff --git a/docs/plans/001-3-longform-pipeline.md b/docs/plans/001-3-longform-pipeline.md index 7439de2..094e896 100644 --- a/docs/plans/001-3-longform-pipeline.md +++ b/docs/plans/001-3-longform-pipeline.md @@ -207,7 +207,7 @@ normalize), `scripts/measure-longform.ts`. - [ ] Run under the lock, old/new interleaved: baseline; long-form guard off; guard variants (`no_repeat_ngram_size 3/5`, `repetition_penalty 1.1`); fallback on if loops remain; VAD `min_silence 2000` vs `160`. Choose defaults by the rollout rule; record the table in D-WS3a. -- [ ] Commit "Add the synthetic meeting fixture and long-form measurement script". +- [x] Commit "Add the synthetic meeting fixture and long-form measurement script". ### Task 7: Worker, protocol, engine and UI wiring diff --git a/src/worker/longform/transformers.ts b/src/worker/longform/transformers.ts index f9552ba..fcc71fd 100644 --- a/src/worker/longform/transformers.ts +++ b/src/worker/longform/transformers.ts @@ -211,6 +211,14 @@ export function createWhisperDeps( }; }; + // generate() keeps only the model's `forward_params` from its kwargs, and + // Whisper's list omits `encoder_outputs` — so a precomputed encoder output + // was silently dropped and the encoder ran again. Admitting it on this + // instance lets generate() skip the encoder (its own `"encoder_outputs" in + // model_inputs` check) when detection already encoded the chunk. + const params = (asr.model as unknown as { forward_params: string[] }).forward_params; + if (!params.includes("encoder_outputs")) params.push("encoder_outputs"); + /** Encoder output of the chunk language detection just ran on. */ let encoded: { audio: Float32Array; hidden: Tensor } | null = null; const decodeReusingEncoder = (audio: Float32Array, request: DecodeRequest) => { diff --git a/tests/live-real-model.test.ts b/tests/live-real-model.test.ts index 9c9ae12..d6f7c2a 100644 --- a/tests/live-real-model.test.ts +++ b/tests/live-real-model.test.ts @@ -352,6 +352,17 @@ describe("long-form file pipeline accuracy (real model, real recordings)", () => // "Auto-detect" in the UI: the chunked path forces English here // (Transformers.js 4.2 has no detection); long-form detects once // on the first speech chunk and pins it (D-WS3a). + // Detection must not cost an extra encoder pass: the first + // chunk's decode reuses the encoder output detection computed. + const model = asr.model as unknown as Record; + const encode = model._prepare_encoder_decoder_kwargs_for_generation as ( + ...args: unknown[] + ) => Promise; + let encoderPasses = 0; + model._prepare_encoder_decoder_kwargs_for_generation = (...args: unknown[]) => { + encoderPasses += 1; + return encode.apply(model, args); + }; const auto = await transcribeLongForm( audio, { @@ -359,8 +370,12 @@ describe("long-form file pipeline accuracy (real model, real recordings)", () => ...createWhisperDeps(asr, { multilingual: true }), }, { ...LONG_FORM_DEFAULTS, task: "transcribe" }, - ); + ).finally(() => { + delete model._prepare_encoder_decoder_kwargs_for_generation; // back to the prototype's + }); expect(auto.language).toBe(clip.language); + // One encoder pass per decoded chunk (fallback is off by default). + expect(encoderPasses).toBe(auto.decodeCalls); expect( wordErrorRate(normalize(reference), normalize(auto.text)), ).toBeLessThanOrEqual(MAX_WER_BY_CLIP[clip.base] ?? MAX_WER);