Reuse the language-detection encoder pass in long-form decodes

Whisper's generate() keeps only forward_params from its kwargs, and that
list omits encoder_outputs, so the precomputed encoder output was dropped
and auto-detect cost three encoder passes for a one-chunk clip. The
real-model test now pins one encoder pass per decoded chunk.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
Lucas Winther
2026-10-30 00:19:08 +01:00
co-authored by Claude Opus 5.5
parent 8ac81baf54
commit 160298197e
3 changed files with 25 additions and 2 deletions
+1 -1
View File
@@ -207,7 +207,7 @@ normalize), `scripts/measure-longform.ts`.
- [ ] Run under the lock, old/new interleaved: baseline; long-form guard off; guard variants
(`no_repeat_ngram_size 3/5`, `repetition_penalty 1.1`); fallback on if loops remain; VAD
`min_silence 2000` vs `160`. Choose defaults by the rollout rule; record the table in D-WS3a.
- [ ] Commit "Add the synthetic meeting fixture and long-form measurement script".
- [x] Commit "Add the synthetic meeting fixture and long-form measurement script".
### Task 7: Worker, protocol, engine and UI wiring
+8
View File
@@ -211,6 +211,14 @@ export function createWhisperDeps(
};
};
// generate() keeps only the model's `forward_params` from its kwargs, and
// Whisper's list omits `encoder_outputs` — so a precomputed encoder output
// was silently dropped and the encoder ran again. Admitting it on this
// instance lets generate() skip the encoder (its own `"encoder_outputs" in
// model_inputs` check) when detection already encoded the chunk.
const params = (asr.model as unknown as { forward_params: string[] }).forward_params;
if (!params.includes("encoder_outputs")) params.push("encoder_outputs");
/** Encoder output of the chunk language detection just ran on. */
let encoded: { audio: Float32Array; hidden: Tensor } | null = null;
const decodeReusingEncoder = (audio: Float32Array, request: DecodeRequest) => {
+16 -1
View File
@@ -352,6 +352,17 @@ describe("long-form file pipeline accuracy (real model, real recordings)", () =>
// "Auto-detect" in the UI: the chunked path forces English here
// (Transformers.js 4.2 has no detection); long-form detects once
// on the first speech chunk and pins it (D-WS3a).
// Detection must not cost an extra encoder pass: the first
// chunk's decode reuses the encoder output detection computed.
const model = asr.model as unknown as Record<string, unknown>;
const encode = model._prepare_encoder_decoder_kwargs_for_generation as (
...args: unknown[]
) => Promise<unknown>;
let encoderPasses = 0;
model._prepare_encoder_decoder_kwargs_for_generation = (...args: unknown[]) => {
encoderPasses += 1;
return encode.apply(model, args);
};
const auto = await transcribeLongForm(
audio,
{
@@ -359,8 +370,12 @@ describe("long-form file pipeline accuracy (real model, real recordings)", () =>
...createWhisperDeps(asr, { multilingual: true }),
},
{ ...LONG_FORM_DEFAULTS, task: "transcribe" },
);
).finally(() => {
delete model._prepare_encoder_decoder_kwargs_for_generation; // back to the prototype's
});
expect(auto.language).toBe(clip.language);
// One encoder pass per decoded chunk (fallback is off by default).
expect(encoderPasses).toBe(auto.decodeCalls);
expect(
wordErrorRate(normalize(reference), normalize(auto.text)),
).toBeLessThanOrEqual(MAX_WER_BY_CLIP[clip.base] ?? MAX_WER);