Reuse the language-detection encoder pass in long-form decodes
Whisper's generate() keeps only forward_params from its kwargs, and that list omits encoder_outputs, so the precomputed encoder output was dropped and auto-detect cost three encoder passes for a one-chunk clip. The real-model test now pins one encoder pass per decoded chunk. Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
8ac81baf54
commit
160298197e
@@ -207,7 +207,7 @@ normalize), `scripts/measure-longform.ts`.
|
||||
- [ ] Run under the lock, old/new interleaved: baseline; long-form guard off; guard variants
|
||||
(`no_repeat_ngram_size 3/5`, `repetition_penalty 1.1`); fallback on if loops remain; VAD
|
||||
`min_silence 2000` vs `160`. Choose defaults by the rollout rule; record the table in D-WS3a.
|
||||
- [ ] Commit "Add the synthetic meeting fixture and long-form measurement script".
|
||||
- [x] Commit "Add the synthetic meeting fixture and long-form measurement script".
|
||||
|
||||
### Task 7: Worker, protocol, engine and UI wiring
|
||||
|
||||
|
||||
@@ -211,6 +211,14 @@ export function createWhisperDeps(
|
||||
};
|
||||
};
|
||||
|
||||
// generate() keeps only the model's `forward_params` from its kwargs, and
|
||||
// Whisper's list omits `encoder_outputs` — so a precomputed encoder output
|
||||
// was silently dropped and the encoder ran again. Admitting it on this
|
||||
// instance lets generate() skip the encoder (its own `"encoder_outputs" in
|
||||
// model_inputs` check) when detection already encoded the chunk.
|
||||
const params = (asr.model as unknown as { forward_params: string[] }).forward_params;
|
||||
if (!params.includes("encoder_outputs")) params.push("encoder_outputs");
|
||||
|
||||
/** Encoder output of the chunk language detection just ran on. */
|
||||
let encoded: { audio: Float32Array; hidden: Tensor } | null = null;
|
||||
const decodeReusingEncoder = (audio: Float32Array, request: DecodeRequest) => {
|
||||
|
||||
@@ -352,6 +352,17 @@ describe("long-form file pipeline accuracy (real model, real recordings)", () =>
|
||||
// "Auto-detect" in the UI: the chunked path forces English here
|
||||
// (Transformers.js 4.2 has no detection); long-form detects once
|
||||
// on the first speech chunk and pins it (D-WS3a).
|
||||
// Detection must not cost an extra encoder pass: the first
|
||||
// chunk's decode reuses the encoder output detection computed.
|
||||
const model = asr.model as unknown as Record<string, unknown>;
|
||||
const encode = model._prepare_encoder_decoder_kwargs_for_generation as (
|
||||
...args: unknown[]
|
||||
) => Promise<unknown>;
|
||||
let encoderPasses = 0;
|
||||
model._prepare_encoder_decoder_kwargs_for_generation = (...args: unknown[]) => {
|
||||
encoderPasses += 1;
|
||||
return encode.apply(model, args);
|
||||
};
|
||||
const auto = await transcribeLongForm(
|
||||
audio,
|
||||
{
|
||||
@@ -359,8 +370,12 @@ describe("long-form file pipeline accuracy (real model, real recordings)", () =>
|
||||
...createWhisperDeps(asr, { multilingual: true }),
|
||||
},
|
||||
{ ...LONG_FORM_DEFAULTS, task: "transcribe" },
|
||||
);
|
||||
).finally(() => {
|
||||
delete model._prepare_encoder_decoder_kwargs_for_generation; // back to the prototype's
|
||||
});
|
||||
expect(auto.language).toBe(clip.language);
|
||||
// One encoder pass per decoded chunk (fallback is off by default).
|
||||
expect(encoderPasses).toBe(auto.decodeCalls);
|
||||
expect(
|
||||
wordErrorRate(normalize(reference), normalize(auto.text)),
|
||||
).toBeLessThanOrEqual(MAX_WER_BY_CLIP[clip.base] ?? MAX_WER);
|
||||
|
||||
Reference in New Issue
Block a user