From dd9dfc628b8da07f507dde4179cbb88f8f6d7393 Mon Sep 17 00:00:00 2001 From: Lucas Winther Date: Fri, 30 Oct 2026 14:35:09 +0100 Subject: [PATCH] Detect the language once for auto-detect file passes on the chunked path (D-053) Transformers.js has no Whisper language detection and silently decoded Danish as English whenever "Skip silence" was off or unavailable. Co-Authored-By: Claude Opus 5.5 --- docs/Architecture.md | 3 +++ docs/Decisions.md | 25 +++++++++++++++++++++++ docs/WorkBreakdown.md | 7 ++++++- src/worker/inference.worker.ts | 21 +++++++++++++++----- src/worker/language.ts | 26 ++++++++++++++++++++++++ tests/live-real-model.test.ts | 36 ++++++++++++++++++++++++++++++++++ 6 files changed, 112 insertions(+), 6 deletions(-) create mode 100644 src/worker/language.ts diff --git a/docs/Architecture.md b/docs/Architecture.md index 517f157..bec1a35 100644 --- a/docs/Architecture.md +++ b/docs/Architecture.md @@ -163,6 +163,9 @@ environment). See `WorkBreakdown.md` for the itemized checklist.) (`VocabularyHint`, stored in localStorage by `lib/hint.ts`), raw as typed (D-048). 5. Worker runs the pipeline (chunked, with timestamps), streaming partial results back via `{type:"partial"}` messages; final result as `{type:"result"}`. + With auto-detect on a multilingual model, it first detects the language on + the first 30 s (`worker/language.ts`, D-053) and pins it — Transformers.js + itself would default to English. A hint becomes `<|startofprev|> hint.` once per pass (`worker/prompt.ts`, cut to a 120-token prompt budget at a whole word) and is prepended to the decoder input of every 30 s window by a wrapper around the model's diff --git a/docs/Decisions.md b/docs/Decisions.md index ea6c02b..a77f07f 100644 --- a/docs/Decisions.md +++ b/docs/Decisions.md @@ -1821,3 +1821,28 @@ chunked path. Live mode is unaffected. **Caveat:** the WER numbers behind this use reference transcripts generated with Whisper (owner, 2026-10-30), so they favour Whisper-like output; re-check once correct transcripts are available. + +## D-053 — "Auto" language on the chunked file path: detect once on the first 30 s, then pin it +**Status:** Accepted (2026-10-30) +Transformers.js 4.3.2 still has no Whisper language detection +(`modeling_whisper.js`: `// TODO: Implement language detection` → "No +language specified - defaulting to English"). With "Auto-detect" selected, +a file transcribed on the chunked path — the fallback whenever "Skip +silence" (D-052) is off or its VAD model is unavailable — was decoded as +English: Danish `da_festuge` came out as a rough English translation (WER +0.94 vs 0.14 with `da`, found in D-049's measurements). +**Decision:** `src/worker/language.ts` `detectFileLanguage` reuses the +long-form pipeline's one-step detector (D-049: encoder pass + one decoder +step restricted to language tokens) on the first 30 s of the file, and the +worker pins the result for the whole chunked pass. Only file passes on +multilingual models without an explicit language; `.en` models and an +explicit language are unchanged. +**Verified:** `tests/live-real-model.test.ts` "auto language on the chunked +file path": da_festuge → `da` (WER within its bar), en_aiproblem and +en_alien → `en` (Large v3 Turbo, onnxruntime-node). +**Cost:** one extra encoder pass per file (the chunked pipeline can't reuse +it). **Not covered:** live passes with "auto" still decode as English — +detecting on every short live pass would add an encoder run per pass; it +needs a once-per-session design (WorkBreakdown). +**Caveat:** leading silence or music in the first 30 s can mislead +detection; long-form detects on the first *speech* chunk instead. diff --git a/docs/WorkBreakdown.md b/docs/WorkBreakdown.md index b35325a..8122e31 100644 --- a/docs/WorkBreakdown.md +++ b/docs/WorkBreakdown.md @@ -441,7 +441,7 @@ Found in spec 001 workstream 6 (D-046), open: this repo has. Found while building the long-form pipeline (spec 001 Goal 3, D-049), 2026-10-29: -- [ ] "Auto-detect" on the default (chunked) file path does not detect: with no +- [x] "Auto-detect" on the default (chunked) file path does not detect: with no `language`, Transformers.js 4.2 logs "No language specified - defaulting to English" and decodes Danish audio as an English translation (da_festuge WER 0.94 vs 0.14 with `da`). Long-form ("Skip silence") detects once and pins @@ -449,6 +449,11 @@ Found while building the long-form pipeline (spec 001 Goal 3, D-049), 2026-10-29 - [ ] Spec 001's measurement fallback mentions a hint list of the names in `en_alien`'s reference; `tests/meetingFixture.ts` returns audio + reference only. Add the list when the vocabulary hint (workstream 2) lands. + — Fixed for file passes: detect once on the first 30 s and pin (D-053). +- [ ] "Auto-detect" on **live** passes (browser engine) still decodes as English: + each live pass would need its own detection (an extra encoder run per + pass). Needs a once-per-session design (detect on the first committed + segment, then pin) — D-053. - [ ] Parakeet v3 also transcribes 24 other European languages, Danish included, but the app offers it English-only (spec 001) — unmeasured on `da_festuge`. Candidate for a later spec. diff --git a/src/worker/inference.worker.ts b/src/worker/inference.worker.ts index 5072019..83e67d9 100644 --- a/src/worker/inference.worker.ts +++ b/src/worker/inference.worker.ts @@ -30,6 +30,7 @@ import { type ParakeetEngine, } from "./parakeetEngine"; import { LONG_FORM_DEFAULTS, transcribeLongForm } from "./longform"; +import { detectFileLanguage } from "./language"; import { createWhisperDeps, loadSileroVad, type SileroVad } from "./longform/transformers"; import { buildPromptPrefix, @@ -411,6 +412,14 @@ function load(message: LoadMessage): Promise { return promise; } +/** Multilingual Whisper checkpoint (has language tokens), per its generation config. */ +function isMultilingual(asr: AutomaticSpeechRecognitionPipeline): boolean { + return Boolean( + (asr.model as unknown as { generation_config?: { is_multilingual?: boolean } }) + .generation_config?.is_multilingual, + ); +} + /** * Silero VAD for long-form file passes (D-049/D-050). Loaded on request * (`load.vad`) once the Whisper pipeline is up — after it, so the Whisper @@ -568,9 +577,14 @@ async function transcribe(message: TranscribeMessage): Promise { ); } // Only meaningful for multilingual models; the UI omits these for `.en` - // variants. "auto" / undefined lets Whisper detect the language. + // variants. Transformers.js has no language detection ("auto" would decode + // as English), so a file pass detects once on its first 30 s and pins it + // (D-053). Live passes keep the old behaviour (see WorkBreakdown). if (message.language !== undefined && message.language !== "auto") { options.language = message.language; + } else if (streaming && isMultilingual(asr)) { + options.language = await detectFileLanguage(asr, message.audio); + console.info(`[whisper] detected language: ${options.language as string}`); } if (message.task !== undefined) { options.task = message.task; @@ -611,10 +625,7 @@ async function transcribeLongFormPass( asr: AutomaticSpeechRecognitionPipeline, vadModel: SileroVad, ): Promise { - const multilingual = Boolean( - (asr.model as unknown as { generation_config?: { is_multilingual?: boolean } }) - .generation_config?.is_multilingual, - ); + const multilingual = isMultilingual(asr); const result = await transcribeLongForm( message.audio, { diff --git a/src/worker/language.ts b/src/worker/language.ts new file mode 100644 index 0000000..b749985 --- /dev/null +++ b/src/worker/language.ts @@ -0,0 +1,26 @@ +/** + * "Auto" language for the chunked file path (D-053). + * + * Transformers.js has no Whisper language detection: with no `language` on a + * multilingual model, `generate()` logs "No language specified - defaulting + * to English" and decodes everything as English (Danish came out as a rough + * English translation, WER 0.94). The long-form path already detects once per + * file (D-049); this reuses its one-step detector on the first 30 s so the + * chunked path can pin the language for the whole file. + */ + +import type { AutomaticSpeechRecognitionPipeline } from "@huggingface/transformers"; +import { createWhisperDeps } from "./longform/transformers"; +import { SAMPLE_RATE } from "./longform/vad"; + +/** Whisper's window: detection looks at no more than the first 30 s. */ +const DETECTION_SECONDS = 30; + +/** Whisper language code ("da", "en", …) of the start of `audio` (16 kHz mono). */ +export async function detectFileLanguage( + asr: AutomaticSpeechRecognitionPipeline, + audio: Float32Array, +): Promise { + const { detectLanguage } = createWhisperDeps(asr, { multilingual: true }); + return detectLanguage!(audio.subarray(0, DETECTION_SECONDS * SAMPLE_RATE)); +} diff --git a/tests/live-real-model.test.ts b/tests/live-real-model.test.ts index dc12389..23a6206 100644 --- a/tests/live-real-model.test.ts +++ b/tests/live-real-model.test.ts @@ -20,6 +20,7 @@ import { import { LONG_FORM_DEFAULTS, transcribeLongForm } from "../src/worker/longform"; import { createWhisperDeps, loadSileroVad } from "../src/worker/longform/transformers"; import { decodeClip } from "./clips"; +import { detectFileLanguage } from "../src/worker/language"; import { wordErrorRate } from "./dsp"; /** @@ -500,3 +501,38 @@ describe("long-form file pipeline accuracy (real model, real recordings)", () => ); } }); + +// --------------------------------------------------------------------------- +// "Auto" language on the default chunked file path: Transformers.js has no +// language detection and silently decodes as English, so the worker detects +// once on the first 30 s (src/worker/language.ts) and pins it. +// --------------------------------------------------------------------------- + +describe("auto language on the chunked file path (real model)", () => { + for (const clip of clips) { + test.skipIf(!available)( + `${clip.base}: detected as ${clip.language}, and the transcript stays close to the reference`, + async () => { + const reference = await Bun.file(join(AUDIO_DIR, clip.transcriptFile)).text(); + const audio = await decodeClip(clip.audioFile); + const asr = await loadAsr(); + const language = await detectFileLanguage(asr, audio); + expect(language).toBe(clip.language); + if (audio.length / RATE > 120) return; // detection is the point; long clips are covered above + const output = await asr(audio, { + chunk_length_s: 30, + stride_length_s: 5, + return_timestamps: true, + max_new_tokens: 440, + language, + task: "transcribe", + }); + const text = ((Array.isArray(output) ? output[0] : output) as { text?: string }).text ?? ""; + expect(wordErrorRate(normalize(reference), normalize(text))).toBeLessThanOrEqual( + MAX_WER_BY_CLIP[clip.base] ?? MAX_WER, + ); + }, + DOWNLOAD_BUDGET_MS + 60 * MS_PER_AUDIO_SECOND, + ); + } +});