From 02a5eab119c61dff775b0f3e0d7a1aa13b992347 Mon Sep 17 00:00:00 2001 From: Lucas Winther Date: Thu, 29 Oct 2026 21:37:28 +0100 Subject: [PATCH] Offer Whisper's Translate to English only on models trained for it (canTranslate) Co-Authored-By: Claude Opus 5.5 --- docs/Architecture.md | 2 +- docs/Decisions.md | 17 ++++++++++++ docs/WorkBreakdown.md | 2 ++ src/App.tsx | 15 ++++++++-- src/components/TranscriptionOptions.tsx | 32 +++++++++++++-------- src/lib/models.ts | 30 ++++++++++++++++++++ tests/models.test.ts | 37 +++++++++++++++++++++++++ tests/transcriptionOptions.test.tsx | 31 +++++++++++++++++++++ 8 files changed, 151 insertions(+), 15 deletions(-) create mode 100644 tests/transcriptionOptions.test.tsx diff --git a/docs/Architecture.md b/docs/Architecture.md index ea5fc28..ffa963b 100644 --- a/docs/Architecture.md +++ b/docs/Architecture.md @@ -141,7 +141,7 @@ environment). See `WorkBreakdown.md` for the itemized checklist.) `AudioContext.decodeAudioData`, downmixes to mono, resamples to 16 kHz, transfers the `Float32Array` to the worker. For multilingual models the `transcribe` message also carries `language` (or auto-detect) and `task` - (transcribe | translate). + (transcribe | translate — translate only for models with `canTranslate`, D-WS4a). 5. Worker runs the pipeline (chunked, with timestamps), streaming partial results back via `{type:"partial"}` messages; final result as `{type:"result"}`. diff --git a/docs/Decisions.md b/docs/Decisions.md index 5dc8f08..342ad0b 100644 --- a/docs/Decisions.md +++ b/docs/Decisions.md @@ -1224,3 +1224,20 @@ whole-file output is equally bad, which is exactly what the absolute bar exists to catch. **Consequence:** full `bun test` ≈ 19 min on CPU (en_alien ≈ 20 min of it on a cold run); the fast suite excludes this file. + +## D-WS4a — Whisper's "Translate to English" only on models trained for it (`canTranslate`) +**Status:** Accepted (2026-10-29) · spec 001 Goal 4a +`WhisperModel` gains `canTranslate: boolean` (`src/lib/models.ts`): **true** for multilingual +tiny / base / small / medium, **false** for every `.en` model (no task token) and for +`whisper-large-v3-turbo` — OpenAI's model card says turbo was fine-tuned on transcription data only +and is not trained for translation, so its "translate" output is unreliable. Any future turbo-based +fine-tune defaults to false until tested. `TranscriptionOptions` renders the Task control only where +true (language stays). **Stale task:** the `task` state is kept, but `taskForModel(modelId, task)` +maps "translate" to "transcribe" for a model that can't translate; App applies it on model switch +(picker and the server-engine auto-switch to turbo) and again when building every file / live run's +options, so a leftover "translate" can never reach the worker or the server. **Not changed:** the +server's `parseLanguageAndTask` (D-032) still accepts "translate" for any model — the browser never +sends it for turbo now, and a direct API caller asking faster-whisper to translate with turbo gets +what faster-whisper does; rejecting it server-side would be a behaviour change for API users with no +UI benefit. Speech in another language than English can still end up in English (or Danish) through +text translation of the finished transcript (D-WS4b), for every model. diff --git a/docs/WorkBreakdown.md b/docs/WorkBreakdown.md index 336b0b4..2ff0310 100644 --- a/docs/WorkBreakdown.md +++ b/docs/WorkBreakdown.md @@ -65,6 +65,8 @@ Notes (M1): surface active backend in UI - [x] Handle model switching (dispose old pipeline, load new) - [x] Language / task options for multilingual models (transcribe vs translate) +- [x] Whisper translate only on models trained for it (`canTranslate`; hidden for + large-v3-turbo and `.en`, stale "translate" resets to transcribe — D-WS4a) - [~] Verify quantized variants (q4/q8/fp16) on both backends; record working combinations in `models.ts` diff --git a/src/App.tsx b/src/App.tsx index e282021..0c22644 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -37,6 +37,7 @@ import { formatModelSize, getModel, resolveDefaultDtype, + taskForModel, } from "./lib/models"; import { shouldWarnBeforeLoad, type MobileMemoryHint } from "./lib/mobileMemoryGuard"; import { @@ -499,6 +500,9 @@ export function App() { function handleModelSelect(nextModelId: string) { if (busy || liveStatus !== "idle" || nextModelId === modelId) return; setModelId(nextModelId); + // A "Translate to English" chosen on a model that can translate must not + // silently carry over to one that cannot (D-WS4a). + setTask((current) => taskForModel(nextModelId, current)); setActiveDevice(null); setActiveThreads(null); setPartialText(""); @@ -675,7 +679,7 @@ export function App() { // Language/task only apply to multilingual models; `.en` variants // take neither. The browser engine ignores modelId (worker holds it). modelId, - ...(multilingual && { language, task }), + ...(multilingual && { language, task: taskForModel(modelId, task) }), onPartial: (fragment) => { if (fileRun.current === run) setPartialText((text) => text + fragment); }, @@ -752,7 +756,10 @@ export function App() { setLivePending(0); setLiveStatus("starting"); - const options = (getModel(modelId)?.multilingual ?? false) ? { language, task } : {}; + const options = + (getModel(modelId)?.multilingual ?? false) + ? { language, task: taskForModel(modelId, task) } + : {}; if (useServer) { const wsUrl = new URL(`/api/transcribe/live?model=${modelId}`, window.location.href); @@ -988,6 +995,7 @@ export function App() { const allowed = ["onnx-community/whisper-large-v3-turbo", "onnx-community/whisper-medium.en_timestamped"]; if (!allowed.includes(modelId) && !busy && liveStatus === "idle") { setModelId("onnx-community/whisper-large-v3-turbo"); + setTask((current) => taskForModel("onnx-community/whisper-large-v3-turbo", current)); setActiveDevice(null); setActiveThreads(null); setPartialText(""); @@ -1036,8 +1044,9 @@ export function App() { )} diff --git a/src/components/TranscriptionOptions.tsx b/src/components/TranscriptionOptions.tsx index badec28..58b9823 100644 --- a/src/components/TranscriptionOptions.tsx +++ b/src/components/TranscriptionOptions.tsx @@ -6,18 +6,26 @@ interface TranscriptionOptionsProps { onLanguageChange: (language: string) => void; task: WhisperTask; onTaskChange: (task: WhisperTask) => void; + /** + * Whether the model supports Whisper's own speech → English translation + * (`WhisperModel.canTranslate`, D-WS4a). When false the Task control is not + * rendered at all — transcribing is the only thing such a model does. + */ + canTranslate: boolean; disabled?: boolean; } /** * Language + task controls for multilingual models. The parent renders this * only when the selected model is multilingual — `.en` models take neither. + * The task control only appears where the model can translate (D-WS4a). */ export function TranscriptionOptions({ language, onLanguageChange, task, onTaskChange, + canTranslate, disabled = false, }: TranscriptionOptionsProps) { return ( @@ -37,17 +45,19 @@ export function TranscriptionOptions({ ))} - + {canTranslate && ( + + )} ); } diff --git a/src/lib/models.ts b/src/lib/models.ts index ad82c0c..cb3218b 100644 --- a/src/lib/models.ts +++ b/src/lib/models.ts @@ -42,6 +42,15 @@ export interface WhisperModel { displayName: string; /** False for `.en` (English-only) variants. */ multilingual: boolean; + /** + * Whether Whisper's own `task: "translate"` (speech → English text) is + * offered (D-WS4a). False for every `.en` model (no task token) and for + * large-v3-turbo, which OpenAI did not train for translation — and for any + * turbo-based fine-tune not tested for it. The UI hides "Translate to + * English" where false; text translation of a finished transcript + * (D-WS4b) works for every model regardless. + */ + canTranslate: boolean; /** Encoder parameter count, in millions (drives size estimates). */ encoderParamsM: number; /** Decoder parameter count, in millions (drives size estimates). */ @@ -174,6 +183,7 @@ export const MODELS: readonly WhisperModel[] = [ fasterWhisperModel: "tiny.en", displayName: "Whisper Tiny (English)", multilingual: false, + canTranslate: false, encoderParamsM: 8, decoderParamsM: 31, defaults: { @@ -188,6 +198,7 @@ export const MODELS: readonly WhisperModel[] = [ fasterWhisperModel: "tiny", displayName: "Whisper Tiny", multilingual: true, + canTranslate: true, encoderParamsM: 8, decoderParamsM: 31, defaults: { wasm: WASM_Q8, webgpu: WEBGPU_FP32, webnn: WEBNN_FP16 }, @@ -197,6 +208,7 @@ export const MODELS: readonly WhisperModel[] = [ fasterWhisperModel: "base.en", displayName: "Whisper Base (English)", multilingual: false, + canTranslate: false, encoderParamsM: 20, decoderParamsM: 54, defaults: { wasm: WASM_Q8, webgpu: WEBGPU_FP32, webnn: WEBNN_FP16 }, @@ -206,6 +218,7 @@ export const MODELS: readonly WhisperModel[] = [ fasterWhisperModel: "base", displayName: "Whisper Base", multilingual: true, + canTranslate: true, encoderParamsM: 20, decoderParamsM: 54, defaults: { wasm: WASM_Q8, webgpu: WEBGPU_FP32, webnn: WEBNN_FP16 }, @@ -215,6 +228,7 @@ export const MODELS: readonly WhisperModel[] = [ fasterWhisperModel: "small.en", displayName: "Whisper Small (English)", multilingual: false, + canTranslate: false, encoderParamsM: 88, decoderParamsM: 156, defaults: { wasm: WASM_Q8, webgpu: WEBGPU_FP16_Q4, webnn: WEBNN_FP16 }, @@ -224,6 +238,7 @@ export const MODELS: readonly WhisperModel[] = [ fasterWhisperModel: "small", displayName: "Whisper Small", multilingual: true, + canTranslate: true, encoderParamsM: 88, decoderParamsM: 156, defaults: { wasm: WASM_Q8, webgpu: WEBGPU_FP16_Q4, webnn: WEBNN_FP16 }, @@ -237,6 +252,7 @@ export const MODELS: readonly WhisperModel[] = [ fasterWhisperModel: "medium.en", displayName: "Whisper Medium (English)", multilingual: false, + canTranslate: false, encoderParamsM: 306, decoderParamsM: 463, defaults: { wasm: WASM_Q4, webgpu: ACCEL_Q4, webnn: ACCEL_Q4 }, @@ -246,6 +262,7 @@ export const MODELS: readonly WhisperModel[] = [ fasterWhisperModel: "medium", displayName: "Whisper Medium", multilingual: true, + canTranslate: true, encoderParamsM: 306, decoderParamsM: 463, defaults: { wasm: WASM_Q4, webgpu: ACCEL_Q4, webnn: ACCEL_Q4 }, @@ -255,6 +272,9 @@ export const MODELS: readonly WhisperModel[] = [ fasterWhisperModel: "large-v3-turbo", displayName: "Whisper Large v3 Turbo", multilingual: true, + // Not trained for translation (OpenAI's turbo model card); its + // "translate" output is unreliable (D-WS4a). + canTranslate: false, encoderParamsM: 635, decoderParamsM: 174, defaults: { wasm: WASM_Q4, webgpu: WEBGPU_FP16_Q4, webnn: WEBNN_FP16 }, @@ -274,6 +294,16 @@ export function getFasterWhisperModel(id: string): string | undefined { return getModel(id)?.fasterWhisperModel; } +/** + * The task a run on `modelId` should actually use (D-WS4a): a "translate" + * left over from a model that could translate falls back to "transcribe" on + * one that cannot (large-v3-turbo, `.en`, unknown ids). App applies it on + * every model switch and again when building each run's options. + */ +export function taskForModel(modelId: string, task: WhisperTask): WhisperTask { + return task === "translate" && getModel(modelId)?.canTranslate !== true ? "transcribe" : task; +} + /** Default dtype for a model on a given device. Unknown ids fall back to q8. */ export function resolveDefaultDtype( modelId: string, diff --git a/tests/models.test.ts b/tests/models.test.ts index 1ef20c3..a082df6 100644 --- a/tests/models.test.ts +++ b/tests/models.test.ts @@ -10,6 +10,7 @@ import { getFasterWhisperModel, getModel, resolveDefaultDtype, + taskForModel, } from "../src/lib/models"; // Pure catalog data (D-006) — no browser, no network. @@ -209,3 +210,39 @@ describe("WHISPER_LANGUAGES", () => { expect(WHISPER_LANGUAGES.some((entry) => entry.code === "auto")).toBe(false); }); }); + +describe("canTranslate (D-WS4a)", () => { + test("Whisper translate is offered only where OpenAI trained it", () => { + const translating = MODELS.filter((model) => model.canTranslate).map((model) => model.id); + // large-v3-turbo was not trained for translation (OpenAI model card); + // `.en` models are English-only and have no task token. + expect(translating).toEqual([ + "onnx-community/whisper-tiny", + "onnx-community/whisper-base", + "onnx-community/whisper-small", + "onnx-community/whisper-medium-ONNX", + ]); + }); + + test("canTranslate implies multilingual", () => { + for (const model of MODELS) { + if (model.canTranslate) expect(model.multilingual).toBe(true); + } + }); +}); + +describe("taskForModel (D-WS4a)", () => { + test("a stale translate resets to transcribe on a model that cannot translate", () => { + expect(taskForModel("onnx-community/whisper-large-v3-turbo", "translate")).toBe("transcribe"); + expect(taskForModel("onnx-community/whisper-small.en", "translate")).toBe("transcribe"); + }); + + test("translate survives on a model that can translate", () => { + expect(taskForModel("onnx-community/whisper-small", "translate")).toBe("translate"); + expect(taskForModel("onnx-community/whisper-small", "transcribe")).toBe("transcribe"); + }); + + test("unknown ids never translate", () => { + expect(taskForModel("someone/unknown", "translate")).toBe("transcribe"); + }); +}); diff --git a/tests/transcriptionOptions.test.tsx b/tests/transcriptionOptions.test.tsx new file mode 100644 index 0000000..a9ada77 --- /dev/null +++ b/tests/transcriptionOptions.test.tsx @@ -0,0 +1,31 @@ +import { describe, expect, test } from "bun:test"; +import { renderToStaticMarkup } from "react-dom/server"; +import { TranscriptionOptions } from "../src/components/TranscriptionOptions"; + +// D-WS4a: Whisper's own "Translate to English" only where the model can do it. +function render(canTranslate: boolean): string { + return renderToStaticMarkup( + {}} + task="transcribe" + onTaskChange={() => {}} + canTranslate={canTranslate} + />, + ); +} + +describe("TranscriptionOptions", () => { + test("offers Translate to English when the model can translate", () => { + const html = render(true); + expect(html).toContain("Translate to English"); + expect(html).toContain("Task"); + }); + + test("hides the task control entirely when the model cannot translate", () => { + const html = render(false); + expect(html).not.toContain("Translate to English"); + expect(html).not.toContain("Task"); + expect(html).toContain("Language"); + }); +});