From 036635e85438e07aae1313a96354c9b6e083c7f2 Mon Sep 17 00:00:00 2001 From: Lucas Winther Date: Thu, 29 Oct 2026 21:35:49 +0100 Subject: [PATCH 01/10] Add workstream 4 plan: canTranslate and Opus-MT transcript translation Co-Authored-By: Claude Opus 5.5 --- docs/plans/001-4-translate.md | 137 ++++++++++++++++++++++++++++++++++ 1 file changed, 137 insertions(+) create mode 100644 docs/plans/001-4-translate.md diff --git a/docs/plans/001-4-translate.md b/docs/plans/001-4-translate.md new file mode 100644 index 0000000..b0a57e7 --- /dev/null +++ b/docs/plans/001-4-translate.md @@ -0,0 +1,137 @@ +--- +status: in-progress +--- + +# 001-4 — Translate: implementation plan + +> **For agentic workers:** REQUIRED SUB-SKILL: superpowers:executing-plans (inline, no subagents — +> coordination plan). Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** (4a) a per-model `canTranslate` flag that hides Whisper's "Translate to English" where it is +unsupported and resets a stale `task: "translate"`; (4b) on-device Opus-MT text translation of any +finished transcript (da→en, en→da), with timestamps kept for `.srt` / timestamped `.txt` exports. + +**Architecture:** `canTranslate` lives in the catalog (`src/lib/models.ts`) with a pure +`taskForModel()` guard used by App on model switch and on every run. Text MT runs in a **dedicated +worker** (`src/worker/translate.worker.ts`) — its own onnxruntime instance with full WASM threads, +independent of the Whisper worker's lifetime (cancel/D-018 terminate, D-025 restart) and of its +accelerator (an ort runtime first initialised for WebGPU is pinned single-threaded, D-023). Pure +planning code (`src/lib/translation.ts`) groups transcript chunks into sentence-ending units, splits +units into sentence pieces under a token budget, and reassembles translated units with the merged +timestamps. UI: a secondary `
` disclosure "Translate transcript…" under the transcript. + +**Tech Stack:** Bun, React 19, Transformers.js 4.2.0 (`translation` pipeline, MarianMT), onnxruntime-web +(WASM) in the browser, onnxruntime-node for measurement. + +**Spec:** `docs/specs/001-transcription-quality-and-speed.md` (Goal 4, call "Text translation (4b) +scope"); coordination: `docs/plans/001-0-coordination.md`. + +## Global Constraints + +- AGENTS.md rules: Bun only; no cross-origin subresources (models come from huggingface.co via + Transformers.js like Whisper; ort from `/ort/`); colours as tokens; errors via `classifyError`. +- D-020 / anti-goal: the MT model is downloaded only when the user clicks Translate; the panel states + the one-time size first. Cached via Transformers.js Cache API (D-005); offline works once cached. +- Nothing leaves the device: translation runs in the browser for every engine (server results too). +- Copy: Whisper's option stays **"Translate to English"**; the new entry is **"Translate transcript…"**. +- `canTranslate`: false for `onnx-community/whisper-large-v3-turbo` and every `.en` model; true for + multilingual tiny/base/small/medium. +- Decision ids: `D-WS4a` (canTranslate), `D-WS4b` (text MT). Ports 3140–3149. Heavy jobs under + `flock /tmp/whisper-heavy.lock`. + +## Review Focus + +1. **Transcript > 512 tokens** (Opus-MT input limit): split on sentence boundaries, order preserved, + nothing dropped → Task 3 test `splitForTranslation keeps every word of en_alien in order with every + piece under budget` + real-model test (Task 4) on all of `en_alien.txt`. +2. **Multi-sentence input to Opus-MT drops later sentences** (observed in the probe: "…Copenhagen. + How are you?" → only the first sentence) → each sentence is its own piece (Task 3 test). +3. **Unpunctuated text** (live output, or a 2-minute run-on): hard-wrap on commas, then words, so no + piece exceeds the budget (Task 3 test). +4. **Stale `task: "translate"` after switching to turbo / a `.en` model / the server auto-switch to + turbo** → reset to transcribe (Task 1 test on `taskForModel`, App wired at every switch + run). +5. **Offline / model not cached** → the Translate button still offers the download; a failed download + surfaces a classified network error with Retry (Task 5, errors test). + +--- + +### Task 1: `canTranslate` + stale-task guard (4a) + +**Files:** Modify `src/lib/models.ts`, `src/components/TranscriptionOptions.tsx`, `src/App.tsx`; +Test `tests/models.test.ts`, new `tests/transcriptionOptions.test.tsx`. + +**Interfaces — Produces:** `WhisperModel.canTranslate: boolean`; +`taskForModel(modelId: string, task: WhisperTask): WhisperTask`; +`TranscriptionOptions` prop `canTranslate: boolean`. + +- [ ] Failing tests: `canTranslate` false for turbo and every `.en`, true for the other multilingual + ids; `taskForModel("onnx-community/whisper-large-v3-turbo", "translate") === "transcribe"`, + unchanged for `whisper-small`; unknown id → `"transcribe"`. `renderToStaticMarkup` of + `TranscriptionOptions` with `canTranslate={false}` contains no "Translate to English" and no Task + select; with `true` contains it. +- [ ] Implement; App: `setTask(t => taskForModel(next, t))` in `handleModelSelect` and the server + auto-switch effect; runs pass `task: taskForModel(modelId, task)`. +- [ ] Typecheck, fast suite; D-WS4a entry; commit. + +### Task 2: error stage for translation + +**Files:** `src/lib/errors.ts`, `tests/errors.test.ts`. +- [ ] Failing test: `classifyError("Out of memory", { stage: "translate" })` → suggestion mentions a + shorter transcript / closing tabs, not "smaller model"; network on `stage: "translate"` retryable. +- [ ] Add `"translate"` to `ErrorStage`; commit with Task 3. + +### Task 3: pure translation planning (`src/lib/translation.ts`) + +**Interfaces — Produces:** +- `TRANSLATION_MODELS: readonly TranslationModel[]` — `{ id, source: "da"|"en", target, label, + approxBytes }` for `Xenova/opus-mt-da-en`, `Xenova/opus-mt-en-da` (q8, ~113 MB each). +- `getTranslationModel(source, target)`, `guessSourceLanguage(text, hint?): "da" | "en"`. +- `buildTranslationUnits(result: { text; chunks }): TranslationUnit[]` — + `{ text: string; timestamp: [number, number | null] | null }`; consecutive chunks merged until the + text ends a sentence (cap 8 chunks); no chunks → one unit per non-empty line. +- `splitForTranslation(text, { maxTokens = 200, countTokens = estimateTokens }): string[]` — + Intl.Segmenter sentences (abbreviation-only fragments merge forward), then comma, then word wrap. +- `assembleTranslation(units, translated: string[]): { text; chunks }`. +- Tests (`tests/translation.test.ts`): unit merging & timestamps, null end, no-chunk fallback, sentence + splitting, Review Focus 1–3, assemble length mismatch throws, `guessSourceLanguage`. +- [ ] Red → green → commit. + +### Task 4: translate worker + protocol + measurement + +**Files:** Create `src/worker/translate.worker.ts`, `src/worker/translateProtocol.ts`, +`tests/translateProtocol.test.ts`, `tests/translate-real-model.test.ts` (skips when the Hub is +unreachable and the model is not cached; run under the lock), `scripts/bench-translate.ts`. +- Protocol: UI→worker `{ type: "translate"; requestId; modelId; units: string[] }`; + worker→UI `translate-load-progress` (file/loaded/total/status), `translate-progress + { done; total }` (pieces), `translate-result { texts: string[] }`, `translate-error + { stage: "load" | "translate"; message }`; guards `isTranslateRequest` / `isTranslateWorkerMessage`. +- Worker: `pipeline("translation", id, { dtype: "q8", device: "wasm", + session_options: { graphOptimizationLevel: "basic" } })` (D-014 — Marian also ties embeddings), + WASM threads as D-023, `wasmPaths "./ort/"`; one cached pipeline per model id; pieces via + `splitForTranslation` with the real tokenizer count; batches of `BATCH_SIZE` (measured). +- Bundle + serve: `src/index.ts` dev route `/translate.worker.js`, `scripts/build.ts` emits and + precaches it. +- [ ] Measure en_alien.txt → da and da_festuge → en on onnxruntime-node (batch 1 vs 4 vs 8); record. +- [ ] Commit. + +### Task 5: UI — `TranslatePanel` + +**Files:** Create `src/components/TranslatePanel.tsx`; modify `src/App.tsx`, +`src/components/TranscriptView.tsx` (`title` prop + `exportSuffix`), `src/components/StorageManager.tsx` +(list MT models), `src/index.css`. +- `
Translate transcript…` with direction + select ("Danish → English", "English → Danish", default from `guessSourceLanguage`), size line when + not cached ("Downloads a translation model (~113 MB, one time)"), Translate / Cancel buttons, + progress ("Downloading translation model… 40%", "Translating… 12 of 80 sentences"), error box via + `classifyError(…, { stage: "translate" | "load" })` with Retry when retryable. +- Result rendered by `TranscriptView` with title "Translation (English)" and exports named + `.en.txt` / `.en.srt`. +- State resets whenever `result` changes; Cancel terminates the translate worker. +- [ ] impeccable polish pass; screenshots light/dark × desktop/mobile; real headless-Chrome run of a + translation (WASM) if feasible. +- [ ] Commit. + +### Task 6: docs + wrap-up +- [ ] D-WS4b entry (models, licences, numbers); Architecture.md (translate worker), AGENTS.md rule + wording (two workers), WorkBreakdown, README if needed, `tests/offline.md` step. +- [ ] typecheck, fast suite, build; `git rm` this plan; commit. From 02a5eab119c61dff775b0f3e0d7a1aa13b992347 Mon Sep 17 00:00:00 2001 From: Lucas Winther Date: Thu, 29 Oct 2026 21:37:28 +0100 Subject: [PATCH 02/10] Offer Whisper's Translate to English only on models trained for it (canTranslate) Co-Authored-By: Claude Opus 5.5 --- docs/Architecture.md | 2 +- docs/Decisions.md | 17 ++++++++++++ docs/WorkBreakdown.md | 2 ++ src/App.tsx | 15 ++++++++-- src/components/TranscriptionOptions.tsx | 32 +++++++++++++-------- src/lib/models.ts | 30 ++++++++++++++++++++ tests/models.test.ts | 37 +++++++++++++++++++++++++ tests/transcriptionOptions.test.tsx | 31 +++++++++++++++++++++ 8 files changed, 151 insertions(+), 15 deletions(-) create mode 100644 tests/transcriptionOptions.test.tsx diff --git a/docs/Architecture.md b/docs/Architecture.md index ea5fc28..ffa963b 100644 --- a/docs/Architecture.md +++ b/docs/Architecture.md @@ -141,7 +141,7 @@ environment). See `WorkBreakdown.md` for the itemized checklist.) `AudioContext.decodeAudioData`, downmixes to mono, resamples to 16 kHz, transfers the `Float32Array` to the worker. For multilingual models the `transcribe` message also carries `language` (or auto-detect) and `task` - (transcribe | translate). + (transcribe | translate — translate only for models with `canTranslate`, D-WS4a). 5. Worker runs the pipeline (chunked, with timestamps), streaming partial results back via `{type:"partial"}` messages; final result as `{type:"result"}`. diff --git a/docs/Decisions.md b/docs/Decisions.md index 5dc8f08..342ad0b 100644 --- a/docs/Decisions.md +++ b/docs/Decisions.md @@ -1224,3 +1224,20 @@ whole-file output is equally bad, which is exactly what the absolute bar exists to catch. **Consequence:** full `bun test` ≈ 19 min on CPU (en_alien ≈ 20 min of it on a cold run); the fast suite excludes this file. + +## D-WS4a — Whisper's "Translate to English" only on models trained for it (`canTranslate`) +**Status:** Accepted (2026-10-29) · spec 001 Goal 4a +`WhisperModel` gains `canTranslate: boolean` (`src/lib/models.ts`): **true** for multilingual +tiny / base / small / medium, **false** for every `.en` model (no task token) and for +`whisper-large-v3-turbo` — OpenAI's model card says turbo was fine-tuned on transcription data only +and is not trained for translation, so its "translate" output is unreliable. Any future turbo-based +fine-tune defaults to false until tested. `TranscriptionOptions` renders the Task control only where +true (language stays). **Stale task:** the `task` state is kept, but `taskForModel(modelId, task)` +maps "translate" to "transcribe" for a model that can't translate; App applies it on model switch +(picker and the server-engine auto-switch to turbo) and again when building every file / live run's +options, so a leftover "translate" can never reach the worker or the server. **Not changed:** the +server's `parseLanguageAndTask` (D-032) still accepts "translate" for any model — the browser never +sends it for turbo now, and a direct API caller asking faster-whisper to translate with turbo gets +what faster-whisper does; rejecting it server-side would be a behaviour change for API users with no +UI benefit. Speech in another language than English can still end up in English (or Danish) through +text translation of the finished transcript (D-WS4b), for every model. diff --git a/docs/WorkBreakdown.md b/docs/WorkBreakdown.md index 336b0b4..2ff0310 100644 --- a/docs/WorkBreakdown.md +++ b/docs/WorkBreakdown.md @@ -65,6 +65,8 @@ Notes (M1): surface active backend in UI - [x] Handle model switching (dispose old pipeline, load new) - [x] Language / task options for multilingual models (transcribe vs translate) +- [x] Whisper translate only on models trained for it (`canTranslate`; hidden for + large-v3-turbo and `.en`, stale "translate" resets to transcribe — D-WS4a) - [~] Verify quantized variants (q4/q8/fp16) on both backends; record working combinations in `models.ts` diff --git a/src/App.tsx b/src/App.tsx index e282021..0c22644 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -37,6 +37,7 @@ import { formatModelSize, getModel, resolveDefaultDtype, + taskForModel, } from "./lib/models"; import { shouldWarnBeforeLoad, type MobileMemoryHint } from "./lib/mobileMemoryGuard"; import { @@ -499,6 +500,9 @@ export function App() { function handleModelSelect(nextModelId: string) { if (busy || liveStatus !== "idle" || nextModelId === modelId) return; setModelId(nextModelId); + // A "Translate to English" chosen on a model that can translate must not + // silently carry over to one that cannot (D-WS4a). + setTask((current) => taskForModel(nextModelId, current)); setActiveDevice(null); setActiveThreads(null); setPartialText(""); @@ -675,7 +679,7 @@ export function App() { // Language/task only apply to multilingual models; `.en` variants // take neither. The browser engine ignores modelId (worker holds it). modelId, - ...(multilingual && { language, task }), + ...(multilingual && { language, task: taskForModel(modelId, task) }), onPartial: (fragment) => { if (fileRun.current === run) setPartialText((text) => text + fragment); }, @@ -752,7 +756,10 @@ export function App() { setLivePending(0); setLiveStatus("starting"); - const options = (getModel(modelId)?.multilingual ?? false) ? { language, task } : {}; + const options = + (getModel(modelId)?.multilingual ?? false) + ? { language, task: taskForModel(modelId, task) } + : {}; if (useServer) { const wsUrl = new URL(`/api/transcribe/live?model=${modelId}`, window.location.href); @@ -988,6 +995,7 @@ export function App() { const allowed = ["onnx-community/whisper-large-v3-turbo", "onnx-community/whisper-medium.en_timestamped"]; if (!allowed.includes(modelId) && !busy && liveStatus === "idle") { setModelId("onnx-community/whisper-large-v3-turbo"); + setTask((current) => taskForModel("onnx-community/whisper-large-v3-turbo", current)); setActiveDevice(null); setActiveThreads(null); setPartialText(""); @@ -1036,8 +1044,9 @@ export function App() { )} diff --git a/src/components/TranscriptionOptions.tsx b/src/components/TranscriptionOptions.tsx index badec28..58b9823 100644 --- a/src/components/TranscriptionOptions.tsx +++ b/src/components/TranscriptionOptions.tsx @@ -6,18 +6,26 @@ interface TranscriptionOptionsProps { onLanguageChange: (language: string) => void; task: WhisperTask; onTaskChange: (task: WhisperTask) => void; + /** + * Whether the model supports Whisper's own speech → English translation + * (`WhisperModel.canTranslate`, D-WS4a). When false the Task control is not + * rendered at all — transcribing is the only thing such a model does. + */ + canTranslate: boolean; disabled?: boolean; } /** * Language + task controls for multilingual models. The parent renders this * only when the selected model is multilingual — `.en` models take neither. + * The task control only appears where the model can translate (D-WS4a). */ export function TranscriptionOptions({ language, onLanguageChange, task, onTaskChange, + canTranslate, disabled = false, }: TranscriptionOptionsProps) { return ( @@ -37,17 +45,19 @@ export function TranscriptionOptions({ ))} - + {canTranslate && ( + + )} ); } diff --git a/src/lib/models.ts b/src/lib/models.ts index ad82c0c..cb3218b 100644 --- a/src/lib/models.ts +++ b/src/lib/models.ts @@ -42,6 +42,15 @@ export interface WhisperModel { displayName: string; /** False for `.en` (English-only) variants. */ multilingual: boolean; + /** + * Whether Whisper's own `task: "translate"` (speech → English text) is + * offered (D-WS4a). False for every `.en` model (no task token) and for + * large-v3-turbo, which OpenAI did not train for translation — and for any + * turbo-based fine-tune not tested for it. The UI hides "Translate to + * English" where false; text translation of a finished transcript + * (D-WS4b) works for every model regardless. + */ + canTranslate: boolean; /** Encoder parameter count, in millions (drives size estimates). */ encoderParamsM: number; /** Decoder parameter count, in millions (drives size estimates). */ @@ -174,6 +183,7 @@ export const MODELS: readonly WhisperModel[] = [ fasterWhisperModel: "tiny.en", displayName: "Whisper Tiny (English)", multilingual: false, + canTranslate: false, encoderParamsM: 8, decoderParamsM: 31, defaults: { @@ -188,6 +198,7 @@ export const MODELS: readonly WhisperModel[] = [ fasterWhisperModel: "tiny", displayName: "Whisper Tiny", multilingual: true, + canTranslate: true, encoderParamsM: 8, decoderParamsM: 31, defaults: { wasm: WASM_Q8, webgpu: WEBGPU_FP32, webnn: WEBNN_FP16 }, @@ -197,6 +208,7 @@ export const MODELS: readonly WhisperModel[] = [ fasterWhisperModel: "base.en", displayName: "Whisper Base (English)", multilingual: false, + canTranslate: false, encoderParamsM: 20, decoderParamsM: 54, defaults: { wasm: WASM_Q8, webgpu: WEBGPU_FP32, webnn: WEBNN_FP16 }, @@ -206,6 +218,7 @@ export const MODELS: readonly WhisperModel[] = [ fasterWhisperModel: "base", displayName: "Whisper Base", multilingual: true, + canTranslate: true, encoderParamsM: 20, decoderParamsM: 54, defaults: { wasm: WASM_Q8, webgpu: WEBGPU_FP32, webnn: WEBNN_FP16 }, @@ -215,6 +228,7 @@ export const MODELS: readonly WhisperModel[] = [ fasterWhisperModel: "small.en", displayName: "Whisper Small (English)", multilingual: false, + canTranslate: false, encoderParamsM: 88, decoderParamsM: 156, defaults: { wasm: WASM_Q8, webgpu: WEBGPU_FP16_Q4, webnn: WEBNN_FP16 }, @@ -224,6 +238,7 @@ export const MODELS: readonly WhisperModel[] = [ fasterWhisperModel: "small", displayName: "Whisper Small", multilingual: true, + canTranslate: true, encoderParamsM: 88, decoderParamsM: 156, defaults: { wasm: WASM_Q8, webgpu: WEBGPU_FP16_Q4, webnn: WEBNN_FP16 }, @@ -237,6 +252,7 @@ export const MODELS: readonly WhisperModel[] = [ fasterWhisperModel: "medium.en", displayName: "Whisper Medium (English)", multilingual: false, + canTranslate: false, encoderParamsM: 306, decoderParamsM: 463, defaults: { wasm: WASM_Q4, webgpu: ACCEL_Q4, webnn: ACCEL_Q4 }, @@ -246,6 +262,7 @@ export const MODELS: readonly WhisperModel[] = [ fasterWhisperModel: "medium", displayName: "Whisper Medium", multilingual: true, + canTranslate: true, encoderParamsM: 306, decoderParamsM: 463, defaults: { wasm: WASM_Q4, webgpu: ACCEL_Q4, webnn: ACCEL_Q4 }, @@ -255,6 +272,9 @@ export const MODELS: readonly WhisperModel[] = [ fasterWhisperModel: "large-v3-turbo", displayName: "Whisper Large v3 Turbo", multilingual: true, + // Not trained for translation (OpenAI's turbo model card); its + // "translate" output is unreliable (D-WS4a). + canTranslate: false, encoderParamsM: 635, decoderParamsM: 174, defaults: { wasm: WASM_Q4, webgpu: WEBGPU_FP16_Q4, webnn: WEBNN_FP16 }, @@ -274,6 +294,16 @@ export function getFasterWhisperModel(id: string): string | undefined { return getModel(id)?.fasterWhisperModel; } +/** + * The task a run on `modelId` should actually use (D-WS4a): a "translate" + * left over from a model that could translate falls back to "transcribe" on + * one that cannot (large-v3-turbo, `.en`, unknown ids). App applies it on + * every model switch and again when building each run's options. + */ +export function taskForModel(modelId: string, task: WhisperTask): WhisperTask { + return task === "translate" && getModel(modelId)?.canTranslate !== true ? "transcribe" : task; +} + /** Default dtype for a model on a given device. Unknown ids fall back to q8. */ export function resolveDefaultDtype( modelId: string, diff --git a/tests/models.test.ts b/tests/models.test.ts index 1ef20c3..a082df6 100644 --- a/tests/models.test.ts +++ b/tests/models.test.ts @@ -10,6 +10,7 @@ import { getFasterWhisperModel, getModel, resolveDefaultDtype, + taskForModel, } from "../src/lib/models"; // Pure catalog data (D-006) — no browser, no network. @@ -209,3 +210,39 @@ describe("WHISPER_LANGUAGES", () => { expect(WHISPER_LANGUAGES.some((entry) => entry.code === "auto")).toBe(false); }); }); + +describe("canTranslate (D-WS4a)", () => { + test("Whisper translate is offered only where OpenAI trained it", () => { + const translating = MODELS.filter((model) => model.canTranslate).map((model) => model.id); + // large-v3-turbo was not trained for translation (OpenAI model card); + // `.en` models are English-only and have no task token. + expect(translating).toEqual([ + "onnx-community/whisper-tiny", + "onnx-community/whisper-base", + "onnx-community/whisper-small", + "onnx-community/whisper-medium-ONNX", + ]); + }); + + test("canTranslate implies multilingual", () => { + for (const model of MODELS) { + if (model.canTranslate) expect(model.multilingual).toBe(true); + } + }); +}); + +describe("taskForModel (D-WS4a)", () => { + test("a stale translate resets to transcribe on a model that cannot translate", () => { + expect(taskForModel("onnx-community/whisper-large-v3-turbo", "translate")).toBe("transcribe"); + expect(taskForModel("onnx-community/whisper-small.en", "translate")).toBe("transcribe"); + }); + + test("translate survives on a model that can translate", () => { + expect(taskForModel("onnx-community/whisper-small", "translate")).toBe("translate"); + expect(taskForModel("onnx-community/whisper-small", "transcribe")).toBe("transcribe"); + }); + + test("unknown ids never translate", () => { + expect(taskForModel("someone/unknown", "translate")).toBe("transcribe"); + }); +}); diff --git a/tests/transcriptionOptions.test.tsx b/tests/transcriptionOptions.test.tsx new file mode 100644 index 0000000..a9ada77 --- /dev/null +++ b/tests/transcriptionOptions.test.tsx @@ -0,0 +1,31 @@ +import { describe, expect, test } from "bun:test"; +import { renderToStaticMarkup } from "react-dom/server"; +import { TranscriptionOptions } from "../src/components/TranscriptionOptions"; + +// D-WS4a: Whisper's own "Translate to English" only where the model can do it. +function render(canTranslate: boolean): string { + return renderToStaticMarkup( + {}} + task="transcribe" + onTaskChange={() => {}} + canTranslate={canTranslate} + />, + ); +} + +describe("TranscriptionOptions", () => { + test("offers Translate to English when the model can translate", () => { + const html = render(true); + expect(html).toContain("Translate to English"); + expect(html).toContain("Task"); + }); + + test("hides the task control entirely when the model cannot translate", () => { + const html = render(false); + expect(html).not.toContain("Translate to English"); + expect(html).not.toContain("Task"); + expect(html).toContain("Language"); + }); +}); From 82ed34c7f9471fbb8699b0ae4c5d5459418ed5bd Mon Sep 17 00:00:00 2001 From: Lucas Winther Date: Thu, 29 Oct 2026 21:39:37 +0100 Subject: [PATCH 03/10] Plan transcript translation units and sentence pieces for Opus-MT Co-Authored-By: Claude Opus 5.5 --- src/lib/errors.ts | 15 ++- src/lib/translation.ts | 273 ++++++++++++++++++++++++++++++++++++++ tests/errors.test.ts | 16 +++ tests/translation.test.ts | 185 ++++++++++++++++++++++++++ 4 files changed, 486 insertions(+), 3 deletions(-) create mode 100644 src/lib/translation.ts create mode 100644 tests/translation.test.ts diff --git a/src/lib/errors.ts b/src/lib/errors.ts index fa6ec30..a0cc89a 100644 --- a/src/lib/errors.ts +++ b/src/lib/errors.ts @@ -16,7 +16,12 @@ * inspect across Web Audio / Transformers.js / onnxruntime-web / fetch. */ -export type ErrorStage = "decode" | "load" | "transcribe" | "worker-crash"; +/** + * `"translate"` covers text translation of a finished transcript (D-WS4b): + * both its one-time model download and the translation pass, which run in + * their own worker — only the suggestions differ from the Whisper stages. + */ +export type ErrorStage = "decode" | "load" | "transcribe" | "worker-crash" | "translate"; export type ErrorKind = "unsupported-file" | "out-of-memory" | "device-lost" | "network" | "generic"; @@ -95,7 +100,9 @@ export function classifyError(raw: unknown, context: ClassifyContext = {}): Clas kind: "out-of-memory", message: trimmed.length > 0 ? message : "The browser ran out of memory and had to stop.", suggestion: - "Try a smaller model (Tiny, Base, or Small) — it needs far less memory to load and run.", + context.stage === "translate" + ? "Close other tabs to free memory, then try translating again." + : "Try a smaller model (Tiny, Base, or Small) — it needs far less memory to load and run.", retryable: false, }; } @@ -117,7 +124,9 @@ export function classifyError(raw: unknown, context: ClassifyContext = {}): Clas suggestion: context.stage === "load" ? "Check your connection, then retry — the model download will restart." - : "Check your connection and try again.", + : context.stage === "translate" + ? "Check your connection, then retry — the translation model download will restart." + : "Check your connection and try again.", retryable: true, }; } diff --git a/src/lib/translation.ts b/src/lib/translation.ts new file mode 100644 index 0000000..eecd042 --- /dev/null +++ b/src/lib/translation.ts @@ -0,0 +1,273 @@ +/** + * Text translation of a finished transcript (D-WS4b) — the pure half. + * + * Opus-MT (MarianMT) runs in `src/worker/translate.worker.ts`; everything + * here is planning that needs no model: which checkpoint serves a language + * pair, how a transcript becomes translation units that keep their + * timestamps, and how a unit is cut into sentence pieces under the model's + * input budget. No browser globals, so `bun test` covers it. + * + * Why per-sentence pieces: Opus-MT is trained on sentence pairs and, given + * several sentences at once, routinely drops the later ones (measured + * 2026-10-29: "…I live in Copenhagen. How are you?" came back as one + * sentence). Its encoder also tops out at 512 positions. + */ + +import type { TranscriptChunk } from "../worker/protocol"; + +export type TranslationLanguage = "da" | "en"; + +export interface TranslationModel { + /** Hugging Face Hub id of the Transformers.js ONNX export. */ + id: string; + source: TranslationLanguage; + target: TranslationLanguage; + /** Direction as shown in the picker, e.g. "Danish → English". */ + label: string; + /** Plain name of the target language, e.g. "English". */ + targetLabel: string; + /** + * One-time download at the q8 dtype the worker loads: encoder + merged + * decoder + tokenizer/configs, summed from the Hub file sizes. + */ + approxBytes: number; +} + +/** + * Xenova's ONNX exports of Helsinki-NLP's Opus-MT checkpoints (Apache-2.0 + * upstream). Verified to load and translate under Transformers.js 4.2.0. + */ +export const TRANSLATION_MODELS: readonly TranslationModel[] = [ + { + id: "Xenova/opus-mt-da-en", + source: "da", + target: "en", + label: "Danish → English", + targetLabel: "English", + approxBytes: 113_000_000, + }, + { + id: "Xenova/opus-mt-en-da", + source: "en", + target: "da", + label: "English → Danish", + targetLabel: "Danish", + approxBytes: 113_000_000, + }, +]; + +export const TRANSLATION_MODEL_IDS: readonly string[] = TRANSLATION_MODELS.map((m) => m.id); + +export function getTranslationModel( + source: TranslationLanguage, + target: TranslationLanguage, +): TranslationModel | undefined { + return TRANSLATION_MODELS.find((m) => m.source === source && m.target === target); +} + +/** Lookup by Hub id (storage panel names, worker request validation). */ +export function getTranslationModelById(id: string): TranslationModel | undefined { + return TRANSLATION_MODELS.find((m) => m.id === id); +} + +// --------------------------------------------------------------------------- +// Units: transcript → sentence-ending groups of chunks with their time span +// --------------------------------------------------------------------------- + +/** A stretch of transcript translated as one cue. */ +export interface TranslationUnit { + text: string; + /** Merged [start, end] of the source chunks; null for untimed text. */ + timestamp: [number, number | null] | null; +} + +const SENTENCE_END = /[.!?…]["'”’)\]]*$/; + +/** Default cap on chunks merged into one unit, so an unpunctuated transcript + * still yields subtitle-sized cues rather than one giant one. */ +export const MAX_CHUNKS_PER_UNIT = 8; + +/** + * Group a transcript into translation units. Whisper's segments often end + * mid-sentence; translating fragments hurts quality, so consecutive chunks + * are merged until one ends a sentence (or `maxChunksPerUnit` is reached) + * and the unit takes the merged time span — `.srt` keeps working, with + * slightly longer cues. Without chunks (nothing timestamped), each non-empty + * line is an untimed unit. + */ +export function buildTranslationUnits( + result: { text: string; chunks: TranscriptChunk[] }, + maxChunksPerUnit: number = MAX_CHUNKS_PER_UNIT, +): TranslationUnit[] { + const chunks = result.chunks.filter((chunk) => chunk.text.trim() !== ""); + if (chunks.length === 0) { + return result.text + .split(/\n+/) + .map((line) => line.trim()) + .filter((line) => line !== "") + .map((text) => ({ text, timestamp: null })); + } + const units: TranslationUnit[] = []; + let group: TranscriptChunk[] = []; + const flush = () => { + const first = group[0]; + const last = group[group.length - 1]; + if (!first || !last) return; + units.push({ + text: group.map((chunk) => chunk.text.trim()).join(" "), + timestamp: [first.timestamp[0], last.timestamp[1]], + }); + group = []; + }; + for (const chunk of chunks) { + group.push(chunk); + if (SENTENCE_END.test(chunk.text.trim()) || group.length >= maxChunksPerUnit) flush(); + } + flush(); + return units; +} + +// --------------------------------------------------------------------------- +// Pieces: unit text → sentences under the model's token budget +// --------------------------------------------------------------------------- + +/** + * Cheap upper-bound token estimate for Opus-MT's SentencePiece vocab, used + * when the real tokenizer isn't at hand (tests, planning). Measured ~0.34 + * tokens/char on en_alien and ~0.26 on da_festuge with the real tokenizers; + * 0.5/char plus the end-of-sentence token stays above both. + */ +export function estimateTokens(text: string): number { + return text.length === 0 ? 0 : Math.ceil(text.length / 2) + 1; +} + +/** + * Default per-piece budget, well inside Opus-MT's 512 positions: quality + * falls off on very long inputs long before the hard limit, and normal + * sentences are 10–60 tokens, so this only bites on unpunctuated run-ons. + */ +export const DEFAULT_MAX_PIECE_TOKENS = 200; + +export interface SplitOptions { + maxTokens?: number; + /** Token counter — the worker passes the real tokenizer. */ + countTokens?: (text: string) => number; +} + +/** "Mr.", "Dr.", "St." — sentence-segmenter false breaks. */ +const ABBREVIATION = /^\p{Lu}\p{Ll}{0,3}\.$/u; + +function sentences(text: string): string[] { + const out: string[] = []; + let carry = ""; + for (const { segment } of new Intl.Segmenter("en", { granularity: "sentence" }).segment(text)) { + const trimmed = segment.trim(); + if (trimmed === "") continue; + if (ABBREVIATION.test(trimmed)) { + carry = carry === "" ? trimmed : `${carry} ${trimmed}`; + continue; + } + out.push(carry === "" ? trimmed : `${carry} ${trimmed}`); + carry = ""; + } + if (carry !== "") out.push(carry); + return out; +} + +/** Greedily pack `parts` (in order) into space-joined pieces under budget. */ +function pack(parts: string[], maxTokens: number, count: (text: string) => number): string[] { + const pieces: string[] = []; + let current = ""; + for (const part of parts) { + const candidate = current === "" ? part : `${current} ${part}`; + if (current !== "" && count(candidate) > maxTokens) { + pieces.push(current); + current = part; + } else { + current = candidate; + } + } + if (current !== "") pieces.push(current); + return pieces; +} + +/** Split one over-budget sentence: commas first, then words. A single word + * longer than the budget is kept whole (nothing is ever dropped). */ +function wrap(sentence: string, maxTokens: number, count: (text: string) => number): string[] { + const clauses = sentence.split(/(?<=,)\s+/); + const parts = clauses.flatMap((clause) => + count(clause) <= maxTokens ? [clause] : pack(clause.split(/\s+/), maxTokens, count), + ); + return pack(parts, maxTokens, count); +} + +/** + * Cut a unit's text into the pieces actually fed to Opus-MT: one per + * sentence, an over-budget sentence wrapped on commas then words. Order is + * preserved and every word survives — joining the pieces with spaces gives + * the input back up to whitespace. + */ +export function splitForTranslation(text: string, options: SplitOptions = {}): string[] { + const maxTokens = options.maxTokens ?? DEFAULT_MAX_PIECE_TOKENS; + const count = options.countTokens ?? estimateTokens; + return sentences(text).flatMap((sentence) => + count(sentence) <= maxTokens ? [sentence] : wrap(sentence, maxTokens, count), + ); +} + +// --------------------------------------------------------------------------- +// Assembly +// --------------------------------------------------------------------------- + +/** + * Pair translated unit texts back with their units' time spans, in the + * `{ text, chunks }` shape the transcript view and exports already take. + */ +export function assembleTranslation( + units: readonly TranslationUnit[], + translated: readonly string[], +): { text: string; chunks: TranscriptChunk[] } { + if (units.length !== translated.length) { + throw new Error( + `Translation returned ${translated.length} parts for ${units.length} — refusing to misalign the transcript.`, + ); + } + const texts = translated.map((text) => text.trim()); + const chunks: TranscriptChunk[] = []; + units.forEach((unit, index) => { + if (unit.timestamp) chunks.push({ text: texts[index] ?? "", timestamp: unit.timestamp }); + }); + const untimed = units.every((unit) => unit.timestamp === null); + return { text: texts.join(untimed ? "\n" : " "), chunks }; +} + +// --------------------------------------------------------------------------- +// Source language guess (default for the direction picker) +// --------------------------------------------------------------------------- + +const DANISH_WORDS = new Set([ + "og", "ikke", "det", "jeg", "vi", "på", "til", "med", "som", "har", "af", "der", + "kan", "skal", "vil", "også", "men", "hvad", "hvordan", "være", "noget", +]); +const ENGLISH_WORDS = new Set([ + "the", "and", "is", "to", "of", "that", "you", "it", "with", "have", "this", "not", + "what", "how", "be", "was", "are", +]); + +/** + * Best guess at which language a transcript is in, for the direction + * picker's default (the user can always change it). An explicit Whisper + * language wins; otherwise Danish letters and function words vs English + * ones decide, defaulting to English. + */ +export function guessSourceLanguage(text: string, whisperLanguage?: string): TranslationLanguage { + if (whisperLanguage === "da" || whisperLanguage === "en") return whisperLanguage; + const lower = text.toLowerCase(); + let danish = (lower.match(/[æøå]/g)?.length ?? 0) * 2; + let english = 0; + for (const word of lower.match(/\p{L}+/gu) ?? []) { + if (DANISH_WORDS.has(word)) danish += 1; + if (ENGLISH_WORDS.has(word)) english += 1; + } + return danish > english ? "da" : "en"; +} diff --git a/tests/errors.test.ts b/tests/errors.test.ts index 8e38b46..9fe8a52 100644 --- a/tests/errors.test.ts +++ b/tests/errors.test.ts @@ -98,3 +98,19 @@ describe("classifyError", () => { expect(classifyError(42).message).toBe("42"); }); }); + +describe("classifyError — transcript translation (D-WS4b)", () => { + test("out of memory while translating suggests closing tabs, not a smaller Whisper model", () => { + const classified = classifyError("RangeError: Out of memory", { stage: "translate" }); + expect(classified.kind).toBe("out-of-memory"); + expect(classified.suggestion).not.toContain("smaller model"); + expect(classified.suggestion).toContain("other tabs"); + }); + + test("a failed translation-model download is retryable and says the download restarts", () => { + const classified = classifyError("Failed to fetch", { stage: "translate" }); + expect(classified.kind).toBe("network"); + expect(classified.retryable).toBe(true); + expect(classified.suggestion).toContain("translation model"); + }); +}); diff --git a/tests/translation.test.ts b/tests/translation.test.ts new file mode 100644 index 0000000..dd96443 --- /dev/null +++ b/tests/translation.test.ts @@ -0,0 +1,185 @@ +import { describe, expect, test } from "bun:test"; +import { join } from "node:path"; +import { + TRANSLATION_MODELS, + assembleTranslation, + buildTranslationUnits, + estimateTokens, + getTranslationModel, + guessSourceLanguage, + splitForTranslation, +} from "../src/lib/translation"; +import type { TranscriptChunk } from "../src/worker/protocol"; + +// Pure planning for transcript translation (D-WS4b) — no model, no network. + +const words = (text: string) => text.split(/\s+/).filter(Boolean); + +describe("TRANSLATION_MODELS", () => { + test("offers da→en and en→da Opus-MT ONNX exports", () => { + expect(TRANSLATION_MODELS.map((m) => m.id)).toEqual([ + "Xenova/opus-mt-da-en", + "Xenova/opus-mt-en-da", + ]); + expect(getTranslationModel("da", "en")?.id).toBe("Xenova/opus-mt-da-en"); + expect(getTranslationModel("en", "da")?.id).toBe("Xenova/opus-mt-en-da"); + for (const model of TRANSLATION_MODELS) { + // q8 encoder + merged decoder + tokenizer, measured on the Hub. + expect(model.approxBytes).toBeGreaterThan(100e6); + expect(model.approxBytes).toBeLessThan(130e6); + } + }); +}); + +describe("buildTranslationUnits", () => { + test("merges chunks until a sentence ends, keeping the merged time span", () => { + const chunks: TranscriptChunk[] = [ + { text: " So the plan is", timestamp: [0, 2] }, + { text: " to ship on Friday.", timestamp: [2, 4.5] }, + { text: " Any questions?", timestamp: [5, 6] }, + ]; + expect(buildTranslationUnits({ text: "", chunks })).toEqual([ + { text: "So the plan is to ship on Friday.", timestamp: [0, 4.5] }, + { text: "Any questions?", timestamp: [5, 6] }, + ]); + }); + + test("an unterminated tail still becomes a unit, with its null end kept", () => { + const chunks: TranscriptChunk[] = [ + { text: " Done.", timestamp: [0, 1] }, + { text: " and then", timestamp: [1, null] }, + ]; + expect(buildTranslationUnits({ text: "", chunks })).toEqual([ + { text: "Done.", timestamp: [0, 1] }, + { text: "and then", timestamp: [1, null] }, + ]); + }); + + test("unpunctuated chunks are capped so cues stay subtitle-sized", () => { + const chunks: TranscriptChunk[] = Array.from({ length: 20 }, (_, i) => ({ + text: ` word${i}`, + timestamp: [i, i + 1] as [number, number], + })); + const units = buildTranslationUnits({ text: "", chunks }, 8); + expect(units.map((u) => u.timestamp)).toEqual([ + [0, 8], + [8, 16], + [16, 20], + ]); + }); + + test("empty chunks are skipped", () => { + const chunks: TranscriptChunk[] = [ + { text: " ", timestamp: [0, 1] }, + { text: " Hi.", timestamp: [1, 2] }, + ]; + expect(buildTranslationUnits({ text: "", chunks })).toEqual([ + { text: "Hi.", timestamp: [1, 2] }, + ]); + }); + + test("without chunks, each non-empty line is an untimed unit", () => { + expect(buildTranslationUnits({ text: "Hej.\n\nHvordan går det?\n", chunks: [] })).toEqual([ + { text: "Hej.", timestamp: null }, + { text: "Hvordan går det?", timestamp: null }, + ]); + }); +}); + +describe("splitForTranslation", () => { + test("every sentence is its own piece (Opus-MT drops trailing sentences otherwise)", () => { + expect( + splitForTranslation("My name is Lucas and I live in Copenhagen. How are you? Fine!"), + ).toEqual(["My name is Lucas and I live in Copenhagen.", "How are you?", "Fine!"]); + }); + + test("abbreviation-only fragments stay with the following sentence", () => { + expect(splitForTranslation("Mr. Smith went home. He slept.")).toEqual([ + "Mr. Smith went home.", + "He slept.", + ]); + }); + + test("blank input yields no pieces", () => { + expect(splitForTranslation(" ")).toEqual([]); + }); + + test("an unpunctuated run-on is wrapped under the budget, words intact and in order", () => { + const runOn = Array.from({ length: 300 }, (_, i) => `ord${i}`).join(" "); + const pieces = splitForTranslation(runOn, { maxTokens: 40, countTokens: (s) => words(s).length }); + expect(pieces.length).toBeGreaterThan(1); + for (const piece of pieces) expect(words(piece).length).toBeLessThanOrEqual(40); + expect(pieces.join(" ")).toBe(runOn); + }); + + test("a long sentence prefers comma boundaries before word wrapping", () => { + const clause = "one two three four five six"; + const sentence = `${clause}, ${clause}, ${clause}, ${clause}.`; + const pieces = splitForTranslation(sentence, { maxTokens: 14, countTokens: (s) => words(s).length }); + expect(pieces).toEqual([`${clause}, ${clause},`, `${clause}, ${clause}.`]); + }); + + test("a transcript far beyond Opus-MT's 512-token input is split with nothing dropped, in order (Review Focus 5)", async () => { + const alien = await Bun.file(join(import.meta.dir, "audio_files/en_alien.txt")).text(); + expect(estimateTokens(alien)).toBeGreaterThan(512); + const pieces = splitForTranslation(alien, { maxTokens: 200 }); + for (const piece of pieces) expect(estimateTokens(piece)).toBeLessThanOrEqual(200); + expect(words(pieces.join(" "))).toEqual(words(alien)); + }); +}); + +describe("estimateTokens", () => { + test("over-estimates rather than under-estimates SentencePiece counts", () => { + // Measured with the Xenova/opus-mt-en-da tokenizer: this sentence is 12 tokens (incl. ). + expect(estimateTokens("My name is Lucas and I live in Copenhagen.")).toBeGreaterThanOrEqual(12); + expect(estimateTokens("")).toBe(0); + }); +}); + +describe("assembleTranslation", () => { + test("pairs translated units with their time spans", () => { + const units = [ + { text: "Hej.", timestamp: [0, 1] as [number, number] }, + { text: "Farvel.", timestamp: [2, null] as [number, null] }, + ]; + expect(assembleTranslation(units, [" Hello. ", "Goodbye."])).toEqual({ + text: "Hello. Goodbye.", + chunks: [ + { text: "Hello.", timestamp: [0, 1] }, + { text: "Goodbye.", timestamp: [2, null] }, + ], + }); + }); + + test("untimed units join as lines and produce no chunks", () => { + const units = [ + { text: "Hej.", timestamp: null }, + { text: "Farvel.", timestamp: null }, + ]; + expect(assembleTranslation(units, ["Hello.", "Goodbye."])).toEqual({ + text: "Hello.\nGoodbye.", + chunks: [], + }); + }); + + test("a count mismatch is an error, never a silently shifted transcript", () => { + expect(() => assembleTranslation([{ text: "a", timestamp: null }], [])).toThrow(); + }); +}); + +describe("guessSourceLanguage", () => { + test("trusts an explicit Whisper language", () => { + expect(guessSourceLanguage("whatever", "da")).toBe("da"); + expect(guessSourceLanguage("whatever", "en")).toBe("en"); + }); + + test("detects Danish from its letters and common words", () => { + expect(guessSourceLanguage("På Aarhus Festuges sekretariat har vi løbet rigtig stærkt.")).toBe("da"); + expect(guessSourceLanguage("Det er ikke noget med at vi skal")).toBe("da"); + }); + + test("defaults to English", () => { + expect(guessSourceLanguage("Guys, I have been addicted to this game.")).toBe("en"); + expect(guessSourceLanguage("", "auto")).toBe("en"); + }); +}); From a0748b5fa6851d4df61c23768871085cd67a0b99 Mon Sep 17 00:00:00 2001 From: Lucas Winther Date: Thu, 29 Oct 2026 22:10:51 +0100 Subject: [PATCH 04/10] Translate transcripts with Opus-MT in a dedicated translation worker Co-Authored-By: Claude Opus 5.5 --- scripts/bench-translate.ts | 53 +++++++++++++++ scripts/build.ts | 15 +++- src/index.ts | 10 ++- src/lib/translation.ts | 63 ++++++++++++++++- src/worker/opusMt.ts | 59 ++++++++++++++++ src/worker/translate.worker.ts | 98 ++++++++++++++++++++++++++ src/worker/translateProtocol.ts | 106 +++++++++++++++++++++++++++++ tests/translate-real-model.test.ts | 97 ++++++++++++++++++++++++++ tests/translateProtocol.test.ts | 49 +++++++++++++ tests/translation.test.ts | 70 +++++++++++++++++++ 10 files changed, 615 insertions(+), 5 deletions(-) create mode 100644 scripts/bench-translate.ts create mode 100644 src/worker/opusMt.ts create mode 100644 src/worker/translate.worker.ts create mode 100644 src/worker/translateProtocol.ts create mode 100644 tests/translate-real-model.test.ts create mode 100644 tests/translateProtocol.test.ts diff --git a/scripts/bench-translate.ts b/scripts/bench-translate.ts new file mode 100644 index 0000000..12f8ac6 --- /dev/null +++ b/scripts/bench-translate.ts @@ -0,0 +1,53 @@ +/** + * Transcript-translation speed on onnxruntime-node (CPU), D-WS4b. + * + * flock /tmp/whisper-heavy.lock bun scripts/bench-translate.ts [batchSizes…] + * + * Translates the reference transcripts in tests/audio_files/ with the same + * Opus-MT loader and translation loop the browser worker uses, once per + * batch size, and prints wall-clock time, pieces, and whether the output + * depends on the batch size (padding). Downloads ~113 MB per direction into + * the Transformers.js cache on first run. Not browser numbers: WASM in a + * tab is slower than onnxruntime-node. + */ + +import { join } from "node:path"; +import { buildTranslationUnits, getTranslationModel, translateUnits } from "../src/lib/translation"; +import { loadOpusMt } from "../src/worker/opusMt"; + +const AUDIO = join(import.meta.dir, "../tests/audio_files"); +const batchSizes = process.argv.slice(2).map(Number).filter((n) => n > 0); +const sizes = batchSizes.length > 0 ? batchSizes : [1, 8]; + +for (const [file, source, target] of [ + ["en_alien.txt", "en", "da"], + ["da_festuge.txt", "da", "en"], +] as const) { + const model = getTranslationModel(source, target)!; + const text = await Bun.file(join(AUDIO, file)).text(); + const units = buildTranslationUnits({ text, chunks: [] }).map((unit) => unit.text); + const loadStart = performance.now(); + const translator = await loadOpusMt(model.id); + const loadMs = performance.now() - loadStart; + console.log(`\n${file} (${text.length} chars, ${units.length} lines) ${model.label} — load ${loadMs.toFixed(0)} ms`); + let baseline: string[] | null = null; + for (const batchSize of sizes) { + let pieces = 0; + const start = performance.now(); + const out = await translateUnits(units, { + batchSize, + countTokens: translator.countTokens, + translateBatch: (batch) => translator.translateBatch(batch), + onProgress: (_done, total) => (pieces = total), + }); + const seconds = (performance.now() - start) / 1000; + const same = baseline === null ? "baseline" : JSON.stringify(out) === JSON.stringify(baseline) ? "identical" : "differs"; + baseline ??= out; + console.log( + ` batch ${String(batchSize).padStart(2)}: ${seconds.toFixed(2)} s, ${pieces} pieces, ` + + `${(text.length / seconds).toFixed(0)} chars/s, output ${same}`, + ); + } + console.log(` sample: ${baseline!.slice(0, 3).join(" | ")}`); + await translator.dispose(); +} diff --git a/scripts/build.ts b/scripts/build.ts index 702ed38..0bd50ef 100644 --- a/scripts/build.ts +++ b/scripts/build.ts @@ -3,7 +3,8 @@ * * Emits a self-contained `dist/`: * 1. `bun build src/index.html` → hashed app-shell assets - * 2. inference worker bundle at the stable /inference.worker.js URL + * 2. inference + translate worker bundles at stable /inference.worker.js and + * /translate.worker.js URLs * 3. manifest + icons + onnxruntime .wasm/.mjs (dist/ort/, offline inference) * 4. sw.js with the precache manifest + a content-hash version injected * @@ -53,6 +54,15 @@ const workerBuild = await Bun.build({ }); const workerScript = await workerBuild.outputs[0]!.text(); await Bun.write(join(DIST, "inference.worker.js"), workerScript); +// Transcript-translation worker (D-WS4b), same mechanism. +const translateWorkerBuild = await Bun.build({ + entrypoints: [join(ROOT, "src/worker/translate.worker.ts")], + target: "browser", + format: "esm", + minify: true, +}); +const translateWorkerScript = await translateWorkerBuild.outputs[0]!.text(); +await Bun.write(join(DIST, "translate.worker.js"), translateWorkerScript); // 3. Static PWA assets + onnxruntime wasm binaries (offline inference — the // worker loads them from /ort/ instead of the jsdelivr CDN). The manifest and @@ -89,6 +99,7 @@ const precachePaths = precachePathsFromOutputs( shellBuild.outputs.map((output) => relative(DIST, output.path)), [ "inference.worker.js", + "translate.worker.js", "manifest.webmanifest", ...ICON_SIZES.map((size) => `icons/icon-${size}.png`), ...fontFiles.map((name) => `fonts/${name}`), @@ -98,7 +109,7 @@ const fontBytes = await Promise.all( fontFiles.map((name) => Bun.file(join(FONTS_DIR, name)).bytes()), ); const version = Bun.hash( - JSON.stringify(precachePaths) + swSource + workerScript + fontBytes.map((b) => Bun.hash(b)).join(), + JSON.stringify(precachePaths) + swSource + workerScript + translateWorkerScript + fontBytes.map((b) => Bun.hash(b)).join(), ).toString(36); await Bun.write(join(DIST, "sw.js"), injectPrecacheManifest(swSource, precachePaths, version)); diff --git a/src/index.ts b/src/index.ts index 328d7f3..e657557 100644 --- a/src/index.ts +++ b/src/index.ts @@ -223,7 +223,8 @@ async function productionServeOptions(): Promise { * Bun's bundler does not currently follow `new Worker(...)` graphs (verified * with Bun 1.3: both `bun build` and the dev server leave the call untouched), * so worker-like entrypoints are bundled at startup and served from stable - * URLs. The UI spawns `new Worker("/inference.worker.js")`; the service + * URLs. The UI spawns `new Worker("/inference.worker.js")` (and + * "/translate.worker.js" for transcript translation, D-WS4b); the service * worker registers as "/sw.js". */ async function bundleEntrypoint(relativePath: string): Promise { @@ -249,6 +250,8 @@ async function developmentServeOptions(): Promise { const { default: index } = await import("./index.html"); const workerScript = await bundleEntrypoint("./worker/inference.worker.ts"); + // Transcript translation runs in its own worker (D-WS4b). + const translateWorkerScript = await bundleEntrypoint("./worker/translate.worker.ts"); // Dev SW: empty precache list — assets are unhashed here, so the shell is // runtime-cached network-first instead (see src/pwa/sw.ts). const swScript = injectPrecacheManifest( @@ -283,6 +286,11 @@ async function developmentServeOptions(): Promise { if (pathname.endsWith("/inference.worker.js")) { return withIsolationHeaders(new Response(workerScript, { headers: scriptHeaders })); } + if (pathname.endsWith("/translate.worker.js")) { + return withIsolationHeaders( + new Response(translateWorkerScript, { headers: scriptHeaders }), + ); + } if (pathname.endsWith("/sw.js")) { return withIsolationHeaders( new Response(swScript, { headers: { ...scriptHeaders, "Cache-Control": "no-cache" } }), diff --git a/src/lib/translation.ts b/src/lib/translation.ts index eecd042..b568734 100644 --- a/src/lib/translation.ts +++ b/src/lib/translation.ts @@ -154,8 +154,13 @@ export interface SplitOptions { countTokens?: (text: string) => number; } -/** "Mr.", "Dr.", "St." — sentence-segmenter false breaks. */ -const ABBREVIATION = /^\p{Lu}\p{Ll}{0,3}\.$/u; +/** + * Title abbreviations (en + da) and single-letter initials ("J.") — the + * sentence segmenter's false breaks. Kept to an explicit list: a broad + * "short capitalised word" rule would glue real one-word sentences + * ("Yes.", "Okay.") onto the next one. + */ +const ABBREVIATION = /^(?:Mr|Mrs|Ms|Dr|Prof|St|Jr|Sr|Hr|Fru|Nr|\p{Lu})\.$/u; function sentences(text: string): string[] { const out: string[] = []; @@ -215,6 +220,60 @@ export function splitForTranslation(text: string, options: SplitOptions = {}): s ); } +// --------------------------------------------------------------------------- +// The translation loop (model injected: worker, benchmark and tests share it) +// --------------------------------------------------------------------------- + +export interface TranslateUnitsDeps extends SplitOptions { + /** Translate a batch of pieces; must answer one string per piece, in order. */ + translateBatch: (pieces: string[]) => Promise; + /** Pieces per model call (padding makes larger batches cheaper per piece). */ + batchSize?: number; + /** Called with (done, total) pieces: once at 0, then after every batch. */ + onProgress?: (done: number, total: number) => void; +} + +/** + * One piece per model call. Batching would be ~4x faster, but Transformers.js + * 4.2's batched Marian generation is broken: every row of a padded batch + * comes back as row 0's translation (measured 2026-10-29 on + * onnxruntime-node, q8 and fp32, any graph optimisation level — D-WS4b). + * Re-measure with `scripts/bench-translate.ts` after a Transformers.js + * upgrade before raising this. + */ +export const DEFAULT_TRANSLATE_BATCH_SIZE = 1; + +/** + * Translate unit texts: split each into sentence pieces, translate all + * pieces in order-preserving batches, rejoin each unit's pieces with spaces. + * Returns one translated string per input unit. + */ +export async function translateUnits( + units: readonly string[], + deps: TranslateUnitsDeps, +): Promise { + const batchSize = Math.max(1, deps.batchSize ?? DEFAULT_TRANSLATE_BATCH_SIZE); + const perUnit = units.map((unit) => splitForTranslation(unit, deps)); + const pieces = perUnit.flat(); + const translated: string[] = []; + deps.onProgress?.(0, pieces.length); + for (let start = 0; start < pieces.length; start += batchSize) { + const batch = pieces.slice(start, start + batchSize); + const out = await deps.translateBatch(batch); + if (out.length !== batch.length) { + throw new Error(`Translation returned ${out.length} sentences for ${batch.length}.`); + } + translated.push(...out.map((text) => text.trim())); + deps.onProgress?.(translated.length, pieces.length); + } + let next = 0; + return perUnit.map((unitPieces) => { + const joined = translated.slice(next, next + unitPieces.length).join(" "); + next += unitPieces.length; + return joined; + }); +} + // --------------------------------------------------------------------------- // Assembly // --------------------------------------------------------------------------- diff --git a/src/worker/opusMt.ts b/src/worker/opusMt.ts new file mode 100644 index 0000000..8bbee00 --- /dev/null +++ b/src/worker/opusMt.ts @@ -0,0 +1,59 @@ +/** + * Opus-MT (MarianMT) through Transformers.js — the model half of transcript + * translation (D-WS4b). Shared by the browser translate worker and the + * onnxruntime-node measurement script / real-model test, so all three run + * the exact same pipeline options. + */ + +import { pipeline, type ProgressInfo, type TranslationPipeline } from "@huggingface/transformers"; + +/** Hard input limit of every Opus-MT checkpoint (max_position_embeddings). */ +export const OPUS_MT_MAX_POSITIONS = 512; + +export interface OpusMtTranslator { + /** One translated string per piece, same order. */ + translateBatch(pieces: string[]): Promise; + /** Real SentencePiece token count (incl. ) for the splitter's budget. */ + countTokens(text: string): number; + dispose(): Promise; +} + +export interface LoadOpusMtOptions { + /** "wasm" in the browser; omit under onnxruntime-node (defaults to CPU). */ + device?: "wasm"; + progressCallback?: (info: ProgressInfo) => void; +} + +export async function loadOpusMt( + modelId: string, + options: LoadOpusMtOptions = {}, +): Promise { + const translator = (await pipeline("translation", modelId, { + // q8: 113 MB per direction (encoder + merged decoder + tokenizer); the + // fp32 export is ~4x larger for no audible gain on transcripts. + dtype: "q8", + ...(options.device && { device: options.device }), + // D-014: the bundled onnxruntime-web's level-2 QDQ transform rejects + // 8-bit decoders with a tied embedding weight — Marian ties them too. + session_options: { graphOptimizationLevel: "basic" as const }, + ...(options.progressCallback && { progress_callback: options.progressCallback }), + })) as TranslationPipeline; + + const countTokens = (text: string) => translator.tokenizer.encode(text).length; + + return { + countTokens, + async translateBatch(pieces) { + if (pieces.length === 0) return []; + // Cap output length relative to the longest input: a translation is + // rarely more than ~2x its source in tokens, and the cap stops a rare + // repetition loop from running to 512 tokens. + const longest = Math.max(...pieces.map(countTokens)); + const output = (await translator(pieces, { + max_new_tokens: Math.min(OPUS_MT_MAX_POSITIONS, 3 * longest + 16), + })) as { translation_text: string }[]; + return output.map((entry) => entry.translation_text); + }, + dispose: () => translator.dispose(), + }; +} diff --git a/src/worker/translate.worker.ts b/src/worker/translate.worker.ts new file mode 100644 index 0000000..6e0ba4e --- /dev/null +++ b/src/worker/translate.worker.ts @@ -0,0 +1,98 @@ +/** + * Transcript-translation Web Worker (D-WS4b): Opus-MT through + * Transformers.js, off the main thread (D-003). + * + * A dedicated worker rather than a message type on the Whisper worker: + * - its own onnxruntime instance gets the full WASM thread pool. ort fixes + * the pool at first init (D-023), and a Whisper worker that initialised + * ort for WebGPU/WebNN runs WASM single-threaded for life; + * - cancelling a translation terminates only this worker — the loaded + * Whisper pipeline survives, and a Whisper cancel/restart (D-018/D-025) + * never kills a translation; + * - the model is downloaded only when the user asks for a translation and + * cached by Transformers.js in the Cache API like Whisper weights + * (D-005/D-020), so a cached model works offline. + * WASM only for now: Opus-MT is small, and WebGPU behaviour can't be + * measured in this environment. + */ + +import { env, type ProgressInfo } from "@huggingface/transformers"; +import { getTranslationModelById, translateUnits } from "../lib/translation"; +import { loadOpusMt, type OpusMtTranslator } from "./opusMt"; +import { + DownloadTally, + isTranslateRequest, + type TranslateRequest, + type TranslateWorkerMessage, +} from "./translateProtocol"; + +// Same-origin onnxruntime binaries (offline, COEP) — see inference.worker.ts. +if (env.backends.onnx.wasm) { + env.backends.onnx.wasm.wasmPaths = "./ort/"; + // D-023: with cross-origin isolation, use most of the machine. + env.backends.onnx.wasm.numThreads = globalThis.crossOriginIsolated + ? Math.max(1, Math.min((navigator.hardwareConcurrency || 4) - 1, 12)) + : 1; +} + +interface DedicatedWorkerScope { + postMessage(message: TranslateWorkerMessage): void; + addEventListener(type: "message", listener: (event: MessageEvent) => void): void; +} +const scope = self as unknown as DedicatedWorkerScope; + +/** One loaded model at a time; switching direction disposes the other. */ +let active: { modelId: string; promise: Promise } | null = null; + +function load(modelId: string, onProgress: (info: ProgressInfo) => void): Promise { + if (active?.modelId === modelId) return active.promise; + const previous = active; + const promise = (async () => { + if (previous) await (await previous.promise.catch(() => null))?.dispose(); + return loadOpusMt(modelId, { device: "wasm", progressCallback: onProgress }); + })(); + active = { modelId, promise }; + promise.catch(() => { + if (active?.promise === promise) active = null; // allow a clean retry + }); + return promise; +} + +async function handle(request: TranslateRequest): Promise { + const { requestId } = request; + if (!getTranslationModelById(request.modelId)) { + throw new Error(`Unknown translation model: ${request.modelId}`); + } + const tally = new DownloadTally(); + const translator = await load(request.modelId, (info) => { + if (info.status === "progress") { + const { loaded, total } = tally.update(info.file, info.loaded, info.total); + scope.postMessage({ type: "translate-download", requestId, loaded, total }); + } + }); + const texts = await translateUnits(request.units, { + translateBatch: (pieces) => translator.translateBatch(pieces), + countTokens: translator.countTokens, + onProgress: (done, total) => + scope.postMessage({ type: "translate-progress", requestId, done, total }), + }); + scope.postMessage({ type: "translate-result", requestId, texts }); +} + +// Requests run one at a time, in arrival order: the UI only ever has one in +// flight, and serialising keeps two passes from sharing the thread pool. +let queue: Promise = Promise.resolve(); + +scope.addEventListener("message", (event) => { + const data = event.data; + if (!isTranslateRequest(data)) return; + queue = queue.then(() => + handle(data).catch((error: unknown) => { + scope.postMessage({ + type: "translate-error", + requestId: data.requestId, + message: error instanceof Error ? error.message : String(error), + }); + }), + ); +}); diff --git a/src/worker/translateProtocol.ts b/src/worker/translateProtocol.ts new file mode 100644 index 0000000..e26c105 --- /dev/null +++ b/src/worker/translateProtocol.ts @@ -0,0 +1,106 @@ +/** + * Typed postMessage protocol between the UI and the transcript-translation + * worker (`src/worker/translate.worker.ts`, D-WS4b). Kept apart from the + * Whisper worker's protocol: separate worker, separate lifetime. Free of + * browser globals so `bun test` covers the guards. + */ + +/** UI → worker: translate these unit texts with this Opus-MT model. */ +export interface TranslateRequest { + type: "translate"; + /** Correlates every reply; the UI ignores replies for stale ids. */ + requestId: string; + /** One of `TRANSLATION_MODEL_IDS` (src/lib/translation.ts). */ + modelId: string; + /** Unit texts (`buildTranslationUnits`), answered 1:1 in `texts`. */ + units: string[]; +} + +/** Model files being fetched (first use) — bytes summed across files. */ +export interface TranslateDownloadMessage { + type: "translate-download"; + requestId: string; + loaded: number; + total: number; +} + +/** Model ready; `done` of `total` sentence pieces translated so far. */ +export interface TranslateProgressMessage { + type: "translate-progress"; + requestId: string; + done: number; + total: number; +} + +export interface TranslateResultMessage { + type: "translate-result"; + requestId: string; + /** One translated string per request unit, same order. */ + texts: string[]; +} + +export interface TranslateErrorMessage { + type: "translate-error"; + requestId: string; + message: string; +} + +export type TranslateWorkerMessage = + | TranslateDownloadMessage + | TranslateProgressMessage + | TranslateResultMessage + | TranslateErrorMessage; + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null; +} + +const isNumber = (value: unknown): value is number => + typeof value === "number" && Number.isFinite(value); + +export function isTranslateRequest(value: unknown): value is TranslateRequest { + return ( + isRecord(value) && + value.type === "translate" && + typeof value.requestId === "string" && + typeof value.modelId === "string" && + Array.isArray(value.units) && + value.units.every((unit) => typeof unit === "string") + ); +} + +export function isTranslateWorkerMessage(value: unknown): value is TranslateWorkerMessage { + if (!isRecord(value) || typeof value.requestId !== "string") return false; + switch (value.type) { + case "translate-download": + return isNumber(value.loaded) && isNumber(value.total); + case "translate-progress": + return isNumber(value.done) && isNumber(value.total); + case "translate-result": + return Array.isArray(value.texts) && value.texts.every((t) => typeof t === "string"); + case "translate-error": + return typeof value.message === "string"; + default: + return false; + } +} + +/** + * Sums Transformers.js per-file download progress into one byte count for + * the UI. A file's total is only known once its download starts, so the + * total grows as files start — fine for a percentage that never goes back. + */ +export class DownloadTally { + private readonly files = new Map(); + + update(file: string, loaded: number, total: number): { loaded: number; total: number } { + this.files.set(file, { loaded, total }); + let sumLoaded = 0; + let sumTotal = 0; + for (const entry of this.files.values()) { + sumLoaded += entry.loaded; + sumTotal += entry.total; + } + return { loaded: sumLoaded, total: sumTotal }; + } +} diff --git a/tests/translate-real-model.test.ts b/tests/translate-real-model.test.ts new file mode 100644 index 0000000..b2b156b --- /dev/null +++ b/tests/translate-real-model.test.ts @@ -0,0 +1,97 @@ +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import { join } from "node:path"; +import { + assembleTranslation, + buildTranslationUnits, + getTranslationModel, + translateUnits, +} from "../src/lib/translation"; +import { loadOpusMt, type OpusMtTranslator } from "../src/worker/opusMt"; + +/** + * Real Opus-MT translation on onnxruntime-node (D-WS4b), opt-in: + * + * MT_INTEGRATION=1 flock /tmp/whisper-heavy.lock bun test tests/translate-real-model.test.ts + * + * Downloads Xenova/opus-mt-en-da and -da-en (~113 MB each, q8) into the + * Transformers.js cache on first run; ~3 min on a 16-core CPU. Same loader + * and loop as the browser worker. + */ +const ENABLED = process.env.MT_INTEGRATION === "1"; +const AUDIO = join(import.meta.dir, "audio_files"); +const words = (text: string) => text.split(/\s+/).filter(Boolean); + +describe.skipIf(!ENABLED)("Opus-MT on a real transcript", () => { + let enDa: OpusMtTranslator; + let daEn: OpusMtTranslator; + + beforeAll(async () => { + enDa = await loadOpusMt(getTranslationModel("en", "da")!.id); + daEn = await loadOpusMt(getTranslationModel("da", "en")!.id); + }, 600_000); + + afterAll(async () => { + await enDa?.dispose(); + await daEn?.dispose(); + }); + + const run = (translator: OpusMtTranslator, units: string[]) => + translateUnits(units, { + countTokens: translator.countTokens, + translateBatch: (pieces) => translator.translateBatch(pieces), + }); + + test( + "all of en_alien → Danish: one translation per line, none empty, none copied from a neighbour", + async () => { + const text = await Bun.file(join(AUDIO, "en_alien.txt")).text(); + const units = buildTranslationUnits({ text, chunks: [] }); + const started = performance.now(); + const out = await run(enDa, units.map((unit) => unit.text)); + console.info(`[mt] en_alien en→da: ${((performance.now() - started) / 1000).toFixed(1)} s`); + const assembled = assembleTranslation(units, out); + expect(out).toHaveLength(units.length); + for (const line of out) expect(line.trim()).not.toBe(""); + // Guards the batched-generation bug (every row = row 0's output). + const distinctSources = new Set(units.map((unit) => unit.text)).size; + expect(new Set(out).size).toBeGreaterThan(distinctSources * 0.9); + // Nothing dropped: a translation keeps roughly the source's length. + const ratio = words(assembled.text).length / words(text).length; + expect(ratio).toBeGreaterThan(0.6); + expect(ratio).toBeLessThan(1.5); + }, + 600_000, + ); + + test( + "an unpunctuated unit far beyond 512 tokens is translated whole (Review Focus 5)", + async () => { + const text = await Bun.file(join(AUDIO, "en_alien.txt")).text(); + // Live-mode-like run-on: no sentence punctuation, one single unit. + const runOn = words(text.replace(/[.!?…]/g, "")).slice(0, 900).join(" "); + expect(enDa.countTokens(runOn)).toBeGreaterThan(512); + const [out] = await run(enDa, [runOn]); + const ratio = words(out ?? "").length / words(runOn).length; + expect(ratio).toBeGreaterThan(0.6); + expect(ratio).toBeLessThan(1.5); + }, + 600_000, + ); + + test( + "da_festuge → English keeps names and dates, timestamps carried over", + async () => { + const text = await Bun.file(join(AUDIO, "da_festuge.txt")).text(); + const chunks = text + .split("\n") + .filter((line) => line.trim() !== "") + .map((line, index) => ({ text: ` ${line}`, timestamp: [index * 5, index * 5 + 5] as [number, number] })); + const units = buildTranslationUnits({ text, chunks }); + const result = assembleTranslation(units, await run(daEn, units.map((unit) => unit.text))); + expect(result.text).toContain("Aarhus"); + expect(result.text).toMatch(/August/); + expect(result.chunks.map((chunk) => chunk.timestamp)).toEqual(chunks.map((chunk) => chunk.timestamp)); + }, + 600_000, + ); +}); diff --git a/tests/translateProtocol.test.ts b/tests/translateProtocol.test.ts new file mode 100644 index 0000000..2f88f01 --- /dev/null +++ b/tests/translateProtocol.test.ts @@ -0,0 +1,49 @@ +import { describe, expect, test } from "bun:test"; +import { + DownloadTally, + isTranslateRequest, + isTranslateWorkerMessage, +} from "../src/worker/translateProtocol"; + +describe("isTranslateRequest", () => { + test("accepts a well-formed request", () => { + expect( + isTranslateRequest({ type: "translate", requestId: "t-1", modelId: "Xenova/opus-mt-da-en", units: ["Hej."] }), + ).toBe(true); + }); + + test("rejects missing ids and non-string units", () => { + expect(isTranslateRequest({ type: "translate", modelId: "m", units: [] })).toBe(false); + expect(isTranslateRequest({ type: "translate", requestId: "t", modelId: "m", units: [1] })).toBe(false); + expect(isTranslateRequest({ type: "load", requestId: "t", modelId: "m", units: [] })).toBe(false); + expect(isTranslateRequest(null)).toBe(false); + }); +}); + +describe("isTranslateWorkerMessage", () => { + test("accepts each reply type", () => { + for (const message of [ + { type: "translate-download", requestId: "t", loaded: 1, total: 2 }, + { type: "translate-progress", requestId: "t", done: 0, total: 3 }, + { type: "translate-result", requestId: "t", texts: ["Hello."] }, + { type: "translate-error", requestId: "t", message: "boom" }, + ]) { + expect(isTranslateWorkerMessage(message)).toBe(true); + } + }); + + test("rejects malformed replies and Whisper-worker messages", () => { + expect(isTranslateWorkerMessage({ type: "translate-progress", requestId: "t", done: "1", total: 3 })).toBe(false); + expect(isTranslateWorkerMessage({ type: "translate-result", texts: [] })).toBe(false); + expect(isTranslateWorkerMessage({ type: "result", requestId: "t", text: "", chunks: [] })).toBe(false); + }); +}); + +describe("DownloadTally", () => { + test("sums bytes across files, updating each file in place", () => { + const tally = new DownloadTally(); + expect(tally.update("onnx/encoder.onnx", 10, 100)).toEqual({ loaded: 10, total: 100 }); + expect(tally.update("onnx/decoder.onnx", 5, 50)).toEqual({ loaded: 15, total: 150 }); + expect(tally.update("onnx/encoder.onnx", 100, 100)).toEqual({ loaded: 105, total: 150 }); + }); +}); diff --git a/tests/translation.test.ts b/tests/translation.test.ts index dd96443..06e3ca2 100644 --- a/tests/translation.test.ts +++ b/tests/translation.test.ts @@ -8,11 +8,73 @@ import { getTranslationModel, guessSourceLanguage, splitForTranslation, + translateUnits, } from "../src/lib/translation"; import type { TranscriptChunk } from "../src/worker/protocol"; // Pure planning for transcript translation (D-WS4b) — no model, no network. +describe("translateUnits", () => { + test("translates every piece in order, batched, and rejoins per unit", async () => { + const batches: string[][] = []; + const out = await translateUnits(["One. Two. Three.", "Four.", "Five. Six."], { + batchSize: 4, + translateBatch: async (pieces) => { + batches.push(pieces); + return pieces.map((p) => p.toUpperCase()); + }, + }); + expect(out).toEqual(["ONE. TWO. THREE.", "FOUR.", "FIVE. SIX."]); + expect(batches).toEqual([["One.", "Two.", "Three.", "Four."], ["Five.", "Six."]]); + }); + + test("defaults to one piece per call (batched Marian generation is broken in 4.2)", async () => { + const sizes: number[] = []; + await translateUnits(["One. Two. Three."], { + translateBatch: async (pieces) => { + sizes.push(pieces.length); + return pieces; + }, + }); + expect(sizes).toEqual([1, 1, 1]); + }); + + test("reports progress in pieces, starting at zero", async () => { + const progress: [number, number][] = []; + await translateUnits(["Yes. No. Maybe."], { + batchSize: 2, + translateBatch: async (pieces) => pieces, + onProgress: (done, total) => progress.push([done, total]), + }); + expect(progress).toEqual([[0, 3], [2, 3], [3, 3]]); + }); + + test("an empty unit stays empty without a model call", async () => { + let calls = 0; + const out = await translateUnits(["", "Hi."], { + translateBatch: async (pieces) => { + calls += 1; + return pieces; + }, + }); + expect(out).toEqual(["", "Hi."]); + expect(calls).toBe(1); + }); + + test("a batch answering with the wrong count is an error", async () => { + await expect( + translateUnits(["Yes. No."], { batchSize: 2, translateBatch: async () => ["only one"] }), + ).rejects.toThrow(); + }); + + test("the whole of en_alien round-trips through an echo translator with nothing dropped (Review Focus 5)", async () => { + const alien = await Bun.file(join(import.meta.dir, "audio_files/en_alien.txt")).text(); + const units = buildTranslationUnits({ text: alien, chunks: [] }).map((u) => u.text); + const out = await translateUnits(units, { translateBatch: async (pieces) => pieces }); + expect(words(out.join(" "))).toEqual(words(alien)); + }); +}); + const words = (text: string) => text.split(/\s+/).filter(Boolean); describe("TRANSLATION_MODELS", () => { @@ -100,6 +162,14 @@ describe("splitForTranslation", () => { ]); }); + test("one-word sentences are not mistaken for abbreviations", () => { + expect(splitForTranslation("Yes. Okay. J. K. Rowling wrote it.")).toEqual([ + "Yes.", + "Okay.", + "J. K. Rowling wrote it.", + ]); + }); + test("blank input yields no pieces", () => { expect(splitForTranslation(" ")).toEqual([]); }); From c110c9203fa9dcecd9418f729ad8e4be664bd13a Mon Sep 17 00:00:00 2001 From: Lucas Winther Date: Thu, 29 Oct 2026 22:10:51 +0100 Subject: [PATCH 05/10] Add the Translate transcript panel with its own exports Co-Authored-By: Claude Opus 5.5 --- src/App.tsx | 25 ++- src/components/StorageManager.tsx | 12 +- src/components/TranscriptView.tsx | 18 +- src/components/TranslatePanel.tsx | 269 ++++++++++++++++++++++++++++++ src/index.css | 119 +++++++++++++ tests/translatePanel.test.tsx | 65 ++++++++ 6 files changed, 501 insertions(+), 7 deletions(-) create mode 100644 src/components/TranslatePanel.tsx create mode 100644 tests/translatePanel.test.tsx diff --git a/src/App.tsx b/src/App.tsx index 0c22644..4985267 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -10,6 +10,7 @@ import { ModelPicker } from "./components/ModelPicker"; import { StorageManager } from "./components/StorageManager"; import { TranscriptView } from "./components/TranscriptView"; import { TranscriptionOptions } from "./components/TranscriptionOptions"; +import { TranslatePanel } from "./components/TranslatePanel"; import { decodeAudioFile } from "./lib/audio"; import { classifyError, type ClassifiedError } from "./lib/errors"; import { @@ -26,6 +27,7 @@ import { useAuth } from "./lib/useAuth"; import { LIVE_TUNING, LiveTranscriber, type LiveTranscribeResult } from "./lib/live"; import { startMicCapture, type MicCapture } from "./lib/mic"; import { getDownloadedModels } from "./lib/modelCache"; +import { TRANSLATION_MODEL_IDS } from "./lib/translation"; import { registerServiceWorker } from "./pwa/register"; import { AUTO_LANGUAGE, @@ -68,6 +70,8 @@ interface AppErrorState { } const MODEL_IDS = MODELS.map((model) => model.id); +// Downloaded badges cover the Opus-MT models too (transcript translation, D-WS4b). +const CACHED_MODEL_IDS = [...MODEL_IDS, ...TRANSLATION_MODEL_IDS]; export function App() { const workerRef = useRef(null); @@ -183,7 +187,7 @@ export function App() { function refreshDownloaded() { // Downloaded badges (D-005): query Cache Storage for cached weights. - getDownloadedModels(MODEL_IDS).then(setDownloaded, () => {}); + getDownloadedModels(CACHED_MODEL_IDS).then(setDownloaded, () => {}); } // Terminate the worker when the app unmounts (also keeps HMR tidy). @@ -1165,6 +1169,25 @@ export function App() { } /> + {/* Text translation of the finished transcript (D-WS4b) — any engine, + file or live; secondary to the transcript itself. */} + {result && !liveActive && ( + + )} + {applyUpdate && (
A new version of Whisper Web is available. diff --git a/src/components/StorageManager.tsx b/src/components/StorageManager.tsx index b5159da..152f767 100644 --- a/src/components/StorageManager.tsx +++ b/src/components/StorageManager.tsx @@ -5,6 +5,7 @@ import { type ModelStorageInfo, } from "../lib/modelCache"; import { MODELS, getModel } from "../lib/models"; +import { TRANSLATION_MODEL_IDS, getTranslationModelById } from "../lib/translation"; interface StorageManagerProps { /** Downloaded-badge set from App — changes re-measure an open panel. */ @@ -15,7 +16,14 @@ interface StorageManagerProps { onChanged: () => void; } -const MODEL_IDS = MODELS.map((model) => model.id); +// Whisper weights plus the Opus-MT translation models (D-WS4b). +const MODEL_IDS = [...MODELS.map((model) => model.id), ...TRANSLATION_MODEL_IDS]; + +function displayName(modelId: string): string { + const translation = getTranslationModelById(modelId); + if (translation) return `Translation, ${translation.label}`; + return getModel(modelId)?.displayName ?? modelId; +} function formatBytes(bytes: number): string { if (bytes >= 1e9) return `${(bytes / 1e9).toFixed(2)} GB`; @@ -82,7 +90,7 @@ export function StorageManager({ downloaded, disabled, onChanged }: StorageManag
  • - {getModel(entry.modelId)?.displayName ?? entry.modelId} + {displayName(entry.modelId)} {entry.modelId}
    diff --git a/src/components/TranscriptView.tsx b/src/components/TranscriptView.tsx index 188b832..d593f37 100644 --- a/src/components/TranscriptView.tsx +++ b/src/components/TranscriptView.tsx @@ -34,6 +34,13 @@ interface TranscriptViewProps { * "talk.txt"/"talk.srt"); null falls back to "transcript.*". */ exportName?: string | null; + /** Heading once the result is final; defaults to "Transcript". */ + title?: string; + /** + * Inserted before each export extension ("en" → "talk.en.txt", + * "talk.en.srt") so a translation's exports sit beside the original's. + */ + exportSuffix?: string; } /** Trigger a browser download of `contents` as `filename`. */ @@ -62,7 +69,10 @@ export function TranscriptView({ progress = null, placeholder = "Listening for the first words…", exportName = null, + title = "Transcript", + exportSuffix, }: TranscriptViewProps) { + const ext = (extension: string) => (exportSuffix ? `${exportSuffix}.${extension}` : extension); const [copied, setCopied] = useState(false); const headingRef = useRef(null); const prevResultRef = useRef(null); @@ -102,7 +112,7 @@ export function TranscriptView({

    - {result ? "Transcript" : activeLabel} + {result ? title : activeLabel} {percent !== null && ( {percent}% )} @@ -115,7 +125,7 @@ export function TranscriptView({ onClick={() => download( transcriptToTxt(result), - exportFilename(exportName, "txt"), + exportFilename(exportName, ext("txt")), "text/plain", ) } @@ -130,7 +140,7 @@ export function TranscriptView({ onClick={() => download( transcriptToSrt(result.chunks), - exportFilename(exportName, "srt"), + exportFilename(exportName, ext("srt")), "application/x-subrip", ) } @@ -146,7 +156,7 @@ export function TranscriptView({ onClick={() => download( transcriptToTimestampedTxt(result.chunks), - exportFilename(exportName, "timestamps.txt"), + exportFilename(exportName, ext("timestamps.txt")), "text/plain", ) } diff --git a/src/components/TranslatePanel.tsx b/src/components/TranslatePanel.tsx new file mode 100644 index 0000000..be60664 --- /dev/null +++ b/src/components/TranslatePanel.tsx @@ -0,0 +1,269 @@ +import { useEffect, useRef, useState } from "react"; +import { classifyError, type ClassifiedError } from "../lib/errors"; +import { formatModelSize } from "../lib/models"; +import { + TRANSLATION_MODELS, + assembleTranslation, + buildTranslationUnits, + guessSourceLanguage, + type TranslationModel, + type TranslationUnit, +} from "../lib/translation"; +import type { TranscriptChunk } from "../worker/protocol"; +import { isTranslateWorkerMessage, type TranslateRequest } from "../worker/translateProtocol"; +import { TranscriptView } from "./TranscriptView"; + +interface TranscriptResult { + text: string; + chunks: TranscriptChunk[]; +} + +/** Where a translation run is; `idle` covers "never started" and "finished". */ +export type TranslatePhase = + | { kind: "idle" } + | { kind: "starting" } + | { kind: "downloading"; loaded: number; total: number } + | { kind: "translating"; done: number; total: number }; + +/** + * Plain-language status for a running translation (PRODUCT.md: say what is + * happening, in everyday words). Null when nothing is running. + */ +export function translateStatusText(phase: TranslatePhase, cached: boolean): string | null { + switch (phase.kind) { + case "idle": + return null; + case "starting": + return cached ? "Loading the translation model…" : "Starting the download…"; + case "downloading": { + const percent = phase.total > 0 ? Math.min(100, Math.round((phase.loaded / phase.total) * 100)) : 0; + return `Downloading the translation model… ${percent}%`; + } + case "translating": + return phase.total === 0 + ? "Translating…" + : `Translating… ${phase.done} of ${phase.total} sentence${phase.total === 1 ? "" : "s"}`; + } +} + +interface TranslatePanelProps { + /** The finished transcript to translate (any engine, file or live). */ + result: TranscriptResult; + /** + * Language Whisper was told the audio is in ("da", "en"), "en" when + * Whisper itself translated, or undefined/"auto" — seeds the direction. + */ + sourceLanguageHint?: string; + /** Hub ids with cached weights (D-005) — decides the download wording. */ + downloaded: ReadonlySet; + /** Called once a model finished loading, to refresh downloaded badges. */ + onModelCached: () => void; + /** Source file name for export names ("talk.m4a" → "talk.en.srt"). */ + exportName: string | null; + /** True while a transcription or live session is running. */ + disabled: boolean; +} + +/** + * Text translation of a finished transcript with Opus-MT (D-WS4b): a + * secondary disclosure under the transcript, separate from Whisper's own + * "Translate to English" task. Runs on-device in its own worker; the model + * is downloaded only when the user clicks Translate (D-020). + */ +export function TranslatePanel({ + result, + sourceLanguageHint, + downloaded, + onModelCached, + exportName, + disabled, +}: TranslatePanelProps) { + const workerRef = useRef(null); + const runRef = useRef(0); + const [modelId, setModelId] = useState(() => defaultModel(result, sourceLanguageHint).id); + const [phase, setPhase] = useState({ kind: "idle" }); + const [error, setError] = useState(null); + const [translation, setTranslation] = useState<{ + model: TranslationModel; + result: TranscriptResult; + } | null>(null); + + const model = TRANSLATION_MODELS.find((m) => m.id === modelId) ?? TRANSLATION_MODELS[0]!; + const cached = downloaded.has(model.id); + const running = phase.kind !== "idle"; + const empty = result.text.trim() === "" && result.chunks.every((c) => c.text.trim() === ""); + + function stopWorker() { + runRef.current++; + workerRef.current?.terminate(); + workerRef.current = null; + } + + // A new transcript (or model switch upstream) invalidates everything here. + useEffect(() => { + stopWorker(); + setPhase({ kind: "idle" }); + setError(null); + setTranslation(null); + setModelId(defaultModel(result, sourceLanguageHint).id); + // Keyed on the transcript only: the language hint seeds the default for + // a new transcript but must not wipe a translation when it changes. + }, [result]); + + useEffect(() => stopWorker, []); + + function translate() { + if (running || empty) return; + const run = ++runRef.current; + const units: TranslationUnit[] = buildTranslationUnits(result); + const target = model; + setError(null); + setTranslation(null); + setPhase({ kind: "starting" }); + + // Separate worker, bundled and served at a stable URL like the Whisper + // worker (src/index.ts, scripts/build.ts). Reused across runs. + const worker = (workerRef.current ??= new Worker("translate.worker.js", { type: "module" })); + const requestId = `translate-${run}`; + const fail = (raw: unknown, stage: "translate" | "worker-crash") => { + if (runRef.current !== run) return; + setPhase({ kind: "idle" }); + const offline = typeof navigator !== "undefined" && !navigator.onLine; + // A crash (e.g. OOM tab-kill) leaves nothing to reuse. + if (stage === "worker-crash") stopWorker(); + setError(classifyError(raw, { stage: "translate", offline })); + }; + worker.onmessage = (event: MessageEvent) => { + const message = event.data; + if (runRef.current !== run || !isTranslateWorkerMessage(message)) return; + if (message.requestId !== requestId) return; + switch (message.type) { + case "translate-download": + setPhase({ kind: "downloading", loaded: message.loaded, total: message.total }); + break; + case "translate-progress": + if (message.done === 0) onModelCached(); + setPhase({ kind: "translating", done: message.done, total: message.total }); + break; + case "translate-result": + try { + setTranslation({ model: target, result: assembleTranslation(units, message.texts) }); + setPhase({ kind: "idle" }); + } catch (assembleError) { + fail(assembleError, "translate"); + } + break; + case "translate-error": + fail(message.message, "translate"); + break; + } + }; + worker.onerror = (event) => fail(event.message ?? "", "worker-crash"); + const request: TranslateRequest = { + type: "translate", + requestId, + modelId: target.id, + units: units.map((unit) => unit.text), + }; + worker.postMessage(request); + } + + function cancel() { + stopWorker(); + setPhase({ kind: "idle" }); + } + + const status = translateStatusText(phase, cached); + const progressValue = + phase.kind === "downloading" + ? { value: phase.loaded, max: Math.max(phase.total, 1) } + : phase.kind === "translating" + ? { value: phase.done, max: Math.max(phase.total, 1) } + : null; + + return ( + <> +
    + Translate transcript… +
    + + {running ? ( + + ) : ( + + )} +
    +

    + {cached + ? "Translation model downloaded — works offline. " + : `Downloads a translation model once (~${formatModelSize(model.approxBytes)}), then works offline. `} + Runs on this device; the text never leaves it. Machine translation — check + names and important details. +

    +

    + Model: {model.id} (Opus-MT) +

    + {status && ( +
    + {status} + {progressValue && ( + + )} +
    + )} + {error && ( +
    +

    {error.message}

    + {error.suggestion &&

    {error.suggestion}

    } + {error.retryable && ( + + )} +
    + )} +
    + {translation && ( + + )} + + ); +} + +function defaultModel(result: TranscriptResult, hint?: string): TranslationModel { + const source = guessSourceLanguage(result.text, hint); + return TRANSLATION_MODELS.find((m) => m.source === source) ?? TRANSLATION_MODELS[0]!; +} diff --git a/src/index.css b/src/index.css index 999d03c..9f444d1 100644 --- a/src/index.css +++ b/src/index.css @@ -1006,3 +1006,122 @@ code { align-items: center; gap: 0.5rem; } + +/* -------------------------------------------------------------------------- + Transcript translation (D-WS4b): a secondary disclosure under the + transcript, same panel language as .storage-manager. + -------------------------------------------------------------------------- */ + +.translate-panel { + background: var(--panel); + border: 1px solid var(--border); + border-radius: 12px; + padding: 0.75rem 1rem; +} + +.translate-panel summary { + cursor: pointer; + font-weight: 600; +} + +.translate-controls { + display: flex; + align-items: center; + flex-wrap: wrap; + gap: 0.5rem 0.75rem; + margin-top: 0.75rem; +} + +.translate-direction { + display: flex; + align-items: center; + gap: 0.5rem; + font-size: 0.85rem; + color: var(--muted); + min-width: 0; +} + +.translate-direction select { + min-width: 0; + max-width: 100%; + background: var(--surface-raised); + color: var(--fg); + border: 1px solid var(--border); + border-radius: 8px; + padding: 0.3rem 0.5rem; + font-size: 0.85rem; +} + +.translate-button, +.translate-cancel { + border: 1px solid var(--border); + background: transparent; + color: inherit; + border-radius: 8px; + padding: 0.3rem 0.9rem; + font-size: 0.85rem; + cursor: pointer; + white-space: nowrap; + transition: border-color 0.15s, color 0.15s; +} + +/* The panel's one action reads as the action: accent outline, like a + focused copy button, without competing with the transcript above. */ +.translate-button { + border-color: var(--accent); + color: var(--accent); +} + +.translate-button:hover:enabled { + background: var(--accent); + color: var(--on-accent); +} + +.translate-button:disabled { + opacity: 0.5; + cursor: default; +} + +.translate-cancel:hover { + border-color: var(--danger-border); + color: var(--danger); +} + +.translate-hint { + margin: 0.6rem 0 0; + max-width: 65ch; + color: var(--muted); + font-size: 0.85rem; +} + +.translate-model-id { + margin-top: 0.25rem; + font-size: 0.75rem; +} + +.translate-model-id code { + overflow-wrap: anywhere; +} + +.translate-progress { + display: flex; + align-items: center; + flex-wrap: wrap; + gap: 0.4rem 0.75rem; + margin-top: 0.75rem; + font-size: 0.85rem; +} + +.translate-status { + font-variant-numeric: tabular-nums; +} + +.translate-progress progress { + flex: 1 1 10rem; + height: 6px; + accent-color: var(--accent); +} + +.translate-error { + margin-top: 0.75rem; +} diff --git a/tests/translatePanel.test.tsx b/tests/translatePanel.test.tsx new file mode 100644 index 0000000..bd5186f --- /dev/null +++ b/tests/translatePanel.test.tsx @@ -0,0 +1,65 @@ +import { describe, expect, test } from "bun:test"; +import { renderToStaticMarkup } from "react-dom/server"; +import { TranslatePanel, translateStatusText } from "../src/components/TranslatePanel"; + +// D-WS4b: the panel's copy, before any worker exists (static render). +const danish = { text: "Det er ikke noget med at vi skal på ferie.", chunks: [] }; + +function render(downloaded: string[] = [], hint?: string) { + return renderToStaticMarkup( + {}} + exportName="talk.m4a" + disabled={false} + />, + ); +} + +describe("TranslatePanel", () => { + test("is a secondary disclosure named in plain words", () => { + const html = render(); + expect(html).toContain(" { + const html = render(); + expect(html).toContain("Downloads a translation model once (~113 MB)"); + expect(html).toContain("never leaves"); + }); + + test("says it works offline once the model is cached", () => { + expect(render(["Xenova/opus-mt-da-en"])).toContain("Translation model downloaded — works offline"); + }); + + test("defaults the direction from the transcript's language", () => { + expect(render()).toMatch(/