diff --git a/AGENTS.md b/AGENTS.md index 7f9e600..b49a62b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -17,6 +17,8 @@ Test all: `bun test` — ~19 min: `tests/live-real-model.test.ts` downloads and weights whenever huggingface.co is reachable. Run it when touching the live pipeline or decoding. Build: `bun run build` (rewrites `dist/`) Opt-in real sidecar test (needs `pip install -r server-py/requirements.txt`): `FW_INTEGRATION=1 bun test tests/transcribe.test.ts` +Opt-in real translation test (~3 min, downloads 2 × 113 MB Opus-MT): `MT_INTEGRATION=1 bun test tests/translate-real-model.test.ts`; +translation speed: `bun scripts/bench-translate.ts` Typecheck: `bun run typecheck` (tsc 7, < 1 s). No linter configured. Browser-only behaviour (offline PWA, mic, WebGPU) has no automated coverage: follow `tests/offline.md`, @@ -35,7 +37,8 @@ step with the commands above, in the same commit. dotenv. Use `Bun.serve()` routes + HTML imports, `bun:sqlite`, built-in `WebSocket`, `Bun.file`, `Bun.$`. Bun API docs: `node_modules/bun-types/docs/`. - **Two engines, two libraries (D-013).** Browser inference = Transformers.js, inside - `src/worker/inference.worker.ts` only — never on the main thread. Server inference = faster-whisper + `src/worker/inference.worker.ts` only (transcript translation: `src/worker/translate.worker.ts`, + D-WS4b) — never on the main thread. Server inference = faster-whisper via `server-py/sidecar.py` — never Transformers.js on the server. - **`@huggingface/transformers` is patched and a devDependency on purpose.** It is pinned by `patches/@huggingface%2Ftransformers@4.3.2.patch` (D-027/D-028, D-WS1a) and kept out of runtime deps so the diff --git a/DESIGN.md b/DESIGN.md index e6dd7c9..069ed80 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -31,3 +31,11 @@ One BEM-ish class block per React component in `src/components/` (`.model-picker `.live-controls`, `.engine-toggle`, `.auth-panel`, `.update-toast`, …), with modifier classes like `-selected`, `-active`, `-disabled`. Interactive states use `:hover:enabled` and `:focus-visible`. `.visually-hidden` exists for screen-reader-only text. +Card headers (`.transcript-header`: title + export/Copy buttons) wrap, so on a phone the buttons +move under the title instead of overflowing the card. + +Secondary disclosures (`.storage-manager`, `.translate-panel`) share one look: a 12px-radius +`--panel` card with a semibold ``, muted 0.85rem hints, outlined 8px buttons. Inside the +translate panel the one action (`.translate-button`) is the only accent-outlined button and fills +with `--accent` / `--on-accent` on hover; a translation renders as a second `.transcript` card below +it rather than a card nested inside the panel. diff --git a/docs/Architecture.md b/docs/Architecture.md index ea5fc28..6afa86e 100644 --- a/docs/Architecture.md +++ b/docs/Architecture.md @@ -27,6 +27,9 @@ Logged-in users may optionally route inference to the server. │ ├─ Device selection: WebNN → WebGPU → WASM fallback │ │ └─ Model weights cached via browser Cache API │ │ │ +│ Translate Web Worker (D-WS4b) │ +│ └─ Transformers.js Opus-MT (MarianMT), WASM, on user request │ +│ │ │ Service Worker (PWA) │ │ ├─ Precached app shell (offline-first) │ │ └─ Works with zero network once installed + model cached │ @@ -38,6 +41,7 @@ Logged-in users may optionally route inference to the server. │ ├─ / → src/index.html (bundled React app) │ │ ├─ /inference.worker.js → worker bundle (Bun.build at startup, │ │ │ since Bun doesn't bundle `new Worker` graphs) │ +│ ├─ /translate.worker.js → translation worker bundle (same) │ │ ├─ /api/auth/* → Passport.js (local strategy) + sessions │ │ ├─ /api/transcribe → server engine (auth required) │ │ └─ /manifest.webmanifest, /sw.js → PWA assets │ @@ -71,9 +75,13 @@ src/ models.ts # model catalog: ids, sizes, dtype defaults per device, languages modelCache.ts # "downloaded?" per model via Cache Storage (D-005; reused by M3 offline indicator) engine.ts # engine abstraction: BrowserEngine | ServerEngine + translation.ts # transcript translation planning: Opus-MT catalog, units, sentence pieces (pure, D-WS4b) worker/ inference.worker.ts # Transformers.js pipeline, WebNN/WebGPU/WASM selection protocol.ts # typed postMessage messages (shared with UI) + translate.worker.ts # transcript translation (Opus-MT), its own worker + ort instance (D-WS4b) + translateProtocol.ts # typed messages for the translate worker + opusMt.ts # Opus-MT loader shared by the worker, bench script and real-model test pwa/ sw.ts # service worker (app-shell precache, /ort runtime cache) precache.ts # placeholder-injection helpers shared by build + dev serve @@ -141,7 +149,7 @@ environment). See `WorkBreakdown.md` for the itemized checklist.) `AudioContext.decodeAudioData`, downmixes to mono, resamples to 16 kHz, transfers the `Float32Array` to the worker. For multilingual models the `transcribe` message also carries `language` (or auto-detect) and `task` - (transcribe | translate). + (transcribe | translate — translate only for models with `canTranslate`, D-WS4a). 5. Worker runs the pipeline (chunked, with timestamps), streaming partial results back via `{type:"partial"}` messages; final result as `{type:"result"}`. @@ -176,6 +184,24 @@ environment). See `WorkBreakdown.md` for the itemized checklist.) only when signed in with the server engine available (`EngineToggle`); anonymous/offline users always get the browser-engine path above. +### 4.1c Transcript translation (any engine — D-WS4b) + +1. Any finished transcript (browser file, browser live, server engine) shows a + "Translate transcript…" disclosure (`TranslatePanel`). Whisper's own + "Translate to English" task stays in `TranscriptionOptions`, only for + models with `canTranslate` (D-WS4a). +2. On **Translate**, the UI groups the transcript's chunks into sentence-ending + units (`buildTranslationUnits`, merged time spans) and posts their texts to + `translate.worker.js` — a separate worker with its own onnxruntime instance. +3. The worker loads the Opus-MT model (q8, ~113 MB, downloaded on this first + request and cached by Transformers.js like Whisper weights), splits every + unit into sentence pieces under a token budget (`splitForTranslation`, + Opus-MT's 512-token input), translates them one by one, and answers one + string per unit. The UI reassembles `{ text, chunks }` with the units' + timestamps and renders it in a second `TranscriptView` with its own + `.txt` / `.srt` exports (`..srt`). The text never leaves the + device, whichever engine produced it. + ### 4.2 Offline (PWA) - Service worker (`pwa/sw.ts`, hand-rolled — D-007) precaches the built app shell (hashed JS/CSS, `/`, worker bundle, manifest, icons, `/fonts/*` Inter subsets) on install into diff --git a/docs/Decisions.md b/docs/Decisions.md index a124be5..38f3da3 100644 --- a/docs/Decisions.md +++ b/docs/Decisions.md @@ -1360,3 +1360,74 @@ check is re-run. **Not covered here:** a real browser's ort-web wasm build (the same `ort-wasm-simd-threaded.*` files, but multithreaded under COOP/COEP) — `tests/offline.md` §2's WASM step is the owner's browser check. + +## D-WS4a — Whisper's "Translate to English" only on models trained for it (`canTranslate`) +**Status:** Accepted (2026-10-29) · spec 001 Goal 4a +`WhisperModel` gains `canTranslate: boolean` (`src/lib/models.ts`): **true** for multilingual +tiny / base / small / medium, **false** for every `.en` model (no task token) and for +`whisper-large-v3-turbo` — OpenAI's model card says turbo was fine-tuned on transcription data only +and is not trained for translation, so its "translate" output is unreliable. Any future turbo-based +fine-tune defaults to false until tested. `TranscriptionOptions` renders the Task control only where +true (language stays). **Stale task:** the `task` state is kept, but `taskForModel(modelId, task)` +maps "translate" to "transcribe" for a model that can't translate; App applies it on model switch +(picker and the server-engine auto-switch to turbo) and again when building every file / live run's +options, so a leftover "translate" can never reach the worker or the server. **Not changed:** the +server's `parseLanguageAndTask` (D-032) still accepts "translate" for any model — the browser never +sends it for turbo now, and a direct API caller asking faster-whisper to translate with turbo gets +what faster-whisper does; rejecting it server-side would be a behaviour change for API users with no +UI benefit. Speech in another language than English can still end up in English (or Danish) through +text translation of the finished transcript (D-WS4b), for every model. + +## D-WS4b — Text translation of finished transcripts with Opus-MT, in its own worker +**Status:** Accepted (2026-10-29) · spec 001 Goal 4b +**What:** any finished transcript — browser file, browser live, server engine — can be translated +da→en or en→da from a secondary "Translate transcript…" disclosure under the transcript +(`TranslatePanel`). Whisper's own task keeps its label "Translate to English" (D-WS4a). The +translation renders as a second transcript card ("Translation (English)") with its own +`.txt` / `.srt` / `.txt+ts` exports named `..`. +**Models:** `Xenova/opus-mt-da-en` and `Xenova/opus-mt-en-da` — Transformers.js ONNX exports of +Helsinki-NLP's Opus-MT (MarianMT, ~77 M params; upstream licence Apache-2.0, the Xenova repos +carry no separate licence). Loaded at **q8**: encoder 49.8 MB + merged decoder 57.1 MB + +tokenizer/config 5.7 MB = **~113 MB per direction**, one-time. Verified to load and translate under +the pinned Transformers.js 4.2.0 (onnxruntime-node; headless-Chrome WASM run below). WASM only, with +`graphOptimizationLevel: "basic"` (D-014 — Marian ties its embeddings like Whisper's 8-bit decoders). +**Where it runs — a dedicated worker** (`src/worker/translate.worker.ts`, served at +`/translate.worker.js`, precached), not a message type on the Whisper worker: onnxruntime fixes its +WASM thread pool at first init (D-023), so a Whisper worker initialised for WebGPU — the owner's +setup — would run MT single-threaded; a separate worker also keeps a Whisper cancel/restart +(D-018/D-025) from killing a translation and vice versa, and keeps workstreams 2/3's edits to +`inference.worker.ts` conflict-free. Nothing goes to the server for any engine (anti-goal: text stays +on the device). **Downloads:** only when the user clicks Translate; the hint states the size first +("Downloads a translation model once (~113 MB), then works offline") and switches to "Translation +model downloaded — works offline" once the Cache API holds it (D-005/D-020). The storage panel lists +and deletes the two models like Whisper weights. +**Chunking (Review Focus 5):** `buildTranslationUnits` merges Whisper chunks until one ends a sentence +(cap 8 chunks) and gives the unit the merged time span, so `.srt` keeps working with slightly longer +cues; untimed text is split per line. Each unit is cut by `splitForTranslation` into **one piece per +sentence** (`Intl.Segmenter`, abbreviation/initial fragments merged forward), an over-budget sentence +wrapped on commas, then words, under 200 real tokens (Opus-MT's input limit is 512; the tokenizer +otherwise *truncates silently*). One sentence per call is required, not just tidy: given two +sentences Opus-MT dropped the second one in the probe ("…I live in Copenhagen. How are you?" → +first sentence only). Order and every word are pinned by unit tests on all of `en_alien.txt` and by +the opt-in real-model test (`MT_INTEGRATION=1`), which also translates a 900-word unpunctuated run-on. +**Batching is off (batch size 1):** Transformers.js 4.2's batched Marian generation is broken — +every row of a padded batch comes back as row 0's translation (q8 and fp32, any optimisation level, +onnxruntime-node). Batch 8 would be ~4× faster; re-measure with `scripts/bench-translate.ts` after +the Transformers.js upgrade (workstream 1) before raising `DEFAULT_TRANSLATE_BATCH_SIZE`. +**Measured (onnxruntime-node, CPU, 16 cores shared with other workstreams; not browser numbers):** +`en_alien.txt` (7 470 chars, 279 lines → 293 sentence pieces) en→da **58 s** (~128 chars/s, +~0.2 s/sentence) in the bench (71 s / ~105 chars/s on a re-run with other workstreams' jobs +running), 76 s in the real-model test under load; `da_festuge.txt` (3 sentences) +da→en 2.8 s incl. warm-up; model load from cache 0.3–0.4 s. Extrapolated, a 90-minute meeting +(~80 000 chars) is ~10 min on this CPU — a background action, hence the sentence-count progress and +Cancel. **Headless Chrome (WASM, `--disable-gpu`, cross-origin isolated, 12 threads):** a real +end-to-end run — Whisper Tiny transcript of `da_festuge.m4a`, "Translate transcript…" → download +`Xenova/opus-mt-en-da` → "Translating… 0 of 1 sentence" → "Translation (Danish)" card with four +exports — took 4–8 s from click to result for that one-sentence transcript (download included); +no browser number for a long transcript was taken. Sample da→en: "At the Aarhus Festival Secretariat we have been running very strong in the +last few months." +**Errors** go through `classifyError` with the new stage `"translate"` (out-of-memory suggests closing +tabs instead of a smaller Whisper model; a failed download is a retryable network error). Cancel +terminates the translate worker only. +**Not done / follow-ups:** WebGPU for Opus-MT (unmeasurable here); more language pairs (the catalog +is two entries, `TRANSLATION_MODELS`); a translation of live text *while* recording. diff --git a/docs/WorkBreakdown.md b/docs/WorkBreakdown.md index cb1963f..2151840 100644 --- a/docs/WorkBreakdown.md +++ b/docs/WorkBreakdown.md @@ -65,6 +65,8 @@ Notes (M1): surface active backend in UI - [x] Handle model switching (dispose old pipeline, load new) - [x] Language / task options for multilingual models (transcribe vs translate) +- [x] Whisper translate only on models trained for it (`canTranslate`; hidden for + large-v3-turbo and `.en`, stale "translate" resets to transcribe — D-WS4a) - [~] Verify quantized variants (q4/q8/fp16) on both backends; record working combinations in `models.ts` @@ -347,6 +349,16 @@ cached. --- +## Spec 001 — Translate (workstream 4) +- [x] `canTranslate` per model; Task control hidden where false; stale + "translate" resets to transcribe (D-WS4a) +- [x] Text translation of any finished transcript, da→en / en→da, Opus-MT + in its own worker, sentence-chunked, timestamps kept for exports (D-WS4b) +- [~] Real-browser checks: offline translation (`tests/offline.md` §5), + WebGPU-machine speed (owner) + +--- + ## Deliberately out of scope (for now) Arbitrary Hub model ids, OAuth providers, multi-user transcript storage, whisper.cpp server backend, i18n of the UI. @@ -378,6 +390,14 @@ Found while bootstrapping agent docs (D-036); all fixed on 2026-10-29. — Light theme via tokens + self-hosted Inter (D-037). Also fixed: the Backend select overflowed its panel on 390 px-wide phones. +Found while building transcript translation (D-WS4b), 2026-10-29: +- [ ] Transformers.js 4.2 batched Marian generation returns row 0's output for + every row of a padded batch, so translation runs one sentence per call + (~4× slower than batch 8 would be). Re-check after the 4.3 upgrade with + `bun scripts/bench-translate.ts` ("output identical" across batch sizes). +- [ ] The server API still accepts `task: "translate"` with large-v3-turbo + (only the UI hides it, D-WS4a). + ## Recurring - [ ] Maintain `AGENTS.md`: - **Add a rule only in response to a real failure.** When an agent repeatedly diff --git a/scripts/bench-translate.ts b/scripts/bench-translate.ts new file mode 100644 index 0000000..12f8ac6 --- /dev/null +++ b/scripts/bench-translate.ts @@ -0,0 +1,53 @@ +/** + * Transcript-translation speed on onnxruntime-node (CPU), D-WS4b. + * + * flock /tmp/whisper-heavy.lock bun scripts/bench-translate.ts [batchSizes…] + * + * Translates the reference transcripts in tests/audio_files/ with the same + * Opus-MT loader and translation loop the browser worker uses, once per + * batch size, and prints wall-clock time, pieces, and whether the output + * depends on the batch size (padding). Downloads ~113 MB per direction into + * the Transformers.js cache on first run. Not browser numbers: WASM in a + * tab is slower than onnxruntime-node. + */ + +import { join } from "node:path"; +import { buildTranslationUnits, getTranslationModel, translateUnits } from "../src/lib/translation"; +import { loadOpusMt } from "../src/worker/opusMt"; + +const AUDIO = join(import.meta.dir, "../tests/audio_files"); +const batchSizes = process.argv.slice(2).map(Number).filter((n) => n > 0); +const sizes = batchSizes.length > 0 ? batchSizes : [1, 8]; + +for (const [file, source, target] of [ + ["en_alien.txt", "en", "da"], + ["da_festuge.txt", "da", "en"], +] as const) { + const model = getTranslationModel(source, target)!; + const text = await Bun.file(join(AUDIO, file)).text(); + const units = buildTranslationUnits({ text, chunks: [] }).map((unit) => unit.text); + const loadStart = performance.now(); + const translator = await loadOpusMt(model.id); + const loadMs = performance.now() - loadStart; + console.log(`\n${file} (${text.length} chars, ${units.length} lines) ${model.label} — load ${loadMs.toFixed(0)} ms`); + let baseline: string[] | null = null; + for (const batchSize of sizes) { + let pieces = 0; + const start = performance.now(); + const out = await translateUnits(units, { + batchSize, + countTokens: translator.countTokens, + translateBatch: (batch) => translator.translateBatch(batch), + onProgress: (_done, total) => (pieces = total), + }); + const seconds = (performance.now() - start) / 1000; + const same = baseline === null ? "baseline" : JSON.stringify(out) === JSON.stringify(baseline) ? "identical" : "differs"; + baseline ??= out; + console.log( + ` batch ${String(batchSize).padStart(2)}: ${seconds.toFixed(2)} s, ${pieces} pieces, ` + + `${(text.length / seconds).toFixed(0)} chars/s, output ${same}`, + ); + } + console.log(` sample: ${baseline!.slice(0, 3).join(" | ")}`); + await translator.dispose(); +} diff --git a/scripts/build.ts b/scripts/build.ts index 702ed38..0bd50ef 100644 --- a/scripts/build.ts +++ b/scripts/build.ts @@ -3,7 +3,8 @@ * * Emits a self-contained `dist/`: * 1. `bun build src/index.html` → hashed app-shell assets - * 2. inference worker bundle at the stable /inference.worker.js URL + * 2. inference + translate worker bundles at stable /inference.worker.js and + * /translate.worker.js URLs * 3. manifest + icons + onnxruntime .wasm/.mjs (dist/ort/, offline inference) * 4. sw.js with the precache manifest + a content-hash version injected * @@ -53,6 +54,15 @@ const workerBuild = await Bun.build({ }); const workerScript = await workerBuild.outputs[0]!.text(); await Bun.write(join(DIST, "inference.worker.js"), workerScript); +// Transcript-translation worker (D-WS4b), same mechanism. +const translateWorkerBuild = await Bun.build({ + entrypoints: [join(ROOT, "src/worker/translate.worker.ts")], + target: "browser", + format: "esm", + minify: true, +}); +const translateWorkerScript = await translateWorkerBuild.outputs[0]!.text(); +await Bun.write(join(DIST, "translate.worker.js"), translateWorkerScript); // 3. Static PWA assets + onnxruntime wasm binaries (offline inference — the // worker loads them from /ort/ instead of the jsdelivr CDN). The manifest and @@ -89,6 +99,7 @@ const precachePaths = precachePathsFromOutputs( shellBuild.outputs.map((output) => relative(DIST, output.path)), [ "inference.worker.js", + "translate.worker.js", "manifest.webmanifest", ...ICON_SIZES.map((size) => `icons/icon-${size}.png`), ...fontFiles.map((name) => `fonts/${name}`), @@ -98,7 +109,7 @@ const fontBytes = await Promise.all( fontFiles.map((name) => Bun.file(join(FONTS_DIR, name)).bytes()), ); const version = Bun.hash( - JSON.stringify(precachePaths) + swSource + workerScript + fontBytes.map((b) => Bun.hash(b)).join(), + JSON.stringify(precachePaths) + swSource + workerScript + translateWorkerScript + fontBytes.map((b) => Bun.hash(b)).join(), ).toString(36); await Bun.write(join(DIST, "sw.js"), injectPrecacheManifest(swSource, precachePaths, version)); diff --git a/src/App.tsx b/src/App.tsx index e282021..4985267 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -10,6 +10,7 @@ import { ModelPicker } from "./components/ModelPicker"; import { StorageManager } from "./components/StorageManager"; import { TranscriptView } from "./components/TranscriptView"; import { TranscriptionOptions } from "./components/TranscriptionOptions"; +import { TranslatePanel } from "./components/TranslatePanel"; import { decodeAudioFile } from "./lib/audio"; import { classifyError, type ClassifiedError } from "./lib/errors"; import { @@ -26,6 +27,7 @@ import { useAuth } from "./lib/useAuth"; import { LIVE_TUNING, LiveTranscriber, type LiveTranscribeResult } from "./lib/live"; import { startMicCapture, type MicCapture } from "./lib/mic"; import { getDownloadedModels } from "./lib/modelCache"; +import { TRANSLATION_MODEL_IDS } from "./lib/translation"; import { registerServiceWorker } from "./pwa/register"; import { AUTO_LANGUAGE, @@ -37,6 +39,7 @@ import { formatModelSize, getModel, resolveDefaultDtype, + taskForModel, } from "./lib/models"; import { shouldWarnBeforeLoad, type MobileMemoryHint } from "./lib/mobileMemoryGuard"; import { @@ -67,6 +70,8 @@ interface AppErrorState { } const MODEL_IDS = MODELS.map((model) => model.id); +// Downloaded badges cover the Opus-MT models too (transcript translation, D-WS4b). +const CACHED_MODEL_IDS = [...MODEL_IDS, ...TRANSLATION_MODEL_IDS]; export function App() { const workerRef = useRef(null); @@ -182,7 +187,7 @@ export function App() { function refreshDownloaded() { // Downloaded badges (D-005): query Cache Storage for cached weights. - getDownloadedModels(MODEL_IDS).then(setDownloaded, () => {}); + getDownloadedModels(CACHED_MODEL_IDS).then(setDownloaded, () => {}); } // Terminate the worker when the app unmounts (also keeps HMR tidy). @@ -499,6 +504,9 @@ export function App() { function handleModelSelect(nextModelId: string) { if (busy || liveStatus !== "idle" || nextModelId === modelId) return; setModelId(nextModelId); + // A "Translate to English" chosen on a model that can translate must not + // silently carry over to one that cannot (D-WS4a). + setTask((current) => taskForModel(nextModelId, current)); setActiveDevice(null); setActiveThreads(null); setPartialText(""); @@ -675,7 +683,7 @@ export function App() { // Language/task only apply to multilingual models; `.en` variants // take neither. The browser engine ignores modelId (worker holds it). modelId, - ...(multilingual && { language, task }), + ...(multilingual && { language, task: taskForModel(modelId, task) }), onPartial: (fragment) => { if (fileRun.current === run) setPartialText((text) => text + fragment); }, @@ -752,7 +760,10 @@ export function App() { setLivePending(0); setLiveStatus("starting"); - const options = (getModel(modelId)?.multilingual ?? false) ? { language, task } : {}; + const options = + (getModel(modelId)?.multilingual ?? false) + ? { language, task: taskForModel(modelId, task) } + : {}; if (useServer) { const wsUrl = new URL(`/api/transcribe/live?model=${modelId}`, window.location.href); @@ -988,6 +999,7 @@ export function App() { const allowed = ["onnx-community/whisper-large-v3-turbo", "onnx-community/whisper-medium.en_timestamped"]; if (!allowed.includes(modelId) && !busy && liveStatus === "idle") { setModelId("onnx-community/whisper-large-v3-turbo"); + setTask((current) => taskForModel("onnx-community/whisper-large-v3-turbo", current)); setActiveDevice(null); setActiveThreads(null); setPartialText(""); @@ -1036,8 +1048,9 @@ export function App() { )} @@ -1156,6 +1169,25 @@ export function App() { } /> + {/* Text translation of the finished transcript (D-WS4b) — any engine, + file or live; secondary to the transcript itself. */} + {result && !liveActive && ( + + )} + {applyUpdate && (
A new version of Whisper Web is available. diff --git a/src/components/StorageManager.tsx b/src/components/StorageManager.tsx index b5159da..152f767 100644 --- a/src/components/StorageManager.tsx +++ b/src/components/StorageManager.tsx @@ -5,6 +5,7 @@ import { type ModelStorageInfo, } from "../lib/modelCache"; import { MODELS, getModel } from "../lib/models"; +import { TRANSLATION_MODEL_IDS, getTranslationModelById } from "../lib/translation"; interface StorageManagerProps { /** Downloaded-badge set from App — changes re-measure an open panel. */ @@ -15,7 +16,14 @@ interface StorageManagerProps { onChanged: () => void; } -const MODEL_IDS = MODELS.map((model) => model.id); +// Whisper weights plus the Opus-MT translation models (D-WS4b). +const MODEL_IDS = [...MODELS.map((model) => model.id), ...TRANSLATION_MODEL_IDS]; + +function displayName(modelId: string): string { + const translation = getTranslationModelById(modelId); + if (translation) return `Translation, ${translation.label}`; + return getModel(modelId)?.displayName ?? modelId; +} function formatBytes(bytes: number): string { if (bytes >= 1e9) return `${(bytes / 1e9).toFixed(2)} GB`; @@ -82,7 +90,7 @@ export function StorageManager({ downloaded, disabled, onChanged }: StorageManag
  • - {getModel(entry.modelId)?.displayName ?? entry.modelId} + {displayName(entry.modelId)} {entry.modelId}
    diff --git a/src/components/TranscriptView.tsx b/src/components/TranscriptView.tsx index 188b832..d593f37 100644 --- a/src/components/TranscriptView.tsx +++ b/src/components/TranscriptView.tsx @@ -34,6 +34,13 @@ interface TranscriptViewProps { * "talk.txt"/"talk.srt"); null falls back to "transcript.*". */ exportName?: string | null; + /** Heading once the result is final; defaults to "Transcript". */ + title?: string; + /** + * Inserted before each export extension ("en" → "talk.en.txt", + * "talk.en.srt") so a translation's exports sit beside the original's. + */ + exportSuffix?: string; } /** Trigger a browser download of `contents` as `filename`. */ @@ -62,7 +69,10 @@ export function TranscriptView({ progress = null, placeholder = "Listening for the first words…", exportName = null, + title = "Transcript", + exportSuffix, }: TranscriptViewProps) { + const ext = (extension: string) => (exportSuffix ? `${exportSuffix}.${extension}` : extension); const [copied, setCopied] = useState(false); const headingRef = useRef(null); const prevResultRef = useRef(null); @@ -102,7 +112,7 @@ export function TranscriptView({

    - {result ? "Transcript" : activeLabel} + {result ? title : activeLabel} {percent !== null && ( {percent}% )} @@ -115,7 +125,7 @@ export function TranscriptView({ onClick={() => download( transcriptToTxt(result), - exportFilename(exportName, "txt"), + exportFilename(exportName, ext("txt")), "text/plain", ) } @@ -130,7 +140,7 @@ export function TranscriptView({ onClick={() => download( transcriptToSrt(result.chunks), - exportFilename(exportName, "srt"), + exportFilename(exportName, ext("srt")), "application/x-subrip", ) } @@ -146,7 +156,7 @@ export function TranscriptView({ onClick={() => download( transcriptToTimestampedTxt(result.chunks), - exportFilename(exportName, "timestamps.txt"), + exportFilename(exportName, ext("timestamps.txt")), "text/plain", ) } diff --git a/src/components/TranscriptionOptions.tsx b/src/components/TranscriptionOptions.tsx index badec28..58b9823 100644 --- a/src/components/TranscriptionOptions.tsx +++ b/src/components/TranscriptionOptions.tsx @@ -6,18 +6,26 @@ interface TranscriptionOptionsProps { onLanguageChange: (language: string) => void; task: WhisperTask; onTaskChange: (task: WhisperTask) => void; + /** + * Whether the model supports Whisper's own speech → English translation + * (`WhisperModel.canTranslate`, D-WS4a). When false the Task control is not + * rendered at all — transcribing is the only thing such a model does. + */ + canTranslate: boolean; disabled?: boolean; } /** * Language + task controls for multilingual models. The parent renders this * only when the selected model is multilingual — `.en` models take neither. + * The task control only appears where the model can translate (D-WS4a). */ export function TranscriptionOptions({ language, onLanguageChange, task, onTaskChange, + canTranslate, disabled = false, }: TranscriptionOptionsProps) { return ( @@ -37,17 +45,19 @@ export function TranscriptionOptions({ ))} - + {canTranslate && ( + + )}

    ); } diff --git a/src/components/TranslatePanel.tsx b/src/components/TranslatePanel.tsx new file mode 100644 index 0000000..539c305 --- /dev/null +++ b/src/components/TranslatePanel.tsx @@ -0,0 +1,275 @@ +import { useEffect, useRef, useState } from "react"; +import { classifyError, type ClassifiedError } from "../lib/errors"; +import { formatModelSize } from "../lib/models"; +import { + TRANSLATION_MODELS, + assembleTranslation, + buildTranslationUnits, + guessSourceLanguage, + type TranslationModel, + type TranslationUnit, +} from "../lib/translation"; +import type { TranscriptChunk } from "../worker/protocol"; +import { isTranslateWorkerMessage, type TranslateRequest } from "../worker/translateProtocol"; +import { TranscriptView } from "./TranscriptView"; + +interface TranscriptResult { + text: string; + chunks: TranscriptChunk[]; +} + +/** Where a translation run is; `idle` covers "never started" and "finished". */ +export type TranslatePhase = + | { kind: "idle" } + | { kind: "starting" } + | { kind: "downloading"; loaded: number; total: number } + | { kind: "translating"; done: number; total: number }; + +/** + * Plain-language status for a running translation (PRODUCT.md: say what is + * happening, in everyday words). Null when nothing is running. + */ +export function translateStatusText(phase: TranslatePhase, cached: boolean): string | null { + switch (phase.kind) { + case "idle": + return null; + case "starting": + return cached ? "Loading the translation model…" : "Starting the download…"; + case "downloading": { + const percent = phase.total > 0 ? Math.min(100, Math.round((phase.loaded / phase.total) * 100)) : 0; + return `Downloading the translation model… ${percent}%`; + } + case "translating": + return phase.total === 0 + ? "Translating…" + : `Translating… ${phase.done} of ${phase.total} sentence${phase.total === 1 ? "" : "s"}`; + } +} + +interface TranslatePanelProps { + /** The finished transcript to translate (any engine, file or live). */ + result: TranscriptResult; + /** + * Language Whisper was told the audio is in ("da", "en"), "en" when + * Whisper itself translated, or undefined/"auto" — seeds the direction. + */ + sourceLanguageHint?: string; + /** Hub ids with cached weights (D-005) — decides the download wording. */ + downloaded: ReadonlySet; + /** Called once a model finished loading, to refresh downloaded badges. */ + onModelCached: () => void; + /** Source file name for export names ("talk.m4a" → "talk.en.srt"). */ + exportName: string | null; + /** True while a transcription or live session is running. */ + disabled: boolean; +} + +/** + * Text translation of a finished transcript with Opus-MT (D-WS4b): a + * secondary disclosure under the transcript, separate from Whisper's own + * "Translate to English" task. Runs on-device in its own worker; the model + * is downloaded only when the user clicks Translate (D-020). + */ +export function TranslatePanel({ + result, + sourceLanguageHint, + downloaded, + onModelCached, + exportName, + disabled, +}: TranslatePanelProps) { + const workerRef = useRef(null); + const runRef = useRef(0); + const [modelId, setModelId] = useState(() => defaultModel(result, sourceLanguageHint).id); + const [phase, setPhase] = useState({ kind: "idle" }); + const [error, setError] = useState(null); + const [translation, setTranslation] = useState<{ + model: TranslationModel; + result: TranscriptResult; + } | null>(null); + + const model = TRANSLATION_MODELS.find((m) => m.id === modelId) ?? TRANSLATION_MODELS[0]!; + const cached = downloaded.has(model.id); + const running = phase.kind !== "idle"; + const empty = result.text.trim() === "" && result.chunks.every((c) => c.text.trim() === ""); + + function stopWorker() { + runRef.current++; + workerRef.current?.terminate(); + workerRef.current = null; + } + + // A new transcript (or model switch upstream) invalidates everything here. + useEffect(() => { + stopWorker(); + setPhase({ kind: "idle" }); + setError(null); + setTranslation(null); + setModelId(defaultModel(result, sourceLanguageHint).id); + // Keyed on the transcript only: the language hint seeds the default for + // a new transcript but must not wipe a translation when it changes. + }, [result]); + + useEffect(() => stopWorker, []); + + function translate() { + if (running || empty) return; + const run = ++runRef.current; + const units: TranslationUnit[] = buildTranslationUnits(result); + const target = model; + setError(null); + setTranslation(null); + setPhase({ kind: "starting" }); + + // Separate worker, bundled and served at a stable URL like the Whisper + // worker (src/index.ts, scripts/build.ts). Reused across runs. + const worker = (workerRef.current ??= new Worker("translate.worker.js", { type: "module" })); + const requestId = `translate-${run}`; + const fail = (raw: unknown, stage: "translate" | "worker-crash") => { + if (runRef.current !== run) return; + setPhase({ kind: "idle" }); + const offline = typeof navigator !== "undefined" && !navigator.onLine; + // A crash (e.g. OOM tab-kill) leaves nothing to reuse. + if (stage === "worker-crash") stopWorker(); + setError(classifyError(raw, { stage: "translate", offline })); + }; + worker.onmessage = (event: MessageEvent) => { + const message = event.data; + if (runRef.current !== run || !isTranslateWorkerMessage(message)) return; + if (message.requestId !== requestId) return; + switch (message.type) { + case "translate-download": + setPhase({ kind: "downloading", loaded: message.loaded, total: message.total }); + break; + case "translate-progress": + if (message.done === 0) onModelCached(); + setPhase({ kind: "translating", done: message.done, total: message.total }); + break; + case "translate-result": + try { + setTranslation({ model: target, result: assembleTranslation(units, message.texts) }); + setPhase({ kind: "idle" }); + } catch (assembleError) { + fail(assembleError, "translate"); + } + break; + case "translate-error": + fail(message.message, "translate"); + break; + } + }; + // An empty crash message is the silent OOM tab-kill signature (D-027, + // D-030) — say so in words the "translate" stage classifies as OOM. + worker.onerror = (event) => + fail( + event.message || "The translation stopped: the browser ran out of memory.", + "worker-crash", + ); + const request: TranslateRequest = { + type: "translate", + requestId, + modelId: target.id, + units: units.map((unit) => unit.text), + }; + worker.postMessage(request); + } + + function cancel() { + stopWorker(); + setPhase({ kind: "idle" }); + } + + const status = translateStatusText(phase, cached); + const progressValue = + phase.kind === "downloading" + ? { value: phase.loaded, max: Math.max(phase.total, 1) } + : phase.kind === "translating" + ? { value: phase.done, max: Math.max(phase.total, 1) } + : null; + + return ( + <> +
    + Translate transcript… +
    + + {running ? ( + + ) : ( + + )} +
    +

    + {cached + ? "Translation model downloaded — works offline. " + : `Downloads a translation model once (~${formatModelSize(model.approxBytes)}), then works offline. `} + Runs on this device; the text never leaves it. Machine translation — check + names and important details. +

    +

    + Model: {model.id} (Opus-MT) +

    + {status && ( +
    + {status} + {progressValue && ( + + )} +
    + )} + {error && ( +
    +

    {error.message}

    + {error.suggestion &&

    {error.suggestion}

    } + {error.retryable && ( + + )} +
    + )} +
    + {translation && ( + + )} + + ); +} + +function defaultModel(result: TranscriptResult, hint?: string): TranslationModel { + const source = guessSourceLanguage(result.text, hint); + return TRANSLATION_MODELS.find((m) => m.source === source) ?? TRANSLATION_MODELS[0]!; +} diff --git a/src/index.css b/src/index.css index 999d03c..bf8f2c2 100644 --- a/src/index.css +++ b/src/index.css @@ -773,7 +773,8 @@ body { display: flex; align-items: center; justify-content: space-between; - gap: 1rem; + flex-wrap: wrap; + gap: 0.5rem 1rem; } .transcript-header h2 { @@ -1004,5 +1005,125 @@ code { .transcript-actions { display: flex; align-items: center; + flex-wrap: wrap; gap: 0.5rem; } + +/* -------------------------------------------------------------------------- + Transcript translation (D-WS4b): a secondary disclosure under the + transcript, same panel language as .storage-manager. + -------------------------------------------------------------------------- */ + +.translate-panel { + background: var(--panel); + border: 1px solid var(--border); + border-radius: 12px; + padding: 0.75rem 1rem; +} + +.translate-panel summary { + cursor: pointer; + font-weight: 600; +} + +.translate-controls { + display: flex; + align-items: center; + flex-wrap: wrap; + gap: 0.5rem 0.75rem; + margin-top: 0.75rem; +} + +.translate-direction { + display: flex; + align-items: center; + gap: 0.5rem; + font-size: 0.85rem; + color: var(--muted); + min-width: 0; +} + +.translate-direction select { + min-width: 0; + max-width: 100%; + background: var(--surface-raised); + color: var(--fg); + border: 1px solid var(--border); + border-radius: 8px; + padding: 0.3rem 0.5rem; + font-size: 0.85rem; +} + +.translate-button, +.translate-cancel { + border: 1px solid var(--border); + background: transparent; + color: inherit; + border-radius: 8px; + padding: 0.3rem 0.9rem; + font-size: 0.85rem; + cursor: pointer; + white-space: nowrap; + transition: border-color 0.15s, color 0.15s, background-color 0.15s; +} + +/* The panel's one action reads as the action: accent outline, like a + focused copy button, without competing with the transcript above. */ +.translate-button { + border-color: var(--accent); + color: var(--accent); +} + +.translate-button:hover:enabled { + background: var(--accent); + color: var(--on-accent); +} + +.translate-button:disabled { + opacity: 0.5; + cursor: default; +} + +.translate-cancel:hover { + border-color: var(--danger-border); + color: var(--danger); +} + +.translate-hint { + margin: 0.6rem 0 0; + max-width: 65ch; + color: var(--muted); + font-size: 0.85rem; +} + +.translate-model-id { + margin-top: 0.25rem; + font-size: 0.75rem; +} + +.translate-model-id code { + overflow-wrap: anywhere; +} + +.translate-progress { + display: flex; + align-items: center; + flex-wrap: wrap; + gap: 0.4rem 0.75rem; + margin-top: 0.75rem; + font-size: 0.85rem; +} + +.translate-status { + font-variant-numeric: tabular-nums; +} + +.translate-progress progress { + flex: 1 1 10rem; + height: 6px; + accent-color: var(--accent); +} + +.translate-error { + margin-top: 0.75rem; +} diff --git a/src/index.ts b/src/index.ts index 328d7f3..e657557 100644 --- a/src/index.ts +++ b/src/index.ts @@ -223,7 +223,8 @@ async function productionServeOptions(): Promise { * Bun's bundler does not currently follow `new Worker(...)` graphs (verified * with Bun 1.3: both `bun build` and the dev server leave the call untouched), * so worker-like entrypoints are bundled at startup and served from stable - * URLs. The UI spawns `new Worker("/inference.worker.js")`; the service + * URLs. The UI spawns `new Worker("/inference.worker.js")` (and + * "/translate.worker.js" for transcript translation, D-WS4b); the service * worker registers as "/sw.js". */ async function bundleEntrypoint(relativePath: string): Promise { @@ -249,6 +250,8 @@ async function developmentServeOptions(): Promise { const { default: index } = await import("./index.html"); const workerScript = await bundleEntrypoint("./worker/inference.worker.ts"); + // Transcript translation runs in its own worker (D-WS4b). + const translateWorkerScript = await bundleEntrypoint("./worker/translate.worker.ts"); // Dev SW: empty precache list — assets are unhashed here, so the shell is // runtime-cached network-first instead (see src/pwa/sw.ts). const swScript = injectPrecacheManifest( @@ -283,6 +286,11 @@ async function developmentServeOptions(): Promise { if (pathname.endsWith("/inference.worker.js")) { return withIsolationHeaders(new Response(workerScript, { headers: scriptHeaders })); } + if (pathname.endsWith("/translate.worker.js")) { + return withIsolationHeaders( + new Response(translateWorkerScript, { headers: scriptHeaders }), + ); + } if (pathname.endsWith("/sw.js")) { return withIsolationHeaders( new Response(swScript, { headers: { ...scriptHeaders, "Cache-Control": "no-cache" } }), diff --git a/src/lib/errors.ts b/src/lib/errors.ts index fa6ec30..a0cc89a 100644 --- a/src/lib/errors.ts +++ b/src/lib/errors.ts @@ -16,7 +16,12 @@ * inspect across Web Audio / Transformers.js / onnxruntime-web / fetch. */ -export type ErrorStage = "decode" | "load" | "transcribe" | "worker-crash"; +/** + * `"translate"` covers text translation of a finished transcript (D-WS4b): + * both its one-time model download and the translation pass, which run in + * their own worker — only the suggestions differ from the Whisper stages. + */ +export type ErrorStage = "decode" | "load" | "transcribe" | "worker-crash" | "translate"; export type ErrorKind = "unsupported-file" | "out-of-memory" | "device-lost" | "network" | "generic"; @@ -95,7 +100,9 @@ export function classifyError(raw: unknown, context: ClassifyContext = {}): Clas kind: "out-of-memory", message: trimmed.length > 0 ? message : "The browser ran out of memory and had to stop.", suggestion: - "Try a smaller model (Tiny, Base, or Small) — it needs far less memory to load and run.", + context.stage === "translate" + ? "Close other tabs to free memory, then try translating again." + : "Try a smaller model (Tiny, Base, or Small) — it needs far less memory to load and run.", retryable: false, }; } @@ -117,7 +124,9 @@ export function classifyError(raw: unknown, context: ClassifyContext = {}): Clas suggestion: context.stage === "load" ? "Check your connection, then retry — the model download will restart." - : "Check your connection and try again.", + : context.stage === "translate" + ? "Check your connection, then retry — the translation model download will restart." + : "Check your connection and try again.", retryable: true, }; } diff --git a/src/lib/models.ts b/src/lib/models.ts index ad82c0c..cb3218b 100644 --- a/src/lib/models.ts +++ b/src/lib/models.ts @@ -42,6 +42,15 @@ export interface WhisperModel { displayName: string; /** False for `.en` (English-only) variants. */ multilingual: boolean; + /** + * Whether Whisper's own `task: "translate"` (speech → English text) is + * offered (D-WS4a). False for every `.en` model (no task token) and for + * large-v3-turbo, which OpenAI did not train for translation — and for any + * turbo-based fine-tune not tested for it. The UI hides "Translate to + * English" where false; text translation of a finished transcript + * (D-WS4b) works for every model regardless. + */ + canTranslate: boolean; /** Encoder parameter count, in millions (drives size estimates). */ encoderParamsM: number; /** Decoder parameter count, in millions (drives size estimates). */ @@ -174,6 +183,7 @@ export const MODELS: readonly WhisperModel[] = [ fasterWhisperModel: "tiny.en", displayName: "Whisper Tiny (English)", multilingual: false, + canTranslate: false, encoderParamsM: 8, decoderParamsM: 31, defaults: { @@ -188,6 +198,7 @@ export const MODELS: readonly WhisperModel[] = [ fasterWhisperModel: "tiny", displayName: "Whisper Tiny", multilingual: true, + canTranslate: true, encoderParamsM: 8, decoderParamsM: 31, defaults: { wasm: WASM_Q8, webgpu: WEBGPU_FP32, webnn: WEBNN_FP16 }, @@ -197,6 +208,7 @@ export const MODELS: readonly WhisperModel[] = [ fasterWhisperModel: "base.en", displayName: "Whisper Base (English)", multilingual: false, + canTranslate: false, encoderParamsM: 20, decoderParamsM: 54, defaults: { wasm: WASM_Q8, webgpu: WEBGPU_FP32, webnn: WEBNN_FP16 }, @@ -206,6 +218,7 @@ export const MODELS: readonly WhisperModel[] = [ fasterWhisperModel: "base", displayName: "Whisper Base", multilingual: true, + canTranslate: true, encoderParamsM: 20, decoderParamsM: 54, defaults: { wasm: WASM_Q8, webgpu: WEBGPU_FP32, webnn: WEBNN_FP16 }, @@ -215,6 +228,7 @@ export const MODELS: readonly WhisperModel[] = [ fasterWhisperModel: "small.en", displayName: "Whisper Small (English)", multilingual: false, + canTranslate: false, encoderParamsM: 88, decoderParamsM: 156, defaults: { wasm: WASM_Q8, webgpu: WEBGPU_FP16_Q4, webnn: WEBNN_FP16 }, @@ -224,6 +238,7 @@ export const MODELS: readonly WhisperModel[] = [ fasterWhisperModel: "small", displayName: "Whisper Small", multilingual: true, + canTranslate: true, encoderParamsM: 88, decoderParamsM: 156, defaults: { wasm: WASM_Q8, webgpu: WEBGPU_FP16_Q4, webnn: WEBNN_FP16 }, @@ -237,6 +252,7 @@ export const MODELS: readonly WhisperModel[] = [ fasterWhisperModel: "medium.en", displayName: "Whisper Medium (English)", multilingual: false, + canTranslate: false, encoderParamsM: 306, decoderParamsM: 463, defaults: { wasm: WASM_Q4, webgpu: ACCEL_Q4, webnn: ACCEL_Q4 }, @@ -246,6 +262,7 @@ export const MODELS: readonly WhisperModel[] = [ fasterWhisperModel: "medium", displayName: "Whisper Medium", multilingual: true, + canTranslate: true, encoderParamsM: 306, decoderParamsM: 463, defaults: { wasm: WASM_Q4, webgpu: ACCEL_Q4, webnn: ACCEL_Q4 }, @@ -255,6 +272,9 @@ export const MODELS: readonly WhisperModel[] = [ fasterWhisperModel: "large-v3-turbo", displayName: "Whisper Large v3 Turbo", multilingual: true, + // Not trained for translation (OpenAI's turbo model card); its + // "translate" output is unreliable (D-WS4a). + canTranslate: false, encoderParamsM: 635, decoderParamsM: 174, defaults: { wasm: WASM_Q4, webgpu: WEBGPU_FP16_Q4, webnn: WEBNN_FP16 }, @@ -274,6 +294,16 @@ export function getFasterWhisperModel(id: string): string | undefined { return getModel(id)?.fasterWhisperModel; } +/** + * The task a run on `modelId` should actually use (D-WS4a): a "translate" + * left over from a model that could translate falls back to "transcribe" on + * one that cannot (large-v3-turbo, `.en`, unknown ids). App applies it on + * every model switch and again when building each run's options. + */ +export function taskForModel(modelId: string, task: WhisperTask): WhisperTask { + return task === "translate" && getModel(modelId)?.canTranslate !== true ? "transcribe" : task; +} + /** Default dtype for a model on a given device. Unknown ids fall back to q8. */ export function resolveDefaultDtype( modelId: string, diff --git a/src/lib/translation.ts b/src/lib/translation.ts new file mode 100644 index 0000000..b568734 --- /dev/null +++ b/src/lib/translation.ts @@ -0,0 +1,332 @@ +/** + * Text translation of a finished transcript (D-WS4b) — the pure half. + * + * Opus-MT (MarianMT) runs in `src/worker/translate.worker.ts`; everything + * here is planning that needs no model: which checkpoint serves a language + * pair, how a transcript becomes translation units that keep their + * timestamps, and how a unit is cut into sentence pieces under the model's + * input budget. No browser globals, so `bun test` covers it. + * + * Why per-sentence pieces: Opus-MT is trained on sentence pairs and, given + * several sentences at once, routinely drops the later ones (measured + * 2026-10-29: "…I live in Copenhagen. How are you?" came back as one + * sentence). Its encoder also tops out at 512 positions. + */ + +import type { TranscriptChunk } from "../worker/protocol"; + +export type TranslationLanguage = "da" | "en"; + +export interface TranslationModel { + /** Hugging Face Hub id of the Transformers.js ONNX export. */ + id: string; + source: TranslationLanguage; + target: TranslationLanguage; + /** Direction as shown in the picker, e.g. "Danish → English". */ + label: string; + /** Plain name of the target language, e.g. "English". */ + targetLabel: string; + /** + * One-time download at the q8 dtype the worker loads: encoder + merged + * decoder + tokenizer/configs, summed from the Hub file sizes. + */ + approxBytes: number; +} + +/** + * Xenova's ONNX exports of Helsinki-NLP's Opus-MT checkpoints (Apache-2.0 + * upstream). Verified to load and translate under Transformers.js 4.2.0. + */ +export const TRANSLATION_MODELS: readonly TranslationModel[] = [ + { + id: "Xenova/opus-mt-da-en", + source: "da", + target: "en", + label: "Danish → English", + targetLabel: "English", + approxBytes: 113_000_000, + }, + { + id: "Xenova/opus-mt-en-da", + source: "en", + target: "da", + label: "English → Danish", + targetLabel: "Danish", + approxBytes: 113_000_000, + }, +]; + +export const TRANSLATION_MODEL_IDS: readonly string[] = TRANSLATION_MODELS.map((m) => m.id); + +export function getTranslationModel( + source: TranslationLanguage, + target: TranslationLanguage, +): TranslationModel | undefined { + return TRANSLATION_MODELS.find((m) => m.source === source && m.target === target); +} + +/** Lookup by Hub id (storage panel names, worker request validation). */ +export function getTranslationModelById(id: string): TranslationModel | undefined { + return TRANSLATION_MODELS.find((m) => m.id === id); +} + +// --------------------------------------------------------------------------- +// Units: transcript → sentence-ending groups of chunks with their time span +// --------------------------------------------------------------------------- + +/** A stretch of transcript translated as one cue. */ +export interface TranslationUnit { + text: string; + /** Merged [start, end] of the source chunks; null for untimed text. */ + timestamp: [number, number | null] | null; +} + +const SENTENCE_END = /[.!?…]["'”’)\]]*$/; + +/** Default cap on chunks merged into one unit, so an unpunctuated transcript + * still yields subtitle-sized cues rather than one giant one. */ +export const MAX_CHUNKS_PER_UNIT = 8; + +/** + * Group a transcript into translation units. Whisper's segments often end + * mid-sentence; translating fragments hurts quality, so consecutive chunks + * are merged until one ends a sentence (or `maxChunksPerUnit` is reached) + * and the unit takes the merged time span — `.srt` keeps working, with + * slightly longer cues. Without chunks (nothing timestamped), each non-empty + * line is an untimed unit. + */ +export function buildTranslationUnits( + result: { text: string; chunks: TranscriptChunk[] }, + maxChunksPerUnit: number = MAX_CHUNKS_PER_UNIT, +): TranslationUnit[] { + const chunks = result.chunks.filter((chunk) => chunk.text.trim() !== ""); + if (chunks.length === 0) { + return result.text + .split(/\n+/) + .map((line) => line.trim()) + .filter((line) => line !== "") + .map((text) => ({ text, timestamp: null })); + } + const units: TranslationUnit[] = []; + let group: TranscriptChunk[] = []; + const flush = () => { + const first = group[0]; + const last = group[group.length - 1]; + if (!first || !last) return; + units.push({ + text: group.map((chunk) => chunk.text.trim()).join(" "), + timestamp: [first.timestamp[0], last.timestamp[1]], + }); + group = []; + }; + for (const chunk of chunks) { + group.push(chunk); + if (SENTENCE_END.test(chunk.text.trim()) || group.length >= maxChunksPerUnit) flush(); + } + flush(); + return units; +} + +// --------------------------------------------------------------------------- +// Pieces: unit text → sentences under the model's token budget +// --------------------------------------------------------------------------- + +/** + * Cheap upper-bound token estimate for Opus-MT's SentencePiece vocab, used + * when the real tokenizer isn't at hand (tests, planning). Measured ~0.34 + * tokens/char on en_alien and ~0.26 on da_festuge with the real tokenizers; + * 0.5/char plus the end-of-sentence token stays above both. + */ +export function estimateTokens(text: string): number { + return text.length === 0 ? 0 : Math.ceil(text.length / 2) + 1; +} + +/** + * Default per-piece budget, well inside Opus-MT's 512 positions: quality + * falls off on very long inputs long before the hard limit, and normal + * sentences are 10–60 tokens, so this only bites on unpunctuated run-ons. + */ +export const DEFAULT_MAX_PIECE_TOKENS = 200; + +export interface SplitOptions { + maxTokens?: number; + /** Token counter — the worker passes the real tokenizer. */ + countTokens?: (text: string) => number; +} + +/** + * Title abbreviations (en + da) and single-letter initials ("J.") — the + * sentence segmenter's false breaks. Kept to an explicit list: a broad + * "short capitalised word" rule would glue real one-word sentences + * ("Yes.", "Okay.") onto the next one. + */ +const ABBREVIATION = /^(?:Mr|Mrs|Ms|Dr|Prof|St|Jr|Sr|Hr|Fru|Nr|\p{Lu})\.$/u; + +function sentences(text: string): string[] { + const out: string[] = []; + let carry = ""; + for (const { segment } of new Intl.Segmenter("en", { granularity: "sentence" }).segment(text)) { + const trimmed = segment.trim(); + if (trimmed === "") continue; + if (ABBREVIATION.test(trimmed)) { + carry = carry === "" ? trimmed : `${carry} ${trimmed}`; + continue; + } + out.push(carry === "" ? trimmed : `${carry} ${trimmed}`); + carry = ""; + } + if (carry !== "") out.push(carry); + return out; +} + +/** Greedily pack `parts` (in order) into space-joined pieces under budget. */ +function pack(parts: string[], maxTokens: number, count: (text: string) => number): string[] { + const pieces: string[] = []; + let current = ""; + for (const part of parts) { + const candidate = current === "" ? part : `${current} ${part}`; + if (current !== "" && count(candidate) > maxTokens) { + pieces.push(current); + current = part; + } else { + current = candidate; + } + } + if (current !== "") pieces.push(current); + return pieces; +} + +/** Split one over-budget sentence: commas first, then words. A single word + * longer than the budget is kept whole (nothing is ever dropped). */ +function wrap(sentence: string, maxTokens: number, count: (text: string) => number): string[] { + const clauses = sentence.split(/(?<=,)\s+/); + const parts = clauses.flatMap((clause) => + count(clause) <= maxTokens ? [clause] : pack(clause.split(/\s+/), maxTokens, count), + ); + return pack(parts, maxTokens, count); +} + +/** + * Cut a unit's text into the pieces actually fed to Opus-MT: one per + * sentence, an over-budget sentence wrapped on commas then words. Order is + * preserved and every word survives — joining the pieces with spaces gives + * the input back up to whitespace. + */ +export function splitForTranslation(text: string, options: SplitOptions = {}): string[] { + const maxTokens = options.maxTokens ?? DEFAULT_MAX_PIECE_TOKENS; + const count = options.countTokens ?? estimateTokens; + return sentences(text).flatMap((sentence) => + count(sentence) <= maxTokens ? [sentence] : wrap(sentence, maxTokens, count), + ); +} + +// --------------------------------------------------------------------------- +// The translation loop (model injected: worker, benchmark and tests share it) +// --------------------------------------------------------------------------- + +export interface TranslateUnitsDeps extends SplitOptions { + /** Translate a batch of pieces; must answer one string per piece, in order. */ + translateBatch: (pieces: string[]) => Promise; + /** Pieces per model call (padding makes larger batches cheaper per piece). */ + batchSize?: number; + /** Called with (done, total) pieces: once at 0, then after every batch. */ + onProgress?: (done: number, total: number) => void; +} + +/** + * One piece per model call. Batching would be ~4x faster, but Transformers.js + * 4.2's batched Marian generation is broken: every row of a padded batch + * comes back as row 0's translation (measured 2026-10-29 on + * onnxruntime-node, q8 and fp32, any graph optimisation level — D-WS4b). + * Re-measure with `scripts/bench-translate.ts` after a Transformers.js + * upgrade before raising this. + */ +export const DEFAULT_TRANSLATE_BATCH_SIZE = 1; + +/** + * Translate unit texts: split each into sentence pieces, translate all + * pieces in order-preserving batches, rejoin each unit's pieces with spaces. + * Returns one translated string per input unit. + */ +export async function translateUnits( + units: readonly string[], + deps: TranslateUnitsDeps, +): Promise { + const batchSize = Math.max(1, deps.batchSize ?? DEFAULT_TRANSLATE_BATCH_SIZE); + const perUnit = units.map((unit) => splitForTranslation(unit, deps)); + const pieces = perUnit.flat(); + const translated: string[] = []; + deps.onProgress?.(0, pieces.length); + for (let start = 0; start < pieces.length; start += batchSize) { + const batch = pieces.slice(start, start + batchSize); + const out = await deps.translateBatch(batch); + if (out.length !== batch.length) { + throw new Error(`Translation returned ${out.length} sentences for ${batch.length}.`); + } + translated.push(...out.map((text) => text.trim())); + deps.onProgress?.(translated.length, pieces.length); + } + let next = 0; + return perUnit.map((unitPieces) => { + const joined = translated.slice(next, next + unitPieces.length).join(" "); + next += unitPieces.length; + return joined; + }); +} + +// --------------------------------------------------------------------------- +// Assembly +// --------------------------------------------------------------------------- + +/** + * Pair translated unit texts back with their units' time spans, in the + * `{ text, chunks }` shape the transcript view and exports already take. + */ +export function assembleTranslation( + units: readonly TranslationUnit[], + translated: readonly string[], +): { text: string; chunks: TranscriptChunk[] } { + if (units.length !== translated.length) { + throw new Error( + `Translation returned ${translated.length} parts for ${units.length} — refusing to misalign the transcript.`, + ); + } + const texts = translated.map((text) => text.trim()); + const chunks: TranscriptChunk[] = []; + units.forEach((unit, index) => { + if (unit.timestamp) chunks.push({ text: texts[index] ?? "", timestamp: unit.timestamp }); + }); + const untimed = units.every((unit) => unit.timestamp === null); + return { text: texts.join(untimed ? "\n" : " "), chunks }; +} + +// --------------------------------------------------------------------------- +// Source language guess (default for the direction picker) +// --------------------------------------------------------------------------- + +const DANISH_WORDS = new Set([ + "og", "ikke", "det", "jeg", "vi", "på", "til", "med", "som", "har", "af", "der", + "kan", "skal", "vil", "også", "men", "hvad", "hvordan", "være", "noget", +]); +const ENGLISH_WORDS = new Set([ + "the", "and", "is", "to", "of", "that", "you", "it", "with", "have", "this", "not", + "what", "how", "be", "was", "are", +]); + +/** + * Best guess at which language a transcript is in, for the direction + * picker's default (the user can always change it). An explicit Whisper + * language wins; otherwise Danish letters and function words vs English + * ones decide, defaulting to English. + */ +export function guessSourceLanguage(text: string, whisperLanguage?: string): TranslationLanguage { + if (whisperLanguage === "da" || whisperLanguage === "en") return whisperLanguage; + const lower = text.toLowerCase(); + let danish = (lower.match(/[æøå]/g)?.length ?? 0) * 2; + let english = 0; + for (const word of lower.match(/\p{L}+/gu) ?? []) { + if (DANISH_WORDS.has(word)) danish += 1; + if (ENGLISH_WORDS.has(word)) english += 1; + } + return danish > english ? "da" : "en"; +} diff --git a/src/worker/opusMt.ts b/src/worker/opusMt.ts new file mode 100644 index 0000000..8bbee00 --- /dev/null +++ b/src/worker/opusMt.ts @@ -0,0 +1,59 @@ +/** + * Opus-MT (MarianMT) through Transformers.js — the model half of transcript + * translation (D-WS4b). Shared by the browser translate worker and the + * onnxruntime-node measurement script / real-model test, so all three run + * the exact same pipeline options. + */ + +import { pipeline, type ProgressInfo, type TranslationPipeline } from "@huggingface/transformers"; + +/** Hard input limit of every Opus-MT checkpoint (max_position_embeddings). */ +export const OPUS_MT_MAX_POSITIONS = 512; + +export interface OpusMtTranslator { + /** One translated string per piece, same order. */ + translateBatch(pieces: string[]): Promise; + /** Real SentencePiece token count (incl. ) for the splitter's budget. */ + countTokens(text: string): number; + dispose(): Promise; +} + +export interface LoadOpusMtOptions { + /** "wasm" in the browser; omit under onnxruntime-node (defaults to CPU). */ + device?: "wasm"; + progressCallback?: (info: ProgressInfo) => void; +} + +export async function loadOpusMt( + modelId: string, + options: LoadOpusMtOptions = {}, +): Promise { + const translator = (await pipeline("translation", modelId, { + // q8: 113 MB per direction (encoder + merged decoder + tokenizer); the + // fp32 export is ~4x larger for no audible gain on transcripts. + dtype: "q8", + ...(options.device && { device: options.device }), + // D-014: the bundled onnxruntime-web's level-2 QDQ transform rejects + // 8-bit decoders with a tied embedding weight — Marian ties them too. + session_options: { graphOptimizationLevel: "basic" as const }, + ...(options.progressCallback && { progress_callback: options.progressCallback }), + })) as TranslationPipeline; + + const countTokens = (text: string) => translator.tokenizer.encode(text).length; + + return { + countTokens, + async translateBatch(pieces) { + if (pieces.length === 0) return []; + // Cap output length relative to the longest input: a translation is + // rarely more than ~2x its source in tokens, and the cap stops a rare + // repetition loop from running to 512 tokens. + const longest = Math.max(...pieces.map(countTokens)); + const output = (await translator(pieces, { + max_new_tokens: Math.min(OPUS_MT_MAX_POSITIONS, 3 * longest + 16), + })) as { translation_text: string }[]; + return output.map((entry) => entry.translation_text); + }, + dispose: () => translator.dispose(), + }; +} diff --git a/src/worker/translate.worker.ts b/src/worker/translate.worker.ts new file mode 100644 index 0000000..6e0ba4e --- /dev/null +++ b/src/worker/translate.worker.ts @@ -0,0 +1,98 @@ +/** + * Transcript-translation Web Worker (D-WS4b): Opus-MT through + * Transformers.js, off the main thread (D-003). + * + * A dedicated worker rather than a message type on the Whisper worker: + * - its own onnxruntime instance gets the full WASM thread pool. ort fixes + * the pool at first init (D-023), and a Whisper worker that initialised + * ort for WebGPU/WebNN runs WASM single-threaded for life; + * - cancelling a translation terminates only this worker — the loaded + * Whisper pipeline survives, and a Whisper cancel/restart (D-018/D-025) + * never kills a translation; + * - the model is downloaded only when the user asks for a translation and + * cached by Transformers.js in the Cache API like Whisper weights + * (D-005/D-020), so a cached model works offline. + * WASM only for now: Opus-MT is small, and WebGPU behaviour can't be + * measured in this environment. + */ + +import { env, type ProgressInfo } from "@huggingface/transformers"; +import { getTranslationModelById, translateUnits } from "../lib/translation"; +import { loadOpusMt, type OpusMtTranslator } from "./opusMt"; +import { + DownloadTally, + isTranslateRequest, + type TranslateRequest, + type TranslateWorkerMessage, +} from "./translateProtocol"; + +// Same-origin onnxruntime binaries (offline, COEP) — see inference.worker.ts. +if (env.backends.onnx.wasm) { + env.backends.onnx.wasm.wasmPaths = "./ort/"; + // D-023: with cross-origin isolation, use most of the machine. + env.backends.onnx.wasm.numThreads = globalThis.crossOriginIsolated + ? Math.max(1, Math.min((navigator.hardwareConcurrency || 4) - 1, 12)) + : 1; +} + +interface DedicatedWorkerScope { + postMessage(message: TranslateWorkerMessage): void; + addEventListener(type: "message", listener: (event: MessageEvent) => void): void; +} +const scope = self as unknown as DedicatedWorkerScope; + +/** One loaded model at a time; switching direction disposes the other. */ +let active: { modelId: string; promise: Promise } | null = null; + +function load(modelId: string, onProgress: (info: ProgressInfo) => void): Promise { + if (active?.modelId === modelId) return active.promise; + const previous = active; + const promise = (async () => { + if (previous) await (await previous.promise.catch(() => null))?.dispose(); + return loadOpusMt(modelId, { device: "wasm", progressCallback: onProgress }); + })(); + active = { modelId, promise }; + promise.catch(() => { + if (active?.promise === promise) active = null; // allow a clean retry + }); + return promise; +} + +async function handle(request: TranslateRequest): Promise { + const { requestId } = request; + if (!getTranslationModelById(request.modelId)) { + throw new Error(`Unknown translation model: ${request.modelId}`); + } + const tally = new DownloadTally(); + const translator = await load(request.modelId, (info) => { + if (info.status === "progress") { + const { loaded, total } = tally.update(info.file, info.loaded, info.total); + scope.postMessage({ type: "translate-download", requestId, loaded, total }); + } + }); + const texts = await translateUnits(request.units, { + translateBatch: (pieces) => translator.translateBatch(pieces), + countTokens: translator.countTokens, + onProgress: (done, total) => + scope.postMessage({ type: "translate-progress", requestId, done, total }), + }); + scope.postMessage({ type: "translate-result", requestId, texts }); +} + +// Requests run one at a time, in arrival order: the UI only ever has one in +// flight, and serialising keeps two passes from sharing the thread pool. +let queue: Promise = Promise.resolve(); + +scope.addEventListener("message", (event) => { + const data = event.data; + if (!isTranslateRequest(data)) return; + queue = queue.then(() => + handle(data).catch((error: unknown) => { + scope.postMessage({ + type: "translate-error", + requestId: data.requestId, + message: error instanceof Error ? error.message : String(error), + }); + }), + ); +}); diff --git a/src/worker/translateProtocol.ts b/src/worker/translateProtocol.ts new file mode 100644 index 0000000..e26c105 --- /dev/null +++ b/src/worker/translateProtocol.ts @@ -0,0 +1,106 @@ +/** + * Typed postMessage protocol between the UI and the transcript-translation + * worker (`src/worker/translate.worker.ts`, D-WS4b). Kept apart from the + * Whisper worker's protocol: separate worker, separate lifetime. Free of + * browser globals so `bun test` covers the guards. + */ + +/** UI → worker: translate these unit texts with this Opus-MT model. */ +export interface TranslateRequest { + type: "translate"; + /** Correlates every reply; the UI ignores replies for stale ids. */ + requestId: string; + /** One of `TRANSLATION_MODEL_IDS` (src/lib/translation.ts). */ + modelId: string; + /** Unit texts (`buildTranslationUnits`), answered 1:1 in `texts`. */ + units: string[]; +} + +/** Model files being fetched (first use) — bytes summed across files. */ +export interface TranslateDownloadMessage { + type: "translate-download"; + requestId: string; + loaded: number; + total: number; +} + +/** Model ready; `done` of `total` sentence pieces translated so far. */ +export interface TranslateProgressMessage { + type: "translate-progress"; + requestId: string; + done: number; + total: number; +} + +export interface TranslateResultMessage { + type: "translate-result"; + requestId: string; + /** One translated string per request unit, same order. */ + texts: string[]; +} + +export interface TranslateErrorMessage { + type: "translate-error"; + requestId: string; + message: string; +} + +export type TranslateWorkerMessage = + | TranslateDownloadMessage + | TranslateProgressMessage + | TranslateResultMessage + | TranslateErrorMessage; + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null; +} + +const isNumber = (value: unknown): value is number => + typeof value === "number" && Number.isFinite(value); + +export function isTranslateRequest(value: unknown): value is TranslateRequest { + return ( + isRecord(value) && + value.type === "translate" && + typeof value.requestId === "string" && + typeof value.modelId === "string" && + Array.isArray(value.units) && + value.units.every((unit) => typeof unit === "string") + ); +} + +export function isTranslateWorkerMessage(value: unknown): value is TranslateWorkerMessage { + if (!isRecord(value) || typeof value.requestId !== "string") return false; + switch (value.type) { + case "translate-download": + return isNumber(value.loaded) && isNumber(value.total); + case "translate-progress": + return isNumber(value.done) && isNumber(value.total); + case "translate-result": + return Array.isArray(value.texts) && value.texts.every((t) => typeof t === "string"); + case "translate-error": + return typeof value.message === "string"; + default: + return false; + } +} + +/** + * Sums Transformers.js per-file download progress into one byte count for + * the UI. A file's total is only known once its download starts, so the + * total grows as files start — fine for a percentage that never goes back. + */ +export class DownloadTally { + private readonly files = new Map(); + + update(file: string, loaded: number, total: number): { loaded: number; total: number } { + this.files.set(file, { loaded, total }); + let sumLoaded = 0; + let sumTotal = 0; + for (const entry of this.files.values()) { + sumLoaded += entry.loaded; + sumTotal += entry.total; + } + return { loaded: sumLoaded, total: sumTotal }; + } +} diff --git a/tests/errors.test.ts b/tests/errors.test.ts index 8e38b46..9fe8a52 100644 --- a/tests/errors.test.ts +++ b/tests/errors.test.ts @@ -98,3 +98,19 @@ describe("classifyError", () => { expect(classifyError(42).message).toBe("42"); }); }); + +describe("classifyError — transcript translation (D-WS4b)", () => { + test("out of memory while translating suggests closing tabs, not a smaller Whisper model", () => { + const classified = classifyError("RangeError: Out of memory", { stage: "translate" }); + expect(classified.kind).toBe("out-of-memory"); + expect(classified.suggestion).not.toContain("smaller model"); + expect(classified.suggestion).toContain("other tabs"); + }); + + test("a failed translation-model download is retryable and says the download restarts", () => { + const classified = classifyError("Failed to fetch", { stage: "translate" }); + expect(classified.kind).toBe("network"); + expect(classified.retryable).toBe(true); + expect(classified.suggestion).toContain("translation model"); + }); +}); diff --git a/tests/models.test.ts b/tests/models.test.ts index 1ef20c3..a082df6 100644 --- a/tests/models.test.ts +++ b/tests/models.test.ts @@ -10,6 +10,7 @@ import { getFasterWhisperModel, getModel, resolveDefaultDtype, + taskForModel, } from "../src/lib/models"; // Pure catalog data (D-006) — no browser, no network. @@ -209,3 +210,39 @@ describe("WHISPER_LANGUAGES", () => { expect(WHISPER_LANGUAGES.some((entry) => entry.code === "auto")).toBe(false); }); }); + +describe("canTranslate (D-WS4a)", () => { + test("Whisper translate is offered only where OpenAI trained it", () => { + const translating = MODELS.filter((model) => model.canTranslate).map((model) => model.id); + // large-v3-turbo was not trained for translation (OpenAI model card); + // `.en` models are English-only and have no task token. + expect(translating).toEqual([ + "onnx-community/whisper-tiny", + "onnx-community/whisper-base", + "onnx-community/whisper-small", + "onnx-community/whisper-medium-ONNX", + ]); + }); + + test("canTranslate implies multilingual", () => { + for (const model of MODELS) { + if (model.canTranslate) expect(model.multilingual).toBe(true); + } + }); +}); + +describe("taskForModel (D-WS4a)", () => { + test("a stale translate resets to transcribe on a model that cannot translate", () => { + expect(taskForModel("onnx-community/whisper-large-v3-turbo", "translate")).toBe("transcribe"); + expect(taskForModel("onnx-community/whisper-small.en", "translate")).toBe("transcribe"); + }); + + test("translate survives on a model that can translate", () => { + expect(taskForModel("onnx-community/whisper-small", "translate")).toBe("translate"); + expect(taskForModel("onnx-community/whisper-small", "transcribe")).toBe("transcribe"); + }); + + test("unknown ids never translate", () => { + expect(taskForModel("someone/unknown", "translate")).toBe("transcribe"); + }); +}); diff --git a/tests/offline.md b/tests/offline.md index 1206587..d49bc87 100644 --- a/tests/offline.md +++ b/tests/offline.md @@ -111,6 +111,24 @@ Result: Works. Result: Works +## 5. Transcript translation offline (D-WS4b) + +1. Online: transcribe anything, open **Translate transcript…**, pick a + direction, click **Translate**. The hint said "Downloads a translation + model once (~113 MB)"; the status shows "Downloading the translation + model… N%", then "Translating… n of m sentences", then a second card + "Translation (English|Danish)" with `.txt` / `.srt` / `.txt+ts` exports + named `..`. +2. Cache Storage → `transformers-cache` now holds `Xenova/opus-mt-…` files; + `whisper-shell-*` holds `/translate.worker.js`. The hint reads + "Translation model downloaded — works offline". +3. Go offline (DevTools → Network → Offline), reload, transcribe with a + cached Whisper model, translate again in the same direction: it must work + with no network request. The other (never downloaded) direction must fail + with "You appear to be offline…" and a Retry button, not hang. + +Result: _pending (needs a real browser)_ + ## Recording the result When this passes in a real browser, flip the `[~]` offline-verification item diff --git a/tests/transcriptionOptions.test.tsx b/tests/transcriptionOptions.test.tsx new file mode 100644 index 0000000..a9ada77 --- /dev/null +++ b/tests/transcriptionOptions.test.tsx @@ -0,0 +1,31 @@ +import { describe, expect, test } from "bun:test"; +import { renderToStaticMarkup } from "react-dom/server"; +import { TranscriptionOptions } from "../src/components/TranscriptionOptions"; + +// D-WS4a: Whisper's own "Translate to English" only where the model can do it. +function render(canTranslate: boolean): string { + return renderToStaticMarkup( + {}} + task="transcribe" + onTaskChange={() => {}} + canTranslate={canTranslate} + />, + ); +} + +describe("TranscriptionOptions", () => { + test("offers Translate to English when the model can translate", () => { + const html = render(true); + expect(html).toContain("Translate to English"); + expect(html).toContain("Task"); + }); + + test("hides the task control entirely when the model cannot translate", () => { + const html = render(false); + expect(html).not.toContain("Translate to English"); + expect(html).not.toContain("Task"); + expect(html).toContain("Language"); + }); +}); diff --git a/tests/translate-real-model.test.ts b/tests/translate-real-model.test.ts new file mode 100644 index 0000000..b2b156b --- /dev/null +++ b/tests/translate-real-model.test.ts @@ -0,0 +1,97 @@ +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import { join } from "node:path"; +import { + assembleTranslation, + buildTranslationUnits, + getTranslationModel, + translateUnits, +} from "../src/lib/translation"; +import { loadOpusMt, type OpusMtTranslator } from "../src/worker/opusMt"; + +/** + * Real Opus-MT translation on onnxruntime-node (D-WS4b), opt-in: + * + * MT_INTEGRATION=1 flock /tmp/whisper-heavy.lock bun test tests/translate-real-model.test.ts + * + * Downloads Xenova/opus-mt-en-da and -da-en (~113 MB each, q8) into the + * Transformers.js cache on first run; ~3 min on a 16-core CPU. Same loader + * and loop as the browser worker. + */ +const ENABLED = process.env.MT_INTEGRATION === "1"; +const AUDIO = join(import.meta.dir, "audio_files"); +const words = (text: string) => text.split(/\s+/).filter(Boolean); + +describe.skipIf(!ENABLED)("Opus-MT on a real transcript", () => { + let enDa: OpusMtTranslator; + let daEn: OpusMtTranslator; + + beforeAll(async () => { + enDa = await loadOpusMt(getTranslationModel("en", "da")!.id); + daEn = await loadOpusMt(getTranslationModel("da", "en")!.id); + }, 600_000); + + afterAll(async () => { + await enDa?.dispose(); + await daEn?.dispose(); + }); + + const run = (translator: OpusMtTranslator, units: string[]) => + translateUnits(units, { + countTokens: translator.countTokens, + translateBatch: (pieces) => translator.translateBatch(pieces), + }); + + test( + "all of en_alien → Danish: one translation per line, none empty, none copied from a neighbour", + async () => { + const text = await Bun.file(join(AUDIO, "en_alien.txt")).text(); + const units = buildTranslationUnits({ text, chunks: [] }); + const started = performance.now(); + const out = await run(enDa, units.map((unit) => unit.text)); + console.info(`[mt] en_alien en→da: ${((performance.now() - started) / 1000).toFixed(1)} s`); + const assembled = assembleTranslation(units, out); + expect(out).toHaveLength(units.length); + for (const line of out) expect(line.trim()).not.toBe(""); + // Guards the batched-generation bug (every row = row 0's output). + const distinctSources = new Set(units.map((unit) => unit.text)).size; + expect(new Set(out).size).toBeGreaterThan(distinctSources * 0.9); + // Nothing dropped: a translation keeps roughly the source's length. + const ratio = words(assembled.text).length / words(text).length; + expect(ratio).toBeGreaterThan(0.6); + expect(ratio).toBeLessThan(1.5); + }, + 600_000, + ); + + test( + "an unpunctuated unit far beyond 512 tokens is translated whole (Review Focus 5)", + async () => { + const text = await Bun.file(join(AUDIO, "en_alien.txt")).text(); + // Live-mode-like run-on: no sentence punctuation, one single unit. + const runOn = words(text.replace(/[.!?…]/g, "")).slice(0, 900).join(" "); + expect(enDa.countTokens(runOn)).toBeGreaterThan(512); + const [out] = await run(enDa, [runOn]); + const ratio = words(out ?? "").length / words(runOn).length; + expect(ratio).toBeGreaterThan(0.6); + expect(ratio).toBeLessThan(1.5); + }, + 600_000, + ); + + test( + "da_festuge → English keeps names and dates, timestamps carried over", + async () => { + const text = await Bun.file(join(AUDIO, "da_festuge.txt")).text(); + const chunks = text + .split("\n") + .filter((line) => line.trim() !== "") + .map((line, index) => ({ text: ` ${line}`, timestamp: [index * 5, index * 5 + 5] as [number, number] })); + const units = buildTranslationUnits({ text, chunks }); + const result = assembleTranslation(units, await run(daEn, units.map((unit) => unit.text))); + expect(result.text).toContain("Aarhus"); + expect(result.text).toMatch(/August/); + expect(result.chunks.map((chunk) => chunk.timestamp)).toEqual(chunks.map((chunk) => chunk.timestamp)); + }, + 600_000, + ); +}); diff --git a/tests/translatePanel.test.tsx b/tests/translatePanel.test.tsx new file mode 100644 index 0000000..bd5186f --- /dev/null +++ b/tests/translatePanel.test.tsx @@ -0,0 +1,65 @@ +import { describe, expect, test } from "bun:test"; +import { renderToStaticMarkup } from "react-dom/server"; +import { TranslatePanel, translateStatusText } from "../src/components/TranslatePanel"; + +// D-WS4b: the panel's copy, before any worker exists (static render). +const danish = { text: "Det er ikke noget med at vi skal på ferie.", chunks: [] }; + +function render(downloaded: string[] = [], hint?: string) { + return renderToStaticMarkup( + {}} + exportName="talk.m4a" + disabled={false} + />, + ); +} + +describe("TranslatePanel", () => { + test("is a secondary disclosure named in plain words", () => { + const html = render(); + expect(html).toContain(" { + const html = render(); + expect(html).toContain("Downloads a translation model once (~113 MB)"); + expect(html).toContain("never leaves"); + }); + + test("says it works offline once the model is cached", () => { + expect(render(["Xenova/opus-mt-da-en"])).toContain("Translation model downloaded — works offline"); + }); + + test("defaults the direction from the transcript's language", () => { + expect(render()).toMatch(/