Merge feat/translate: canTranslate + Opus-MT transcript translation (spec 001 workstream 4)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

# Conflicts:
#	docs/Decisions.md
This commit is contained in:
Lucas Winther committed 2026-10-30 01:15:05 +01:00
commit c94689e2e0
28 files changed
+1888 -30

No files matched your search

+16
View File
@@ -98,3 +98,19 @@ describe("classifyError", () => {
expect(classifyError(42).message).toBe("42");
});
});
describe("classifyError — transcript translation (D-WS4b)", () => {
test("out of memory while translating suggests closing tabs, not a smaller Whisper model", () => {
const classified = classifyError("RangeError: Out of memory", { stage: "translate" });
expect(classified.kind).toBe("out-of-memory");
expect(classified.suggestion).not.toContain("smaller model");
expect(classified.suggestion).toContain("other tabs");
});
test("a failed translation-model download is retryable and says the download restarts", () => {
const classified = classifyError("Failed to fetch", { stage: "translate" });
expect(classified.kind).toBe("network");
expect(classified.retryable).toBe(true);
expect(classified.suggestion).toContain("translation model");
});
});
+37
View File
@@ -10,6 +10,7 @@ import {
getFasterWhisperModel,
getModel,
resolveDefaultDtype,
taskForModel,
} from "../src/lib/models";
// Pure catalog data (D-006) — no browser, no network.
@@ -209,3 +210,39 @@ describe("WHISPER_LANGUAGES", () => {
expect(WHISPER_LANGUAGES.some((entry) => entry.code === "auto")).toBe(false);
});
});
describe("canTranslate (D-WS4a)", () => {
test("Whisper translate is offered only where OpenAI trained it", () => {
const translating = MODELS.filter((model) => model.canTranslate).map((model) => model.id);
// large-v3-turbo was not trained for translation (OpenAI model card);
// `.en` models are English-only and have no task token.
expect(translating).toEqual([
"onnx-community/whisper-tiny",
"onnx-community/whisper-base",
"onnx-community/whisper-small",
"onnx-community/whisper-medium-ONNX",
]);
});
test("canTranslate implies multilingual", () => {
for (const model of MODELS) {
if (model.canTranslate) expect(model.multilingual).toBe(true);
}
});
});
describe("taskForModel (D-WS4a)", () => {
test("a stale translate resets to transcribe on a model that cannot translate", () => {
expect(taskForModel("onnx-community/whisper-large-v3-turbo", "translate")).toBe("transcribe");
expect(taskForModel("onnx-community/whisper-small.en", "translate")).toBe("transcribe");
});
test("translate survives on a model that can translate", () => {
expect(taskForModel("onnx-community/whisper-small", "translate")).toBe("translate");
expect(taskForModel("onnx-community/whisper-small", "transcribe")).toBe("transcribe");
});
test("unknown ids never translate", () => {
expect(taskForModel("someone/unknown", "translate")).toBe("transcribe");
});
});
+18
View File
@@ -111,6 +111,24 @@ Result: Works.
Result: Works
## 5. Transcript translation offline (D-WS4b)
1. Online: transcribe anything, open **Translate transcript…**, pick a
direction, click **Translate**. The hint said "Downloads a translation
model once (~113 MB)"; the status shows "Downloading the translation
model… N%", then "Translating… n of m sentences", then a second card
"Translation (English|Danish)" with `.txt` / `.srt` / `.txt+ts` exports
named `<file>.<en|da>.<ext>`.
2. Cache Storage → `transformers-cache` now holds `Xenova/opus-mt-…` files;
`whisper-shell-*` holds `/translate.worker.js`. The hint reads
"Translation model downloaded — works offline".
3. Go offline (DevTools → Network → Offline), reload, transcribe with a
cached Whisper model, translate again in the same direction: it must work
with no network request. The other (never downloaded) direction must fail
with "You appear to be offline…" and a Retry button, not hang.
Result: _pending (needs a real browser)_
## Recording the result
When this passes in a real browser, flip the `[~]` offline-verification item
+31
View File
@@ -0,0 +1,31 @@
import { describe, expect, test } from "bun:test";
import { renderToStaticMarkup } from "react-dom/server";
import { TranscriptionOptions } from "../src/components/TranscriptionOptions";
// D-WS4a: Whisper's own "Translate to English" only where the model can do it.
function render(canTranslate: boolean): string {
return renderToStaticMarkup(
<TranscriptionOptions
language="auto"
onLanguageChange={() => {}}
task="transcribe"
onTaskChange={() => {}}
canTranslate={canTranslate}
/>,
);
}
describe("TranscriptionOptions", () => {
test("offers Translate to English when the model can translate", () => {
const html = render(true);
expect(html).toContain("Translate to English");
expect(html).toContain("Task");
});
test("hides the task control entirely when the model cannot translate", () => {
const html = render(false);
expect(html).not.toContain("Translate to English");
expect(html).not.toContain("Task");
expect(html).toContain("Language");
});
});
+97
View File
@@ -0,0 +1,97 @@
import { afterAll, beforeAll, describe, expect, test } from "bun:test";
import { join } from "node:path";
import {
assembleTranslation,
buildTranslationUnits,
getTranslationModel,
translateUnits,
} from "../src/lib/translation";
import { loadOpusMt, type OpusMtTranslator } from "../src/worker/opusMt";
/**
* Real Opus-MT translation on onnxruntime-node (D-WS4b), opt-in:
*
* MT_INTEGRATION=1 flock /tmp/whisper-heavy.lock bun test tests/translate-real-model.test.ts
*
* Downloads Xenova/opus-mt-en-da and -da-en (~113 MB each, q8) into the
* Transformers.js cache on first run; ~3 min on a 16-core CPU. Same loader
* and loop as the browser worker.
*/
const ENABLED = process.env.MT_INTEGRATION === "1";
const AUDIO = join(import.meta.dir, "audio_files");
const words = (text: string) => text.split(/\s+/).filter(Boolean);
describe.skipIf(!ENABLED)("Opus-MT on a real transcript", () => {
let enDa: OpusMtTranslator;
let daEn: OpusMtTranslator;
beforeAll(async () => {
enDa = await loadOpusMt(getTranslationModel("en", "da")!.id);
daEn = await loadOpusMt(getTranslationModel("da", "en")!.id);
}, 600_000);
afterAll(async () => {
await enDa?.dispose();
await daEn?.dispose();
});
const run = (translator: OpusMtTranslator, units: string[]) =>
translateUnits(units, {
countTokens: translator.countTokens,
translateBatch: (pieces) => translator.translateBatch(pieces),
});
test(
"all of en_alien → Danish: one translation per line, none empty, none copied from a neighbour",
async () => {
const text = await Bun.file(join(AUDIO, "en_alien.txt")).text();
const units = buildTranslationUnits({ text, chunks: [] });
const started = performance.now();
const out = await run(enDa, units.map((unit) => unit.text));
console.info(`[mt] en_alien en→da: ${((performance.now() - started) / 1000).toFixed(1)} s`);
const assembled = assembleTranslation(units, out);
expect(out).toHaveLength(units.length);
for (const line of out) expect(line.trim()).not.toBe("");
// Guards the batched-generation bug (every row = row 0's output).
const distinctSources = new Set(units.map((unit) => unit.text)).size;
expect(new Set(out).size).toBeGreaterThan(distinctSources * 0.9);
// Nothing dropped: a translation keeps roughly the source's length.
const ratio = words(assembled.text).length / words(text).length;
expect(ratio).toBeGreaterThan(0.6);
expect(ratio).toBeLessThan(1.5);
},
600_000,
);
test(
"an unpunctuated unit far beyond 512 tokens is translated whole (Review Focus 5)",
async () => {
const text = await Bun.file(join(AUDIO, "en_alien.txt")).text();
// Live-mode-like run-on: no sentence punctuation, one single unit.
const runOn = words(text.replace(/[.!?…]/g, "")).slice(0, 900).join(" ");
expect(enDa.countTokens(runOn)).toBeGreaterThan(512);
const [out] = await run(enDa, [runOn]);
const ratio = words(out ?? "").length / words(runOn).length;
expect(ratio).toBeGreaterThan(0.6);
expect(ratio).toBeLessThan(1.5);
},
600_000,
);
test(
"da_festuge → English keeps names and dates, timestamps carried over",
async () => {
const text = await Bun.file(join(AUDIO, "da_festuge.txt")).text();
const chunks = text
.split("\n")
.filter((line) => line.trim() !== "")
.map((line, index) => ({ text: ` ${line}`, timestamp: [index * 5, index * 5 + 5] as [number, number] }));
const units = buildTranslationUnits({ text, chunks });
const result = assembleTranslation(units, await run(daEn, units.map((unit) => unit.text)));
expect(result.text).toContain("Aarhus");
expect(result.text).toMatch(/August/);
expect(result.chunks.map((chunk) => chunk.timestamp)).toEqual(chunks.map((chunk) => chunk.timestamp));
},
600_000,
);
});
+65
View File
@@ -0,0 +1,65 @@
import { describe, expect, test } from "bun:test";
import { renderToStaticMarkup } from "react-dom/server";
import { TranslatePanel, translateStatusText } from "../src/components/TranslatePanel";
// D-WS4b: the panel's copy, before any worker exists (static render).
const danish = { text: "Det er ikke noget med at vi skal på ferie.", chunks: [] };
function render(downloaded: string[] = [], hint?: string) {
return renderToStaticMarkup(
<TranslatePanel
result={danish}
sourceLanguageHint={hint}
downloaded={new Set(downloaded)}
onModelCached={() => {}}
exportName="talk.m4a"
disabled={false}
/>,
);
}
describe("TranslatePanel", () => {
test("is a secondary disclosure named in plain words", () => {
const html = render();
expect(html).toContain("<details");
expect(html).toContain("Translate transcript…");
});
test("states the one-time download size before anything downloads (D-020)", () => {
const html = render();
expect(html).toContain("Downloads a translation model once (~113 MB)");
expect(html).toContain("never leaves");
});
test("says it works offline once the model is cached", () => {
expect(render(["Xenova/opus-mt-da-en"])).toContain("Translation model downloaded — works offline");
});
test("defaults the direction from the transcript's language", () => {
expect(render()).toMatch(/<option value="Xenova\/opus-mt-da-en" selected="">/);
expect(render([], "en")).toMatch(/<option value="Xenova\/opus-mt-en-da" selected="">/);
});
});
describe("translateStatusText", () => {
test("idle says nothing", () => {
expect(translateStatusText({ kind: "idle" }, false)).toBeNull();
});
test("never claims a download for a cached model (D-033 spirit)", () => {
expect(translateStatusText({ kind: "starting" }, true)).toBe("Loading the translation model…");
expect(translateStatusText({ kind: "starting" }, false)).toBe("Starting the download…");
});
test("download percent and sentence counts", () => {
expect(translateStatusText({ kind: "downloading", loaded: 40, total: 100 }, false)).toBe(
"Downloading the translation model… 40%",
);
expect(translateStatusText({ kind: "translating", done: 12, total: 80 }, true)).toBe(
"Translating… 12 of 80 sentences",
);
expect(translateStatusText({ kind: "translating", done: 0, total: 1 }, true)).toBe(
"Translating… 0 of 1 sentence",
);
});
});
+49
View File
@@ -0,0 +1,49 @@
import { describe, expect, test } from "bun:test";
import {
DownloadTally,
isTranslateRequest,
isTranslateWorkerMessage,
} from "../src/worker/translateProtocol";
describe("isTranslateRequest", () => {
test("accepts a well-formed request", () => {
expect(
isTranslateRequest({ type: "translate", requestId: "t-1", modelId: "Xenova/opus-mt-da-en", units: ["Hej."] }),
).toBe(true);
});
test("rejects missing ids and non-string units", () => {
expect(isTranslateRequest({ type: "translate", modelId: "m", units: [] })).toBe(false);
expect(isTranslateRequest({ type: "translate", requestId: "t", modelId: "m", units: [1] })).toBe(false);
expect(isTranslateRequest({ type: "load", requestId: "t", modelId: "m", units: [] })).toBe(false);
expect(isTranslateRequest(null)).toBe(false);
});
});
describe("isTranslateWorkerMessage", () => {
test("accepts each reply type", () => {
for (const message of [
{ type: "translate-download", requestId: "t", loaded: 1, total: 2 },
{ type: "translate-progress", requestId: "t", done: 0, total: 3 },
{ type: "translate-result", requestId: "t", texts: ["Hello."] },
{ type: "translate-error", requestId: "t", message: "boom" },
]) {
expect(isTranslateWorkerMessage(message)).toBe(true);
}
});
test("rejects malformed replies and Whisper-worker messages", () => {
expect(isTranslateWorkerMessage({ type: "translate-progress", requestId: "t", done: "1", total: 3 })).toBe(false);
expect(isTranslateWorkerMessage({ type: "translate-result", texts: [] })).toBe(false);
expect(isTranslateWorkerMessage({ type: "result", requestId: "t", text: "", chunks: [] })).toBe(false);
});
});
describe("DownloadTally", () => {
test("sums bytes across files, updating each file in place", () => {
const tally = new DownloadTally();
expect(tally.update("onnx/encoder.onnx", 10, 100)).toEqual({ loaded: 10, total: 100 });
expect(tally.update("onnx/decoder.onnx", 5, 50)).toEqual({ loaded: 15, total: 150 });
expect(tally.update("onnx/encoder.onnx", 100, 100)).toEqual({ loaded: 105, total: 150 });
});
});
+255
View File
@@ -0,0 +1,255 @@
import { describe, expect, test } from "bun:test";
import { join } from "node:path";
import {
TRANSLATION_MODELS,
assembleTranslation,
buildTranslationUnits,
estimateTokens,
getTranslationModel,
guessSourceLanguage,
splitForTranslation,
translateUnits,
} from "../src/lib/translation";
import type { TranscriptChunk } from "../src/worker/protocol";
// Pure planning for transcript translation (D-WS4b) — no model, no network.
describe("translateUnits", () => {
test("translates every piece in order, batched, and rejoins per unit", async () => {
const batches: string[][] = [];
const out = await translateUnits(["One. Two. Three.", "Four.", "Five. Six."], {
batchSize: 4,
translateBatch: async (pieces) => {
batches.push(pieces);
return pieces.map((p) => p.toUpperCase());
},
});
expect(out).toEqual(["ONE. TWO. THREE.", "FOUR.", "FIVE. SIX."]);
expect(batches).toEqual([["One.", "Two.", "Three.", "Four."], ["Five.", "Six."]]);
});
test("defaults to one piece per call (batched Marian generation is broken in 4.2)", async () => {
const sizes: number[] = [];
await translateUnits(["One. Two. Three."], {
translateBatch: async (pieces) => {
sizes.push(pieces.length);
return pieces;
},
});
expect(sizes).toEqual([1, 1, 1]);
});
test("reports progress in pieces, starting at zero", async () => {
const progress: [number, number][] = [];
await translateUnits(["Yes. No. Maybe."], {
batchSize: 2,
translateBatch: async (pieces) => pieces,
onProgress: (done, total) => progress.push([done, total]),
});
expect(progress).toEqual([[0, 3], [2, 3], [3, 3]]);
});
test("an empty unit stays empty without a model call", async () => {
let calls = 0;
const out = await translateUnits(["", "Hi."], {
translateBatch: async (pieces) => {
calls += 1;
return pieces;
},
});
expect(out).toEqual(["", "Hi."]);
expect(calls).toBe(1);
});
test("a batch answering with the wrong count is an error", async () => {
await expect(
translateUnits(["Yes. No."], { batchSize: 2, translateBatch: async () => ["only one"] }),
).rejects.toThrow();
});
test("the whole of en_alien round-trips through an echo translator with nothing dropped (Review Focus 5)", async () => {
const alien = await Bun.file(join(import.meta.dir, "audio_files/en_alien.txt")).text();
const units = buildTranslationUnits({ text: alien, chunks: [] }).map((u) => u.text);
const out = await translateUnits(units, { translateBatch: async (pieces) => pieces });
expect(words(out.join(" "))).toEqual(words(alien));
});
});
const words = (text: string) => text.split(/\s+/).filter(Boolean);
describe("TRANSLATION_MODELS", () => {
test("offers da→en and en→da Opus-MT ONNX exports", () => {
expect(TRANSLATION_MODELS.map((m) => m.id)).toEqual([
"Xenova/opus-mt-da-en",
"Xenova/opus-mt-en-da",
]);
expect(getTranslationModel("da", "en")?.id).toBe("Xenova/opus-mt-da-en");
expect(getTranslationModel("en", "da")?.id).toBe("Xenova/opus-mt-en-da");
for (const model of TRANSLATION_MODELS) {
// q8 encoder + merged decoder + tokenizer, measured on the Hub.
expect(model.approxBytes).toBeGreaterThan(100e6);
expect(model.approxBytes).toBeLessThan(130e6);
}
});
});
describe("buildTranslationUnits", () => {
test("merges chunks until a sentence ends, keeping the merged time span", () => {
const chunks: TranscriptChunk[] = [
{ text: " So the plan is", timestamp: [0, 2] },
{ text: " to ship on Friday.", timestamp: [2, 4.5] },
{ text: " Any questions?", timestamp: [5, 6] },
];
expect(buildTranslationUnits({ text: "", chunks })).toEqual([
{ text: "So the plan is to ship on Friday.", timestamp: [0, 4.5] },
{ text: "Any questions?", timestamp: [5, 6] },
]);
});
test("an unterminated tail still becomes a unit, with its null end kept", () => {
const chunks: TranscriptChunk[] = [
{ text: " Done.", timestamp: [0, 1] },
{ text: " and then", timestamp: [1, null] },
];
expect(buildTranslationUnits({ text: "", chunks })).toEqual([
{ text: "Done.", timestamp: [0, 1] },
{ text: "and then", timestamp: [1, null] },
]);
});
test("unpunctuated chunks are capped so cues stay subtitle-sized", () => {
const chunks: TranscriptChunk[] = Array.from({ length: 20 }, (_, i) => ({
text: ` word${i}`,
timestamp: [i, i + 1] as [number, number],
}));
const units = buildTranslationUnits({ text: "", chunks }, 8);
expect(units.map((u) => u.timestamp)).toEqual([
[0, 8],
[8, 16],
[16, 20],
]);
});
test("empty chunks are skipped", () => {
const chunks: TranscriptChunk[] = [
{ text: " ", timestamp: [0, 1] },
{ text: " Hi.", timestamp: [1, 2] },
];
expect(buildTranslationUnits({ text: "", chunks })).toEqual([
{ text: "Hi.", timestamp: [1, 2] },
]);
});
test("without chunks, each non-empty line is an untimed unit", () => {
expect(buildTranslationUnits({ text: "Hej.\n\nHvordan går det?\n", chunks: [] })).toEqual([
{ text: "Hej.", timestamp: null },
{ text: "Hvordan går det?", timestamp: null },
]);
});
});
describe("splitForTranslation", () => {
test("every sentence is its own piece (Opus-MT drops trailing sentences otherwise)", () => {
expect(
splitForTranslation("My name is Lucas and I live in Copenhagen. How are you? Fine!"),
).toEqual(["My name is Lucas and I live in Copenhagen.", "How are you?", "Fine!"]);
});
test("abbreviation-only fragments stay with the following sentence", () => {
expect(splitForTranslation("Mr. Smith went home. He slept.")).toEqual([
"Mr. Smith went home.",
"He slept.",
]);
});
test("one-word sentences are not mistaken for abbreviations", () => {
expect(splitForTranslation("Yes. Okay. J. K. Rowling wrote it.")).toEqual([
"Yes.",
"Okay.",
"J. K. Rowling wrote it.",
]);
});
test("blank input yields no pieces", () => {
expect(splitForTranslation(" ")).toEqual([]);
});
test("an unpunctuated run-on is wrapped under the budget, words intact and in order", () => {
const runOn = Array.from({ length: 300 }, (_, i) => `ord${i}`).join(" ");
const pieces = splitForTranslation(runOn, { maxTokens: 40, countTokens: (s) => words(s).length });
expect(pieces.length).toBeGreaterThan(1);
for (const piece of pieces) expect(words(piece).length).toBeLessThanOrEqual(40);
expect(pieces.join(" ")).toBe(runOn);
});
test("a long sentence prefers comma boundaries before word wrapping", () => {
const clause = "one two three four five six";
const sentence = `${clause}, ${clause}, ${clause}, ${clause}.`;
const pieces = splitForTranslation(sentence, { maxTokens: 14, countTokens: (s) => words(s).length });
expect(pieces).toEqual([`${clause}, ${clause},`, `${clause}, ${clause}.`]);
});
test("a transcript far beyond Opus-MT's 512-token input is split with nothing dropped, in order (Review Focus 5)", async () => {
const alien = await Bun.file(join(import.meta.dir, "audio_files/en_alien.txt")).text();
expect(estimateTokens(alien)).toBeGreaterThan(512);
const pieces = splitForTranslation(alien, { maxTokens: 200 });
for (const piece of pieces) expect(estimateTokens(piece)).toBeLessThanOrEqual(200);
expect(words(pieces.join(" "))).toEqual(words(alien));
});
});
describe("estimateTokens", () => {
test("over-estimates rather than under-estimates SentencePiece counts", () => {
// Measured with the Xenova/opus-mt-en-da tokenizer: this sentence is 12 tokens (incl. </s>).
expect(estimateTokens("My name is Lucas and I live in Copenhagen.")).toBeGreaterThanOrEqual(12);
expect(estimateTokens("")).toBe(0);
});
});
describe("assembleTranslation", () => {
test("pairs translated units with their time spans", () => {
const units = [
{ text: "Hej.", timestamp: [0, 1] as [number, number] },
{ text: "Farvel.", timestamp: [2, null] as [number, null] },
];
expect(assembleTranslation(units, [" Hello. ", "Goodbye."])).toEqual({
text: "Hello. Goodbye.",
chunks: [
{ text: "Hello.", timestamp: [0, 1] },
{ text: "Goodbye.", timestamp: [2, null] },
],
});
});
test("untimed units join as lines and produce no chunks", () => {
const units = [
{ text: "Hej.", timestamp: null },
{ text: "Farvel.", timestamp: null },
];
expect(assembleTranslation(units, ["Hello.", "Goodbye."])).toEqual({
text: "Hello.\nGoodbye.",
chunks: [],
});
});
test("a count mismatch is an error, never a silently shifted transcript", () => {
expect(() => assembleTranslation([{ text: "a", timestamp: null }], [])).toThrow();
});
});
describe("guessSourceLanguage", () => {
test("trusts an explicit Whisper language", () => {
expect(guessSourceLanguage("whatever", "da")).toBe("da");
expect(guessSourceLanguage("whatever", "en")).toBe("en");
});
test("detects Danish from its letters and common words", () => {
expect(guessSourceLanguage("På Aarhus Festuges sekretariat har vi løbet rigtig stærkt.")).toBe("da");
expect(guessSourceLanguage("Det er ikke noget med at vi skal")).toBe("da");
});
test("defaults to English", () => {
expect(guessSourceLanguage("Guys, I have been addicted to this game.")).toBe("en");
expect(guessSourceLanguage("", "auto")).toBe("en");
});
});