From 82ed34c7f9471fbb8699b0ae4c5d5459418ed5bd Mon Sep 17 00:00:00 2001 From: Lucas Winther Date: Thu, 29 Oct 2026 21:39:37 +0100 Subject: [PATCH] Plan transcript translation units and sentence pieces for Opus-MT Co-Authored-By: Claude Opus 5.5 --- src/lib/errors.ts | 15 ++- src/lib/translation.ts | 273 ++++++++++++++++++++++++++++++++++++++ tests/errors.test.ts | 16 +++ tests/translation.test.ts | 185 ++++++++++++++++++++++++++ 4 files changed, 486 insertions(+), 3 deletions(-) create mode 100644 src/lib/translation.ts create mode 100644 tests/translation.test.ts diff --git a/src/lib/errors.ts b/src/lib/errors.ts index fa6ec30..a0cc89a 100644 --- a/src/lib/errors.ts +++ b/src/lib/errors.ts @@ -16,7 +16,12 @@ * inspect across Web Audio / Transformers.js / onnxruntime-web / fetch. */ -export type ErrorStage = "decode" | "load" | "transcribe" | "worker-crash"; +/** + * `"translate"` covers text translation of a finished transcript (D-WS4b): + * both its one-time model download and the translation pass, which run in + * their own worker — only the suggestions differ from the Whisper stages. + */ +export type ErrorStage = "decode" | "load" | "transcribe" | "worker-crash" | "translate"; export type ErrorKind = "unsupported-file" | "out-of-memory" | "device-lost" | "network" | "generic"; @@ -95,7 +100,9 @@ export function classifyError(raw: unknown, context: ClassifyContext = {}): Clas kind: "out-of-memory", message: trimmed.length > 0 ? message : "The browser ran out of memory and had to stop.", suggestion: - "Try a smaller model (Tiny, Base, or Small) — it needs far less memory to load and run.", + context.stage === "translate" + ? "Close other tabs to free memory, then try translating again." + : "Try a smaller model (Tiny, Base, or Small) — it needs far less memory to load and run.", retryable: false, }; } @@ -117,7 +124,9 @@ export function classifyError(raw: unknown, context: ClassifyContext = {}): Clas suggestion: context.stage === "load" ? "Check your connection, then retry — the model download will restart." - : "Check your connection and try again.", + : context.stage === "translate" + ? "Check your connection, then retry — the translation model download will restart." + : "Check your connection and try again.", retryable: true, }; } diff --git a/src/lib/translation.ts b/src/lib/translation.ts new file mode 100644 index 0000000..eecd042 --- /dev/null +++ b/src/lib/translation.ts @@ -0,0 +1,273 @@ +/** + * Text translation of a finished transcript (D-WS4b) — the pure half. + * + * Opus-MT (MarianMT) runs in `src/worker/translate.worker.ts`; everything + * here is planning that needs no model: which checkpoint serves a language + * pair, how a transcript becomes translation units that keep their + * timestamps, and how a unit is cut into sentence pieces under the model's + * input budget. No browser globals, so `bun test` covers it. + * + * Why per-sentence pieces: Opus-MT is trained on sentence pairs and, given + * several sentences at once, routinely drops the later ones (measured + * 2026-10-29: "…I live in Copenhagen. How are you?" came back as one + * sentence). Its encoder also tops out at 512 positions. + */ + +import type { TranscriptChunk } from "../worker/protocol"; + +export type TranslationLanguage = "da" | "en"; + +export interface TranslationModel { + /** Hugging Face Hub id of the Transformers.js ONNX export. */ + id: string; + source: TranslationLanguage; + target: TranslationLanguage; + /** Direction as shown in the picker, e.g. "Danish → English". */ + label: string; + /** Plain name of the target language, e.g. "English". */ + targetLabel: string; + /** + * One-time download at the q8 dtype the worker loads: encoder + merged + * decoder + tokenizer/configs, summed from the Hub file sizes. + */ + approxBytes: number; +} + +/** + * Xenova's ONNX exports of Helsinki-NLP's Opus-MT checkpoints (Apache-2.0 + * upstream). Verified to load and translate under Transformers.js 4.2.0. + */ +export const TRANSLATION_MODELS: readonly TranslationModel[] = [ + { + id: "Xenova/opus-mt-da-en", + source: "da", + target: "en", + label: "Danish → English", + targetLabel: "English", + approxBytes: 113_000_000, + }, + { + id: "Xenova/opus-mt-en-da", + source: "en", + target: "da", + label: "English → Danish", + targetLabel: "Danish", + approxBytes: 113_000_000, + }, +]; + +export const TRANSLATION_MODEL_IDS: readonly string[] = TRANSLATION_MODELS.map((m) => m.id); + +export function getTranslationModel( + source: TranslationLanguage, + target: TranslationLanguage, +): TranslationModel | undefined { + return TRANSLATION_MODELS.find((m) => m.source === source && m.target === target); +} + +/** Lookup by Hub id (storage panel names, worker request validation). */ +export function getTranslationModelById(id: string): TranslationModel | undefined { + return TRANSLATION_MODELS.find((m) => m.id === id); +} + +// --------------------------------------------------------------------------- +// Units: transcript → sentence-ending groups of chunks with their time span +// --------------------------------------------------------------------------- + +/** A stretch of transcript translated as one cue. */ +export interface TranslationUnit { + text: string; + /** Merged [start, end] of the source chunks; null for untimed text. */ + timestamp: [number, number | null] | null; +} + +const SENTENCE_END = /[.!?…]["'”’)\]]*$/; + +/** Default cap on chunks merged into one unit, so an unpunctuated transcript + * still yields subtitle-sized cues rather than one giant one. */ +export const MAX_CHUNKS_PER_UNIT = 8; + +/** + * Group a transcript into translation units. Whisper's segments often end + * mid-sentence; translating fragments hurts quality, so consecutive chunks + * are merged until one ends a sentence (or `maxChunksPerUnit` is reached) + * and the unit takes the merged time span — `.srt` keeps working, with + * slightly longer cues. Without chunks (nothing timestamped), each non-empty + * line is an untimed unit. + */ +export function buildTranslationUnits( + result: { text: string; chunks: TranscriptChunk[] }, + maxChunksPerUnit: number = MAX_CHUNKS_PER_UNIT, +): TranslationUnit[] { + const chunks = result.chunks.filter((chunk) => chunk.text.trim() !== ""); + if (chunks.length === 0) { + return result.text + .split(/\n+/) + .map((line) => line.trim()) + .filter((line) => line !== "") + .map((text) => ({ text, timestamp: null })); + } + const units: TranslationUnit[] = []; + let group: TranscriptChunk[] = []; + const flush = () => { + const first = group[0]; + const last = group[group.length - 1]; + if (!first || !last) return; + units.push({ + text: group.map((chunk) => chunk.text.trim()).join(" "), + timestamp: [first.timestamp[0], last.timestamp[1]], + }); + group = []; + }; + for (const chunk of chunks) { + group.push(chunk); + if (SENTENCE_END.test(chunk.text.trim()) || group.length >= maxChunksPerUnit) flush(); + } + flush(); + return units; +} + +// --------------------------------------------------------------------------- +// Pieces: unit text → sentences under the model's token budget +// --------------------------------------------------------------------------- + +/** + * Cheap upper-bound token estimate for Opus-MT's SentencePiece vocab, used + * when the real tokenizer isn't at hand (tests, planning). Measured ~0.34 + * tokens/char on en_alien and ~0.26 on da_festuge with the real tokenizers; + * 0.5/char plus the end-of-sentence token stays above both. + */ +export function estimateTokens(text: string): number { + return text.length === 0 ? 0 : Math.ceil(text.length / 2) + 1; +} + +/** + * Default per-piece budget, well inside Opus-MT's 512 positions: quality + * falls off on very long inputs long before the hard limit, and normal + * sentences are 10–60 tokens, so this only bites on unpunctuated run-ons. + */ +export const DEFAULT_MAX_PIECE_TOKENS = 200; + +export interface SplitOptions { + maxTokens?: number; + /** Token counter — the worker passes the real tokenizer. */ + countTokens?: (text: string) => number; +} + +/** "Mr.", "Dr.", "St." — sentence-segmenter false breaks. */ +const ABBREVIATION = /^\p{Lu}\p{Ll}{0,3}\.$/u; + +function sentences(text: string): string[] { + const out: string[] = []; + let carry = ""; + for (const { segment } of new Intl.Segmenter("en", { granularity: "sentence" }).segment(text)) { + const trimmed = segment.trim(); + if (trimmed === "") continue; + if (ABBREVIATION.test(trimmed)) { + carry = carry === "" ? trimmed : `${carry} ${trimmed}`; + continue; + } + out.push(carry === "" ? trimmed : `${carry} ${trimmed}`); + carry = ""; + } + if (carry !== "") out.push(carry); + return out; +} + +/** Greedily pack `parts` (in order) into space-joined pieces under budget. */ +function pack(parts: string[], maxTokens: number, count: (text: string) => number): string[] { + const pieces: string[] = []; + let current = ""; + for (const part of parts) { + const candidate = current === "" ? part : `${current} ${part}`; + if (current !== "" && count(candidate) > maxTokens) { + pieces.push(current); + current = part; + } else { + current = candidate; + } + } + if (current !== "") pieces.push(current); + return pieces; +} + +/** Split one over-budget sentence: commas first, then words. A single word + * longer than the budget is kept whole (nothing is ever dropped). */ +function wrap(sentence: string, maxTokens: number, count: (text: string) => number): string[] { + const clauses = sentence.split(/(?<=,)\s+/); + const parts = clauses.flatMap((clause) => + count(clause) <= maxTokens ? [clause] : pack(clause.split(/\s+/), maxTokens, count), + ); + return pack(parts, maxTokens, count); +} + +/** + * Cut a unit's text into the pieces actually fed to Opus-MT: one per + * sentence, an over-budget sentence wrapped on commas then words. Order is + * preserved and every word survives — joining the pieces with spaces gives + * the input back up to whitespace. + */ +export function splitForTranslation(text: string, options: SplitOptions = {}): string[] { + const maxTokens = options.maxTokens ?? DEFAULT_MAX_PIECE_TOKENS; + const count = options.countTokens ?? estimateTokens; + return sentences(text).flatMap((sentence) => + count(sentence) <= maxTokens ? [sentence] : wrap(sentence, maxTokens, count), + ); +} + +// --------------------------------------------------------------------------- +// Assembly +// --------------------------------------------------------------------------- + +/** + * Pair translated unit texts back with their units' time spans, in the + * `{ text, chunks }` shape the transcript view and exports already take. + */ +export function assembleTranslation( + units: readonly TranslationUnit[], + translated: readonly string[], +): { text: string; chunks: TranscriptChunk[] } { + if (units.length !== translated.length) { + throw new Error( + `Translation returned ${translated.length} parts for ${units.length} — refusing to misalign the transcript.`, + ); + } + const texts = translated.map((text) => text.trim()); + const chunks: TranscriptChunk[] = []; + units.forEach((unit, index) => { + if (unit.timestamp) chunks.push({ text: texts[index] ?? "", timestamp: unit.timestamp }); + }); + const untimed = units.every((unit) => unit.timestamp === null); + return { text: texts.join(untimed ? "\n" : " "), chunks }; +} + +// --------------------------------------------------------------------------- +// Source language guess (default for the direction picker) +// --------------------------------------------------------------------------- + +const DANISH_WORDS = new Set([ + "og", "ikke", "det", "jeg", "vi", "på", "til", "med", "som", "har", "af", "der", + "kan", "skal", "vil", "også", "men", "hvad", "hvordan", "være", "noget", +]); +const ENGLISH_WORDS = new Set([ + "the", "and", "is", "to", "of", "that", "you", "it", "with", "have", "this", "not", + "what", "how", "be", "was", "are", +]); + +/** + * Best guess at which language a transcript is in, for the direction + * picker's default (the user can always change it). An explicit Whisper + * language wins; otherwise Danish letters and function words vs English + * ones decide, defaulting to English. + */ +export function guessSourceLanguage(text: string, whisperLanguage?: string): TranslationLanguage { + if (whisperLanguage === "da" || whisperLanguage === "en") return whisperLanguage; + const lower = text.toLowerCase(); + let danish = (lower.match(/[æøå]/g)?.length ?? 0) * 2; + let english = 0; + for (const word of lower.match(/\p{L}+/gu) ?? []) { + if (DANISH_WORDS.has(word)) danish += 1; + if (ENGLISH_WORDS.has(word)) english += 1; + } + return danish > english ? "da" : "en"; +} diff --git a/tests/errors.test.ts b/tests/errors.test.ts index 8e38b46..9fe8a52 100644 --- a/tests/errors.test.ts +++ b/tests/errors.test.ts @@ -98,3 +98,19 @@ describe("classifyError", () => { expect(classifyError(42).message).toBe("42"); }); }); + +describe("classifyError — transcript translation (D-WS4b)", () => { + test("out of memory while translating suggests closing tabs, not a smaller Whisper model", () => { + const classified = classifyError("RangeError: Out of memory", { stage: "translate" }); + expect(classified.kind).toBe("out-of-memory"); + expect(classified.suggestion).not.toContain("smaller model"); + expect(classified.suggestion).toContain("other tabs"); + }); + + test("a failed translation-model download is retryable and says the download restarts", () => { + const classified = classifyError("Failed to fetch", { stage: "translate" }); + expect(classified.kind).toBe("network"); + expect(classified.retryable).toBe(true); + expect(classified.suggestion).toContain("translation model"); + }); +}); diff --git a/tests/translation.test.ts b/tests/translation.test.ts new file mode 100644 index 0000000..dd96443 --- /dev/null +++ b/tests/translation.test.ts @@ -0,0 +1,185 @@ +import { describe, expect, test } from "bun:test"; +import { join } from "node:path"; +import { + TRANSLATION_MODELS, + assembleTranslation, + buildTranslationUnits, + estimateTokens, + getTranslationModel, + guessSourceLanguage, + splitForTranslation, +} from "../src/lib/translation"; +import type { TranscriptChunk } from "../src/worker/protocol"; + +// Pure planning for transcript translation (D-WS4b) — no model, no network. + +const words = (text: string) => text.split(/\s+/).filter(Boolean); + +describe("TRANSLATION_MODELS", () => { + test("offers da→en and en→da Opus-MT ONNX exports", () => { + expect(TRANSLATION_MODELS.map((m) => m.id)).toEqual([ + "Xenova/opus-mt-da-en", + "Xenova/opus-mt-en-da", + ]); + expect(getTranslationModel("da", "en")?.id).toBe("Xenova/opus-mt-da-en"); + expect(getTranslationModel("en", "da")?.id).toBe("Xenova/opus-mt-en-da"); + for (const model of TRANSLATION_MODELS) { + // q8 encoder + merged decoder + tokenizer, measured on the Hub. + expect(model.approxBytes).toBeGreaterThan(100e6); + expect(model.approxBytes).toBeLessThan(130e6); + } + }); +}); + +describe("buildTranslationUnits", () => { + test("merges chunks until a sentence ends, keeping the merged time span", () => { + const chunks: TranscriptChunk[] = [ + { text: " So the plan is", timestamp: [0, 2] }, + { text: " to ship on Friday.", timestamp: [2, 4.5] }, + { text: " Any questions?", timestamp: [5, 6] }, + ]; + expect(buildTranslationUnits({ text: "", chunks })).toEqual([ + { text: "So the plan is to ship on Friday.", timestamp: [0, 4.5] }, + { text: "Any questions?", timestamp: [5, 6] }, + ]); + }); + + test("an unterminated tail still becomes a unit, with its null end kept", () => { + const chunks: TranscriptChunk[] = [ + { text: " Done.", timestamp: [0, 1] }, + { text: " and then", timestamp: [1, null] }, + ]; + expect(buildTranslationUnits({ text: "", chunks })).toEqual([ + { text: "Done.", timestamp: [0, 1] }, + { text: "and then", timestamp: [1, null] }, + ]); + }); + + test("unpunctuated chunks are capped so cues stay subtitle-sized", () => { + const chunks: TranscriptChunk[] = Array.from({ length: 20 }, (_, i) => ({ + text: ` word${i}`, + timestamp: [i, i + 1] as [number, number], + })); + const units = buildTranslationUnits({ text: "", chunks }, 8); + expect(units.map((u) => u.timestamp)).toEqual([ + [0, 8], + [8, 16], + [16, 20], + ]); + }); + + test("empty chunks are skipped", () => { + const chunks: TranscriptChunk[] = [ + { text: " ", timestamp: [0, 1] }, + { text: " Hi.", timestamp: [1, 2] }, + ]; + expect(buildTranslationUnits({ text: "", chunks })).toEqual([ + { text: "Hi.", timestamp: [1, 2] }, + ]); + }); + + test("without chunks, each non-empty line is an untimed unit", () => { + expect(buildTranslationUnits({ text: "Hej.\n\nHvordan går det?\n", chunks: [] })).toEqual([ + { text: "Hej.", timestamp: null }, + { text: "Hvordan går det?", timestamp: null }, + ]); + }); +}); + +describe("splitForTranslation", () => { + test("every sentence is its own piece (Opus-MT drops trailing sentences otherwise)", () => { + expect( + splitForTranslation("My name is Lucas and I live in Copenhagen. How are you? Fine!"), + ).toEqual(["My name is Lucas and I live in Copenhagen.", "How are you?", "Fine!"]); + }); + + test("abbreviation-only fragments stay with the following sentence", () => { + expect(splitForTranslation("Mr. Smith went home. He slept.")).toEqual([ + "Mr. Smith went home.", + "He slept.", + ]); + }); + + test("blank input yields no pieces", () => { + expect(splitForTranslation(" ")).toEqual([]); + }); + + test("an unpunctuated run-on is wrapped under the budget, words intact and in order", () => { + const runOn = Array.from({ length: 300 }, (_, i) => `ord${i}`).join(" "); + const pieces = splitForTranslation(runOn, { maxTokens: 40, countTokens: (s) => words(s).length }); + expect(pieces.length).toBeGreaterThan(1); + for (const piece of pieces) expect(words(piece).length).toBeLessThanOrEqual(40); + expect(pieces.join(" ")).toBe(runOn); + }); + + test("a long sentence prefers comma boundaries before word wrapping", () => { + const clause = "one two three four five six"; + const sentence = `${clause}, ${clause}, ${clause}, ${clause}.`; + const pieces = splitForTranslation(sentence, { maxTokens: 14, countTokens: (s) => words(s).length }); + expect(pieces).toEqual([`${clause}, ${clause},`, `${clause}, ${clause}.`]); + }); + + test("a transcript far beyond Opus-MT's 512-token input is split with nothing dropped, in order (Review Focus 5)", async () => { + const alien = await Bun.file(join(import.meta.dir, "audio_files/en_alien.txt")).text(); + expect(estimateTokens(alien)).toBeGreaterThan(512); + const pieces = splitForTranslation(alien, { maxTokens: 200 }); + for (const piece of pieces) expect(estimateTokens(piece)).toBeLessThanOrEqual(200); + expect(words(pieces.join(" "))).toEqual(words(alien)); + }); +}); + +describe("estimateTokens", () => { + test("over-estimates rather than under-estimates SentencePiece counts", () => { + // Measured with the Xenova/opus-mt-en-da tokenizer: this sentence is 12 tokens (incl. ). + expect(estimateTokens("My name is Lucas and I live in Copenhagen.")).toBeGreaterThanOrEqual(12); + expect(estimateTokens("")).toBe(0); + }); +}); + +describe("assembleTranslation", () => { + test("pairs translated units with their time spans", () => { + const units = [ + { text: "Hej.", timestamp: [0, 1] as [number, number] }, + { text: "Farvel.", timestamp: [2, null] as [number, null] }, + ]; + expect(assembleTranslation(units, [" Hello. ", "Goodbye."])).toEqual({ + text: "Hello. Goodbye.", + chunks: [ + { text: "Hello.", timestamp: [0, 1] }, + { text: "Goodbye.", timestamp: [2, null] }, + ], + }); + }); + + test("untimed units join as lines and produce no chunks", () => { + const units = [ + { text: "Hej.", timestamp: null }, + { text: "Farvel.", timestamp: null }, + ]; + expect(assembleTranslation(units, ["Hello.", "Goodbye."])).toEqual({ + text: "Hello.\nGoodbye.", + chunks: [], + }); + }); + + test("a count mismatch is an error, never a silently shifted transcript", () => { + expect(() => assembleTranslation([{ text: "a", timestamp: null }], [])).toThrow(); + }); +}); + +describe("guessSourceLanguage", () => { + test("trusts an explicit Whisper language", () => { + expect(guessSourceLanguage("whatever", "da")).toBe("da"); + expect(guessSourceLanguage("whatever", "en")).toBe("en"); + }); + + test("detects Danish from its letters and common words", () => { + expect(guessSourceLanguage("På Aarhus Festuges sekretariat har vi løbet rigtig stærkt.")).toBe("da"); + expect(guessSourceLanguage("Det er ikke noget med at vi skal")).toBe("da"); + }); + + test("defaults to English", () => { + expect(guessSourceLanguage("Guys, I have been addicted to this game.")).toBe("en"); + expect(guessSourceLanguage("", "auto")).toBe("en"); + }); +});