Translate transcripts with Opus-MT in a dedicated translation worker

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

src/index.ts

@@ -223,7 +223,8 @@ async function productionServeOptions(): Promise<ServeOptions> {
  * Bun's bundler does not currently follow `new Worker(...)` graphs (verified
  * with Bun 1.3: both `bun build` and the dev server leave the call untouched),
  * so worker-like entrypoints are bundled at startup and served from stable
- * URLs. The UI spawns `new Worker("/inference.worker.js")`; the service
+ * URLs. The UI spawns `new Worker("/inference.worker.js")` (and
+ * "/translate.worker.js" for transcript translation, D-WS4b); the service
  * worker registers as "/sw.js".
  */
 async function bundleEntrypoint(relativePath: string): Promise<string> {
@@ -249,6 +250,8 @@ async function developmentServeOptions(): Promise<ServeOptions> {
   const { default: index } = await import("./index.html");
 
   const workerScript = await bundleEntrypoint("./worker/inference.worker.ts");
+  // Transcript translation runs in its own worker (D-WS4b).
+  const translateWorkerScript = await bundleEntrypoint("./worker/translate.worker.ts");
   // Dev SW: empty precache list — assets are unhashed here, so the shell is
   // runtime-cached network-first instead (see src/pwa/sw.ts).
   const swScript = injectPrecacheManifest(
@@ -283,6 +286,11 @@ async function developmentServeOptions(): Promise<ServeOptions> {
       if (pathname.endsWith("/inference.worker.js")) {
         return withIsolationHeaders(new Response(workerScript, { headers: scriptHeaders }));
       }
+      if (pathname.endsWith("/translate.worker.js")) {
+        return withIsolationHeaders(
+          new Response(translateWorkerScript, { headers: scriptHeaders }),
+        );
+      }
       if (pathname.endsWith("/sw.js")) {
         return withIsolationHeaders(
           new Response(swScript, { headers: { ...scriptHeaders, "Cache-Control": "no-cache" } }),

src/lib/translation.ts

@@ -154,8 +154,13 @@ export interface SplitOptions {
   countTokens?: (text: string) => number;
 }
 
-/** "Mr.", "Dr.", "St." — sentence-segmenter false breaks. */
-const ABBREVIATION = /^\p{Lu}\p{Ll}{0,3}\.$/u;
+/**
+ * Title abbreviations (en + da) and single-letter initials ("J.") — the
+ * sentence segmenter's false breaks. Kept to an explicit list: a broad
+ * "short capitalised word" rule would glue real one-word sentences
+ * ("Yes.", "Okay.") onto the next one.
+ */
+const ABBREVIATION = /^(?:Mr|Mrs|Ms|Dr|Prof|St|Jr|Sr|Hr|Fru|Nr|\p{Lu})\.$/u;
 
 function sentences(text: string): string[] {
   const out: string[] = [];
@@ -215,6 +220,60 @@ export function splitForTranslation(text: string, options: SplitOptions = {}): s
   );
 }
 
+// ---------------------------------------------------------------------------
+// The translation loop (model injected: worker, benchmark and tests share it)
+// ---------------------------------------------------------------------------
+
+export interface TranslateUnitsDeps extends SplitOptions {
+  /** Translate a batch of pieces; must answer one string per piece, in order. */
+  translateBatch: (pieces: string[]) => Promise<string[]>;
+  /** Pieces per model call (padding makes larger batches cheaper per piece). */
+  batchSize?: number;
+  /** Called with (done, total) pieces: once at 0, then after every batch. */
+  onProgress?: (done: number, total: number) => void;
+}
+
+/**
+ * One piece per model call. Batching would be ~4x faster, but Transformers.js
+ * 4.2's batched Marian generation is broken: every row of a padded batch
+ * comes back as row 0's translation (measured 2026-10-29 on
+ * onnxruntime-node, q8 and fp32, any graph optimisation level — D-WS4b).
+ * Re-measure with `scripts/bench-translate.ts` after a Transformers.js
+ * upgrade before raising this.
+ */
+export const DEFAULT_TRANSLATE_BATCH_SIZE = 1;
+
+/**
+ * Translate unit texts: split each into sentence pieces, translate all
+ * pieces in order-preserving batches, rejoin each unit's pieces with spaces.
+ * Returns one translated string per input unit.
+ */
+export async function translateUnits(
+  units: readonly string[],
+  deps: TranslateUnitsDeps,
+): Promise<string[]> {
+  const batchSize = Math.max(1, deps.batchSize ?? DEFAULT_TRANSLATE_BATCH_SIZE);
+  const perUnit = units.map((unit) => splitForTranslation(unit, deps));
+  const pieces = perUnit.flat();
+  const translated: string[] = [];
+  deps.onProgress?.(0, pieces.length);
+  for (let start = 0; start < pieces.length; start += batchSize) {
+    const batch = pieces.slice(start, start + batchSize);
+    const out = await deps.translateBatch(batch);
+    if (out.length !== batch.length) {
+      throw new Error(`Translation returned ${out.length} sentences for ${batch.length}.`);
+    }
+    translated.push(...out.map((text) => text.trim()));
+    deps.onProgress?.(translated.length, pieces.length);
+  }
+  let next = 0;
+  return perUnit.map((unitPieces) => {
+    const joined = translated.slice(next, next + unitPieces.length).join(" ");
+    next += unitPieces.length;
+    return joined;
+  });
+}
+
 // ---------------------------------------------------------------------------
 // Assembly
 // ---------------------------------------------------------------------------

src/worker/opusMt.ts

@@ -0,0 +1,59 @@
+/**
+ * Opus-MT (MarianMT) through Transformers.js — the model half of transcript
+ * translation (D-WS4b). Shared by the browser translate worker and the
+ * onnxruntime-node measurement script / real-model test, so all three run
+ * the exact same pipeline options.
+ */
+
+import { pipeline, type ProgressInfo, type TranslationPipeline } from "@huggingface/transformers";
+
+/** Hard input limit of every Opus-MT checkpoint (max_position_embeddings). */
+export const OPUS_MT_MAX_POSITIONS = 512;
+
+export interface OpusMtTranslator {
+  /** One translated string per piece, same order. */
+  translateBatch(pieces: string[]): Promise<string[]>;
+  /** Real SentencePiece token count (incl. </s>) for the splitter's budget. */
+  countTokens(text: string): number;
+  dispose(): Promise<void>;
+}
+
+export interface LoadOpusMtOptions {
+  /** "wasm" in the browser; omit under onnxruntime-node (defaults to CPU). */
+  device?: "wasm";
+  progressCallback?: (info: ProgressInfo) => void;
+}
+
+export async function loadOpusMt(
+  modelId: string,
+  options: LoadOpusMtOptions = {},
+): Promise<OpusMtTranslator> {
+  const translator = (await pipeline("translation", modelId, {
+    // q8: 113 MB per direction (encoder + merged decoder + tokenizer); the
+    // fp32 export is ~4x larger for no audible gain on transcripts.
+    dtype: "q8",
+    ...(options.device && { device: options.device }),
+    // D-014: the bundled onnxruntime-web's level-2 QDQ transform rejects
+    // 8-bit decoders with a tied embedding weight — Marian ties them too.
+    session_options: { graphOptimizationLevel: "basic" as const },
+    ...(options.progressCallback && { progress_callback: options.progressCallback }),
+  })) as TranslationPipeline;
+
+  const countTokens = (text: string) => translator.tokenizer.encode(text).length;
+
+  return {
+    countTokens,
+    async translateBatch(pieces) {
+      if (pieces.length === 0) return [];
+      // Cap output length relative to the longest input: a translation is
+      // rarely more than ~2x its source in tokens, and the cap stops a rare
+      // repetition loop from running to 512 tokens.
+      const longest = Math.max(...pieces.map(countTokens));
+      const output = (await translator(pieces, {
+        max_new_tokens: Math.min(OPUS_MT_MAX_POSITIONS, 3 * longest + 16),
+      })) as { translation_text: string }[];
+      return output.map((entry) => entry.translation_text);
+    },
+    dispose: () => translator.dispose(),
+  };
+}

src/worker/translate.worker.ts

@@ -0,0 +1,98 @@
+/**
+ * Transcript-translation Web Worker (D-WS4b): Opus-MT through
+ * Transformers.js, off the main thread (D-003).
+ *
+ * A dedicated worker rather than a message type on the Whisper worker:
+ * - its own onnxruntime instance gets the full WASM thread pool. ort fixes
+ *   the pool at first init (D-023), and a Whisper worker that initialised
+ *   ort for WebGPU/WebNN runs WASM single-threaded for life;
+ * - cancelling a translation terminates only this worker — the loaded
+ *   Whisper pipeline survives, and a Whisper cancel/restart (D-018/D-025)
+ *   never kills a translation;
+ * - the model is downloaded only when the user asks for a translation and
+ *   cached by Transformers.js in the Cache API like Whisper weights
+ *   (D-005/D-020), so a cached model works offline.
+ * WASM only for now: Opus-MT is small, and WebGPU behaviour can't be
+ * measured in this environment.
+ */
+
+import { env, type ProgressInfo } from "@huggingface/transformers";
+import { getTranslationModelById, translateUnits } from "../lib/translation";
+import { loadOpusMt, type OpusMtTranslator } from "./opusMt";
+import {
+  DownloadTally,
+  isTranslateRequest,
+  type TranslateRequest,
+  type TranslateWorkerMessage,
+} from "./translateProtocol";
+
+// Same-origin onnxruntime binaries (offline, COEP) — see inference.worker.ts.
+if (env.backends.onnx.wasm) {
+  env.backends.onnx.wasm.wasmPaths = "./ort/";
+  // D-023: with cross-origin isolation, use most of the machine.
+  env.backends.onnx.wasm.numThreads = globalThis.crossOriginIsolated
+    ? Math.max(1, Math.min((navigator.hardwareConcurrency || 4) - 1, 12))
+    : 1;
+}
+
+interface DedicatedWorkerScope {
+  postMessage(message: TranslateWorkerMessage): void;
+  addEventListener(type: "message", listener: (event: MessageEvent<unknown>) => void): void;
+}
+const scope = self as unknown as DedicatedWorkerScope;
+
+/** One loaded model at a time; switching direction disposes the other. */
+let active: { modelId: string; promise: Promise<OpusMtTranslator> } | null = null;
+
+function load(modelId: string, onProgress: (info: ProgressInfo) => void): Promise<OpusMtTranslator> {
+  if (active?.modelId === modelId) return active.promise;
+  const previous = active;
+  const promise = (async () => {
+    if (previous) await (await previous.promise.catch(() => null))?.dispose();
+    return loadOpusMt(modelId, { device: "wasm", progressCallback: onProgress });
+  })();
+  active = { modelId, promise };
+  promise.catch(() => {
+    if (active?.promise === promise) active = null; // allow a clean retry
+  });
+  return promise;
+}
+
+async function handle(request: TranslateRequest): Promise<void> {
+  const { requestId } = request;
+  if (!getTranslationModelById(request.modelId)) {
+    throw new Error(`Unknown translation model: ${request.modelId}`);
+  }
+  const tally = new DownloadTally();
+  const translator = await load(request.modelId, (info) => {
+    if (info.status === "progress") {
+      const { loaded, total } = tally.update(info.file, info.loaded, info.total);
+      scope.postMessage({ type: "translate-download", requestId, loaded, total });
+    }
+  });
+  const texts = await translateUnits(request.units, {
+    translateBatch: (pieces) => translator.translateBatch(pieces),
+    countTokens: translator.countTokens,
+    onProgress: (done, total) =>
+      scope.postMessage({ type: "translate-progress", requestId, done, total }),
+  });
+  scope.postMessage({ type: "translate-result", requestId, texts });
+}
+
+// Requests run one at a time, in arrival order: the UI only ever has one in
+// flight, and serialising keeps two passes from sharing the thread pool.
+let queue: Promise<void> = Promise.resolve();
+
+scope.addEventListener("message", (event) => {
+  const data = event.data;
+  if (!isTranslateRequest(data)) return;
+  queue = queue.then(() =>
+    handle(data).catch((error: unknown) => {
+      scope.postMessage({
+        type: "translate-error",
+        requestId: data.requestId,
+        message: error instanceof Error ? error.message : String(error),
+      });
+    }),
+  );
+});

src/worker/translateProtocol.ts

@@ -0,0 +1,106 @@
+/**
+ * Typed postMessage protocol between the UI and the transcript-translation
+ * worker (`src/worker/translate.worker.ts`, D-WS4b). Kept apart from the
+ * Whisper worker's protocol: separate worker, separate lifetime. Free of
+ * browser globals so `bun test` covers the guards.
+ */
+
+/** UI → worker: translate these unit texts with this Opus-MT model. */
+export interface TranslateRequest {
+  type: "translate";
+  /** Correlates every reply; the UI ignores replies for stale ids. */
+  requestId: string;
+  /** One of `TRANSLATION_MODEL_IDS` (src/lib/translation.ts). */
+  modelId: string;
+  /** Unit texts (`buildTranslationUnits`), answered 1:1 in `texts`. */
+  units: string[];
+}
+
+/** Model files being fetched (first use) — bytes summed across files. */
+export interface TranslateDownloadMessage {
+  type: "translate-download";
+  requestId: string;
+  loaded: number;
+  total: number;
+}
+
+/** Model ready; `done` of `total` sentence pieces translated so far. */
+export interface TranslateProgressMessage {
+  type: "translate-progress";
+  requestId: string;
+  done: number;
+  total: number;
+}
+
+export interface TranslateResultMessage {
+  type: "translate-result";
+  requestId: string;
+  /** One translated string per request unit, same order. */
+  texts: string[];
+}
+
+export interface TranslateErrorMessage {
+  type: "translate-error";
+  requestId: string;
+  message: string;
+}
+
+export type TranslateWorkerMessage =
+  | TranslateDownloadMessage
+  | TranslateProgressMessage
+  | TranslateResultMessage
+  | TranslateErrorMessage;
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null;
+}
+
+const isNumber = (value: unknown): value is number =>
+  typeof value === "number" && Number.isFinite(value);
+
+export function isTranslateRequest(value: unknown): value is TranslateRequest {
+  return (
+    isRecord(value) &&
+    value.type === "translate" &&
+    typeof value.requestId === "string" &&
+    typeof value.modelId === "string" &&
+    Array.isArray(value.units) &&
+    value.units.every((unit) => typeof unit === "string")
+  );
+}
+
+export function isTranslateWorkerMessage(value: unknown): value is TranslateWorkerMessage {
+  if (!isRecord(value) || typeof value.requestId !== "string") return false;
+  switch (value.type) {
+    case "translate-download":
+      return isNumber(value.loaded) && isNumber(value.total);
+    case "translate-progress":
+      return isNumber(value.done) && isNumber(value.total);
+    case "translate-result":
+      return Array.isArray(value.texts) && value.texts.every((t) => typeof t === "string");
+    case "translate-error":
+      return typeof value.message === "string";
+    default:
+      return false;
+  }
+}
+
+/**
+ * Sums Transformers.js per-file download progress into one byte count for
+ * the UI. A file's total is only known once its download starts, so the
+ * total grows as files start — fine for a percentage that never goes back.
+ */
+export class DownloadTally {
+  private readonly files = new Map<string, { loaded: number; total: number }>();
+
+  update(file: string, loaded: number, total: number): { loaded: number; total: number } {
+    this.files.set(file, { loaded, total });
+    let sumLoaded = 0;
+    let sumTotal = 0;
+    for (const entry of this.files.values()) {
+      sumLoaded += entry.loaded;
+      sumTotal += entry.total;
+    }
+    return { loaded: sumLoaded, total: sumTotal };
+  }
+}