WIP: uncommitted work at pause (session limit)

Co-Authored-By: Claude Opus 5.5 <[email protected]>

docs/Architecture.md

@@ -63,7 +63,7 @@ src/
   index.html               # HTML entry, imports frontend.tsx
   frontend.tsx             # React root
   App.tsx                  # top-level UI composition
-  components/              # ModelPicker, TranscriptionOptions, FileDropzone, LiveControls, TranscriptView, EngineToggle, AuthForms
+  components/              # ModelPicker, TranscriptionOptions, VocabularyHint, FileDropzone, LiveControls, TranscriptView, EngineToggle, AuthForms
   lib/
     audio.ts               # decode + resample to 16 kHz mono Float32Array (Web Audio API)
     live.ts                # live-mode segment loop: RMS gate, LiveSegmenter, LiveTranscriber (pure, D-016)
@@ -71,9 +71,11 @@ src/
     models.ts              # model catalog: ids, sizes, dtype defaults per device, languages
     modelCache.ts          # "downloaded?" per model via Cache Storage (D-005; reused by M3 offline indicator)
     engine.ts              # engine abstraction: BrowserEngine | ServerEngine
+    hint.ts                # "Names and terms" hint: normalize, localStorage, prompt truncation rule (D-WS2a)
   worker/
     inference.worker.ts    # Transformers.js pipeline, WebNN/WebGPU/WASM selection
     protocol.ts            # typed postMessage messages (shared with UI)
+    prompt.ts              # decoder prompt (<|startofprev|> hint) via a generate() wrapper (D-WS2a)
   pwa/
     sw.ts                  # service worker (app-shell precache, /ort runtime cache)
     precache.ts            # placeholder-injection helpers shared by build + dev serve
@@ -141,9 +143,16 @@ environment). See `WorkBreakdown.md` for the itemized checklist.)
    `AudioContext.decodeAudioData`, downmixes to mono, resamples to 16 kHz,
    transfers the `Float32Array` to the worker. For multilingual models the
    `transcribe` message also carries `language` (or auto-detect) and `task`
-   (transcribe | translate).
+   (transcribe | translate). For every model it may carry `prompt`: the
+   optional "Names and terms" hint (`VocabularyHint`, stored in localStorage
+   by `lib/hint.ts`), raw as typed (D-WS2a).
 5. Worker runs the pipeline (chunked, with timestamps), streaming partial results
    back via `{type:"partial"}` messages; final result as `{type:"result"}`.
+   A hint becomes `<|startofprev|> hint.` once per pass (`worker/prompt.ts`,
+   cut to Whisper's 223-token prompt budget at a whole word) and is prepended
+   to the decoder input of every 30 s window by a wrapper around the model's
+   `generate()`, which also strips it from the output again (D-WS2a).
+   Live passes (4.1b) carry the same `prompt`.
 
 ### 4.1b Live microphone transcription (anonymous, browser engine — D-016)
 1. `LiveControls` starts a session: `getUserMedia` → `AudioContext` at the
@@ -204,7 +213,11 @@ environment). See `WorkBreakdown.md` for the itemized checklist.)
    (`server-py/`) that Bun spawns and supervises (D-013; faster-whisper has no
    browser/WASM build, so it is server-only — the browser stays on Transformers.js).
    `src/server/transcribe.ts` bridges the sidecar's segment stream to the streamed
-   JSON-lines HTTP response (D-011). The shared catalog maps model ids to
+   JSON-lines HTTP response (D-011). `language`, `task` and the vocabulary hint
+   (`prompt`) travel as query params (also on the live WebSocket) and are
+   validated by `parseLanguageAndTask` (D-032, D-WS2a); the sidecar cuts the
+   hint with the same rule as the browser and passes it to faster-whisper
+   (D-WS2a). The shared catalog maps model ids to
    `Systran/faster-whisper-*` CTranslate2 conversions. If the sidecar is absent,
    the server engine reports unavailable; the browser/anonymous flow is unaffected.
 4. Both engines implement the same `Engine` interface in `src/lib/engine.ts`, so the

tests/live-real-model.test.ts

@@ -10,6 +10,13 @@ import {
   type LiveTranscribeFn,
 } from "../src/lib/live";
 import { resolveDefaultDtype } from "../src/lib/models";
+import {
+  buildPromptPrefix,
+  installPromptSupport,
+  PROMPT_PREFIX_KWARG,
+  type PromptableModel,
+  type PromptTokenizer,
+} from "../src/worker/prompt";
 import { wordErrorRate } from "./dsp";
 
 /**
@@ -266,6 +273,10 @@ describe("live pipeline accuracy (real model, real recordings)", () => {
           normalize((await transcribe(whole)).text),
         );
 
+        // File-mode accuracy gate (spec 001): the upload flow is held to the
+        // same per-clip bar as live, so a file-mode regression fails here too.
+        expect(fileWer).toBeLessThanOrEqual(MAX_WER_BY_CLIP[clip.base] ?? MAX_WER);
+
         // The live session, with the injected clock advancing with the audio
         // fed so pass scheduling behaves as in real time.
         let fedSamples = 0;
@@ -310,3 +321,103 @@ describe("live pipeline accuracy (real model, real recordings)", () => {
     );
   }
 });
+
+// ---------------------------------------------------------------------------
+// Vocabulary hint (D-WS2a): the worker's real prompt code on real weights
+// ---------------------------------------------------------------------------
+
+const HINT = "Maxwell, Voices of the Void, Slenderman, RuneScape, UFO";
+
+/** Decode the first `seconds` of a clip straight to 16 kHz (no mic path needed). */
+async function decode16k(file: string, seconds?: number): Promise<Float32Array> {
+  const proc = Bun.spawn(
+    [
+      ffmpegPath as unknown as string,
+      ...["-hide_banner", "-loglevel", "error", "-i", join(AUDIO_DIR, file)],
+      ...(seconds ? ["-t", String(seconds)] : []),
+      ...["-ac", "1", "-ar", String(RATE), "-f", "f32le", "-"],
+    ],
+    { stdout: "pipe", stderr: "pipe" },
+  );
+  const [bytes, exitCode] = await Promise.all([new Response(proc.stdout).arrayBuffer(), proc.exited]);
+  if (exitCode !== 0) throw new Error(`ffmpeg failed on ${file}`);
+  return new Float32Array(bytes);
+}
+
+/** One pass with the inference worker's options and (optionally) its prompt. */
+async function transcribeWithHint(
+  audio: Float32Array,
+  hint: string,
+  language = "en",
+): Promise<string> {
+  const asr = await loadAsr();
+  installPromptSupport(asr.model as unknown as PromptableModel);
+  const prefix = buildPromptPrefix(asr.tokenizer as unknown as PromptTokenizer, hint);
+  const output = await asr(audio, {
+    chunk_length_s: 30,
+    stride_length_s: 5,
+    return_timestamps: true,
+    max_new_tokens: 440,
+    language,
+    task: "transcribe",
+    ...(prefix.length > 0 && { [PROMPT_PREFIX_KWARG]: prefix }),
+  } as Record<string, unknown>);
+  const single = (Array.isArray(output) ? output[0] : output) as { text?: string };
+  return single.text ?? "";
+}
+
+describe("vocabulary hint (real model, D-WS2a)", () => {
+  test.skipIf(!available)(
+    "a hint keeps the transcript intact, keeps the hinted names, and never leaks into the text",
+    async () => {
+      // The first 2 min of en_alien: four windows, "Voices of the Void" and
+      // several "Maxwell"s.
+      const audio = await decode16k("en_alien.opus", 120);
+      const plain = await transcribeWithHint(audio, "");
+      const hinted = await transcribeWithHint(audio, HINT);
+      const count = (text: string, term: string) => normalize(text).split(term).length - 1;
+
+      expect(count(hinted, "maxwell")).toBeGreaterThanOrEqual(count(plain, "maxwell"));
+      expect(count(hinted, "voices of the void")).toBeGreaterThanOrEqual(1);
+      // Not garbled: no repetition loop, no dropped windows (measured
+      // difference between the two runs ≈ punctuation-level).
+      expect(wordErrorRate(normalize(plain), normalize(hinted))).toBeLessThanOrEqual(0.15);
+      // The prompt itself is never part of the transcript.
+      expect(normalize(hinted).startsWith(normalize(HINT))).toBe(false);
+      expect(normalize(hinted)).not.toContain("slenderman runescape");
+    },
+    DOWNLOAD_BUDGET_MS + 120 * 2 * MS_PER_AUDIO_SECOND,
+  );
+
+  test.skipIf(!available)(
+    "a hint longer than Whisper's prompt budget is cut, not an error or a garbled window",
+    async () => {
+      const longHint = Array.from({ length: 200 }, (_, i) => `Project${i} Codename${i}`)
+        .join(", ")
+        .slice(0, 1000);
+      const reference = await Bun.file(join(AUDIO_DIR, "en_aiproblem.txt")).text();
+      const text = await transcribeWithHint(await decode16k("en_aiproblem.m4a"), longHint);
+      expect(wordErrorRate(normalize(reference), normalize(text))).toBeLessThanOrEqual(MAX_WER);
+      expect(text).not.toContain("Codename");
+    },
+    DOWNLOAD_BUDGET_MS + 60 * MS_PER_AUDIO_SECOND,
+  );
+
+  test.skipIf(!available)(
+    "a short, near-silent live window with a hint does not transcribe the hint",
+    async () => {
+      // 3 s of faint noise — what a live pass sees in a pause that the
+      // segmenter did not drop (its near-silence gate is far lower).
+      let seed = 1;
+      const noise = new Float32Array(3 * RATE).map(() => {
+        seed = (seed * 1103515245 + 12345) % 2 ** 31;
+        return (seed / 2 ** 31 - 0.5) * 0.01;
+      });
+      const text = normalize(await transcribeWithHint(noise, HINT));
+      for (const term of ["maxwell", "voices of the void", "slenderman", "runescape"]) {
+        expect(text).not.toContain(term);
+      }
+    },
+    DOWNLOAD_BUDGET_MS + 30 * MS_PER_AUDIO_SECOND,
+  );
+});