From 225634c165ca60bd547ff8d27378eefe15414443 Mon Sep 17 00:00:00 2001 From: Lucas Winther Date: Thu, 29 Oct 2026 21:41:45 +0100 Subject: [PATCH 01/17] Plan workstream 2: vocabulary hint Co-Authored-By: Claude Opus 5.5 --- docs/plans/001-2-vocabulary-hint.md | 206 ++++++++++++++++++++++++++++ 1 file changed, 206 insertions(+) create mode 100644 docs/plans/001-2-vocabulary-hint.md diff --git a/docs/plans/001-2-vocabulary-hint.md b/docs/plans/001-2-vocabulary-hint.md new file mode 100644 index 0000000..74cd37d --- /dev/null +++ b/docs/plans/001-2-vocabulary-hint.md @@ -0,0 +1,206 @@ +--- +status: in-progress +--- + +# Vocabulary hint ("Names and terms") Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: superpowers:executing-plans (inline, no nested +> subagents — coordination plan 001-0). Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** an optional "Names and terms" hint, remembered in localStorage, fed to Whisper as a +decoder prompt on every window of every browser pass (file + live) and to faster-whisper on the +server engine. + +**Architecture:** the UI sends the raw hint as `prompt?: string` (shared interface, 001-0). A pure +`src/lib/hint.ts` owns normalization, storage and the truncation rule; the worker turns the hint +into `<|startofprev|> + ids` once per transcribe message and passes it through the pipeline as a +private generate kwarg, which a one-time wrapper around `model.generate` (`src/worker/prompt.ts`) +turns into `decoder_input_ids` and strips from the output again. Transformers.js 4.2.0 **and** +4.3.2 accept `prompt_ids` in the config but never use it (commented out in +`modeling_whisper.js`), and `_decode_asr` would print prompt tokens as transcript text — hence the +wrapper. Server: `parseLanguageAndTask` validates `prompt` for both routes (D-032 pattern), the +sidecar applies the same truncation rule (Python twin, parity-tested) and passes token ids as +`initial_prompt` (+ `hotwords` only if measured to help). + +**Tech Stack:** Bun 1.3, React 19, Transformers.js 4.2.0 (patched), faster-whisper 1.2.1 venv at +`/home/user/venvs/faster-whisper` (measurements only). + +**Spec:** `docs/specs/001-transcription-quality-and-speed.md` (Goal 2; file-mode accuracy gate); +`docs/plans/001-0-coordination.md` (shared interfaces, global constraints). + +## Global Constraints + +- Field name `prompt?: string` on the transcribe message, raw user text; tokenized/truncated in the + worker. Live and server paths carry the same name. +- Decision entry `## D-WS2a — …`, cited as `D-WS2a` in code comments. No real `D-###`. +- Heavy jobs (real-model tests, full `bun test`, measurements, downloads) under + `flock /tmp/whisper-heavy.lock`. Ports 3120–3129. No push/publish. +- Plain-language UI, optional, never required (PRODUCT.md); colour tokens only (DESIGN.md). +- `max_new_tokens: 440` (D-021) keeps the single-pass path; with a prompt it is capped so prompt + + SOT + new tokens ≤ 448 decoder positions. +- Measurements are onnxruntime-node (CPU) numbers; say so. WebGPU effects → owner's browser check. + +## Decisions fixed by this plan + +- Storage key `whisper-vocabulary-hint`; `HINT_MAX_CHARS = 1000` (textarea `maxLength`, server 400 + above it — URL-safe since it travels as a query param). +- `normalizeHint`: drop control chars, one term per line or comma-separated → lines trimmed, empty + lines dropped, joined with `", "`, whitespace collapsed. Empty → `""` (= no prompt). +- `PROMPT_TOKEN_BUDGET = 223` (= 448/2 − 1, the openai/faster-whisper prompt cap). +- **Truncation rule:** encode `" " + hint`; if > budget, keep the longest prefix of whole + (space-separated) words that fits, trailing `,;` stripped from the kept text; if not even the + first word fits, its first 223 tokens. The start of the list is kept, the end dropped. +- Prompt prefix computed once per transcribe message (not per window); per-window cost is the + extra decoder positions only. + +## Review Focus + +1. **Very long hint (> 223 tokens)** — truncated at a word boundary, never an error, first window + not garbled. Pinned by Task 1 (unit), Task 2 (cap of `max_new_tokens`), Task 7 (real model, + 1000-char hint). +2. **Short / near-silent live windows with a hint** — must not emit the hint text (Whisper's + "transcribe the prompt" failure). Task 7 real-model test on low-noise audio. +3. **Prompt tokens leaking into the transcript** (`_decode_asr` treats them as text) — Task 2 unit + test + Task 7 real-model check that output does not start with the hint. +4. **Non-ASCII hints on the server path** (Danish names in a query param) — Task 4 handler test + with `Århus, Ærø` round-trips to the sidecar request. +5. **`.en` models** (no language/task tokens) — Task 2 test with init tokens `[SOT]` only; the hint + applies to every catalog model, multilingual or not. + +--- + +### Task 1: `src/lib/hint.ts` — normalize, store, truncate + +**Files:** Create `src/lib/hint.ts`; Test `tests/hint.test.ts` + +**Interfaces — Produces:** +- `HINT_STORAGE_KEY`, `HINT_MAX_CHARS = 1000`, `PROMPT_TOKEN_BUDGET = 223` +- `normalizeHint(raw: string): string` +- `loadHint(storage?: StorageLike | null): string`, `saveHint(hint: string, storage?)` (StorageLike + from `src/lib/engine.ts`; never throw) +- `fitHintToBudget(hint: string, encode: (text: string) => number[], budget = PROMPT_TOKEN_BUDGET): number[]` + (normalizes internally; `[]` for empty) + +- [ ] Step 1: failing tests — `normalizeHint(" Maxwell\n\n Voices of the Void \n")` === + `"Maxwell, Voices of the Void"`; control chars removed; `"a, b"` unchanged; whitespace-only → `""`. + `loadHint`/`saveHint` round-trip with a Map-backed storage, throwing storage → `""`/no throw. + `fitHintToBudget` with fake `encode = (t) => [...t].map(c => c.charCodeAt(0))`: short hint → + encode(" Maxwell, Voices of the Void"); budget 10 on `"alpha, beta gamma"` → ids of `" alpha"` + (trailing comma stripped, whole words only); single 50-char word with budget 10 → first 10 ids; + 5000-word hint with budget 223 → length ≤ 223 and decodes to a word prefix of the hint. +- [ ] Step 2: `bun test tests/hint.test.ts` → FAIL (module missing). +- [ ] Step 3: implement; binary search over word count (encode is monotone in prefix length). +- [ ] Step 4: tests PASS. Commit. + +### Task 2: `src/worker/prompt.ts` — decoder prompt through the pipeline + +**Files:** Create `src/worker/prompt.ts`; Test `tests/prompt.test.ts` + +**Interfaces — Consumes:** `fitHintToBudget`. **Produces:** +- `PROMPT_PREFIX_KWARG = "whisper_web_prompt_prefix"`, `WHISPER_DECODER_POSITIONS = 448` +- `buildPromptPrefix(tokenizer: { encode(t: string, o: {add_special_tokens: false}): number[]; convert_tokens_to_ids(t: string): number }, hint: string): number[]` + → `[]` or `[<|startofprev|>, ...ids]` +- `installPromptSupport(model: PromptableModel): void` — idempotent; wraps `model.generate`: pops + the kwarg; empty → original call unchanged; else `decoder_input_ids = [...prefix, + ...model._retrieve_init_tokens(model._prepare_generation_config(null, rest))]`, `max_new_tokens = + min(rest.max_new_tokens ?? 440, 448 − decoder_input_ids.length)`, and returns the output Tensor + sliced `[:, prefix.length:]`. + +- [ ] Step 1: failing tests with a fake model (records kwargs, returns a real `Tensor` int64 of + `[decoder_input_ids..., generated...]`): no kwarg → kwargs passed through untouched and no + `decoder_input_ids`; with prefix `[50362, 1, 2]` and init `[50258, 50259, 50360]` → + decoder_input_ids equal concatenation, private kwarg absent, output tokens start at 50258; + `.en`-style init `[50257]` works; 224-token prefix + 440 requested → max_new_tokens = 448 − 227; + installing twice wraps once. `buildPromptPrefix` with a fake tokenizer: `""` → `[]`; `"Maxwell"` + → `[prevId, ...encode(" Maxwell")]`. +- [ ] Step 2: FAIL. Step 3: implement. Step 4: PASS. Commit. + +### Task 3: protocol + worker + engine + live plumbing + +**Files:** Modify `src/worker/protocol.ts`, `src/worker/inference.worker.ts`, `src/lib/engine.ts`; +Tests `tests/protocol.test.ts`, `tests/engine.test.ts` + +**Interfaces — Produces:** `TranscribeMessage.prompt?: string`; `EngineTranscribeOptions.prompt?: +string`; BrowserEngine forwards non-empty `prompt`; ServerEngine adds `prompt` query param when +non-empty. + +- [ ] Step 1: failing tests — guard accepts string `prompt`, rejects `prompt: 5`; BrowserEngine + posts `prompt` (and omits it when `""`/undefined); ServerEngine URL carries + `prompt=Maxwell%2C+Voices…` (URLSearchParams encoding) and none when empty. +- [ ] Step 2: FAIL. Step 3: implement; worker: `installPromptSupport(asr.model)` once after + pipeline creation (and lazily in transcribe), `options[PROMPT_PREFIX_KWARG] = + buildPromptPrefix(asr.tokenizer, message.prompt ?? "")` when non-empty; cite D-WS2a. Step 4: PASS; + typecheck. Commit. + +### Task 4: server validation, sidecar request, live WS, sidecar.py + +**Files:** Modify `src/server/transcribe.ts`, `src/server/live.ts`, `src/index.ts`, +`server-py/sidecar.py`, `tests/fixtures/fake_sidecar.py`; Tests `tests/transcribe.test.ts`, +`tests/server-live.test.ts`, `tests/hint-parity.test.ts` + +**Interfaces — Produces:** `parseLanguageAndTask(url)` → `{ language?, task, prompt? }` | error +string (`prompt` = `normalizeHint(raw)` or absent; raw > 1000 chars → `"prompt too long (max 1000 +characters)"`); `SidecarTranscribeRequest.prompt?: string`; sidecar JSON `"prompt": str | null`; +`LiveSessionData.prompt?: string`; Python `fit_prompt_ids(hint, encode, budget=223) -> list[int]` +(+ `normalize_hint`). + +- [ ] Step 1: failing tests — parse: `?prompt=Maxwell%0AVoid` → `"Maxwell, Void"`; empty/whitespace + → no prompt; 1001 chars → error string; POST with `prompt=Århus, Ærø` reaches sidecar request as + that string (Review Focus 4); live session passes `prompt` to the sidecar; createSidecar + pass-through via fake sidecar echo (`prompt=` appended only when present); parity: Python + `fit_prompt_ids` == TS `fitHintToBudget` with the char-code fake encoder over a table of cases + (spawn `python3 -I -c` importing `server-py/sidecar.py`, which imports faster-whisper only in + `main()`). +- [ ] Step 2: FAIL. Step 3: implement; sidecar encodes with + `cached_model.hf_tokenizer.encode(text, add_special_tokens=False).ids`, passes + `initial_prompt=ids` (hotwords decided in Task 6). Step 4: PASS. Commit. + +### Task 5: UI — `VocabularyHint` component + App wiring (impeccable) + +**Files:** Create `src/components/VocabularyHint.tsx`; Modify `src/App.tsx`, `src/index.css`, +`DESIGN.md` + +- [ ] Step 1: render below the language/task options for **every** model: label "Names and terms + (optional)", 2-row textarea (`maxLength={HINT_MAX_CHARS}`), placeholder with an example, one line + of help (what it does, that it is remembered in this browser, long lists are cut at the end). + Disabled while busy/live. State initialized from `loadHint()`, saved on change. +- [ ] Step 2: pass `prompt: hint` in the file run, `transcribeLiveSegment` options, and the server + live WS URL (`prompt` param). Typecheck + fast suite. +- [ ] Step 3: impeccable polish/harden on the component; tokens only; DESIGN.md component line. +- [ ] Step 4: build, serve `PORT=3120 NODE_ENV=production bun src/index.ts`, headless Chrome + screenshots light/dark × 390 px/1280 px; fix what looks wrong. Commit. + +### Task 6: measurements (under the lock) → D-WS2a + +Scratch scripts in `/tmp/ws2/` (not committed), importing the worktree's modules. + +- [ ] Browser path (onnxruntime-node, catalog dtypes, `wasm` defaults): en_alien whole file, turbo + q4 and one small model (base.en or tiny.en, downloaded under the lock), without hint vs hint + `"Maxwell, Voices of the Void"`: WER (test normalizer), count of `maxwell` and `voices of the + void` vs reference counts, wall-clock, punctuation count (a prompt without a final full stop may + suppress punctuation — if so, `normalizeHint` appends `.` and Task 1 tests are updated). +- [ ] Short-window hallucination: 1–3 s low-noise and 2 s speech snippets with/without hint, turbo + + small model; output must not contain hint words that are absent from the audio. +- [ ] Server path (venv, `mobiuslabsgmbh/faster-whisper-large-v3-turbo` cached, int8 CPU): + en_alien none / initial_prompt / hotwords / both. Keep hotwords only if it measurably beats + initial_prompt alone on hinted-word recall without WER loss. Implement the outcome in sidecar.py. +- [ ] Write D-WS2a (bottom of `docs/Decisions.md`) with numbers labelled CPU/onnxruntime-node, + truncation rule, wrapper rationale, 4.3.2 check. Architecture.md data-flow note; WorkBreakdown + tick/noticed. Commit. + +### Task 7: real-model tests (`tests/live-real-model.test.ts`) + +- [ ] Whole-file assertion in its own small block after `fileWer` is computed: + `expect(fileWer).toBeLessThanOrEqual(MAX_WER_BY_CLIP[clip.base] ?? MAX_WER)` (file-mode gate, + shared with workstream 3 — keep it separate for an easy merge). +- [ ] New `describe("vocabulary hint (real model)")` using the worker's `installPromptSupport` / + `buildPromptPrefix`: (a) first 120 s of en_alien with the hint → hinted words present at least + as often as without, WER ≤ 0.35, output does not begin with the hint; (b) en_aiproblem with a + 1000-char hint → no error, WER ≤ MAX_WER; (c) 2 s low-noise audio with the hint → output contains + neither "Maxwell" nor "Void". +- [ ] Run the file under the lock; green. Commit. + +### Task 8: finish + +- [ ] `bun run typecheck`, fast suite, `bun run build` green; `git status` clean. +- [ ] `git rm docs/plans/001-2-vocabulary-hint.md`, commit last. From 942799d3cb9d9c1b7bd18d3124ca6383ec583b55 Mon Sep 17 00:00:00 2001 From: Lucas Winther Date: Thu, 29 Oct 2026 21:42:25 +0100 Subject: [PATCH 02/17] Add the vocabulary hint's normalization, storage and prompt truncation rule Co-Authored-By: Claude Opus 5.5 --- src/lib/hint.ts | 108 ++++++++++++++++++++++++++++++++++++++++++++ tests/hint.test.ts | 109 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 217 insertions(+) create mode 100644 src/lib/hint.ts create mode 100644 tests/hint.test.ts diff --git a/src/lib/hint.ts b/src/lib/hint.ts new file mode 100644 index 0000000..f66a6fe --- /dev/null +++ b/src/lib/hint.ts @@ -0,0 +1,108 @@ +/** + * The optional "Names and terms" vocabulary hint (D-WS2a): free text the user + * types once, remembered in this browser, and fed to Whisper as a decoder + * prompt (`<|startofprev|> hint …`) so names and jargon come out spelled the + * way the user spells them. + * + * Pure and dependency-free so the UI, the worker (src/worker/prompt.ts) and + * the server (src/server/transcribe.ts) share one normalization rule, and the + * truncation rule below has a Python twin in server-py/sidecar.py + * (`fit_prompt_ids`, parity-tested in tests/hint-parity.test.ts). + */ + +import type { StorageLike } from "./engine"; + +export const HINT_STORAGE_KEY = "whisper-vocabulary-hint"; + +/** + * Longest raw hint accepted (textarea `maxLength`; the server answers 400 + * above it). The hint travels as a URL query parameter on the server engine, + * so it must stay well below URL limits even fully percent-encoded. + */ +export const HINT_MAX_CHARS = 1000; + +/** + * Whisper's prompt budget: half its 448 decoder positions, minus one for + * `<|startofprev|>` — the cap openai-whisper and faster-whisper use too. + */ +export const PROMPT_TOKEN_BUDGET = 223; + +/** C0 controls + DEL, except \n and \r (line separators, handled below). Tab stays whitespace. */ +const CONTROL_CHARS = /[\u0000-\u0008\u000b\u000c\u000e-\u001f\u007f]/g; + +/** + * Canonical prompt text: one term per line or comma-separated, both become + * "a, b, c"; control characters dropped; whitespace collapsed. "" = no hint. + */ +export function normalizeHint(raw: string): string { + return raw + .replace(CONTROL_CHARS, "") + .split(/\r?\n|\r/) + .map((line) => line.replace(/\s+/g, " ").trim()) + .filter((line) => line.length > 0) + .join(", "); +} + +/** Saved hint (raw, as typed); "" when nothing is stored or storage is unavailable. */ +export function loadHint(storage: StorageLike | null = defaultStorage()): string { + try { + return storage?.getItem(HINT_STORAGE_KEY) ?? ""; + } catch { + return ""; + } +} + +export function saveHint(hint: string, storage: StorageLike | null = defaultStorage()): void { + try { + storage?.setItem(HINT_STORAGE_KEY, hint); + } catch { + // Not persistable (private mode / quota) — the hint just won't stick. + } +} + +function defaultStorage(): StorageLike | null { + try { + return (globalThis as { localStorage?: StorageLike }).localStorage ?? null; + } catch { + return null; + } +} + +/** + * Token ids for the hint as a Whisper prompt (without `<|startofprev|>`), + * truncated predictably to `budget` tokens (D-WS2a): + * + * 1. Encode `" " + normalizeHint(hint)` (Whisper prompts start with a space). + * 2. Over budget → keep the longest prefix of whole space-separated words that + * fits, with a dangling "," / ";" removed: the start of the list is kept, + * the end dropped, never mid-word. + * 3. Not even the first word fits → its first `budget` tokens. + * + * `encode` must add no special tokens. Mirrored by `fit_prompt_ids` in + * server-py/sidecar.py — change both together. + */ +export function fitHintToBudget( + hint: string, + encode: (text: string) => number[], + budget: number = PROMPT_TOKEN_BUDGET, +): number[] { + const text = normalizeHint(hint); + if (text.length === 0) return []; + const full = encode(` ${text}`); + if (full.length <= budget) return full; + + const words = text.split(" "); + const encodePrefix = (count: number) => + encode(` ${words.slice(0, count).join(" ").replace(/[,;]+$/, "")}`); + // Largest word count whose encoding fits; token count grows with the + // prefix, so binary search needs O(log n) encodes even for huge pastes. + let low = 0; + let high = words.length - 1; // the full text is already known not to fit + while (low < high) { + const mid = Math.ceil((low + high) / 2); + if (encodePrefix(mid).length <= budget) low = mid; + else high = mid - 1; + } + if (low === 0) return encode(` ${words[0]}`).slice(0, budget); + return encodePrefix(low); +} diff --git a/tests/hint.test.ts b/tests/hint.test.ts new file mode 100644 index 0000000..fdfb5ed --- /dev/null +++ b/tests/hint.test.ts @@ -0,0 +1,109 @@ +import { describe, expect, test } from "bun:test"; +import { + HINT_STORAGE_KEY, + PROMPT_TOKEN_BUDGET, + fitHintToBudget, + loadHint, + normalizeHint, + saveHint, +} from "../src/lib/hint"; + +/** One "token" per UTF-16 unit: deterministic, monotone, easy to reason about. */ +const charEncode = (text: string) => [...text].map((c) => c.charCodeAt(0)); +const charDecode = (ids: number[]) => String.fromCharCode(...ids); + +function mapStorage(initial: Record = {}) { + const map = new Map(Object.entries(initial)); + return { + getItem: (key: string) => map.get(key) ?? null, + setItem: (key: string, value: string) => void map.set(key, value), + map, + }; +} + +const throwingStorage = { + getItem(): string | null { + throw new Error("denied"); + }, + setItem(): void { + throw new Error("denied"); + }, +}; + +describe("normalizeHint", () => { + test("one term per line becomes a comma-separated list", () => { + expect(normalizeHint(" Maxwell\n\n Voices of the Void \n")).toBe( + "Maxwell, Voices of the Void", + ); + }); + + test("a comma-separated list is unchanged", () => { + expect(normalizeHint("Maxwell, Voices of the Void")).toBe("Maxwell, Voices of the Void"); + }); + + test("control characters are removed and whitespace-only means no hint", () => { + expect(normalizeHint("Max\u0000well\u0007")).toBe("Maxwell"); + expect(normalizeHint(" \n\t \r\n")).toBe(""); + }); + + test("Windows line endings and tabs are handled", () => { + expect(normalizeHint("Århus\r\nÆrø\tFestuge")).toBe("Århus, Ærø Festuge"); + }); +}); + +describe("hint storage", () => { + test("round-trips through storage", () => { + const storage = mapStorage(); + expect(loadHint(storage)).toBe(""); + saveHint("Maxwell\nVoid", storage); + expect(storage.map.get(HINT_STORAGE_KEY)).toBe("Maxwell\nVoid"); + expect(loadHint(storage)).toBe("Maxwell\nVoid"); + }); + + test("unavailable storage never throws", () => { + expect(loadHint(throwingStorage)).toBe(""); + expect(() => saveHint("x", throwingStorage)).not.toThrow(); + expect(loadHint(null)).toBe(""); + }); +}); + +describe("fitHintToBudget (D-WS2a truncation rule)", () => { + test("empty hint means no prompt", () => { + expect(fitHintToBudget(" \n", charEncode)).toEqual([]); + }); + + test("a hint within budget is encoded with Whisper's leading space", () => { + expect(fitHintToBudget("Maxwell\nVoices of the Void", charEncode)).toEqual( + charEncode(" Maxwell, Voices of the Void"), + ); + }); + + test("over budget: keeps whole leading words and drops a dangling comma", () => { + expect(charDecode(fitHintToBudget("alpha, beta gamma", charEncode, 10))).toBe(" alpha"); + }); + + test("a single word longer than the budget is cut at the budget", () => { + const ids = fitHintToBudget("x".repeat(50), charEncode, 10); + expect(ids).toEqual(charEncode(" xxxxxxxxx")); + expect(ids.length).toBe(10); + }); + + test("a very long hint fits the 223-token budget and is a word prefix of the hint", () => { + const words = Array.from({ length: 5000 }, (_, i) => `term${i}`); + const hint = words.join(", "); + const ids = fitHintToBudget(hint, charEncode); + expect(PROMPT_TOKEN_BUDGET).toBe(223); + expect(ids.length).toBeLessThanOrEqual(223); + expect(ids.length).toBeGreaterThan(200); + const kept = charDecode(ids); + expect(` ${hint}`.startsWith(kept)).toBe(true); + // Ends on a whole term: the next character of the hint is a separator. + expect(kept.endsWith(",")).toBe(false); + expect([",", " "]).toContain(` ${hint}`[kept.length]); + }); + + test("is deterministic", () => { + const hint = "Maxwell, ".repeat(200); + expect(fitHintToBudget(hint, charEncode)).toEqual(fitHintToBudget(hint, charEncode)); + }); +}); From e6de097cfd3f19afe37cba9528e72087c0644519 Mon Sep 17 00:00:00 2001 From: Lucas Winther Date: Thu, 29 Oct 2026 21:43:11 +0100 Subject: [PATCH 03/17] Feed Whisper a decoder prompt through the ASR pipeline and strip it from the output Co-Authored-By: Claude Opus 5.5 --- src/worker/prompt.ts | 101 +++++++++++++++++++++++++++++++++++ tests/prompt.test.ts | 124 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 225 insertions(+) create mode 100644 src/worker/prompt.ts create mode 100644 tests/prompt.test.ts diff --git a/src/worker/prompt.ts b/src/worker/prompt.ts new file mode 100644 index 0000000..c9514f4 --- /dev/null +++ b/src/worker/prompt.ts @@ -0,0 +1,101 @@ +/** + * Whisper decoder prompts (the "Names and terms" hint, D-WS2a) on top of the + * Transformers.js ASR pipeline. + * + * Transformers.js 4.2.0 (and 4.3.2) declare `prompt_ids` in the Whisper + * generation config but never use it — the parameter is commented out in + * `WhisperForConditionalGeneration.generate`. What generate() does honor is + * an explicit `decoder_input_ids`, so a prompt is expressed as + * `<|startofprev|> hint… <|startoftranscript|> <|lang|> <|task|>`. Two things + * then need fixing up around the call, which is what the wrapper does: + * + * - generate() returns the whole decoder sequence, and the pipeline's + * `_decode_asr` treats every non-special token as transcript text — the + * hint would be printed at the start of every window. The wrapper slices + * the prompt off the output again. + * - `max_new_tokens: 440` (D-021) assumes a 4-token prefix; with a prompt the + * total must still fit Whisper's 448 decoder positions. + * + * The prefix travels through the pipeline as a private kwarg: the ASR + * pipeline copies its call options into every per-window generate() call, so + * every 30 s window of a file and every live pass gets the same prompt, and + * concurrent passes cannot see each other's prompt (no shared mutable state). + * The `WhisperTextStreamer`'s `skip_prompt` drops the first `put()` — the + * whole decoder input, prompt included — so streamed partials stay clean. + */ + +import { fitHintToBudget } from "../lib/hint"; + +/** Private generate() kwarg carrying `[<|startofprev|>, ...hint ids]`. */ +export const PROMPT_PREFIX_KWARG = "whisper_web_prompt_prefix"; + +/** Whisper's decoder context (max_target_positions) for every catalog model. */ +export const WHISPER_DECODER_POSITIONS = 448; + +/** D-021's single-pass budget, used when a caller passes none. */ +const DEFAULT_MAX_NEW_TOKENS = 440; + +/** The slice of a Transformers.js Whisper model the wrapper relies on. */ +export interface PromptableModel { + generate(kwargs: Record): Promise; + _prepare_generation_config(config: null, kwargs: Record): unknown; + _retrieve_init_tokens(generationConfig: unknown): number[]; +} + +/** The slice of the Whisper tokenizer `buildPromptPrefix` needs. */ +export interface PromptTokenizer { + encode(text: string, options: { add_special_tokens: false }): number[]; + convert_tokens_to_ids(token: string): number; +} + +interface SliceableTensor { + slice(...slices: (number[] | null)[]): unknown; +} + +const WRAPPED = Symbol("whisper-web-prompt-support"); + +/** + * `[<|startofprev|>, ...ids]` for a raw user hint, truncated to Whisper's + * prompt budget (src/lib/hint.ts), or `[]` when the hint is empty. + */ +export function buildPromptPrefix(tokenizer: PromptTokenizer, hint: string): number[] { + const ids = fitHintToBudget(hint, (text) => + tokenizer.encode(text, { add_special_tokens: false }), + ); + if (ids.length === 0) return []; + return [tokenizer.convert_tokens_to_ids("<|startofprev|>"), ...ids]; +} + +/** + * Teach `model.generate` the `PROMPT_PREFIX_KWARG` option. Idempotent; calls + * without the kwarg (or with an empty prefix) are passed through untouched. + */ +export function installPromptSupport(model: PromptableModel): void { + const marked = model as PromptableModel & { [WRAPPED]?: true }; + if (marked[WRAPPED]) return; + marked[WRAPPED] = true; + + const original = model.generate.bind(model); + model.generate = async (kwargs) => { + const { [PROMPT_PREFIX_KWARG]: rawPrefix, ...rest } = kwargs; + const prefix = Array.isArray(rawPrefix) ? (rawPrefix as number[]) : []; + if (prefix.length === 0) return original(rest); + + // The same init tokens generate() would pick itself (language/task/ + // timestamps), with the prompt in front. + const initTokens = model._retrieve_init_tokens( + model._prepare_generation_config(null, rest), + ); + const decoderInputIds = [...prefix, ...initTokens]; + const requested = + typeof rest.max_new_tokens === "number" ? rest.max_new_tokens : DEFAULT_MAX_NEW_TOKENS; + const output = await original({ + ...rest, + decoder_input_ids: decoderInputIds, + max_new_tokens: Math.min(requested, WHISPER_DECODER_POSITIONS - decoderInputIds.length), + }); + // [batch=1, tokens] → drop the prompt columns; the sequence then starts + // at <|startoftranscript|> exactly as an unprompted pass would. + return (output as SliceableTensor).slice(null, [prefix.length, null]); + }; +} diff --git a/tests/prompt.test.ts b/tests/prompt.test.ts new file mode 100644 index 0000000..a91981e --- /dev/null +++ b/tests/prompt.test.ts @@ -0,0 +1,124 @@ +import { describe, expect, test } from "bun:test"; +import { Tensor } from "@huggingface/transformers"; +import { + PROMPT_PREFIX_KWARG, + WHISPER_DECODER_POSITIONS, + buildPromptPrefix, + installPromptSupport, + type PromptableModel, +} from "../src/worker/prompt"; + +const PREV = 50362; + +/** + * Stand-in for a Transformers.js Whisper model: records the kwargs each + * generate() call received and, like the real one, returns the decoder input + * followed by the "generated" tokens as an int64 [1, n] tensor. + */ +function fakeModel(initTokens: number[], generated: number[] = [50365, 400, 401, 50400]) { + const calls: Record[] = []; + const model: PromptableModel & { calls: typeof calls } = { + calls, + _prepare_generation_config: (_config, kwargs) => ({ fromKwargs: kwargs }), + _retrieve_init_tokens: () => [...initTokens], + async generate(kwargs) { + calls.push(kwargs); + const input = (kwargs.decoder_input_ids as number[] | undefined) ?? initTokens; + const sequence = [...input, ...generated]; + return new Tensor("int64", BigInt64Array.from(sequence.map(BigInt)), [1, sequence.length]); + }, + }; + return model; +} + +function tokens(output: unknown): number[] { + return (output as Tensor).tolist()[0]!.map(Number); +} + +describe("installPromptSupport (D-WS2a)", () => { + test("without a prompt kwarg, generate is called exactly as before", async () => { + const model = fakeModel([50258, 50259, 50360]); + installPromptSupport(model); + const kwargs = { max_new_tokens: 440, return_timestamps: true }; + const output = await model.generate({ ...kwargs }); + expect(model.calls).toEqual([kwargs]); + expect(tokens(output)).toEqual([50258, 50259, 50360, 50365, 400, 401, 50400]); + }); + + test("an empty prefix is the same as no prompt", async () => { + const model = fakeModel([50258]); + installPromptSupport(model); + await model.generate({ max_new_tokens: 440, [PROMPT_PREFIX_KWARG]: [] }); + expect(model.calls).toEqual([{ max_new_tokens: 440 }]); + }); + + test("a prefix is prepended to the init tokens and stripped from the output", async () => { + const model = fakeModel([50258, 50259, 50360]); + installPromptSupport(model); + const output = await model.generate({ + max_new_tokens: 440, + language: "en", + [PROMPT_PREFIX_KWARG]: [PREV, 1, 2], + }); + const call = model.calls[0]!; + expect(call.decoder_input_ids).toEqual([PREV, 1, 2, 50258, 50259, 50360]); + expect(call).not.toHaveProperty(PROMPT_PREFIX_KWARG); + expect(call.language).toBe("en"); + // Prompt tokens would otherwise reach _decode_asr as transcript text. + expect(tokens(output)).toEqual([50258, 50259, 50360, 50365, 400, 401, 50400]); + }); + + test("works for English-only models (init tokens are just <|startoftranscript|>)", async () => { + const model = fakeModel([50257]); + installPromptSupport(model); + const output = await model.generate({ max_new_tokens: 440, [PROMPT_PREFIX_KWARG]: [50361, 7] }); + expect(model.calls[0]!.decoder_input_ids).toEqual([50361, 7, 50257]); + expect(tokens(output)[0]).toBe(50257); + }); + + test("caps max_new_tokens so prompt + init + new tokens fit Whisper's 448 positions", async () => { + const model = fakeModel([50258, 50259, 50360]); + installPromptSupport(model); + const prefix = [PREV, ...Array.from({ length: 223 }, (_, i) => 1000 + i)]; + await model.generate({ max_new_tokens: 440, [PROMPT_PREFIX_KWARG]: prefix }); + expect(WHISPER_DECODER_POSITIONS).toBe(448); + expect(model.calls[0]!.max_new_tokens).toBe(448 - 227); + // A short prompt keeps the D-021 value. + await model.generate({ max_new_tokens: 440, [PROMPT_PREFIX_KWARG]: [PREV, 5] }); + expect(model.calls[1]!.max_new_tokens).toBe(440); + }); + + test("installing twice wraps once", async () => { + const model = fakeModel([50258]); + installPromptSupport(model); + installPromptSupport(model); + await model.generate({ max_new_tokens: 440, [PROMPT_PREFIX_KWARG]: [PREV, 5] }); + expect(model.calls).toHaveLength(1); + expect(model.calls[0]!.decoder_input_ids).toEqual([PREV, 5, 50258]); + }); +}); + +describe("buildPromptPrefix", () => { + const tokenizer = { + encode: (text: string, _options: { add_special_tokens: false }) => + [...text].map((c) => c.charCodeAt(0)), + convert_tokens_to_ids: (token: string) => (token === "<|startofprev|>" ? PREV : -1), + }; + + test("no hint → no prefix", () => { + expect(buildPromptPrefix(tokenizer, "")).toEqual([]); + expect(buildPromptPrefix(tokenizer, " \n ")).toEqual([]); + }); + + test("a hint → <|startofprev|> then the space-led hint tokens", () => { + expect(buildPromptPrefix(tokenizer, "Max\nVoid")).toEqual([ + PREV, + ...[..." Max, Void"].map((c) => c.charCodeAt(0)), + ]); + }); + + test("a huge hint stays within the 223-token budget", () => { + const prefix = buildPromptPrefix(tokenizer, "Maxwell, ".repeat(500)); + expect(prefix.length).toBeLessThanOrEqual(224); + }); +}); From 87617bef4c24429777db7d1c0cee7ba2f5d07568 Mon Sep 17 00:00:00 2001 From: Lucas Winther Date: Thu, 29 Oct 2026 21:43:19 +0100 Subject: [PATCH 04/17] Fix a type error in the hint truncation test Co-Authored-By: Claude Opus 5.5 --- tests/hint.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/hint.test.ts b/tests/hint.test.ts index fdfb5ed..f818adc 100644 --- a/tests/hint.test.ts +++ b/tests/hint.test.ts @@ -99,7 +99,7 @@ describe("fitHintToBudget (D-WS2a truncation rule)", () => { expect(` ${hint}`.startsWith(kept)).toBe(true); // Ends on a whole term: the next character of the hint is a separator. expect(kept.endsWith(",")).toBe(false); - expect([",", " "]).toContain(` ${hint}`[kept.length]); + expect([",", " "]).toContain(` ${hint}`.charAt(kept.length)); }); test("is deterministic", () => { From 99b9cd71cd1ea0858b4821b029538080889e4f55 Mon Sep 17 00:00:00 2001 From: Lucas Winther Date: Thu, 29 Oct 2026 21:43:29 +0100 Subject: [PATCH 05/17] Type the prompt wrapper's tensor slice for open-ended ranges Co-Authored-By: Claude Opus 5.5 --- src/worker/prompt.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/worker/prompt.ts b/src/worker/prompt.ts index c9514f4..a964f83 100644 --- a/src/worker/prompt.ts +++ b/src/worker/prompt.ts @@ -49,7 +49,7 @@ export interface PromptTokenizer { } interface SliceableTensor { - slice(...slices: (number[] | null)[]): unknown; + slice(...slices: ((number | null)[] | null)[]): unknown; } const WRAPPED = Symbol("whisper-web-prompt-support"); From 313892b6df9516a6743a24c599c4b8602db913ec Mon Sep 17 00:00:00 2001 From: Lucas Winther Date: Thu, 29 Oct 2026 21:44:06 +0100 Subject: [PATCH 06/17] Carry the vocabulary hint as `prompt` from the engines through the worker into every window Co-Authored-By: Claude Opus 5.5 --- src/lib/engine.ts | 8 ++++++++ src/worker/inference.worker.ts | 18 ++++++++++++++++++ src/worker/protocol.ts | 10 +++++++++- tests/engine.test.ts | 28 ++++++++++++++++++++++++++++ tests/protocol.test.ts | 4 ++++ 5 files changed, 67 insertions(+), 1 deletion(-) diff --git a/src/lib/engine.ts b/src/lib/engine.ts index 5c60e6b..5a692c8 100644 --- a/src/lib/engine.ts +++ b/src/lib/engine.ts @@ -37,6 +37,11 @@ export interface EngineTranscribeOptions { language?: string; /** transcribe | translate (multilingual models only). */ task?: WhisperTask; + /** + * "Names and terms" vocabulary hint, raw user text (D-WS2a). Applies to + * every Whisper model, `.en` ones included; empty/undefined = no prompt. + */ + prompt?: string; /** * Streaming transcript fragments — append each to the text shown so far. * Browser engine: token fragments; server engine: whole segments. @@ -140,6 +145,7 @@ export class BrowserEngine implements Engine { ...(options.language !== undefined && options.language !== "auto" && { language: options.language }), ...(options.task !== undefined && { task: options.task }), + ...(options.prompt && { prompt: options.prompt }), }; // Ownership of the audio moves to the worker — zero-copy transfer. this.worker.postMessage(message, [audio.buffer as ArrayBuffer]); @@ -282,6 +288,8 @@ export class ServerEngine implements Engine { params.set("language", options.language); } if (options.task !== undefined) params.set("task", options.task); + // Validated + normalized server-side (parseLanguageAndTask, D-WS2a). + if (options.prompt) params.set("prompt", options.prompt); let response: Response; try { diff --git a/src/worker/inference.worker.ts b/src/worker/inference.worker.ts index 2d5b4b2..8591f38 100644 --- a/src/worker/inference.worker.ts +++ b/src/worker/inference.worker.ts @@ -17,6 +17,13 @@ import { type ProgressInfo, } from "@huggingface/transformers"; import { resolveDefaultDtype } from "../lib/models"; +import { + buildPromptPrefix, + installPromptSupport, + PROMPT_PREFIX_KWARG, + type PromptableModel, + type PromptTokenizer, +} from "./prompt"; import { deviceAttempts, dtypeRequiresF16, @@ -410,6 +417,17 @@ async function transcribe(message: TranscribeMessage): Promise { if (message.task !== undefined) { options.task = message.task; } + // "Names and terms" hint (D-WS2a): one prefix per pass, applied by the + // generate() wrapper to every 30 s window (Transformers.js ignores + // `prompt_ids`, see src/worker/prompt.ts). The wrapper is idempotent. + const promptPrefix = buildPromptPrefix( + asr.tokenizer as unknown as PromptTokenizer, + message.prompt ?? "", + ); + if (promptPrefix.length > 0) { + installPromptSupport(asr.model as unknown as PromptableModel); + options[PROMPT_PREFIX_KWARG] = promptPrefix; + } const output = await asr(message.audio, options); diff --git a/src/worker/protocol.ts b/src/worker/protocol.ts index 688333a..b8f5bc2 100644 --- a/src/worker/protocol.ts +++ b/src/worker/protocol.ts @@ -87,6 +87,13 @@ export interface TranscribeMessage { * engine (which correlates via requestId, D-010) still streams. */ stream?: boolean; + /** + * Optional "Names and terms" vocabulary hint, raw as the user typed it + * (D-WS2a). The worker normalizes it, truncates it to Whisper's prompt + * budget and feeds it as a decoder prompt to every window of this pass + * (src/worker/prompt.ts). Empty/omitted = no prompt. + */ + prompt?: string; } export type UiToWorkerMessage = LoadMessage | TranscribeMessage; @@ -318,7 +325,8 @@ export function isUiToWorkerMessage(value: unknown): value is UiToWorkerMessage value.task === "transcribe" || value.task === "translate") && isOptionalString(value.requestId) && - (value.stream === undefined || typeof value.stream === "boolean") + (value.stream === undefined || typeof value.stream === "boolean") && + isOptionalString(value.prompt) ); default: return false; diff --git a/tests/engine.test.ts b/tests/engine.test.ts index d174aa7..23d8de2 100644 --- a/tests/engine.test.ts +++ b/tests/engine.test.ts @@ -77,6 +77,17 @@ describe("BrowserEngine", () => { expect("modelId" in message).toBe(false); }); + test("forwards the vocabulary hint as `prompt`, and omits an empty one (D-WS2a)", () => { + const worker = new FakeWorker(); + const engine = new BrowserEngine(worker); + void engine.transcribe(new Float32Array(4), { prompt: "Maxwell\nVoid" }); + void engine.transcribe(new Float32Array(4), { prompt: "" }); + void engine.transcribe(new Float32Array(4)); + expect(worker.posted[0]!.message.prompt).toBe("Maxwell\nVoid"); + expect("prompt" in worker.posted[1]!.message).toBe(false); + expect("prompt" in worker.posted[2]!.message).toBe(false); + }); + test("asks the worker to stream only when a consumer listens", () => { const worker = new FakeWorker(); const engine = new BrowserEngine(worker); @@ -241,6 +252,23 @@ describe("ServerEngine", () => { expect(url).not.toContain("language"); }); + test("sends the vocabulary hint as the `prompt` query param, only when non-empty", async () => { + const urls: string[] = []; + const engine = new ServerEngine(async (input) => { + urls.push(input); + return ndjsonResponse([{ type: "result", text: "", chunks: [] }]); + }); + await engine.transcribe(new Float32Array(4), { + modelId: "onnx-community/whisper-tiny", + prompt: "Århus, Voices of the Void", + }); + await engine.transcribe(new Float32Array(4), { modelId: "onnx-community/whisper-tiny", prompt: "" }); + expect(new URL(urls[0]!, "http://localhost").searchParams.get("prompt")).toBe( + "Århus, Voices of the Void", + ); + expect(urls[1]).not.toContain("prompt"); + }); + test("requires a modelId", () => { const engine = new ServerEngine(async () => ndjsonResponse([])); expect(engine.transcribe(new Float32Array(4))).rejects.toThrow(/model id/); diff --git a/tests/protocol.test.ts b/tests/protocol.test.ts index 43bf74c..be42ff2 100644 --- a/tests/protocol.test.ts +++ b/tests/protocol.test.ts @@ -35,6 +35,7 @@ describe("isUiToWorkerMessage", () => { { type: "transcribe", audio: new Float32Array(8), requestId: "live-3" }, { type: "transcribe", audio: new Float32Array(8), requestId: "engine-1-1", stream: true }, { type: "transcribe", audio: new Float32Array(8), stream: false }, + { type: "transcribe", audio: new Float32Array(8), prompt: "Maxwell, Voices of the Void" }, ]; for (const message of [...loads, ...transcribes]) { expect(isUiToWorkerMessage(message)).toBe(true); @@ -65,6 +66,9 @@ describe("isUiToWorkerMessage", () => { expect( isUiToWorkerMessage({ type: "transcribe", audio: new Float32Array(4), stream: "yes" }), ).toBe(false); + expect( + isUiToWorkerMessage({ type: "transcribe", audio: new Float32Array(4), prompt: 5 }), + ).toBe(false); expect(isUiToWorkerMessage({ type: "result", text: "", chunks: [] })).toBe(false); }); }); From 1c5952f21207e33f1d070d3f513c41dabf7f1090 Mon Sep 17 00:00:00 2001 From: Lucas Winther Date: Thu, 29 Oct 2026 21:45:39 +0100 Subject: [PATCH 07/17] Validate the vocabulary hint server-side and pass it to faster-whisper as initial_prompt Co-Authored-By: Claude Opus 5.5 --- server-py/sidecar.py | 62 ++++++++++++++++++++++++++++++- src/index.ts | 1 + src/server/live.ts | 3 ++ src/server/transcribe.ts | 29 ++++++++++++--- tests/fixtures/fake_sidecar.py | 11 ++++-- tests/hint-parity.test.ts | 60 ++++++++++++++++++++++++++++++ tests/server-live.test.ts | 15 ++++++++ tests/transcribe.test.ts | 68 ++++++++++++++++++++++++++++++++++ 8 files changed, 239 insertions(+), 10 deletions(-) create mode 100644 tests/hint-parity.test.ts diff --git a/server-py/sidecar.py b/server-py/sidecar.py index 95f8d3b..349c135 100644 --- a/server-py/sidecar.py +++ b/server-py/sidecar.py @@ -9,7 +9,8 @@ stdio — one JSON object per line, no framing beyond that: stdin -> {"type": "transcribe", "id": "1", "model": "tiny.en", "audio_path": "/tmp/...", "language": "en" | null, - "task": "transcribe" | "translate"} + "task": "transcribe" | "translate", + "prompt": "Maxwell, Voices of the Void" | null} stdout <- {"type": "segment", "id": "1", "start": 0.0, "end": 4.2, "text": " Hello world."} # 0..n of these stdout <- {"type": "done", "id": "1", "cancelled": false} @@ -43,11 +44,56 @@ as a "fatal" line, which the Bun side surfaces as "server engine unavailable" import json import os import queue +import re import sys import threading SAMPLE_RATE = 16000 +# "Names and terms" vocabulary hint (D-WS2a). Twins of normalizeHint / +# fitHintToBudget in src/lib/hint.ts — change both together; parity is tested +# in tests/hint-parity.test.ts. Whisper's prompt budget is half of its 448 +# decoder positions minus one (<|startofprev|>). +PROMPT_TOKEN_BUDGET = 223 +_CONTROL_CHARS = re.compile("[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]") + + +def normalize_hint(raw): + """One term per line or comma-separated -> "a, b, c"; "" means no hint.""" + lines = re.split(r"\r?\n|\r", _CONTROL_CHARS.sub("", raw)) + parts = [re.sub(r"\s+", " ", line).strip() for line in lines] + return ", ".join(part for part in parts if part) + + +def fit_prompt_ids(hint, encode, budget=PROMPT_TOKEN_BUDGET): + """Prompt token ids for `hint`, cut to `budget` tokens at a whole word. + + The start of the list is kept and the end dropped; a dangling ","/";" is + removed; a first word longer than the budget is cut at the budget. + `encode` must add no special tokens. + """ + text = normalize_hint(hint) + if not text: + return [] + full = encode(" " + text) + if len(full) <= budget: + return full + words = text.split(" ") + + def encode_prefix(count): + return encode(" " + re.sub(r"[,;]+$", "", " ".join(words[:count]))) + + low, high = 0, len(words) - 1 + while low < high: + mid = (low + high + 1) // 2 + if len(encode_prefix(mid)) <= budget: + low = mid + else: + high = mid - 1 + if low == 0: + return encode(" " + words[0])[:budget] + return encode_prefix(low) + def emit(message): """Write one JSON line to stdout. Only the main thread emits.""" @@ -115,6 +161,7 @@ def main(): audio_path = request["audio_path"] language = request.get("language") or None task = request.get("task") or "transcribe" + prompt = request.get("prompt") or "" if model_name != cached_name: cached_model = None # release before loading the replacement @@ -127,8 +174,19 @@ def main(): cached_name = model_name audio = np.fromfile(audio_path, dtype=np.float32) + options = {} + prompt_ids = fit_prompt_ids( + prompt, + lambda text: cached_model.hf_tokenizer.encode( + text, add_special_tokens=False + ).ids, + ) + if prompt_ids: + # D-WS2a: token ids, already cut to the budget by the same + # rule as the browser engine. + options["initial_prompt"] = prompt_ids segments, _info = cached_model.transcribe( - audio, language=language, task=task + audio, language=language, task=task, **options ) was_cancelled = False diff --git a/src/index.ts b/src/index.ts index 328d7f3..12fade9 100644 --- a/src/index.ts +++ b/src/index.ts @@ -153,6 +153,7 @@ async function handleApiRequest(request: Request, server: any): Promise { text.push(segment.text); diff --git a/src/server/transcribe.ts b/src/server/transcribe.ts index eb95835..d625a2a 100644 --- a/src/server/transcribe.ts +++ b/src/server/transcribe.ts @@ -26,6 +26,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import type { TranscribeStreamLine } from "../lib/engine"; +import { HINT_MAX_CHARS, normalizeHint } from "../lib/hint"; import { getFasterWhisperModel } from "../lib/models"; import type { TranscriptChunk, WhisperTask } from "../worker/protocol"; import { getSessionUser as appGetSessionUser, type SessionUser } from "./auth.ts"; @@ -47,6 +48,11 @@ export interface SidecarTranscribeRequest { audioPath: string; language?: string; task?: WhisperTask; + /** + * Normalized "Names and terms" hint (D-WS2a); the sidecar truncates it to + * Whisper's prompt budget with faster-whisper's own tokenizer. + */ + prompt?: string; } export interface SidecarStatus { @@ -265,6 +271,7 @@ export function createSidecar(command?: readonly string[]): Sidecar { audio_path: request.audioPath, language: request.language ?? null, task: request.task ?? "transcribe", + prompt: request.prompt ?? null, }); } catch (error) { pending.delete(id); @@ -322,18 +329,23 @@ interface TranscribeParams { fasterWhisperModel: string; language?: string; task: WhisperTask; + prompt?: string; } /** - * Parse + validate the `language`/`task` query params shared by both the - * HTTP (`POST /api/transcribe`) and WebSocket (`GET /api/transcribe/live`, + * Parse + validate the `language`/`task`/`prompt` query params shared by both + * the HTTP (`POST /api/transcribe`) and WebSocket (`GET /api/transcribe/live`, * src/server/live.ts) entry points, so a live session can't forward an * unvalidated string into the sidecar protocol just because it skips - * `parseParams` below. A string return is the 400 error text. + * `parseParams` below (D-032). A string return is the 400 error text. + * + * `prompt` is the raw "Names and terms" hint (D-WS2a): at most + * HINT_MAX_CHARS (the UI's textarea limit), normalized exactly as the browser + * worker does; empty after normalization = no prompt. */ export function parseLanguageAndTask( url: URL, -): { language?: string; task: WhisperTask } | string { +): { language?: string; task: WhisperTask; prompt?: string } | string { const rawLanguage = url.searchParams.get("language"); let language: string | undefined; if (rawLanguage !== null && rawLanguage !== "" && rawLanguage !== "auto") { @@ -346,7 +358,13 @@ export function parseLanguageAndTask( return `invalid task: ${rawTask}`; } - return { language, task: rawTask }; + const rawPrompt = url.searchParams.get("prompt") ?? ""; + if (rawPrompt.length > HINT_MAX_CHARS) { + return `prompt too long (max ${HINT_MAX_CHARS} characters)`; + } + const prompt = normalizeHint(rawPrompt); + + return { language, task: rawTask, ...(prompt && { prompt }) }; } /** Parse + validate query params; a string is the 400 error text. */ @@ -461,6 +479,7 @@ export function createTranscribeHandlers(deps: TranscribeDeps): TranscribeHandle audioPath, language: params.language, task: params.task, + prompt: params.prompt, }, (segment) => { chunks.push({ diff --git a/tests/fixtures/fake_sidecar.py b/tests/fixtures/fake_sidecar.py index ac4f3e6..66feba4 100644 --- a/tests/fixtures/fake_sidecar.py +++ b/tests/fixtures/fake_sidecar.py @@ -5,7 +5,8 @@ Used by tests/transcribe.test.ts to exercise the real Bun-side supervisor requested model name: "ok" two segments (the second reports the audio byte count, - language and task for pass-through assertions), then done + language, task and - when sent - prompt for pass-through + assertions), then done "boom" an error line "die" exit(1) mid-request (crash simulation) "wait-for-cancel" one segment, then block until the next stdin line @@ -52,7 +53,11 @@ for line in sys.stdin: size = os.path.getsize(message["audio_path"]) emit({"type": "segment", "id": request_id, "start": 0.0, "end": 1.5, "text": " Hello"}) + prompt = message.get("prompt") + echo = "model=%s bytes=%d lang=%s task=%s" % ( + model, size, message.get("language"), message.get("task")) + if prompt is not None: + echo += " prompt=%s" % prompt emit({"type": "segment", "id": request_id, "start": 1.5, "end": 3.0, - "text": " world. [model=%s bytes=%d lang=%s task=%s]" - % (model, size, message.get("language"), message.get("task"))}) + "text": " world. [%s]" % echo}) emit({"type": "done", "id": request_id, "cancelled": False}) diff --git a/tests/hint-parity.test.ts b/tests/hint-parity.test.ts new file mode 100644 index 0000000..b540805 --- /dev/null +++ b/tests/hint-parity.test.ts @@ -0,0 +1,60 @@ +import { describe, expect, test } from "bun:test"; +import { join } from "node:path"; +import { fitHintToBudget, normalizeHint } from "../src/lib/hint"; + +// D-WS2a: the server engine truncates the hint in Python (server-py/sidecar.py, +// with faster-whisper's tokenizer) and the browser engine in TypeScript +// (src/lib/hint.ts, with the Transformers.js tokenizer). Both must apply the +// same rule; with the same deterministic fake tokenizer they must agree +// exactly. sidecar.py imports faster-whisper only inside main(), so the +// helpers import without it. + +const SIDECAR = join(import.meta.dir, "../server-py/sidecar.py"); + +const CASES: { hint: string; budget: number }[] = [ + { hint: "", budget: 223 }, + { hint: " \n\t ", budget: 223 }, + { hint: "Maxwell\nVoices of the Void", budget: 223 }, + { hint: " Århus\r\n\r\nÆrø, Festuge \n", budget: 223 }, + { hint: "Max\u0000well\u0007", budget: 223 }, + { hint: "alpha, beta gamma", budget: 10 }, + { hint: "alpha; beta; gamma; delta", budget: 14 }, + { hint: "x".repeat(50), budget: 10 }, + { hint: "Maxwell, ".repeat(300), budget: 223 }, + { hint: Array.from({ length: 400 }, (_, i) => `term${i}`).join("\n"), budget: 223 }, +]; + +/** One token per UTF-16 code unit — the same function on both sides. */ +const charEncode = (text: string) => [...text].map((c) => c.charCodeAt(0)); + +const PYTHON = ` +import importlib.util, json, sys +spec = importlib.util.spec_from_file_location("sidecar", sys.argv[1]) +sidecar = importlib.util.module_from_spec(spec) +spec.loader.exec_module(sidecar) +encode = lambda text: [ord(c) for c in text] +cases = json.load(sys.stdin) +print(json.dumps([ + {"normalized": sidecar.normalize_hint(c["hint"]), + "ids": sidecar.fit_prompt_ids(c["hint"], encode, c["budget"])} + for c in cases +])) +`; + +describe("hint truncation parity: TypeScript vs sidecar.py", () => { + test("normalize_hint / fit_prompt_ids match normalizeHint / fitHintToBudget", () => { + const proc = Bun.spawnSync(["python3", "-I", "-c", PYTHON, SIDECAR], { + stdin: new TextEncoder().encode(JSON.stringify(CASES)), + stdout: "pipe", + stderr: "pipe", + }); + expect(proc.stderr.toString()).toBe(""); + const python = JSON.parse(proc.stdout.toString()) as { normalized: string; ids: number[] }[]; + expect(python).toEqual( + CASES.map(({ hint, budget }) => ({ + normalized: normalizeHint(hint), + ids: fitHintToBudget(hint, charEncode, budget), + })), + ); + }); +}); diff --git a/tests/server-live.test.ts b/tests/server-live.test.ts index 3c41acc..49ef20b 100644 --- a/tests/server-live.test.ts +++ b/tests/server-live.test.ts @@ -139,6 +139,21 @@ describe("createLiveWebSocket", () => { await waitFor(async () => (await readdir(tempDir)).length === 0); }); + test("every pass of a session carries the session's vocabulary hint (D-WS2a)", async () => { + const tempDir = await makeTempDir(); + const sidecar = fakeSidecar(); + const handler = createLiveWebSocket({ sidecar }); + const ws = fakeSocket(sessionData({ prompt: "Maxwell, Voices of the Void", tempDir })); + handler.open(ws); + handler.message(ws, toBuffer(speech(1))); + handler.message(ws, toBuffer(silence(1.5))); + await waitFor(() => sidecar.requests.length > 0); + for (const request of sidecar.requests) { + expect(request.prompt).toBe("Maxwell, Voices of the Void"); + } + await waitFor(async () => (await readdir(tempDir)).length === 0); + }); + test("a sidecar error surfaces on the update stream instead of crashing the session", async () => { const sidecar = fakeSidecar({ error: "model load failed" }); const handler = createLiveWebSocket({ sidecar }); diff --git a/tests/transcribe.test.ts b/tests/transcribe.test.ts index 2b54b66..a375c9f 100644 --- a/tests/transcribe.test.ts +++ b/tests/transcribe.test.ts @@ -165,6 +165,36 @@ describe("parseLanguageAndTask", () => { }); }); +describe("parseLanguageAndTask: vocabulary hint (D-WS2a)", () => { + const parse = (query: string) => parseLanguageAndTask(new URL(`http://localhost/x${query}`)); + + test("normalizes the hint like the worker does", () => { + expect(parse("?prompt=Maxwell%0A%0AVoices%20of%20the%20Void%0A")).toEqual({ + language: undefined, + task: "transcribe", + prompt: "Maxwell, Voices of the Void", + }); + }); + + test("absent, empty, or whitespace-only means no prompt", () => { + for (const query of ["", "?prompt=", "?prompt=%20%0A%09"]) { + const parsed = parse(query); + expect(typeof parsed).toBe("object"); + expect("prompt" in (parsed as object)).toBe(false); + } + }); + + test("keeps non-ASCII names intact", () => { + const params = new URLSearchParams({ prompt: "Århus, Ærø, Festuge" }); + expect(parse(`?${params}`)).toMatchObject({ prompt: "Århus, Ærø, Festuge" }); + }); + + test("rejects a hint longer than the UI allows", () => { + expect(parse(`?prompt=${"a".repeat(1000)}`)).toMatchObject({ prompt: "a".repeat(1000) }); + expect(parse(`?prompt=${"a".repeat(1001)}`)).toBe("prompt too long (max 1000 characters)"); + }); +}); + describe("GET /api/transcribe/status", () => { test("401 without a session", async () => { const handlers = createTranscribeHandlers({ @@ -323,6 +353,35 @@ describe("POST /api/transcribe", () => { expect(await readdir(tempDir)).toEqual([]); }); + test("passes the vocabulary hint to the sidecar; 400 for an over-long one (D-WS2a)", async () => { + const sidecar = fakeSidecar(); + const handlers = createTranscribeHandlers({ + sidecar, + getSessionUser: sessionFromCookie, + tempDir: await makeTempDir(), + }); + const ok = new URLSearchParams({ + model: "onnx-community/whisper-base", + prompt: "Århus\nÆrø", + }); + const response = await handlers.post( + postRequest(`?${ok}`, pcmBody([0.1, 0.2]), { cookie: "sid=ok" }), + ); + expect(response.status).toBe(200); + await readStreamLines(response); + expect(sidecar.requests[0]!.prompt).toBe("Århus, Ærø"); + + const tooLong = await handlers.post( + postRequest( + `?model=onnx-community/whisper-base&prompt=${"x".repeat(1001)}`, + pcmBody([0.1]), + { cookie: "sid=ok" }, + ), + ); + expect(tooLong.status).toBe(400); + expect(sidecar.requests).toHaveLength(1); + }); + test("sidecar errors travel in-stream after the 200 header", async () => { const tempDir = await makeTempDir(); const handlers = createTranscribeHandlers({ @@ -387,6 +446,15 @@ describe("createSidecar (fake python sidecar)", () => { expect(segments[0]).toEqual({ start: 0, end: 1.5, text: " Hello" }); // The fixture echoes what it received (audio bytes, language, task). expect(segments[1]!.text).toContain("model=ok bytes=12 lang=da task=translate"); + expect(segments[1]!.text).not.toContain("prompt="); + + // The vocabulary hint crosses the JSON-lines protocol intact (D-WS2a). + const prompted: SidecarSegment[] = []; + await sidecar.transcribe( + { model: "ok", audioPath, prompt: "Århus, Voices of the Void" }, + (segment) => prompted.push(segment), + ); + expect(prompted[1]!.text).toContain("prompt=Århus, Voices of the Void]"); }); test("sidecar 'error' lines reject that request only", async () => { From 8236ea3504be913a07aa376cf1c8d36cd6e4832f Mon Sep 17 00:00:00 2001 From: Lucas Winther Date: Thu, 29 Oct 2026 21:55:48 +0100 Subject: [PATCH 08/17] End the hint prompt in a full stop: an unterminated list made Whisper drop punctuation and loop Co-Authored-By: Claude Opus 5.5 --- server-py/sidecar.py | 10 ++++++++-- src/lib/hint.ts | 17 ++++++++++++----- tests/hint.test.ts | 25 +++++++++++++++++-------- tests/prompt.test.ts | 4 ++-- 4 files changed, 39 insertions(+), 17 deletions(-) diff --git a/server-py/sidecar.py b/server-py/sidecar.py index 349c135..65810f2 100644 --- a/server-py/sidecar.py +++ b/server-py/sidecar.py @@ -65,9 +65,15 @@ def normalize_hint(raw): return ", ".join(part for part in parts if part) +def _prompt_sentence(text): + return " " + text if re.search(r"[.?!]$", text) else " " + text + "." + + def fit_prompt_ids(hint, encode, budget=PROMPT_TOKEN_BUDGET): """Prompt token ids for `hint`, cut to `budget` tokens at a whole word. + The prompt is a space-led sentence ending in a full stop (unless it ends + in . ? ! already) - an unterminated list drops punctuation and loops. The start of the list is kept and the end dropped; a dangling ","/";" is removed; a first word longer than the budget is cut at the budget. `encode` must add no special tokens. @@ -75,13 +81,13 @@ def fit_prompt_ids(hint, encode, budget=PROMPT_TOKEN_BUDGET): text = normalize_hint(hint) if not text: return [] - full = encode(" " + text) + full = encode(_prompt_sentence(text)) if len(full) <= budget: return full words = text.split(" ") def encode_prefix(count): - return encode(" " + re.sub(r"[,;]+$", "", " ".join(words[:count]))) + return encode(_prompt_sentence(re.sub(r"[,;]+$", "", " ".join(words[:count])))) low, high = 0, len(words) - 1 while low < high: diff --git a/src/lib/hint.ts b/src/lib/hint.ts index f66a6fe..ec41312 100644 --- a/src/lib/hint.ts +++ b/src/lib/hint.ts @@ -72,10 +72,13 @@ function defaultStorage(): StorageLike | null { * Token ids for the hint as a Whisper prompt (without `<|startofprev|>`), * truncated predictably to `budget` tokens (D-WS2a): * - * 1. Encode `" " + normalizeHint(hint)` (Whisper prompts start with a space). + * 1. Prompt text = `" " + normalizeHint(hint)` (Whisper prompts start with a + * space), ending in a full stop unless it already ends in . ? or ! — + * measured: an unterminated list makes Whisper drop punctuation and, on + * smaller models, fall into repetition loops (D-WS2a). * 2. Over budget → keep the longest prefix of whole space-separated words that - * fits, with a dangling "," / ";" removed: the start of the list is kept, - * the end dropped, never mid-word. + * fits (full stop included), a dangling "," / ";" removed: the start of the + * list is kept, the end dropped, never mid-word. * 3. Not even the first word fits → its first `budget` tokens. * * `encode` must add no special tokens. Mirrored by `fit_prompt_ids` in @@ -88,12 +91,12 @@ export function fitHintToBudget( ): number[] { const text = normalizeHint(hint); if (text.length === 0) return []; - const full = encode(` ${text}`); + const full = encode(promptSentence(text)); if (full.length <= budget) return full; const words = text.split(" "); const encodePrefix = (count: number) => - encode(` ${words.slice(0, count).join(" ").replace(/[,;]+$/, "")}`); + encode(promptSentence(words.slice(0, count).join(" ").replace(/[,;]+$/, ""))); // Largest word count whose encoding fits; token count grows with the // prefix, so binary search needs O(log n) encodes even for huge pastes. let low = 0; @@ -106,3 +109,7 @@ export function fitHintToBudget( if (low === 0) return encode(` ${words[0]}`).slice(0, budget); return encodePrefix(low); } + +function promptSentence(text: string): string { + return /[.?!]$/.test(text) ? ` ${text}` : ` ${text}.`; +} diff --git a/tests/hint.test.ts b/tests/hint.test.ts index f818adc..6c5726a 100644 --- a/tests/hint.test.ts +++ b/tests/hint.test.ts @@ -72,14 +72,22 @@ describe("fitHintToBudget (D-WS2a truncation rule)", () => { expect(fitHintToBudget(" \n", charEncode)).toEqual([]); }); - test("a hint within budget is encoded with Whisper's leading space", () => { + test("a hint within budget is encoded as a space-led sentence ending in a full stop", () => { + // Measured (D-WS2a): an unterminated list prompt suppresses punctuation + // and triggers repetition loops on smaller models; a full stop does not. expect(fitHintToBudget("Maxwell\nVoices of the Void", charEncode)).toEqual( - charEncode(" Maxwell, Voices of the Void"), + charEncode(" Maxwell, Voices of the Void."), ); }); - test("over budget: keeps whole leading words and drops a dangling comma", () => { - expect(charDecode(fitHintToBudget("alpha, beta gamma", charEncode, 10))).toBe(" alpha"); + test("a hint already ending in . ? or ! gets no extra full stop", () => { + expect(charDecode(fitHintToBudget("Is it Maxwell?", charEncode))).toBe(" Is it Maxwell?"); + expect(charDecode(fitHintToBudget("Dr. Bao.", charEncode))).toBe(" Dr. Bao."); + }); + + test("over budget: keeps whole leading words, drops a dangling comma, ends in a full stop", () => { + expect(charDecode(fitHintToBudget("alpha, beta gamma", charEncode, 10))).toBe(" alpha."); + expect(charDecode(fitHintToBudget("alpha, beta gamma", charEncode, 13))).toBe(" alpha, beta."); }); test("a single word longer than the budget is cut at the budget", () => { @@ -96,10 +104,11 @@ describe("fitHintToBudget (D-WS2a truncation rule)", () => { expect(ids.length).toBeLessThanOrEqual(223); expect(ids.length).toBeGreaterThan(200); const kept = charDecode(ids); - expect(` ${hint}`.startsWith(kept)).toBe(true); - // Ends on a whole term: the next character of the hint is a separator. - expect(kept.endsWith(",")).toBe(false); - expect([",", " "]).toContain(` ${hint}`.charAt(kept.length)); + // Ends on a whole term plus the full stop: the hint continues with a separator there. + expect(kept.endsWith(".")).toBe(true); + const body = kept.slice(0, -1); + expect(` ${hint}`.startsWith(body)).toBe(true); + expect([",", " "]).toContain(` ${hint}`.charAt(body.length)); }); test("is deterministic", () => { diff --git a/tests/prompt.test.ts b/tests/prompt.test.ts index a91981e..34d5d44 100644 --- a/tests/prompt.test.ts +++ b/tests/prompt.test.ts @@ -110,10 +110,10 @@ describe("buildPromptPrefix", () => { expect(buildPromptPrefix(tokenizer, " \n ")).toEqual([]); }); - test("a hint → <|startofprev|> then the space-led hint tokens", () => { + test("a hint → <|startofprev|> then the space-led, full-stopped hint tokens", () => { expect(buildPromptPrefix(tokenizer, "Max\nVoid")).toEqual([ PREV, - ...[..." Max, Void"].map((c) => c.charCodeAt(0)), + ...[..." Max, Void."].map((c) => c.charCodeAt(0)), ]); }); From d6c3d1ff70e321ad9b0334b216e5eba57643ea64 Mon Sep 17 00:00:00 2001 From: Lucas Winther Date: Thu, 29 Oct 2026 21:57:30 +0100 Subject: [PATCH 09/17] Add the optional "Names and terms" field and send it with every file and live pass Co-Authored-By: Claude Opus 5.5 --- DESIGN.md | 2 +- src/App.tsx | 24 ++++++++++++-- src/components/VocabularyHint.tsx | 49 ++++++++++++++++++++++++++++ src/index.css | 54 +++++++++++++++++++++++++++++++ 4 files changed, 126 insertions(+), 3 deletions(-) create mode 100644 src/components/VocabularyHint.tsx diff --git a/DESIGN.md b/DESIGN.md index e6dd7c9..ad62958 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -28,6 +28,6 @@ exists, not an aspiration. Update in the same commit as visual changes. ## Components One BEM-ish class block per React component in `src/components/` (`.model-picker`, `.dropzone`, -`.live-controls`, `.engine-toggle`, `.auth-panel`, `.update-toast`, …), with modifier classes like +`.live-controls`, `.engine-toggle`, `.auth-panel`, `.update-toast`, `.vocabulary-hint`, …), with modifier classes like `-selected`, `-active`, `-disabled`. Interactive states use `:hover:enabled` and `:focus-visible`. `.visually-hidden` exists for screen-reader-only text. diff --git a/src/App.tsx b/src/App.tsx index e282021..2962df0 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -10,6 +10,8 @@ import { ModelPicker } from "./components/ModelPicker"; import { StorageManager } from "./components/StorageManager"; import { TranscriptView } from "./components/TranscriptView"; import { TranscriptionOptions } from "./components/TranscriptionOptions"; +import { VocabularyHint } from "./components/VocabularyHint"; +import { loadHint, saveHint } from "./lib/hint"; import { decodeAudioFile } from "./lib/audio"; import { classifyError, type ClassifiedError } from "./lib/errors"; import { @@ -164,6 +166,9 @@ export function App() { const [activeThreads, setActiveThreads] = useState(null); const [language, setLanguage] = useState(AUTO_LANGUAGE); const [task, setTask] = useState(DEFAULT_TASK); + // "Names and terms" vocabulary hint (D-WS2a): raw text, remembered in this + // browser, sent with every pass for every model (`.en` included). + const [hint, setHint] = useState(() => loadHint()); const [downloaded, setDownloaded] = useState>(new Set()); // PWA state (M3, D-007): is a service worker controlling this page (app @@ -676,6 +681,7 @@ export function App() { // take neither. The browser engine ignores modelId (worker holds it). modelId, ...(multilingual && { language, task }), + prompt: hint, onPartial: (fragment) => { if (fileRun.current === run) setPartialText((text) => text + fragment); }, @@ -713,7 +719,7 @@ export function App() { /** One inference pass over a live segment, correlated via requestId. */ async function transcribeLiveSegment( audio: Float32Array, - options: { language?: string; task?: WhisperTask }, + options: { language?: string; task?: WhisperTask; prompt?: string }, ): Promise { // Wait for the pipeline before posting: a load-restart (D-025) may swap // the worker while the model loads, and a pass queued in the old worker @@ -752,12 +758,17 @@ export function App() { setLivePending(0); setLiveStatus("starting"); - const options = (getModel(modelId)?.multilingual ?? false) ? { language, task } : {}; + const options = { + ...((getModel(modelId)?.multilingual ?? false) && { language, task }), + // Same hint for every live pass of the session (D-WS2a). + ...(hint && { prompt: hint }), + }; if (useServer) { const wsUrl = new URL(`/api/transcribe/live?model=${modelId}`, window.location.href); wsUrl.protocol = wsUrl.protocol === "https:" ? "wss:" : "ws:"; if (options.language) wsUrl.searchParams.set("language", options.language); + if (options.prompt) wsUrl.searchParams.set("prompt", options.prompt); if (options.task) wsUrl.searchParams.set("task", options.task); const ws = new WebSocket(wsUrl); @@ -1042,6 +1053,15 @@ export function App() { /> )} + { + setHint(value); + saveHint(value); + }} + disabled={busy || liveActive} + /> + {/* Engine toggle (M5, D-010): signed-in users choose where uploaded files are transcribed. Live mode always stays on-device (D-016). */} {auth.user && ( diff --git a/src/components/VocabularyHint.tsx b/src/components/VocabularyHint.tsx new file mode 100644 index 0000000..03e616a --- /dev/null +++ b/src/components/VocabularyHint.tsx @@ -0,0 +1,49 @@ +import { useId } from "react"; +import { HINT_MAX_CHARS, normalizeHint } from "../lib/hint"; + +interface VocabularyHintProps { + value: string; + onChange: (value: string) => void; + disabled?: boolean; +} + +/** + * Past this many characters the hint likely exceeds Whisper's prompt budget + * (223 tokens ≈ 150 English words) and gets cut at the end (D-WS2a) — say so + * rather than silently ignoring the tail. + */ +const LONG_HINT_CHARS = 600; + +/** + * Optional "Names and terms" vocabulary hint (D-WS2a). Shown for every model; + * the parent persists the value (src/lib/hint.ts) and sends it with each + * transcription as the decoder prompt. + */ +export function VocabularyHint({ value, onChange, disabled = false }: VocabularyHintProps) { + const id = useId(); + const long = normalizeHint(value).length > LONG_HINT_CHARS; + return ( +
+ +