diff --git a/DESIGN.md b/DESIGN.md index 069ed80..79166ff 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -28,7 +28,7 @@ exists, not an aspiration. Update in the same commit as visual changes. ## Components One BEM-ish class block per React component in `src/components/` (`.model-picker`, `.dropzone`, -`.live-controls`, `.engine-toggle`, `.auth-panel`, `.update-toast`, …), with modifier classes like +`.live-controls`, `.engine-toggle`, `.auth-panel`, `.update-toast`, `.vocabulary-hint`, …), with modifier classes like `-selected`, `-active`, `-disabled`. Interactive states use `:hover:enabled` and `:focus-visible`. `.visually-hidden` exists for screen-reader-only text. Card headers (`.transcript-header`: title + export/Copy buttons) wrap, so on a phone the buttons diff --git a/docs/Architecture.md b/docs/Architecture.md index aa98f13..2725f62 100644 --- a/docs/Architecture.md +++ b/docs/Architecture.md @@ -67,7 +67,7 @@ src/ index.html # HTML entry, imports frontend.tsx frontend.tsx # React root App.tsx # top-level UI composition - components/ # ModelPicker, TranscriptionOptions, FileDropzone, LiveControls, TranscriptView, EngineToggle, AuthForms + components/ # ModelPicker, TranscriptionOptions, VocabularyHint, FileDropzone, LiveControls, TranscriptView, EngineToggle, AuthForms lib/ audio.ts # decode + resample to 16 kHz mono Float32Array (Web Audio API) live.ts # live-mode segment loop: RMS gate, LiveSegmenter, LiveTranscriber (pure, D-016) @@ -76,9 +76,11 @@ src/ modelCache.ts # "downloaded?" per model via Cache Storage (D-005; reused by M3 offline indicator) engine.ts # engine abstraction: BrowserEngine | ServerEngine translation.ts # transcript translation planning: Opus-MT catalog, units, sentence pieces (pure, D-WS4b) + hint.ts # "Names and terms" hint: normalize, localStorage, prompt truncation rule (D-WS2a) worker/ inference.worker.ts # Transformers.js pipeline, WebNN/WebGPU/WASM selection protocol.ts # typed postMessage messages (shared with UI) + prompt.ts # decoder prompt (<|startofprev|> hint) via a generate() wrapper (D-WS2a) translate.worker.ts # transcript translation (Opus-MT), its own worker + ort instance (D-WS4b) translateProtocol.ts # typed messages for the translate worker opusMt.ts # Opus-MT loader shared by the worker, bench script and real-model test @@ -150,8 +152,15 @@ environment). See `WorkBreakdown.md` for the itemized checklist.) transfers the `Float32Array` to the worker. For multilingual models the `transcribe` message also carries `language` (or auto-detect) and `task` (transcribe | translate — translate only for models with `canTranslate`, D-WS4a). + For every model it may carry `prompt`: the optional "Names and terms" hint + (`VocabularyHint`, stored in localStorage by `lib/hint.ts`), raw as typed (D-WS2a). 5. Worker runs the pipeline (chunked, with timestamps), streaming partial results back via `{type:"partial"}` messages; final result as `{type:"result"}`. + A hint becomes `<|startofprev|> hint.` once per pass (`worker/prompt.ts`, + cut to a 120-token prompt budget at a whole word) and is prepended + to the decoder input of every 30 s window by a wrapper around the model's + `generate()`, which also strips it from the output again (D-WS2a). + Live passes (4.1b) carry the same `prompt`. ### 4.1b Live microphone transcription (anonymous, browser engine — D-016) 1. `LiveControls` starts a session: `getUserMedia` → `AudioContext` at the @@ -230,7 +239,11 @@ environment). See `WorkBreakdown.md` for the itemized checklist.) (`server-py/`) that Bun spawns and supervises (D-013; faster-whisper has no browser/WASM build, so it is server-only — the browser stays on Transformers.js). `src/server/transcribe.ts` bridges the sidecar's segment stream to the streamed - JSON-lines HTTP response (D-011). File requests (`mode: "file"`) decode with + JSON-lines HTTP response (D-011). `language`, `task` and the vocabulary hint + (`prompt`) travel as query params (also on the live WebSocket) and are + validated by `parseLanguageAndTask` (D-032, D-WS2a); the sidecar cuts the + hint with the same rule as the browser and passes it to faster-whisper as + `initial_prompt` (D-WS2a). File requests (`mode: "file"`) decode with faster-whisper's batched pipeline over Silero-VAD speech chunks; server-live passes send `mode: "live"` and stay sequential (D-WS5a). The shared catalog maps model ids to `Systran/faster-whisper-*` CTranslate2 conversions. If the sidecar is absent, diff --git a/docs/Decisions.md b/docs/Decisions.md index 01acac5..9ea8b51 100644 --- a/docs/Decisions.md +++ b/docs/Decisions.md @@ -1528,3 +1528,94 @@ WER here (0.144); SRT/VTT cues from the server engine are correspondingly long. Segments arrive in bursts of 8 chunks (≈ 4 min of speech) and the first burst can take ~15–25 s on a long file; the D-021 heartbeat covers it, and cancel lands at the next burst boundary. No new env vars. + +## D-WS2a — "Names and terms" vocabulary hint as a Whisper decoder prompt, cut to 120 tokens +**Status:** Accepted (2026-10-29) — spec 001, Goal 2. Placeholder id; the +integrator renumbers it. +**Context:** users want names and jargon spelled their way. Whisper takes a +"previous text" prompt (`<|startofprev|> … <|startoftranscript|>`). +Transformers.js 4.2.0 — and 4.3.2, checked in workstream 1's worktree — +declare `prompt_ids` in the Whisper generation config but never use it (the +parameter is commented out in `WhisperForConditionalGeneration.generate`), +and the pipeline's `_decode_asr` prints every non-special decoder token, so +a prompt passed as `decoder_input_ids` would appear in the transcript. +**Decision:** +- UI: an optional "Names and terms" textarea for every model + (`VocabularyHint`), raw text kept in localStorage (`whisper-vocabulary-hint`), + ≤ 1000 characters, sent as `prompt` on the worker's transcribe message, on + the server engine's query string, and on the live WebSocket URL. +- `src/lib/hint.ts` normalizes (one term per line or comma-separated → + `"a, b, c"`, control characters dropped) and owns the **truncation rule**: + prompt text is `" " + hint` ending in a full stop (unless it ends in + `.?!`); over budget → the longest prefix of whole space-separated words + that fits (dangling `,`/`;` removed, full stop re-added) — the start of the + list is kept, the end dropped; if not even the first word fits, its first + `budget` tokens. Budget **120 tokens** (Whisper allows 223). The Python twin + `fit_prompt_ids` in `server-py/sidecar.py` is parity-tested. +- Browser: `src/worker/prompt.ts` builds `<|startofprev|> + ids` once per pass + and passes it through the ASR pipeline as a private generate kwarg; a + one-time wrapper around `model.generate` prepends it to the init tokens of + every 30 s window, caps `max_new_tokens` so prompt + init + new ≤ 448 + decoder positions (D-021's 440 otherwise), and slices it off the output. + Live passes use the same path. +- Server: `parseLanguageAndTask` validates `prompt` (400 above 1000 chars) for + the HTTP and WebSocket routes; the sidecar passes the cut token ids as + faster-whisper's `initial_prompt`. **No `hotwords`.** +**Measurements** (en_alien, 888 s gaming livestream, whole file; WER vs +`tests/audio_files/en_alien.txt`, single runs; hinted terms counted as +maxwell / voices of the void / slenderman / runescape, reference 9/1/1/1; +hint "Maxwell, Voices of the Void, Slenderman, RuneScape, UFO" = 21 tokens): +- Browser path, **onnxruntime-node CPU**, catalog WASM dtypes (turbo q4, + base.en q8), the worker's prompt code: + + | model | hint | prompt tokens | WER | terms | + |---|---|---|---|---| + | large-v3-turbo | none | 0 | 0.255 | 9/1/0/1 | + | large-v3-turbo | short list | 21 | **0.120** | 9/1/1/1 | + | large-v3-turbo | 54-name realistic list | 120 | 0.253 | 9/1/1/1 | + | large-v3-turbo | 54-name realistic list | 221 | 0.205 | 10/1/1/1 | + | large-v3-turbo | 300 synthetic "TermN" | 120 | 0.266 | 9/1/1/1 | + | base.en | none | 0 | 0.339 | 7/1/0/1 | + | base.en | short list, no final full stop | 21 | 0.566 | 4/1/1/0 | + | base.en | short list | 21 | 0.319 | 7/1/1/1 | + | base.en | realistic list | 32 / 64 / 96 / 120 | 0.286 / 0.306 / 0.305 / 0.284 | 9/1/1/1 at 32 and 120 | + | base.en | realistic list | 160 / 218 | 0.386 / **0.476** | 8/1/1/1, 6/1/1/0 | + | base.en | 300 synthetic "TermN" | 96 / 120 / 223 | 0.530 / 0.474 / **1.050** | | + + Wall-clock per whole-file pass (turbo 141 s without a hint, 129–206 s + with) varied with machine load more than with the hint; the extra cost is + only the prompt's decoder positions per window. +- Without a final full stop the prompt made base.en drop punctuation and loop + (punctuation 239 → 183) — hence the full stop. +- Short windows (live): 10 cases × 2 models (silence 1/3 s, noise 1–3 s, + 2 s speech snippets from en_aiproblem and da_festuge), with and without the + short hint: **0 / 20** outputs contained a hint word the no-hint output + lacked. One 2 s Danish snippet on turbo (language pinned to `da`) came out + as English words with the English hint ("I have been at an order June + August." vs "at den orde tøde i august."). +- Server path, faster-whisper 1.2.1 large-v3-turbo int8 CPU, short hint: + none 0.118 (9/1/0/1), `initial_prompt` 0.085, `hotwords` 0.077, both 0.112 + — all with 9/1/1/1. Hotwords did not improve hinted-word recall over + `initial_prompt` and combining them lost WER, so the plan's bar ("only if it + measurably helps") is not met. +**Why 120 tokens:** with prompts near Whisper's 223-token cap the small +model degraded badly (base.en 0.339 → 0.476 realistic, 1.05 synthetic — +repetition and dropped text), while at ≤ 120 tokens a realistic list stayed +at or below the no-hint WER on both models. A junk list still hurts base.en +at 96–120 tokens (0.47–0.53); the UI says "put the most important names +first" once the text passes ~300 characters (≈ 120 tokens of names), and +short lists measured best everywhere. +**Rejected:** Transformers.js `prompt_ids` (unused in 4.2.0/4.3.2); +patching `_decode_asr` (a wrapper avoids touching the transformers patch); +the full 223-token budget (above); per-model budgets (one rule, one parity +test; revisit if a larger list is wanted on turbo). +**Consequence:** every window pays up to 121 extra decoder positions with a +long hint. WebGPU behaviour of the prompt path is unmeasured here (CPU +only) — needs the owner's browser check. + +**Integration note (2026-10-30, merge with D-WS5a):** faster-whisper's +`BatchedInferencePipeline` (server file mode) re-encodes `initial_prompt` +itself and accepts only text, so the sidecar passes it the budget-cut ids +decoded back to text (re-encoding gives the same ids, checked with the real +tokenizer incl. "Århus, Ærø"); server-live passes keep the ids. Pinned by the +stubbed-sidecar test "the vocabulary hint reaches the batched pipeline as text". diff --git a/docs/WorkBreakdown.md b/docs/WorkBreakdown.md index 67fb229..40b0ecb 100644 --- a/docs/WorkBreakdown.md +++ b/docs/WorkBreakdown.md @@ -355,6 +355,10 @@ cached. for 4.3.2 and guarded by `tests/transformers-runtime.test.ts` (D-WS1a). - [x] WASM sessions back at the default graph optimization level: ort-web 1.31.0-dev loads the q8 decoders at every level (D-WS1c supersedes D-014). +- [x] Vocabulary hint "Names and terms" (D-WS2a) — optional field for every + model, remembered in localStorage, sent as `prompt` on file and live + passes (browser and server engines); cut to 120 tokens at a whole word. + faster-whisper gets `initial_prompt` only (hotwords measured, no gain). --- @@ -398,6 +402,9 @@ Found while bootstrapping agent docs (D-036); all fixed on 2026-10-29. palette; `Inter` is listed first in the font stack but never loaded. — Light theme via tokens + self-hosted Inter (D-037). Also fixed: the Backend select overflowed its panel on 390 px-wide phones. +- [ ] An English "Names and terms" hint can pull a short Danish live window + toward English words (one 2 s da_festuge snippet on turbo, D-WS2a). + Not addressed: suggest same-language hints if it shows up in use. Found while building transcript translation (D-WS4b), 2026-10-29: - [ ] Transformers.js 4.2 batched Marian generation returns row 0's output for diff --git a/server-py/sidecar.py b/server-py/sidecar.py index 125d337..bb247c8 100644 --- a/server-py/sidecar.py +++ b/server-py/sidecar.py @@ -10,7 +10,8 @@ stdio — one JSON object per line, no framing beyond that: stdin -> {"type": "transcribe", "id": "1", "model": "tiny.en", "audio_path": "/tmp/...", "language": "en" | null, "task": "transcribe" | "translate", - "mode": "file" | "live"} # optional, default "file" + "mode": "file" | "live", # optional, default "file" + "prompt": "Maxwell, Voices of the Void" | null} # optional (D-WS2a) stdout <- {"type": "segment", "id": "1", "start": 0.0, "end": 4.2, "text": " Hello world."} # 0..n of these stdout <- {"type": "done", "id": "1", "cancelled": false} @@ -56,6 +57,7 @@ as a "fatal" line, which the Bun side surfaces as "server engine unavailable" import json import os import queue +import re import sys import threading @@ -68,6 +70,56 @@ BATCH_SIZE = 8 # D-WS5a: VAD chunks decoded per batched forward pass # 2000 ms default (a dict replaces the pipeline's 160 ms preset). VAD_PARAMETERS = {"threshold": 0.2, "speech_pad_ms": 800} +# "Names and terms" vocabulary hint (D-WS2a). Twins of normalizeHint / +# fitHintToBudget in src/lib/hint.ts — change both together; parity is tested +# in tests/hint-parity.test.ts. Whisper would allow 223 tokens; 120 is the +# measured cap at which small models are not garbled (D-WS2a). +PROMPT_TOKEN_BUDGET = 120 +_CONTROL_CHARS = re.compile("[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]") + + +def normalize_hint(raw): + """One term per line or comma-separated -> "a, b, c"; "" means no hint.""" + lines = re.split(r"\r?\n|\r", _CONTROL_CHARS.sub("", raw)) + parts = [re.sub(r"\s+", " ", line).strip() for line in lines] + return ", ".join(part for part in parts if part) + + +def _prompt_sentence(text): + return " " + text if re.search(r"[.?!]$", text) else " " + text + "." + + +def fit_prompt_ids(hint, encode, budget=PROMPT_TOKEN_BUDGET): + """Prompt token ids for `hint`, cut to `budget` tokens at a whole word. + + The prompt is a space-led sentence ending in a full stop (unless it ends + in . ? ! already) - an unterminated list drops punctuation and loops. + The start of the list is kept and the end dropped; a dangling ","/";" is + removed; a first word longer than the budget is cut at the budget. + `encode` must add no special tokens. + """ + text = normalize_hint(hint) + if not text: + return [] + full = encode(_prompt_sentence(text)) + if len(full) <= budget: + return full + words = text.split(" ") + + def encode_prefix(count): + return encode(_prompt_sentence(re.sub(r"[,;]+$", "", " ".join(words[:count])))) + + low, high = 0, len(words) - 1 + while low < high: + mid = (low + high + 1) // 2 + if len(encode_prefix(mid)) <= budget: + low = mid + else: + high = mid - 1 + if low == 0: + return encode(" " + words[0])[:budget] + return encode_prefix(low) + def emit(message): """Write one JSON line to stdout. Only the main thread emits.""" @@ -137,6 +189,7 @@ def main(): language = request.get("language") or None task = request.get("task") or "transcribe" mode = request.get("mode") or "file" + prompt = request.get("prompt") or "" if model_name != cached_name: # Release both before loading the replacement: one model resident. @@ -153,11 +206,27 @@ def main(): cached_name = model_name audio = np.fromfile(audio_path, dtype=np.float32) + options = {} + prompt_ids = fit_prompt_ids( + prompt, + lambda text: cached_model.hf_tokenizer.encode( + text, add_special_tokens=False + ).ids, + ) if mode == "live": + if prompt_ids: + # D-WS2a: token ids, already cut to the budget by the + # same rule as the browser engine. + options["initial_prompt"] = prompt_ids segments, _info = cached_model.transcribe( - audio, language=language, task=task + audio, language=language, task=task, **options ) else: + if prompt_ids: + # The batched pipeline re-encodes initial_prompt itself + # and only accepts text, so hand it the budget-cut ids + # decoded back (same tokens after re-encoding). + options["initial_prompt"] = cached_model.hf_tokenizer.decode(prompt_ids) segments, _info = cached_pipeline.transcribe( audio, language=language, @@ -165,6 +234,7 @@ def main(): vad_filter=True, vad_parameters=dict(VAD_PARAMETERS), # fresh: faster-whisper may mutate it batch_size=BATCH_SIZE, + **options, ) was_cancelled = False diff --git a/src/App.tsx b/src/App.tsx index 4985267..7e4f5fd 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -11,6 +11,8 @@ import { StorageManager } from "./components/StorageManager"; import { TranscriptView } from "./components/TranscriptView"; import { TranscriptionOptions } from "./components/TranscriptionOptions"; import { TranslatePanel } from "./components/TranslatePanel"; +import { VocabularyHint } from "./components/VocabularyHint"; +import { loadHint, saveHint } from "./lib/hint"; import { decodeAudioFile } from "./lib/audio"; import { classifyError, type ClassifiedError } from "./lib/errors"; import { @@ -169,6 +171,9 @@ export function App() { const [activeThreads, setActiveThreads] = useState(null); const [language, setLanguage] = useState(AUTO_LANGUAGE); const [task, setTask] = useState(DEFAULT_TASK); + // "Names and terms" vocabulary hint (D-WS2a): raw text, remembered in this + // browser, sent with every pass for every model (`.en` included). + const [hint, setHint] = useState(() => loadHint()); const [downloaded, setDownloaded] = useState>(new Set()); // PWA state (M3, D-007): is a service worker controlling this page (app @@ -684,6 +689,7 @@ export function App() { // take neither. The browser engine ignores modelId (worker holds it). modelId, ...(multilingual && { language, task: taskForModel(modelId, task) }), + prompt: hint, onPartial: (fragment) => { if (fileRun.current === run) setPartialText((text) => text + fragment); }, @@ -721,7 +727,7 @@ export function App() { /** One inference pass over a live segment, correlated via requestId. */ async function transcribeLiveSegment( audio: Float32Array, - options: { language?: string; task?: WhisperTask }, + options: { language?: string; task?: WhisperTask; prompt?: string }, ): Promise { // Wait for the pipeline before posting: a load-restart (D-025) may swap // the worker while the model loads, and a pass queued in the old worker @@ -760,15 +766,20 @@ export function App() { setLivePending(0); setLiveStatus("starting"); - const options = - (getModel(modelId)?.multilingual ?? false) - ? { language, task: taskForModel(modelId, task) } - : {}; + const options = { + ...((getModel(modelId)?.multilingual ?? false) && { + language, + task: taskForModel(modelId, task), + }), + // Same hint for every live pass of the session (D-WS2a). + ...(hint && { prompt: hint }), + }; if (useServer) { const wsUrl = new URL(`/api/transcribe/live?model=${modelId}`, window.location.href); wsUrl.protocol = wsUrl.protocol === "https:" ? "wss:" : "ws:"; if (options.language) wsUrl.searchParams.set("language", options.language); + if (options.prompt) wsUrl.searchParams.set("prompt", options.prompt); if (options.task) wsUrl.searchParams.set("task", options.task); const ws = new WebSocket(wsUrl); @@ -1055,6 +1066,15 @@ export function App() { /> )} + { + setHint(value); + saveHint(value); + }} + disabled={busy || liveActive} + /> + {/* Engine toggle (M5, D-010): signed-in users choose where uploaded files are transcribed. Live mode always stays on-device (D-016). */} {auth.user && ( diff --git a/src/components/VocabularyHint.tsx b/src/components/VocabularyHint.tsx new file mode 100644 index 0000000..83a8279 --- /dev/null +++ b/src/components/VocabularyHint.tsx @@ -0,0 +1,49 @@ +import { useId } from "react"; +import { HINT_MAX_CHARS, normalizeHint } from "../lib/hint"; + +interface VocabularyHintProps { + value: string; + onChange: (value: string) => void; + disabled?: boolean; +} + +/** + * Past this many characters the hint likely exceeds the 120-token prompt + * budget (a list of names runs ≈ 2.8 characters per token, measured) and gets + * cut at the end (D-WS2a) — say so rather than silently ignoring the tail. + */ +const LONG_HINT_CHARS = 300; + +/** + * Optional "Names and terms" vocabulary hint (D-WS2a). Shown for every model; + * the parent persists the value (src/lib/hint.ts) and sends it with each + * transcription as the decoder prompt. + */ +export function VocabularyHint({ value, onChange, disabled = false }: VocabularyHintProps) { + const id = useId(); + const long = normalizeHint(value).length > LONG_HINT_CHARS; + return ( +
+ +