diff --git a/docs/Architecture.md b/docs/Architecture.md index ea5fc28..2d46237 100644 --- a/docs/Architecture.md +++ b/docs/Architecture.md @@ -63,7 +63,7 @@ src/ index.html # HTML entry, imports frontend.tsx frontend.tsx # React root App.tsx # top-level UI composition - components/ # ModelPicker, TranscriptionOptions, FileDropzone, LiveControls, TranscriptView, EngineToggle, AuthForms + components/ # ModelPicker, TranscriptionOptions, VocabularyHint, FileDropzone, LiveControls, TranscriptView, EngineToggle, AuthForms lib/ audio.ts # decode + resample to 16 kHz mono Float32Array (Web Audio API) live.ts # live-mode segment loop: RMS gate, LiveSegmenter, LiveTranscriber (pure, D-016) @@ -71,9 +71,11 @@ src/ models.ts # model catalog: ids, sizes, dtype defaults per device, languages modelCache.ts # "downloaded?" per model via Cache Storage (D-005; reused by M3 offline indicator) engine.ts # engine abstraction: BrowserEngine | ServerEngine + hint.ts # "Names and terms" hint: normalize, localStorage, prompt truncation rule (D-WS2a) worker/ inference.worker.ts # Transformers.js pipeline, WebNN/WebGPU/WASM selection protocol.ts # typed postMessage messages (shared with UI) + prompt.ts # decoder prompt (<|startofprev|> hint) via a generate() wrapper (D-WS2a) pwa/ sw.ts # service worker (app-shell precache, /ort runtime cache) precache.ts # placeholder-injection helpers shared by build + dev serve @@ -141,9 +143,16 @@ environment). See `WorkBreakdown.md` for the itemized checklist.) `AudioContext.decodeAudioData`, downmixes to mono, resamples to 16 kHz, transfers the `Float32Array` to the worker. For multilingual models the `transcribe` message also carries `language` (or auto-detect) and `task` - (transcribe | translate). + (transcribe | translate). For every model it may carry `prompt`: the + optional "Names and terms" hint (`VocabularyHint`, stored in localStorage + by `lib/hint.ts`), raw as typed (D-WS2a). 5. Worker runs the pipeline (chunked, with timestamps), streaming partial results back via `{type:"partial"}` messages; final result as `{type:"result"}`. + A hint becomes `<|startofprev|> hint.` once per pass (`worker/prompt.ts`, + cut to Whisper's 223-token prompt budget at a whole word) and is prepended + to the decoder input of every 30 s window by a wrapper around the model's + `generate()`, which also strips it from the output again (D-WS2a). + Live passes (4.1b) carry the same `prompt`. ### 4.1b Live microphone transcription (anonymous, browser engine — D-016) 1. `LiveControls` starts a session: `getUserMedia` → `AudioContext` at the @@ -204,7 +213,11 @@ environment). See `WorkBreakdown.md` for the itemized checklist.) (`server-py/`) that Bun spawns and supervises (D-013; faster-whisper has no browser/WASM build, so it is server-only — the browser stays on Transformers.js). `src/server/transcribe.ts` bridges the sidecar's segment stream to the streamed - JSON-lines HTTP response (D-011). The shared catalog maps model ids to + JSON-lines HTTP response (D-011). `language`, `task` and the vocabulary hint + (`prompt`) travel as query params (also on the live WebSocket) and are + validated by `parseLanguageAndTask` (D-032, D-WS2a); the sidecar cuts the + hint with the same rule as the browser and passes it to faster-whisper + (D-WS2a). The shared catalog maps model ids to `Systran/faster-whisper-*` CTranslate2 conversions. If the sidecar is absent, the server engine reports unavailable; the browser/anonymous flow is unaffected. 4. Both engines implement the same `Engine` interface in `src/lib/engine.ts`, so the diff --git a/tests/live-real-model.test.ts b/tests/live-real-model.test.ts index 3003937..c4c5722 100644 --- a/tests/live-real-model.test.ts +++ b/tests/live-real-model.test.ts @@ -10,6 +10,13 @@ import { type LiveTranscribeFn, } from "../src/lib/live"; import { resolveDefaultDtype } from "../src/lib/models"; +import { + buildPromptPrefix, + installPromptSupport, + PROMPT_PREFIX_KWARG, + type PromptableModel, + type PromptTokenizer, +} from "../src/worker/prompt"; import { wordErrorRate } from "./dsp"; /** @@ -266,6 +273,10 @@ describe("live pipeline accuracy (real model, real recordings)", () => { normalize((await transcribe(whole)).text), ); + // File-mode accuracy gate (spec 001): the upload flow is held to the + // same per-clip bar as live, so a file-mode regression fails here too. + expect(fileWer).toBeLessThanOrEqual(MAX_WER_BY_CLIP[clip.base] ?? MAX_WER); + // The live session, with the injected clock advancing with the audio // fed so pass scheduling behaves as in real time. let fedSamples = 0; @@ -310,3 +321,103 @@ describe("live pipeline accuracy (real model, real recordings)", () => { ); } }); + +// --------------------------------------------------------------------------- +// Vocabulary hint (D-WS2a): the worker's real prompt code on real weights +// --------------------------------------------------------------------------- + +const HINT = "Maxwell, Voices of the Void, Slenderman, RuneScape, UFO"; + +/** Decode the first `seconds` of a clip straight to 16 kHz (no mic path needed). */ +async function decode16k(file: string, seconds?: number): Promise { + const proc = Bun.spawn( + [ + ffmpegPath as unknown as string, + ...["-hide_banner", "-loglevel", "error", "-i", join(AUDIO_DIR, file)], + ...(seconds ? ["-t", String(seconds)] : []), + ...["-ac", "1", "-ar", String(RATE), "-f", "f32le", "-"], + ], + { stdout: "pipe", stderr: "pipe" }, + ); + const [bytes, exitCode] = await Promise.all([new Response(proc.stdout).arrayBuffer(), proc.exited]); + if (exitCode !== 0) throw new Error(`ffmpeg failed on ${file}`); + return new Float32Array(bytes); +} + +/** One pass with the inference worker's options and (optionally) its prompt. */ +async function transcribeWithHint( + audio: Float32Array, + hint: string, + language = "en", +): Promise { + const asr = await loadAsr(); + installPromptSupport(asr.model as unknown as PromptableModel); + const prefix = buildPromptPrefix(asr.tokenizer as unknown as PromptTokenizer, hint); + const output = await asr(audio, { + chunk_length_s: 30, + stride_length_s: 5, + return_timestamps: true, + max_new_tokens: 440, + language, + task: "transcribe", + ...(prefix.length > 0 && { [PROMPT_PREFIX_KWARG]: prefix }), + } as Record); + const single = (Array.isArray(output) ? output[0] : output) as { text?: string }; + return single.text ?? ""; +} + +describe("vocabulary hint (real model, D-WS2a)", () => { + test.skipIf(!available)( + "a hint keeps the transcript intact, keeps the hinted names, and never leaks into the text", + async () => { + // The first 2 min of en_alien: four windows, "Voices of the Void" and + // several "Maxwell"s. + const audio = await decode16k("en_alien.opus", 120); + const plain = await transcribeWithHint(audio, ""); + const hinted = await transcribeWithHint(audio, HINT); + const count = (text: string, term: string) => normalize(text).split(term).length - 1; + + expect(count(hinted, "maxwell")).toBeGreaterThanOrEqual(count(plain, "maxwell")); + expect(count(hinted, "voices of the void")).toBeGreaterThanOrEqual(1); + // Not garbled: no repetition loop, no dropped windows (measured + // difference between the two runs ≈ punctuation-level). + expect(wordErrorRate(normalize(plain), normalize(hinted))).toBeLessThanOrEqual(0.15); + // The prompt itself is never part of the transcript. + expect(normalize(hinted).startsWith(normalize(HINT))).toBe(false); + expect(normalize(hinted)).not.toContain("slenderman runescape"); + }, + DOWNLOAD_BUDGET_MS + 120 * 2 * MS_PER_AUDIO_SECOND, + ); + + test.skipIf(!available)( + "a hint longer than Whisper's prompt budget is cut, not an error or a garbled window", + async () => { + const longHint = Array.from({ length: 200 }, (_, i) => `Project${i} Codename${i}`) + .join(", ") + .slice(0, 1000); + const reference = await Bun.file(join(AUDIO_DIR, "en_aiproblem.txt")).text(); + const text = await transcribeWithHint(await decode16k("en_aiproblem.m4a"), longHint); + expect(wordErrorRate(normalize(reference), normalize(text))).toBeLessThanOrEqual(MAX_WER); + expect(text).not.toContain("Codename"); + }, + DOWNLOAD_BUDGET_MS + 60 * MS_PER_AUDIO_SECOND, + ); + + test.skipIf(!available)( + "a short, near-silent live window with a hint does not transcribe the hint", + async () => { + // 3 s of faint noise — what a live pass sees in a pause that the + // segmenter did not drop (its near-silence gate is far lower). + let seed = 1; + const noise = new Float32Array(3 * RATE).map(() => { + seed = (seed * 1103515245 + 12345) % 2 ** 31; + return (seed / 2 ** 31 - 0.5) * 0.01; + }); + const text = normalize(await transcribeWithHint(noise, HINT)); + for (const term of ["maxwell", "voices of the void", "slenderman", "runescape"]) { + expect(text).not.toContain(term); + } + }, + DOWNLOAD_BUDGET_MS + 30 * MS_PER_AUDIO_SECOND, + ); +});