From 1c5952f21207e33f1d070d3f513c41dabf7f1090 Mon Sep 17 00:00:00 2001 From: Lucas Winther Date: Thu, 29 Oct 2026 21:45:39 +0100 Subject: [PATCH] Validate the vocabulary hint server-side and pass it to faster-whisper as initial_prompt Co-Authored-By: Claude Opus 5.5 --- server-py/sidecar.py | 62 ++++++++++++++++++++++++++++++- src/index.ts | 1 + src/server/live.ts | 3 ++ src/server/transcribe.ts | 29 ++++++++++++--- tests/fixtures/fake_sidecar.py | 11 ++++-- tests/hint-parity.test.ts | 60 ++++++++++++++++++++++++++++++ tests/server-live.test.ts | 15 ++++++++ tests/transcribe.test.ts | 68 ++++++++++++++++++++++++++++++++++ 8 files changed, 239 insertions(+), 10 deletions(-) create mode 100644 tests/hint-parity.test.ts diff --git a/server-py/sidecar.py b/server-py/sidecar.py index 95f8d3b..349c135 100644 --- a/server-py/sidecar.py +++ b/server-py/sidecar.py @@ -9,7 +9,8 @@ stdio — one JSON object per line, no framing beyond that: stdin -> {"type": "transcribe", "id": "1", "model": "tiny.en", "audio_path": "/tmp/...", "language": "en" | null, - "task": "transcribe" | "translate"} + "task": "transcribe" | "translate", + "prompt": "Maxwell, Voices of the Void" | null} stdout <- {"type": "segment", "id": "1", "start": 0.0, "end": 4.2, "text": " Hello world."} # 0..n of these stdout <- {"type": "done", "id": "1", "cancelled": false} @@ -43,11 +44,56 @@ as a "fatal" line, which the Bun side surfaces as "server engine unavailable" import json import os import queue +import re import sys import threading SAMPLE_RATE = 16000 +# "Names and terms" vocabulary hint (D-WS2a). Twins of normalizeHint / +# fitHintToBudget in src/lib/hint.ts — change both together; parity is tested +# in tests/hint-parity.test.ts. Whisper's prompt budget is half of its 448 +# decoder positions minus one (<|startofprev|>). +PROMPT_TOKEN_BUDGET = 223 +_CONTROL_CHARS = re.compile("[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]") + + +def normalize_hint(raw): + """One term per line or comma-separated -> "a, b, c"; "" means no hint.""" + lines = re.split(r"\r?\n|\r", _CONTROL_CHARS.sub("", raw)) + parts = [re.sub(r"\s+", " ", line).strip() for line in lines] + return ", ".join(part for part in parts if part) + + +def fit_prompt_ids(hint, encode, budget=PROMPT_TOKEN_BUDGET): + """Prompt token ids for `hint`, cut to `budget` tokens at a whole word. + + The start of the list is kept and the end dropped; a dangling ","/";" is + removed; a first word longer than the budget is cut at the budget. + `encode` must add no special tokens. + """ + text = normalize_hint(hint) + if not text: + return [] + full = encode(" " + text) + if len(full) <= budget: + return full + words = text.split(" ") + + def encode_prefix(count): + return encode(" " + re.sub(r"[,;]+$", "", " ".join(words[:count]))) + + low, high = 0, len(words) - 1 + while low < high: + mid = (low + high + 1) // 2 + if len(encode_prefix(mid)) <= budget: + low = mid + else: + high = mid - 1 + if low == 0: + return encode(" " + words[0])[:budget] + return encode_prefix(low) + def emit(message): """Write one JSON line to stdout. Only the main thread emits.""" @@ -115,6 +161,7 @@ def main(): audio_path = request["audio_path"] language = request.get("language") or None task = request.get("task") or "transcribe" + prompt = request.get("prompt") or "" if model_name != cached_name: cached_model = None # release before loading the replacement @@ -127,8 +174,19 @@ def main(): cached_name = model_name audio = np.fromfile(audio_path, dtype=np.float32) + options = {} + prompt_ids = fit_prompt_ids( + prompt, + lambda text: cached_model.hf_tokenizer.encode( + text, add_special_tokens=False + ).ids, + ) + if prompt_ids: + # D-WS2a: token ids, already cut to the budget by the same + # rule as the browser engine. + options["initial_prompt"] = prompt_ids segments, _info = cached_model.transcribe( - audio, language=language, task=task + audio, language=language, task=task, **options ) was_cancelled = False diff --git a/src/index.ts b/src/index.ts index 328d7f3..12fade9 100644 --- a/src/index.ts +++ b/src/index.ts @@ -153,6 +153,7 @@ async function handleApiRequest(request: Request, server: any): Promise { text.push(segment.text); diff --git a/src/server/transcribe.ts b/src/server/transcribe.ts index eb95835..d625a2a 100644 --- a/src/server/transcribe.ts +++ b/src/server/transcribe.ts @@ -26,6 +26,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import type { TranscribeStreamLine } from "../lib/engine"; +import { HINT_MAX_CHARS, normalizeHint } from "../lib/hint"; import { getFasterWhisperModel } from "../lib/models"; import type { TranscriptChunk, WhisperTask } from "../worker/protocol"; import { getSessionUser as appGetSessionUser, type SessionUser } from "./auth.ts"; @@ -47,6 +48,11 @@ export interface SidecarTranscribeRequest { audioPath: string; language?: string; task?: WhisperTask; + /** + * Normalized "Names and terms" hint (D-WS2a); the sidecar truncates it to + * Whisper's prompt budget with faster-whisper's own tokenizer. + */ + prompt?: string; } export interface SidecarStatus { @@ -265,6 +271,7 @@ export function createSidecar(command?: readonly string[]): Sidecar { audio_path: request.audioPath, language: request.language ?? null, task: request.task ?? "transcribe", + prompt: request.prompt ?? null, }); } catch (error) { pending.delete(id); @@ -322,18 +329,23 @@ interface TranscribeParams { fasterWhisperModel: string; language?: string; task: WhisperTask; + prompt?: string; } /** - * Parse + validate the `language`/`task` query params shared by both the - * HTTP (`POST /api/transcribe`) and WebSocket (`GET /api/transcribe/live`, + * Parse + validate the `language`/`task`/`prompt` query params shared by both + * the HTTP (`POST /api/transcribe`) and WebSocket (`GET /api/transcribe/live`, * src/server/live.ts) entry points, so a live session can't forward an * unvalidated string into the sidecar protocol just because it skips - * `parseParams` below. A string return is the 400 error text. + * `parseParams` below (D-032). A string return is the 400 error text. + * + * `prompt` is the raw "Names and terms" hint (D-WS2a): at most + * HINT_MAX_CHARS (the UI's textarea limit), normalized exactly as the browser + * worker does; empty after normalization = no prompt. */ export function parseLanguageAndTask( url: URL, -): { language?: string; task: WhisperTask } | string { +): { language?: string; task: WhisperTask; prompt?: string } | string { const rawLanguage = url.searchParams.get("language"); let language: string | undefined; if (rawLanguage !== null && rawLanguage !== "" && rawLanguage !== "auto") { @@ -346,7 +358,13 @@ export function parseLanguageAndTask( return `invalid task: ${rawTask}`; } - return { language, task: rawTask }; + const rawPrompt = url.searchParams.get("prompt") ?? ""; + if (rawPrompt.length > HINT_MAX_CHARS) { + return `prompt too long (max ${HINT_MAX_CHARS} characters)`; + } + const prompt = normalizeHint(rawPrompt); + + return { language, task: rawTask, ...(prompt && { prompt }) }; } /** Parse + validate query params; a string is the 400 error text. */ @@ -461,6 +479,7 @@ export function createTranscribeHandlers(deps: TranscribeDeps): TranscribeHandle audioPath, language: params.language, task: params.task, + prompt: params.prompt, }, (segment) => { chunks.push({ diff --git a/tests/fixtures/fake_sidecar.py b/tests/fixtures/fake_sidecar.py index ac4f3e6..66feba4 100644 --- a/tests/fixtures/fake_sidecar.py +++ b/tests/fixtures/fake_sidecar.py @@ -5,7 +5,8 @@ Used by tests/transcribe.test.ts to exercise the real Bun-side supervisor requested model name: "ok" two segments (the second reports the audio byte count, - language and task for pass-through assertions), then done + language, task and - when sent - prompt for pass-through + assertions), then done "boom" an error line "die" exit(1) mid-request (crash simulation) "wait-for-cancel" one segment, then block until the next stdin line @@ -52,7 +53,11 @@ for line in sys.stdin: size = os.path.getsize(message["audio_path"]) emit({"type": "segment", "id": request_id, "start": 0.0, "end": 1.5, "text": " Hello"}) + prompt = message.get("prompt") + echo = "model=%s bytes=%d lang=%s task=%s" % ( + model, size, message.get("language"), message.get("task")) + if prompt is not None: + echo += " prompt=%s" % prompt emit({"type": "segment", "id": request_id, "start": 1.5, "end": 3.0, - "text": " world. [model=%s bytes=%d lang=%s task=%s]" - % (model, size, message.get("language"), message.get("task"))}) + "text": " world. [%s]" % echo}) emit({"type": "done", "id": request_id, "cancelled": False}) diff --git a/tests/hint-parity.test.ts b/tests/hint-parity.test.ts new file mode 100644 index 0000000..b540805 --- /dev/null +++ b/tests/hint-parity.test.ts @@ -0,0 +1,60 @@ +import { describe, expect, test } from "bun:test"; +import { join } from "node:path"; +import { fitHintToBudget, normalizeHint } from "../src/lib/hint"; + +// D-WS2a: the server engine truncates the hint in Python (server-py/sidecar.py, +// with faster-whisper's tokenizer) and the browser engine in TypeScript +// (src/lib/hint.ts, with the Transformers.js tokenizer). Both must apply the +// same rule; with the same deterministic fake tokenizer they must agree +// exactly. sidecar.py imports faster-whisper only inside main(), so the +// helpers import without it. + +const SIDECAR = join(import.meta.dir, "../server-py/sidecar.py"); + +const CASES: { hint: string; budget: number }[] = [ + { hint: "", budget: 223 }, + { hint: " \n\t ", budget: 223 }, + { hint: "Maxwell\nVoices of the Void", budget: 223 }, + { hint: " Århus\r\n\r\nÆrø, Festuge \n", budget: 223 }, + { hint: "Max\u0000well\u0007", budget: 223 }, + { hint: "alpha, beta gamma", budget: 10 }, + { hint: "alpha; beta; gamma; delta", budget: 14 }, + { hint: "x".repeat(50), budget: 10 }, + { hint: "Maxwell, ".repeat(300), budget: 223 }, + { hint: Array.from({ length: 400 }, (_, i) => `term${i}`).join("\n"), budget: 223 }, +]; + +/** One token per UTF-16 code unit — the same function on both sides. */ +const charEncode = (text: string) => [...text].map((c) => c.charCodeAt(0)); + +const PYTHON = ` +import importlib.util, json, sys +spec = importlib.util.spec_from_file_location("sidecar", sys.argv[1]) +sidecar = importlib.util.module_from_spec(spec) +spec.loader.exec_module(sidecar) +encode = lambda text: [ord(c) for c in text] +cases = json.load(sys.stdin) +print(json.dumps([ + {"normalized": sidecar.normalize_hint(c["hint"]), + "ids": sidecar.fit_prompt_ids(c["hint"], encode, c["budget"])} + for c in cases +])) +`; + +describe("hint truncation parity: TypeScript vs sidecar.py", () => { + test("normalize_hint / fit_prompt_ids match normalizeHint / fitHintToBudget", () => { + const proc = Bun.spawnSync(["python3", "-I", "-c", PYTHON, SIDECAR], { + stdin: new TextEncoder().encode(JSON.stringify(CASES)), + stdout: "pipe", + stderr: "pipe", + }); + expect(proc.stderr.toString()).toBe(""); + const python = JSON.parse(proc.stdout.toString()) as { normalized: string; ids: number[] }[]; + expect(python).toEqual( + CASES.map(({ hint, budget }) => ({ + normalized: normalizeHint(hint), + ids: fitHintToBudget(hint, charEncode, budget), + })), + ); + }); +}); diff --git a/tests/server-live.test.ts b/tests/server-live.test.ts index 3c41acc..49ef20b 100644 --- a/tests/server-live.test.ts +++ b/tests/server-live.test.ts @@ -139,6 +139,21 @@ describe("createLiveWebSocket", () => { await waitFor(async () => (await readdir(tempDir)).length === 0); }); + test("every pass of a session carries the session's vocabulary hint (D-WS2a)", async () => { + const tempDir = await makeTempDir(); + const sidecar = fakeSidecar(); + const handler = createLiveWebSocket({ sidecar }); + const ws = fakeSocket(sessionData({ prompt: "Maxwell, Voices of the Void", tempDir })); + handler.open(ws); + handler.message(ws, toBuffer(speech(1))); + handler.message(ws, toBuffer(silence(1.5))); + await waitFor(() => sidecar.requests.length > 0); + for (const request of sidecar.requests) { + expect(request.prompt).toBe("Maxwell, Voices of the Void"); + } + await waitFor(async () => (await readdir(tempDir)).length === 0); + }); + test("a sidecar error surfaces on the update stream instead of crashing the session", async () => { const sidecar = fakeSidecar({ error: "model load failed" }); const handler = createLiveWebSocket({ sidecar }); diff --git a/tests/transcribe.test.ts b/tests/transcribe.test.ts index 2b54b66..a375c9f 100644 --- a/tests/transcribe.test.ts +++ b/tests/transcribe.test.ts @@ -165,6 +165,36 @@ describe("parseLanguageAndTask", () => { }); }); +describe("parseLanguageAndTask: vocabulary hint (D-WS2a)", () => { + const parse = (query: string) => parseLanguageAndTask(new URL(`http://localhost/x${query}`)); + + test("normalizes the hint like the worker does", () => { + expect(parse("?prompt=Maxwell%0A%0AVoices%20of%20the%20Void%0A")).toEqual({ + language: undefined, + task: "transcribe", + prompt: "Maxwell, Voices of the Void", + }); + }); + + test("absent, empty, or whitespace-only means no prompt", () => { + for (const query of ["", "?prompt=", "?prompt=%20%0A%09"]) { + const parsed = parse(query); + expect(typeof parsed).toBe("object"); + expect("prompt" in (parsed as object)).toBe(false); + } + }); + + test("keeps non-ASCII names intact", () => { + const params = new URLSearchParams({ prompt: "Århus, Ærø, Festuge" }); + expect(parse(`?${params}`)).toMatchObject({ prompt: "Århus, Ærø, Festuge" }); + }); + + test("rejects a hint longer than the UI allows", () => { + expect(parse(`?prompt=${"a".repeat(1000)}`)).toMatchObject({ prompt: "a".repeat(1000) }); + expect(parse(`?prompt=${"a".repeat(1001)}`)).toBe("prompt too long (max 1000 characters)"); + }); +}); + describe("GET /api/transcribe/status", () => { test("401 without a session", async () => { const handlers = createTranscribeHandlers({ @@ -323,6 +353,35 @@ describe("POST /api/transcribe", () => { expect(await readdir(tempDir)).toEqual([]); }); + test("passes the vocabulary hint to the sidecar; 400 for an over-long one (D-WS2a)", async () => { + const sidecar = fakeSidecar(); + const handlers = createTranscribeHandlers({ + sidecar, + getSessionUser: sessionFromCookie, + tempDir: await makeTempDir(), + }); + const ok = new URLSearchParams({ + model: "onnx-community/whisper-base", + prompt: "Århus\nÆrø", + }); + const response = await handlers.post( + postRequest(`?${ok}`, pcmBody([0.1, 0.2]), { cookie: "sid=ok" }), + ); + expect(response.status).toBe(200); + await readStreamLines(response); + expect(sidecar.requests[0]!.prompt).toBe("Århus, Ærø"); + + const tooLong = await handlers.post( + postRequest( + `?model=onnx-community/whisper-base&prompt=${"x".repeat(1001)}`, + pcmBody([0.1]), + { cookie: "sid=ok" }, + ), + ); + expect(tooLong.status).toBe(400); + expect(sidecar.requests).toHaveLength(1); + }); + test("sidecar errors travel in-stream after the 200 header", async () => { const tempDir = await makeTempDir(); const handlers = createTranscribeHandlers({ @@ -387,6 +446,15 @@ describe("createSidecar (fake python sidecar)", () => { expect(segments[0]).toEqual({ start: 0, end: 1.5, text: " Hello" }); // The fixture echoes what it received (audio bytes, language, task). expect(segments[1]!.text).toContain("model=ok bytes=12 lang=da task=translate"); + expect(segments[1]!.text).not.toContain("prompt="); + + // The vocabulary hint crosses the JSON-lines protocol intact (D-WS2a). + const prompted: SidecarSegment[] = []; + await sidecar.transcribe( + { model: "ok", audioPath, prompt: "Århus, Voices of the Void" }, + (segment) => prompted.push(segment), + ); + expect(prompted[1]!.text).toContain("prompt=Århus, Voices of the Void]"); }); test("sidecar 'error' lines reject that request only", async () => {