From 313892b6df9516a6743a24c599c4b8602db913ec Mon Sep 17 00:00:00 2001 From: Lucas Winther Date: Thu, 29 Oct 2026 21:44:06 +0100 Subject: [PATCH] Carry the vocabulary hint as `prompt` from the engines through the worker into every window Co-Authored-By: Claude Opus 5.5 --- src/lib/engine.ts | 8 ++++++++ src/worker/inference.worker.ts | 18 ++++++++++++++++++ src/worker/protocol.ts | 10 +++++++++- tests/engine.test.ts | 28 ++++++++++++++++++++++++++++ tests/protocol.test.ts | 4 ++++ 5 files changed, 67 insertions(+), 1 deletion(-) diff --git a/src/lib/engine.ts b/src/lib/engine.ts index 5c60e6b..5a692c8 100644 --- a/src/lib/engine.ts +++ b/src/lib/engine.ts @@ -37,6 +37,11 @@ export interface EngineTranscribeOptions { language?: string; /** transcribe | translate (multilingual models only). */ task?: WhisperTask; + /** + * "Names and terms" vocabulary hint, raw user text (D-WS2a). Applies to + * every Whisper model, `.en` ones included; empty/undefined = no prompt. + */ + prompt?: string; /** * Streaming transcript fragments — append each to the text shown so far. * Browser engine: token fragments; server engine: whole segments. @@ -140,6 +145,7 @@ export class BrowserEngine implements Engine { ...(options.language !== undefined && options.language !== "auto" && { language: options.language }), ...(options.task !== undefined && { task: options.task }), + ...(options.prompt && { prompt: options.prompt }), }; // Ownership of the audio moves to the worker — zero-copy transfer. this.worker.postMessage(message, [audio.buffer as ArrayBuffer]); @@ -282,6 +288,8 @@ export class ServerEngine implements Engine { params.set("language", options.language); } if (options.task !== undefined) params.set("task", options.task); + // Validated + normalized server-side (parseLanguageAndTask, D-WS2a). + if (options.prompt) params.set("prompt", options.prompt); let response: Response; try { diff --git a/src/worker/inference.worker.ts b/src/worker/inference.worker.ts index 2d5b4b2..8591f38 100644 --- a/src/worker/inference.worker.ts +++ b/src/worker/inference.worker.ts @@ -17,6 +17,13 @@ import { type ProgressInfo, } from "@huggingface/transformers"; import { resolveDefaultDtype } from "../lib/models"; +import { + buildPromptPrefix, + installPromptSupport, + PROMPT_PREFIX_KWARG, + type PromptableModel, + type PromptTokenizer, +} from "./prompt"; import { deviceAttempts, dtypeRequiresF16, @@ -410,6 +417,17 @@ async function transcribe(message: TranscribeMessage): Promise { if (message.task !== undefined) { options.task = message.task; } + // "Names and terms" hint (D-WS2a): one prefix per pass, applied by the + // generate() wrapper to every 30 s window (Transformers.js ignores + // `prompt_ids`, see src/worker/prompt.ts). The wrapper is idempotent. + const promptPrefix = buildPromptPrefix( + asr.tokenizer as unknown as PromptTokenizer, + message.prompt ?? "", + ); + if (promptPrefix.length > 0) { + installPromptSupport(asr.model as unknown as PromptableModel); + options[PROMPT_PREFIX_KWARG] = promptPrefix; + } const output = await asr(message.audio, options); diff --git a/src/worker/protocol.ts b/src/worker/protocol.ts index 688333a..b8f5bc2 100644 --- a/src/worker/protocol.ts +++ b/src/worker/protocol.ts @@ -87,6 +87,13 @@ export interface TranscribeMessage { * engine (which correlates via requestId, D-010) still streams. */ stream?: boolean; + /** + * Optional "Names and terms" vocabulary hint, raw as the user typed it + * (D-WS2a). The worker normalizes it, truncates it to Whisper's prompt + * budget and feeds it as a decoder prompt to every window of this pass + * (src/worker/prompt.ts). Empty/omitted = no prompt. + */ + prompt?: string; } export type UiToWorkerMessage = LoadMessage | TranscribeMessage; @@ -318,7 +325,8 @@ export function isUiToWorkerMessage(value: unknown): value is UiToWorkerMessage value.task === "transcribe" || value.task === "translate") && isOptionalString(value.requestId) && - (value.stream === undefined || typeof value.stream === "boolean") + (value.stream === undefined || typeof value.stream === "boolean") && + isOptionalString(value.prompt) ); default: return false; diff --git a/tests/engine.test.ts b/tests/engine.test.ts index d174aa7..23d8de2 100644 --- a/tests/engine.test.ts +++ b/tests/engine.test.ts @@ -77,6 +77,17 @@ describe("BrowserEngine", () => { expect("modelId" in message).toBe(false); }); + test("forwards the vocabulary hint as `prompt`, and omits an empty one (D-WS2a)", () => { + const worker = new FakeWorker(); + const engine = new BrowserEngine(worker); + void engine.transcribe(new Float32Array(4), { prompt: "Maxwell\nVoid" }); + void engine.transcribe(new Float32Array(4), { prompt: "" }); + void engine.transcribe(new Float32Array(4)); + expect(worker.posted[0]!.message.prompt).toBe("Maxwell\nVoid"); + expect("prompt" in worker.posted[1]!.message).toBe(false); + expect("prompt" in worker.posted[2]!.message).toBe(false); + }); + test("asks the worker to stream only when a consumer listens", () => { const worker = new FakeWorker(); const engine = new BrowserEngine(worker); @@ -241,6 +252,23 @@ describe("ServerEngine", () => { expect(url).not.toContain("language"); }); + test("sends the vocabulary hint as the `prompt` query param, only when non-empty", async () => { + const urls: string[] = []; + const engine = new ServerEngine(async (input) => { + urls.push(input); + return ndjsonResponse([{ type: "result", text: "", chunks: [] }]); + }); + await engine.transcribe(new Float32Array(4), { + modelId: "onnx-community/whisper-tiny", + prompt: "Århus, Voices of the Void", + }); + await engine.transcribe(new Float32Array(4), { modelId: "onnx-community/whisper-tiny", prompt: "" }); + expect(new URL(urls[0]!, "http://localhost").searchParams.get("prompt")).toBe( + "Århus, Voices of the Void", + ); + expect(urls[1]).not.toContain("prompt"); + }); + test("requires a modelId", () => { const engine = new ServerEngine(async () => ndjsonResponse([])); expect(engine.transcribe(new Float32Array(4))).rejects.toThrow(/model id/); diff --git a/tests/protocol.test.ts b/tests/protocol.test.ts index 43bf74c..be42ff2 100644 --- a/tests/protocol.test.ts +++ b/tests/protocol.test.ts @@ -35,6 +35,7 @@ describe("isUiToWorkerMessage", () => { { type: "transcribe", audio: new Float32Array(8), requestId: "live-3" }, { type: "transcribe", audio: new Float32Array(8), requestId: "engine-1-1", stream: true }, { type: "transcribe", audio: new Float32Array(8), stream: false }, + { type: "transcribe", audio: new Float32Array(8), prompt: "Maxwell, Voices of the Void" }, ]; for (const message of [...loads, ...transcribes]) { expect(isUiToWorkerMessage(message)).toBe(true); @@ -65,6 +66,9 @@ describe("isUiToWorkerMessage", () => { expect( isUiToWorkerMessage({ type: "transcribe", audio: new Float32Array(4), stream: "yes" }), ).toBe(false); + expect( + isUiToWorkerMessage({ type: "transcribe", audio: new Float32Array(4), prompt: 5 }), + ).toBe(false); expect(isUiToWorkerMessage({ type: "result", text: "", chunks: [] })).toBe(false); }); });