Merge feat/vocabulary-hint: names-and-terms decoder prompt (spec 001 workstream 2)
Co-Authored-By: Claude Opus 5.5 <[email protected]> # Conflicts: # docs/Architecture.md # docs/Decisions.md # docs/WorkBreakdown.md # server-py/sidecar.py # src/App.tsx # src/server/live.ts # src/server/transcribe.ts # tests/fixtures/fake_sidecar.py # tests/transcribe.test.ts
This commit is contained in:
Lucas Winther
@@ -28,7 +28,7 @@ exists, not an aspiration. Update in the same commit as visual changes.
|
||||
## Components
|
||||
|
||||
One BEM-ish class block per React component in `src/components/` (`.model-picker`, `.dropzone`,
|
||||
`.live-controls`, `.engine-toggle`, `.auth-panel`, `.update-toast`, …), with modifier classes like
|
||||
`.live-controls`, `.engine-toggle`, `.auth-panel`, `.update-toast`, `.vocabulary-hint`, …), with modifier classes like
|
||||
`-selected`, `-active`, `-disabled`. Interactive states use `:hover:enabled` and `:focus-visible`.
|
||||
`.visually-hidden` exists for screen-reader-only text.
|
||||
Card headers (`.transcript-header`: title + export/Copy buttons) wrap, so on a phone the buttons
|
||||
|
||||
+15
-2
@@ -67,7 +67,7 @@ src/
|
||||
index.html # HTML entry, imports frontend.tsx
|
||||
frontend.tsx # React root
|
||||
App.tsx # top-level UI composition
|
||||
components/ # ModelPicker, TranscriptionOptions, FileDropzone, LiveControls, TranscriptView, EngineToggle, AuthForms
|
||||
components/ # ModelPicker, TranscriptionOptions, VocabularyHint, FileDropzone, LiveControls, TranscriptView, EngineToggle, AuthForms
|
||||
lib/
|
||||
audio.ts # decode + resample to 16 kHz mono Float32Array (Web Audio API)
|
||||
live.ts # live-mode segment loop: RMS gate, LiveSegmenter, LiveTranscriber (pure, D-016)
|
||||
@@ -76,9 +76,11 @@ src/
|
||||
modelCache.ts # "downloaded?" per model via Cache Storage (D-005; reused by M3 offline indicator)
|
||||
engine.ts # engine abstraction: BrowserEngine | ServerEngine
|
||||
translation.ts # transcript translation planning: Opus-MT catalog, units, sentence pieces (pure, D-WS4b)
|
||||
hint.ts # "Names and terms" hint: normalize, localStorage, prompt truncation rule (D-WS2a)
|
||||
worker/
|
||||
inference.worker.ts # Transformers.js pipeline, WebNN/WebGPU/WASM selection
|
||||
protocol.ts # typed postMessage messages (shared with UI)
|
||||
prompt.ts # decoder prompt (<|startofprev|> hint) via a generate() wrapper (D-WS2a)
|
||||
translate.worker.ts # transcript translation (Opus-MT), its own worker + ort instance (D-WS4b)
|
||||
translateProtocol.ts # typed messages for the translate worker
|
||||
opusMt.ts # Opus-MT loader shared by the worker, bench script and real-model test
|
||||
@@ -150,8 +152,15 @@ environment). See `WorkBreakdown.md` for the itemized checklist.)
|
||||
transfers the `Float32Array` to the worker. For multilingual models the
|
||||
`transcribe` message also carries `language` (or auto-detect) and `task`
|
||||
(transcribe | translate — translate only for models with `canTranslate`, D-WS4a).
|
||||
For every model it may carry `prompt`: the optional "Names and terms" hint
|
||||
(`VocabularyHint`, stored in localStorage by `lib/hint.ts`), raw as typed (D-WS2a).
|
||||
5. Worker runs the pipeline (chunked, with timestamps), streaming partial results
|
||||
back via `{type:"partial"}` messages; final result as `{type:"result"}`.
|
||||
A hint becomes `<|startofprev|> hint.` once per pass (`worker/prompt.ts`,
|
||||
cut to a 120-token prompt budget at a whole word) and is prepended
|
||||
to the decoder input of every 30 s window by a wrapper around the model's
|
||||
`generate()`, which also strips it from the output again (D-WS2a).
|
||||
Live passes (4.1b) carry the same `prompt`.
|
||||
|
||||
### 4.1b Live microphone transcription (anonymous, browser engine — D-016)
|
||||
1. `LiveControls` starts a session: `getUserMedia` → `AudioContext` at the
|
||||
@@ -230,7 +239,11 @@ environment). See `WorkBreakdown.md` for the itemized checklist.)
|
||||
(`server-py/`) that Bun spawns and supervises (D-013; faster-whisper has no
|
||||
browser/WASM build, so it is server-only — the browser stays on Transformers.js).
|
||||
`src/server/transcribe.ts` bridges the sidecar's segment stream to the streamed
|
||||
JSON-lines HTTP response (D-011). File requests (`mode: "file"`) decode with
|
||||
JSON-lines HTTP response (D-011). `language`, `task` and the vocabulary hint
|
||||
(`prompt`) travel as query params (also on the live WebSocket) and are
|
||||
validated by `parseLanguageAndTask` (D-032, D-WS2a); the sidecar cuts the
|
||||
hint with the same rule as the browser and passes it to faster-whisper as
|
||||
`initial_prompt` (D-WS2a). File requests (`mode: "file"`) decode with
|
||||
faster-whisper's batched pipeline over Silero-VAD speech chunks; server-live
|
||||
passes send `mode: "live"` and stay sequential (D-WS5a). The shared catalog maps model ids to
|
||||
`Systran/faster-whisper-*` CTranslate2 conversions. If the sidecar is absent,
|
||||
|
||||
@@ -1528,3 +1528,94 @@ WER here (0.144); SRT/VTT cues from the server engine are correspondingly
|
||||
long. Segments arrive in bursts of 8 chunks (≈ 4 min of speech) and the first
|
||||
burst can take ~15–25 s on a long file; the D-021 heartbeat covers it, and
|
||||
cancel lands at the next burst boundary. No new env vars.
|
||||
|
||||
## D-WS2a — "Names and terms" vocabulary hint as a Whisper decoder prompt, cut to 120 tokens
|
||||
**Status:** Accepted (2026-10-29) — spec 001, Goal 2. Placeholder id; the
|
||||
integrator renumbers it.
|
||||
**Context:** users want names and jargon spelled their way. Whisper takes a
|
||||
"previous text" prompt (`<|startofprev|> … <|startoftranscript|>`).
|
||||
Transformers.js 4.2.0 — and 4.3.2, checked in workstream 1's worktree —
|
||||
declare `prompt_ids` in the Whisper generation config but never use it (the
|
||||
parameter is commented out in `WhisperForConditionalGeneration.generate`),
|
||||
and the pipeline's `_decode_asr` prints every non-special decoder token, so
|
||||
a prompt passed as `decoder_input_ids` would appear in the transcript.
|
||||
**Decision:**
|
||||
- UI: an optional "Names and terms" textarea for every model
|
||||
(`VocabularyHint`), raw text kept in localStorage (`whisper-vocabulary-hint`),
|
||||
≤ 1000 characters, sent as `prompt` on the worker's transcribe message, on
|
||||
the server engine's query string, and on the live WebSocket URL.
|
||||
- `src/lib/hint.ts` normalizes (one term per line or comma-separated →
|
||||
`"a, b, c"`, control characters dropped) and owns the **truncation rule**:
|
||||
prompt text is `" " + hint` ending in a full stop (unless it ends in
|
||||
`.?!`); over budget → the longest prefix of whole space-separated words
|
||||
that fits (dangling `,`/`;` removed, full stop re-added) — the start of the
|
||||
list is kept, the end dropped; if not even the first word fits, its first
|
||||
`budget` tokens. Budget **120 tokens** (Whisper allows 223). The Python twin
|
||||
`fit_prompt_ids` in `server-py/sidecar.py` is parity-tested.
|
||||
- Browser: `src/worker/prompt.ts` builds `<|startofprev|> + ids` once per pass
|
||||
and passes it through the ASR pipeline as a private generate kwarg; a
|
||||
one-time wrapper around `model.generate` prepends it to the init tokens of
|
||||
every 30 s window, caps `max_new_tokens` so prompt + init + new ≤ 448
|
||||
decoder positions (D-021's 440 otherwise), and slices it off the output.
|
||||
Live passes use the same path.
|
||||
- Server: `parseLanguageAndTask` validates `prompt` (400 above 1000 chars) for
|
||||
the HTTP and WebSocket routes; the sidecar passes the cut token ids as
|
||||
faster-whisper's `initial_prompt`. **No `hotwords`.**
|
||||
**Measurements** (en_alien, 888 s gaming livestream, whole file; WER vs
|
||||
`tests/audio_files/en_alien.txt`, single runs; hinted terms counted as
|
||||
maxwell / voices of the void / slenderman / runescape, reference 9/1/1/1;
|
||||
hint "Maxwell, Voices of the Void, Slenderman, RuneScape, UFO" = 21 tokens):
|
||||
- Browser path, **onnxruntime-node CPU**, catalog WASM dtypes (turbo q4,
|
||||
base.en q8), the worker's prompt code:
|
||||
|
||||
| model | hint | prompt tokens | WER | terms |
|
||||
|---|---|---|---|---|
|
||||
| large-v3-turbo | none | 0 | 0.255 | 9/1/0/1 |
|
||||
| large-v3-turbo | short list | 21 | **0.120** | 9/1/1/1 |
|
||||
| large-v3-turbo | 54-name realistic list | 120 | 0.253 | 9/1/1/1 |
|
||||
| large-v3-turbo | 54-name realistic list | 221 | 0.205 | 10/1/1/1 |
|
||||
| large-v3-turbo | 300 synthetic "TermN" | 120 | 0.266 | 9/1/1/1 |
|
||||
| base.en | none | 0 | 0.339 | 7/1/0/1 |
|
||||
| base.en | short list, no final full stop | 21 | 0.566 | 4/1/1/0 |
|
||||
| base.en | short list | 21 | 0.319 | 7/1/1/1 |
|
||||
| base.en | realistic list | 32 / 64 / 96 / 120 | 0.286 / 0.306 / 0.305 / 0.284 | 9/1/1/1 at 32 and 120 |
|
||||
| base.en | realistic list | 160 / 218 | 0.386 / **0.476** | 8/1/1/1, 6/1/1/0 |
|
||||
| base.en | 300 synthetic "TermN" | 96 / 120 / 223 | 0.530 / 0.474 / **1.050** | |
|
||||
|
||||
Wall-clock per whole-file pass (turbo 141 s without a hint, 129–206 s
|
||||
with) varied with machine load more than with the hint; the extra cost is
|
||||
only the prompt's decoder positions per window.
|
||||
- Without a final full stop the prompt made base.en drop punctuation and loop
|
||||
(punctuation 239 → 183) — hence the full stop.
|
||||
- Short windows (live): 10 cases × 2 models (silence 1/3 s, noise 1–3 s,
|
||||
2 s speech snippets from en_aiproblem and da_festuge), with and without the
|
||||
short hint: **0 / 20** outputs contained a hint word the no-hint output
|
||||
lacked. One 2 s Danish snippet on turbo (language pinned to `da`) came out
|
||||
as English words with the English hint ("I have been at an order June
|
||||
August." vs "at den orde tøde i august.").
|
||||
- Server path, faster-whisper 1.2.1 large-v3-turbo int8 CPU, short hint:
|
||||
none 0.118 (9/1/0/1), `initial_prompt` 0.085, `hotwords` 0.077, both 0.112
|
||||
— all with 9/1/1/1. Hotwords did not improve hinted-word recall over
|
||||
`initial_prompt` and combining them lost WER, so the plan's bar ("only if it
|
||||
measurably helps") is not met.
|
||||
**Why 120 tokens:** with prompts near Whisper's 223-token cap the small
|
||||
model degraded badly (base.en 0.339 → 0.476 realistic, 1.05 synthetic —
|
||||
repetition and dropped text), while at ≤ 120 tokens a realistic list stayed
|
||||
at or below the no-hint WER on both models. A junk list still hurts base.en
|
||||
at 96–120 tokens (0.47–0.53); the UI says "put the most important names
|
||||
first" once the text passes ~300 characters (≈ 120 tokens of names), and
|
||||
short lists measured best everywhere.
|
||||
**Rejected:** Transformers.js `prompt_ids` (unused in 4.2.0/4.3.2);
|
||||
patching `_decode_asr` (a wrapper avoids touching the transformers patch);
|
||||
the full 223-token budget (above); per-model budgets (one rule, one parity
|
||||
test; revisit if a larger list is wanted on turbo).
|
||||
**Consequence:** every window pays up to 121 extra decoder positions with a
|
||||
long hint. WebGPU behaviour of the prompt path is unmeasured here (CPU
|
||||
only) — needs the owner's browser check.
|
||||
|
||||
**Integration note (2026-10-30, merge with D-WS5a):** faster-whisper's
|
||||
`BatchedInferencePipeline` (server file mode) re-encodes `initial_prompt`
|
||||
itself and accepts only text, so the sidecar passes it the budget-cut ids
|
||||
decoded back to text (re-encoding gives the same ids, checked with the real
|
||||
tokenizer incl. "Århus, Ærø"); server-live passes keep the ids. Pinned by the
|
||||
stubbed-sidecar test "the vocabulary hint reaches the batched pipeline as text".
|
||||
|
||||
@@ -355,6 +355,10 @@ cached.
|
||||
for 4.3.2 and guarded by `tests/transformers-runtime.test.ts` (D-WS1a).
|
||||
- [x] WASM sessions back at the default graph optimization level: ort-web
|
||||
1.31.0-dev loads the q8 decoders at every level (D-WS1c supersedes D-014).
|
||||
- [x] Vocabulary hint "Names and terms" (D-WS2a) — optional field for every
|
||||
model, remembered in localStorage, sent as `prompt` on file and live
|
||||
passes (browser and server engines); cut to 120 tokens at a whole word.
|
||||
faster-whisper gets `initial_prompt` only (hotwords measured, no gain).
|
||||
|
||||
---
|
||||
|
||||
@@ -398,6 +402,9 @@ Found while bootstrapping agent docs (D-036); all fixed on 2026-10-29.
|
||||
palette; `Inter` is listed first in the font stack but never loaded.
|
||||
— Light theme via tokens + self-hosted Inter (D-037). Also fixed: the
|
||||
Backend select overflowed its panel on 390 px-wide phones.
|
||||
- [ ] An English "Names and terms" hint can pull a short Danish live window
|
||||
toward English words (one 2 s da_festuge snippet on turbo, D-WS2a).
|
||||
Not addressed: suggest same-language hints if it shows up in use.
|
||||
|
||||
Found while building transcript translation (D-WS4b), 2026-10-29:
|
||||
- [ ] Transformers.js 4.2 batched Marian generation returns row 0's output for
|
||||
|
||||
+72
-2
@@ -10,7 +10,8 @@ stdio — one JSON object per line, no framing beyond that:
|
||||
stdin -> {"type": "transcribe", "id": "1", "model": "tiny.en",
|
||||
"audio_path": "/tmp/...", "language": "en" | null,
|
||||
"task": "transcribe" | "translate",
|
||||
"mode": "file" | "live"} # optional, default "file"
|
||||
"mode": "file" | "live", # optional, default "file"
|
||||
"prompt": "Maxwell, Voices of the Void" | null} # optional (D-WS2a)
|
||||
stdout <- {"type": "segment", "id": "1", "start": 0.0, "end": 4.2,
|
||||
"text": " Hello world."} # 0..n of these
|
||||
stdout <- {"type": "done", "id": "1", "cancelled": false}
|
||||
@@ -56,6 +57,7 @@ as a "fatal" line, which the Bun side surfaces as "server engine unavailable"
|
||||
import json
|
||||
import os
|
||||
import queue
|
||||
import re
|
||||
import sys
|
||||
import threading
|
||||
|
||||
@@ -68,6 +70,56 @@ BATCH_SIZE = 8 # D-WS5a: VAD chunks decoded per batched forward pass
|
||||
# 2000 ms default (a dict replaces the pipeline's 160 ms preset).
|
||||
VAD_PARAMETERS = {"threshold": 0.2, "speech_pad_ms": 800}
|
||||
|
||||
# "Names and terms" vocabulary hint (D-WS2a). Twins of normalizeHint /
|
||||
# fitHintToBudget in src/lib/hint.ts — change both together; parity is tested
|
||||
# in tests/hint-parity.test.ts. Whisper would allow 223 tokens; 120 is the
|
||||
# measured cap at which small models are not garbled (D-WS2a).
|
||||
PROMPT_TOKEN_BUDGET = 120
|
||||
_CONTROL_CHARS = re.compile("[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]")
|
||||
|
||||
|
||||
def normalize_hint(raw):
|
||||
"""One term per line or comma-separated -> "a, b, c"; "" means no hint."""
|
||||
lines = re.split(r"\r?\n|\r", _CONTROL_CHARS.sub("", raw))
|
||||
parts = [re.sub(r"\s+", " ", line).strip() for line in lines]
|
||||
return ", ".join(part for part in parts if part)
|
||||
|
||||
|
||||
def _prompt_sentence(text):
|
||||
return " " + text if re.search(r"[.?!]$", text) else " " + text + "."
|
||||
|
||||
|
||||
def fit_prompt_ids(hint, encode, budget=PROMPT_TOKEN_BUDGET):
|
||||
"""Prompt token ids for `hint`, cut to `budget` tokens at a whole word.
|
||||
|
||||
The prompt is a space-led sentence ending in a full stop (unless it ends
|
||||
in . ? ! already) - an unterminated list drops punctuation and loops.
|
||||
The start of the list is kept and the end dropped; a dangling ","/";" is
|
||||
removed; a first word longer than the budget is cut at the budget.
|
||||
`encode` must add no special tokens.
|
||||
"""
|
||||
text = normalize_hint(hint)
|
||||
if not text:
|
||||
return []
|
||||
full = encode(_prompt_sentence(text))
|
||||
if len(full) <= budget:
|
||||
return full
|
||||
words = text.split(" ")
|
||||
|
||||
def encode_prefix(count):
|
||||
return encode(_prompt_sentence(re.sub(r"[,;]+$", "", " ".join(words[:count]))))
|
||||
|
||||
low, high = 0, len(words) - 1
|
||||
while low < high:
|
||||
mid = (low + high + 1) // 2
|
||||
if len(encode_prefix(mid)) <= budget:
|
||||
low = mid
|
||||
else:
|
||||
high = mid - 1
|
||||
if low == 0:
|
||||
return encode(" " + words[0])[:budget]
|
||||
return encode_prefix(low)
|
||||
|
||||
|
||||
def emit(message):
|
||||
"""Write one JSON line to stdout. Only the main thread emits."""
|
||||
@@ -137,6 +189,7 @@ def main():
|
||||
language = request.get("language") or None
|
||||
task = request.get("task") or "transcribe"
|
||||
mode = request.get("mode") or "file"
|
||||
prompt = request.get("prompt") or ""
|
||||
|
||||
if model_name != cached_name:
|
||||
# Release both before loading the replacement: one model resident.
|
||||
@@ -153,11 +206,27 @@ def main():
|
||||
cached_name = model_name
|
||||
|
||||
audio = np.fromfile(audio_path, dtype=np.float32)
|
||||
options = {}
|
||||
prompt_ids = fit_prompt_ids(
|
||||
prompt,
|
||||
lambda text: cached_model.hf_tokenizer.encode(
|
||||
text, add_special_tokens=False
|
||||
).ids,
|
||||
)
|
||||
if mode == "live":
|
||||
if prompt_ids:
|
||||
# D-WS2a: token ids, already cut to the budget by the
|
||||
# same rule as the browser engine.
|
||||
options["initial_prompt"] = prompt_ids
|
||||
segments, _info = cached_model.transcribe(
|
||||
audio, language=language, task=task
|
||||
audio, language=language, task=task, **options
|
||||
)
|
||||
else:
|
||||
if prompt_ids:
|
||||
# The batched pipeline re-encodes initial_prompt itself
|
||||
# and only accepts text, so hand it the budget-cut ids
|
||||
# decoded back (same tokens after re-encoding).
|
||||
options["initial_prompt"] = cached_model.hf_tokenizer.decode(prompt_ids)
|
||||
segments, _info = cached_pipeline.transcribe(
|
||||
audio,
|
||||
language=language,
|
||||
@@ -165,6 +234,7 @@ def main():
|
||||
vad_filter=True,
|
||||
vad_parameters=dict(VAD_PARAMETERS), # fresh: faster-whisper may mutate it
|
||||
batch_size=BATCH_SIZE,
|
||||
**options,
|
||||
)
|
||||
|
||||
was_cancelled = False
|
||||
|
||||
+25
-5
@@ -11,6 +11,8 @@ import { StorageManager } from "./components/StorageManager";
|
||||
import { TranscriptView } from "./components/TranscriptView";
|
||||
import { TranscriptionOptions } from "./components/TranscriptionOptions";
|
||||
import { TranslatePanel } from "./components/TranslatePanel";
|
||||
import { VocabularyHint } from "./components/VocabularyHint";
|
||||
import { loadHint, saveHint } from "./lib/hint";
|
||||
import { decodeAudioFile } from "./lib/audio";
|
||||
import { classifyError, type ClassifiedError } from "./lib/errors";
|
||||
import {
|
||||
@@ -169,6 +171,9 @@ export function App() {
|
||||
const [activeThreads, setActiveThreads] = useState<number | null>(null);
|
||||
const [language, setLanguage] = useState(AUTO_LANGUAGE);
|
||||
const [task, setTask] = useState<WhisperTask>(DEFAULT_TASK);
|
||||
// "Names and terms" vocabulary hint (D-WS2a): raw text, remembered in this
|
||||
// browser, sent with every pass for every model (`.en` included).
|
||||
const [hint, setHint] = useState(() => loadHint());
|
||||
const [downloaded, setDownloaded] = useState<ReadonlySet<string>>(new Set());
|
||||
|
||||
// PWA state (M3, D-007): is a service worker controlling this page (app
|
||||
@@ -684,6 +689,7 @@ export function App() {
|
||||
// take neither. The browser engine ignores modelId (worker holds it).
|
||||
modelId,
|
||||
...(multilingual && { language, task: taskForModel(modelId, task) }),
|
||||
prompt: hint,
|
||||
onPartial: (fragment) => {
|
||||
if (fileRun.current === run) setPartialText((text) => text + fragment);
|
||||
},
|
||||
@@ -721,7 +727,7 @@ export function App() {
|
||||
/** One inference pass over a live segment, correlated via requestId. */
|
||||
async function transcribeLiveSegment(
|
||||
audio: Float32Array,
|
||||
options: { language?: string; task?: WhisperTask },
|
||||
options: { language?: string; task?: WhisperTask; prompt?: string },
|
||||
): Promise<LiveTranscribeResult> {
|
||||
// Wait for the pipeline before posting: a load-restart (D-025) may swap
|
||||
// the worker while the model loads, and a pass queued in the old worker
|
||||
@@ -760,15 +766,20 @@ export function App() {
|
||||
setLivePending(0);
|
||||
setLiveStatus("starting");
|
||||
|
||||
const options =
|
||||
(getModel(modelId)?.multilingual ?? false)
|
||||
? { language, task: taskForModel(modelId, task) }
|
||||
: {};
|
||||
const options = {
|
||||
...((getModel(modelId)?.multilingual ?? false) && {
|
||||
language,
|
||||
task: taskForModel(modelId, task),
|
||||
}),
|
||||
// Same hint for every live pass of the session (D-WS2a).
|
||||
...(hint && { prompt: hint }),
|
||||
};
|
||||
|
||||
if (useServer) {
|
||||
const wsUrl = new URL(`/api/transcribe/live?model=${modelId}`, window.location.href);
|
||||
wsUrl.protocol = wsUrl.protocol === "https:" ? "wss:" : "ws:";
|
||||
if (options.language) wsUrl.searchParams.set("language", options.language);
|
||||
if (options.prompt) wsUrl.searchParams.set("prompt", options.prompt);
|
||||
if (options.task) wsUrl.searchParams.set("task", options.task);
|
||||
|
||||
const ws = new WebSocket(wsUrl);
|
||||
@@ -1055,6 +1066,15 @@ export function App() {
|
||||
/>
|
||||
)}
|
||||
|
||||
<VocabularyHint
|
||||
value={hint}
|
||||
onChange={(value) => {
|
||||
setHint(value);
|
||||
saveHint(value);
|
||||
}}
|
||||
disabled={busy || liveActive}
|
||||
/>
|
||||
|
||||
{/* Engine toggle (M5, D-010): signed-in users choose where uploaded
|
||||
files are transcribed. Live mode always stays on-device (D-016). */}
|
||||
{auth.user && (
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
import { useId } from "react";
|
||||
import { HINT_MAX_CHARS, normalizeHint } from "../lib/hint";
|
||||
|
||||
interface VocabularyHintProps {
|
||||
value: string;
|
||||
onChange: (value: string) => void;
|
||||
disabled?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Past this many characters the hint likely exceeds the 120-token prompt
|
||||
* budget (a list of names runs ≈ 2.8 characters per token, measured) and gets
|
||||
* cut at the end (D-WS2a) — say so rather than silently ignoring the tail.
|
||||
*/
|
||||
const LONG_HINT_CHARS = 300;
|
||||
|
||||
/**
|
||||
* Optional "Names and terms" vocabulary hint (D-WS2a). Shown for every model;
|
||||
* the parent persists the value (src/lib/hint.ts) and sends it with each
|
||||
* transcription as the decoder prompt.
|
||||
*/
|
||||
export function VocabularyHint({ value, onChange, disabled = false }: VocabularyHintProps) {
|
||||
const id = useId();
|
||||
const long = normalizeHint(value).length > LONG_HINT_CHARS;
|
||||
return (
|
||||
<div className="vocabulary-hint">
|
||||
<label htmlFor={`${id}-input`} className="vocabulary-hint-label">
|
||||
Names and terms <span className="vocabulary-hint-optional">(optional)</span>
|
||||
</label>
|
||||
<textarea
|
||||
id={`${id}-input`}
|
||||
className="vocabulary-hint-input"
|
||||
rows={2}
|
||||
maxLength={HINT_MAX_CHARS}
|
||||
value={value}
|
||||
disabled={disabled}
|
||||
spellCheck={false}
|
||||
placeholder="e.g. Maxwell, Voices of the Void, Aarhus"
|
||||
aria-describedby={`${id}-help`}
|
||||
onChange={(event) => onChange(event.target.value)}
|
||||
/>
|
||||
<p id={`${id}-help`} className="vocabulary-hint-help">
|
||||
{long
|
||||
? "That's a long list — only the start of it will be used, so put the most important names first."
|
||||
: "Helps spell names, places and special words. Separate them with commas or new lines. Saved in this browser."}
|
||||
</p>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -519,6 +519,60 @@ body {
|
||||
flex-wrap: wrap;
|
||||
}
|
||||
|
||||
/* --- VocabularyHint (D-WS2a) --- */
|
||||
|
||||
.vocabulary-hint {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 0.35rem;
|
||||
}
|
||||
|
||||
.vocabulary-hint-label {
|
||||
font-size: 0.85rem;
|
||||
color: var(--muted);
|
||||
}
|
||||
|
||||
.vocabulary-hint-optional {
|
||||
font-weight: 400;
|
||||
opacity: 0.85;
|
||||
}
|
||||
|
||||
.vocabulary-hint-input {
|
||||
width: 100%;
|
||||
box-sizing: border-box;
|
||||
min-height: 2.9rem;
|
||||
resize: vertical;
|
||||
background: var(--surface-raised);
|
||||
color: var(--fg);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 8px;
|
||||
padding: 0.45rem 0.6rem;
|
||||
font: inherit;
|
||||
font-size: 0.9rem;
|
||||
line-height: 1.4;
|
||||
transition: border-color 0.15s;
|
||||
}
|
||||
|
||||
.vocabulary-hint-input::placeholder {
|
||||
color: var(--muted);
|
||||
}
|
||||
|
||||
.vocabulary-hint-input:focus-visible {
|
||||
border-color: var(--accent);
|
||||
outline: none;
|
||||
}
|
||||
|
||||
.vocabulary-hint-input:disabled {
|
||||
opacity: 0.6;
|
||||
cursor: not-allowed;
|
||||
}
|
||||
|
||||
.vocabulary-hint-help {
|
||||
margin: 0;
|
||||
font-size: 0.8rem;
|
||||
color: var(--muted);
|
||||
}
|
||||
|
||||
/* --- EngineToggle (M5, D-010) --- */
|
||||
|
||||
.engine-toggle {
|
||||
|
||||
@@ -153,6 +153,7 @@ async function handleApiRequest(request: Request, server: any): Promise<Response
|
||||
model: fasterWhisperModel,
|
||||
language: options.language,
|
||||
task: options.task,
|
||||
prompt: options.prompt,
|
||||
tempDir: tmpdir(),
|
||||
},
|
||||
});
|
||||
|
||||
@@ -37,6 +37,11 @@ export interface EngineTranscribeOptions {
|
||||
language?: string;
|
||||
/** transcribe | translate (multilingual models only). */
|
||||
task?: WhisperTask;
|
||||
/**
|
||||
* "Names and terms" vocabulary hint, raw user text (D-WS2a). Applies to
|
||||
* every Whisper model, `.en` ones included; empty/undefined = no prompt.
|
||||
*/
|
||||
prompt?: string;
|
||||
/**
|
||||
* Streaming transcript fragments — append each to the text shown so far.
|
||||
* Browser engine: token fragments; server engine: whole segments.
|
||||
@@ -140,6 +145,7 @@ export class BrowserEngine implements Engine {
|
||||
...(options.language !== undefined &&
|
||||
options.language !== "auto" && { language: options.language }),
|
||||
...(options.task !== undefined && { task: options.task }),
|
||||
...(options.prompt && { prompt: options.prompt }),
|
||||
};
|
||||
// Ownership of the audio moves to the worker — zero-copy transfer.
|
||||
this.worker.postMessage(message, [audio.buffer as ArrayBuffer]);
|
||||
@@ -282,6 +288,8 @@ export class ServerEngine implements Engine {
|
||||
params.set("language", options.language);
|
||||
}
|
||||
if (options.task !== undefined) params.set("task", options.task);
|
||||
// Validated + normalized server-side (parseLanguageAndTask, D-WS2a).
|
||||
if (options.prompt) params.set("prompt", options.prompt);
|
||||
|
||||
let response: Response;
|
||||
try {
|
||||
|
||||
+119
@@ -0,0 +1,119 @@
|
||||
/**
|
||||
* The optional "Names and terms" vocabulary hint (D-WS2a): free text the user
|
||||
* types once, remembered in this browser, and fed to Whisper as a decoder
|
||||
* prompt (`<|startofprev|> hint …`) so names and jargon come out spelled the
|
||||
* way the user spells them.
|
||||
*
|
||||
* Pure and dependency-free so the UI, the worker (src/worker/prompt.ts) and
|
||||
* the server (src/server/transcribe.ts) share one normalization rule, and the
|
||||
* truncation rule below has a Python twin in server-py/sidecar.py
|
||||
* (`fit_prompt_ids`, parity-tested in tests/hint-parity.test.ts).
|
||||
*/
|
||||
|
||||
import type { StorageLike } from "./engine";
|
||||
|
||||
export const HINT_STORAGE_KEY = "whisper-vocabulary-hint";
|
||||
|
||||
/**
|
||||
* Longest raw hint accepted (textarea `maxLength`; the server answers 400
|
||||
* above it). The hint travels as a URL query parameter on the server engine,
|
||||
* so it must stay well below URL limits even fully percent-encoded.
|
||||
*/
|
||||
export const HINT_MAX_CHARS = 1000;
|
||||
|
||||
/**
|
||||
* Prompt budget in tokens (without `<|startofprev|>`). Whisper itself allows
|
||||
* 223 (half its 448 decoder positions, minus one — the openai-whisper and
|
||||
* faster-whisper cap), but measured on en_alien a prompt that long garbles
|
||||
* small models: base.en WER 0.339 → 0.476 with a realistic 218-token list and
|
||||
* → 1.05 with a synthetic one, while ≤ 120 tokens stays at or below the
|
||||
* no-hint WER (D-WS2a).
|
||||
*/
|
||||
export const PROMPT_TOKEN_BUDGET = 120;
|
||||
|
||||
/** C0 controls + DEL, except \n and \r (line separators, handled below). Tab stays whitespace. */
|
||||
const CONTROL_CHARS = /[\u0000-\u0008\u000b\u000c\u000e-\u001f\u007f]/g;
|
||||
|
||||
/**
|
||||
* Canonical prompt text: one term per line or comma-separated, both become
|
||||
* "a, b, c"; control characters dropped; whitespace collapsed. "" = no hint.
|
||||
*/
|
||||
export function normalizeHint(raw: string): string {
|
||||
return raw
|
||||
.replace(CONTROL_CHARS, "")
|
||||
.split(/\r?\n|\r/)
|
||||
.map((line) => line.replace(/\s+/g, " ").trim())
|
||||
.filter((line) => line.length > 0)
|
||||
.join(", ");
|
||||
}
|
||||
|
||||
/** Saved hint (raw, as typed); "" when nothing is stored or storage is unavailable. */
|
||||
export function loadHint(storage: StorageLike | null = defaultStorage()): string {
|
||||
try {
|
||||
return storage?.getItem(HINT_STORAGE_KEY) ?? "";
|
||||
} catch {
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
export function saveHint(hint: string, storage: StorageLike | null = defaultStorage()): void {
|
||||
try {
|
||||
storage?.setItem(HINT_STORAGE_KEY, hint);
|
||||
} catch {
|
||||
// Not persistable (private mode / quota) — the hint just won't stick.
|
||||
}
|
||||
}
|
||||
|
||||
function defaultStorage(): StorageLike | null {
|
||||
try {
|
||||
return (globalThis as { localStorage?: StorageLike }).localStorage ?? null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Token ids for the hint as a Whisper prompt (without `<|startofprev|>`),
|
||||
* truncated predictably to `budget` tokens (D-WS2a):
|
||||
*
|
||||
* 1. Prompt text = `" " + normalizeHint(hint)` (Whisper prompts start with a
|
||||
* space), ending in a full stop unless it already ends in . ? or ! —
|
||||
* measured: an unterminated list makes Whisper drop punctuation and, on
|
||||
* smaller models, fall into repetition loops (D-WS2a).
|
||||
* 2. Over budget → keep the longest prefix of whole space-separated words that
|
||||
* fits (full stop included), a dangling "," / ";" removed: the start of the
|
||||
* list is kept, the end dropped, never mid-word.
|
||||
* 3. Not even the first word fits → its first `budget` tokens.
|
||||
*
|
||||
* `encode` must add no special tokens. Mirrored by `fit_prompt_ids` in
|
||||
* server-py/sidecar.py — change both together.
|
||||
*/
|
||||
export function fitHintToBudget(
|
||||
hint: string,
|
||||
encode: (text: string) => number[],
|
||||
budget: number = PROMPT_TOKEN_BUDGET,
|
||||
): number[] {
|
||||
const text = normalizeHint(hint);
|
||||
if (text.length === 0) return [];
|
||||
const full = encode(promptSentence(text));
|
||||
if (full.length <= budget) return full;
|
||||
|
||||
const words = text.split(" ");
|
||||
const encodePrefix = (count: number) =>
|
||||
encode(promptSentence(words.slice(0, count).join(" ").replace(/[,;]+$/, "")));
|
||||
// Largest word count whose encoding fits; token count grows with the
|
||||
// prefix, so binary search needs O(log n) encodes even for huge pastes.
|
||||
let low = 0;
|
||||
let high = words.length - 1; // the full text is already known not to fit
|
||||
while (low < high) {
|
||||
const mid = Math.ceil((low + high) / 2);
|
||||
if (encodePrefix(mid).length <= budget) low = mid;
|
||||
else high = mid - 1;
|
||||
}
|
||||
if (low === 0) return encode(` ${words[0]}`).slice(0, budget);
|
||||
return encodePrefix(low);
|
||||
}
|
||||
|
||||
function promptSentence(text: string): string {
|
||||
return /[.?!]$/.test(text) ? ` ${text}` : ` ${text}.`;
|
||||
}
|
||||
@@ -25,6 +25,8 @@ export interface LiveSessionData {
|
||||
model: string;
|
||||
language?: string;
|
||||
task: WhisperTask;
|
||||
/** Normalized vocabulary hint, applied to every pass (D-WS2a). */
|
||||
prompt?: string;
|
||||
tempDir: string;
|
||||
}
|
||||
|
||||
@@ -62,6 +64,7 @@ export function createLiveWebSocket(deps: LiveWebSocketDeps) {
|
||||
task: ws.data.task,
|
||||
// Short passes: sequential decode, no batching/VAD (D-WS5a).
|
||||
mode: "live",
|
||||
prompt: ws.data.prompt,
|
||||
},
|
||||
(segment) => {
|
||||
text.push(segment.text);
|
||||
|
||||
@@ -26,6 +26,7 @@
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { TranscribeStreamLine } from "../lib/engine";
|
||||
import { HINT_MAX_CHARS, normalizeHint } from "../lib/hint";
|
||||
import { getFasterWhisperModel } from "../lib/models";
|
||||
import type { TranscriptChunk, WhisperTask } from "../worker/protocol";
|
||||
import { getSessionUser as appGetSessionUser, type SessionUser } from "./auth.ts";
|
||||
@@ -52,6 +53,11 @@ export interface SidecarTranscribeRequest {
|
||||
* chunks. "live": short server-live passes — sequential, no VAD (D-WS5a).
|
||||
*/
|
||||
mode?: SidecarMode;
|
||||
/**
|
||||
* Normalized "Names and terms" hint (D-WS2a); the sidecar truncates it to
|
||||
* Whisper's prompt budget with faster-whisper's own tokenizer.
|
||||
*/
|
||||
prompt?: string;
|
||||
}
|
||||
|
||||
export type SidecarMode = "file" | "live";
|
||||
@@ -273,6 +279,7 @@ export function createSidecar(command?: readonly string[]): Sidecar {
|
||||
language: request.language ?? null,
|
||||
task: request.task ?? "transcribe",
|
||||
mode: request.mode ?? "file",
|
||||
prompt: request.prompt ?? null,
|
||||
});
|
||||
} catch (error) {
|
||||
pending.delete(id);
|
||||
@@ -332,18 +339,23 @@ interface TranscribeParams {
|
||||
fasterWhisperModel: string;
|
||||
language?: string;
|
||||
task: WhisperTask;
|
||||
prompt?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse + validate the `language`/`task` query params shared by both the
|
||||
* HTTP (`POST /api/transcribe`) and WebSocket (`GET /api/transcribe/live`,
|
||||
* Parse + validate the `language`/`task`/`prompt` query params shared by both
|
||||
* the HTTP (`POST /api/transcribe`) and WebSocket (`GET /api/transcribe/live`,
|
||||
* src/server/live.ts) entry points, so a live session can't forward an
|
||||
* unvalidated string into the sidecar protocol just because it skips
|
||||
* `parseParams` below. A string return is the 400 error text.
|
||||
* `parseParams` below (D-032). A string return is the 400 error text.
|
||||
*
|
||||
* `prompt` is the raw "Names and terms" hint (D-WS2a): at most
|
||||
* HINT_MAX_CHARS (the UI's textarea limit), normalized exactly as the browser
|
||||
* worker does; empty after normalization = no prompt.
|
||||
*/
|
||||
export function parseLanguageAndTask(
|
||||
url: URL,
|
||||
): { language?: string; task: WhisperTask } | string {
|
||||
): { language?: string; task: WhisperTask; prompt?: string } | string {
|
||||
const rawLanguage = url.searchParams.get("language");
|
||||
let language: string | undefined;
|
||||
if (rawLanguage !== null && rawLanguage !== "" && rawLanguage !== "auto") {
|
||||
@@ -356,7 +368,13 @@ export function parseLanguageAndTask(
|
||||
return `invalid task: ${rawTask}`;
|
||||
}
|
||||
|
||||
return { language, task: rawTask };
|
||||
const rawPrompt = url.searchParams.get("prompt") ?? "";
|
||||
if (rawPrompt.length > HINT_MAX_CHARS) {
|
||||
return `prompt too long (max ${HINT_MAX_CHARS} characters)`;
|
||||
}
|
||||
const prompt = normalizeHint(rawPrompt);
|
||||
|
||||
return { language, task: rawTask, ...(prompt && { prompt }) };
|
||||
}
|
||||
|
||||
/** Parse + validate query params; a string is the 400 error text. */
|
||||
@@ -472,6 +490,7 @@ export function createTranscribeHandlers(deps: TranscribeDeps): TranscribeHandle
|
||||
audioPath,
|
||||
language: params.language,
|
||||
task: params.task,
|
||||
prompt: params.prompt,
|
||||
},
|
||||
(segment) => {
|
||||
chunks.push({
|
||||
|
||||
@@ -17,6 +17,13 @@ import {
|
||||
type ProgressInfo,
|
||||
} from "@huggingface/transformers";
|
||||
import { resolveDefaultDtype } from "../lib/models";
|
||||
import {
|
||||
buildPromptPrefix,
|
||||
installPromptSupport,
|
||||
PROMPT_PREFIX_KWARG,
|
||||
type PromptableModel,
|
||||
type PromptTokenizer,
|
||||
} from "./prompt";
|
||||
import {
|
||||
deviceAttempts,
|
||||
dtypeRequiresF16,
|
||||
@@ -409,6 +416,17 @@ async function transcribe(message: TranscribeMessage): Promise<void> {
|
||||
if (message.task !== undefined) {
|
||||
options.task = message.task;
|
||||
}
|
||||
// "Names and terms" hint (D-WS2a): one prefix per pass, applied by the
|
||||
// generate() wrapper to every 30 s window (Transformers.js ignores
|
||||
// `prompt_ids`, see src/worker/prompt.ts). The wrapper is idempotent.
|
||||
const promptPrefix = buildPromptPrefix(
|
||||
asr.tokenizer as unknown as PromptTokenizer,
|
||||
message.prompt ?? "",
|
||||
);
|
||||
if (promptPrefix.length > 0) {
|
||||
installPromptSupport(asr.model as unknown as PromptableModel);
|
||||
options[PROMPT_PREFIX_KWARG] = promptPrefix;
|
||||
}
|
||||
|
||||
const output = await asr(message.audio, options);
|
||||
|
||||
|
||||
@@ -0,0 +1,101 @@
|
||||
/**
|
||||
* Whisper decoder prompts (the "Names and terms" hint, D-WS2a) on top of the
|
||||
* Transformers.js ASR pipeline.
|
||||
*
|
||||
* Transformers.js 4.2.0 (and 4.3.2) declare `prompt_ids` in the Whisper
|
||||
* generation config but never use it — the parameter is commented out in
|
||||
* `WhisperForConditionalGeneration.generate`. What generate() does honor is
|
||||
* an explicit `decoder_input_ids`, so a prompt is expressed as
|
||||
* `<|startofprev|> hint… <|startoftranscript|> <|lang|> <|task|>`. Two things
|
||||
* then need fixing up around the call, which is what the wrapper does:
|
||||
*
|
||||
* - generate() returns the whole decoder sequence, and the pipeline's
|
||||
* `_decode_asr` treats every non-special token as transcript text — the
|
||||
* hint would be printed at the start of every window. The wrapper slices
|
||||
* the prompt off the output again.
|
||||
* - `max_new_tokens: 440` (D-021) assumes a 4-token prefix; with a prompt the
|
||||
* total must still fit Whisper's 448 decoder positions.
|
||||
*
|
||||
* The prefix travels through the pipeline as a private kwarg: the ASR
|
||||
* pipeline copies its call options into every per-window generate() call, so
|
||||
* every 30 s window of a file and every live pass gets the same prompt, and
|
||||
* concurrent passes cannot see each other's prompt (no shared mutable state).
|
||||
* The `WhisperTextStreamer`'s `skip_prompt` drops the first `put()` — the
|
||||
* whole decoder input, prompt included — so streamed partials stay clean.
|
||||
*/
|
||||
|
||||
import { fitHintToBudget } from "../lib/hint";
|
||||
|
||||
/** Private generate() kwarg carrying `[<|startofprev|>, ...hint ids]`. */
|
||||
export const PROMPT_PREFIX_KWARG = "whisper_web_prompt_prefix";
|
||||
|
||||
/** Whisper's decoder context (max_target_positions) for every catalog model. */
|
||||
export const WHISPER_DECODER_POSITIONS = 448;
|
||||
|
||||
/** D-021's single-pass budget, used when a caller passes none. */
|
||||
const DEFAULT_MAX_NEW_TOKENS = 440;
|
||||
|
||||
/** The slice of a Transformers.js Whisper model the wrapper relies on. */
|
||||
export interface PromptableModel {
|
||||
generate(kwargs: Record<string, unknown>): Promise<unknown>;
|
||||
_prepare_generation_config(config: null, kwargs: Record<string, unknown>): unknown;
|
||||
_retrieve_init_tokens(generationConfig: unknown): number[];
|
||||
}
|
||||
|
||||
/** The slice of the Whisper tokenizer `buildPromptPrefix` needs. */
|
||||
export interface PromptTokenizer {
|
||||
encode(text: string, options: { add_special_tokens: false }): number[];
|
||||
convert_tokens_to_ids(token: string): number;
|
||||
}
|
||||
|
||||
interface SliceableTensor {
|
||||
slice(...slices: ((number | null)[] | null)[]): unknown;
|
||||
}
|
||||
|
||||
const WRAPPED = Symbol("whisper-web-prompt-support");
|
||||
|
||||
/**
|
||||
* `[<|startofprev|>, ...ids]` for a raw user hint, truncated to Whisper's
|
||||
* prompt budget (src/lib/hint.ts), or `[]` when the hint is empty.
|
||||
*/
|
||||
export function buildPromptPrefix(tokenizer: PromptTokenizer, hint: string): number[] {
|
||||
const ids = fitHintToBudget(hint, (text) =>
|
||||
tokenizer.encode(text, { add_special_tokens: false }),
|
||||
);
|
||||
if (ids.length === 0) return [];
|
||||
return [tokenizer.convert_tokens_to_ids("<|startofprev|>"), ...ids];
|
||||
}
|
||||
|
||||
/**
|
||||
* Teach `model.generate` the `PROMPT_PREFIX_KWARG` option. Idempotent; calls
|
||||
* without the kwarg (or with an empty prefix) are passed through untouched.
|
||||
*/
|
||||
export function installPromptSupport(model: PromptableModel): void {
|
||||
const marked = model as PromptableModel & { [WRAPPED]?: true };
|
||||
if (marked[WRAPPED]) return;
|
||||
marked[WRAPPED] = true;
|
||||
|
||||
const original = model.generate.bind(model);
|
||||
model.generate = async (kwargs) => {
|
||||
const { [PROMPT_PREFIX_KWARG]: rawPrefix, ...rest } = kwargs;
|
||||
const prefix = Array.isArray(rawPrefix) ? (rawPrefix as number[]) : [];
|
||||
if (prefix.length === 0) return original(rest);
|
||||
|
||||
// The same init tokens generate() would pick itself (language/task/
|
||||
// timestamps), with the prompt in front.
|
||||
const initTokens = model._retrieve_init_tokens(
|
||||
model._prepare_generation_config(null, rest),
|
||||
);
|
||||
const decoderInputIds = [...prefix, ...initTokens];
|
||||
const requested =
|
||||
typeof rest.max_new_tokens === "number" ? rest.max_new_tokens : DEFAULT_MAX_NEW_TOKENS;
|
||||
const output = await original({
|
||||
...rest,
|
||||
decoder_input_ids: decoderInputIds,
|
||||
max_new_tokens: Math.min(requested, WHISPER_DECODER_POSITIONS - decoderInputIds.length),
|
||||
});
|
||||
// [batch=1, tokens] → drop the prompt columns; the sequence then starts
|
||||
// at <|startoftranscript|> exactly as an unprompted pass would.
|
||||
return (output as SliceableTensor).slice(null, [prefix.length, null]);
|
||||
};
|
||||
}
|
||||
@@ -87,6 +87,13 @@ export interface TranscribeMessage {
|
||||
* engine (which correlates via requestId, D-010) still streams.
|
||||
*/
|
||||
stream?: boolean;
|
||||
/**
|
||||
* Optional "Names and terms" vocabulary hint, raw as the user typed it
|
||||
* (D-WS2a). The worker normalizes it, truncates it to Whisper's prompt
|
||||
* budget and feeds it as a decoder prompt to every window of this pass
|
||||
* (src/worker/prompt.ts). Empty/omitted = no prompt.
|
||||
*/
|
||||
prompt?: string;
|
||||
}
|
||||
|
||||
export type UiToWorkerMessage = LoadMessage | TranscribeMessage;
|
||||
@@ -318,7 +325,8 @@ export function isUiToWorkerMessage(value: unknown): value is UiToWorkerMessage
|
||||
value.task === "transcribe" ||
|
||||
value.task === "translate") &&
|
||||
isOptionalString(value.requestId) &&
|
||||
(value.stream === undefined || typeof value.stream === "boolean")
|
||||
(value.stream === undefined || typeof value.stream === "boolean") &&
|
||||
isOptionalString(value.prompt)
|
||||
);
|
||||
default:
|
||||
return false;
|
||||
|
||||
@@ -77,6 +77,17 @@ describe("BrowserEngine", () => {
|
||||
expect("modelId" in message).toBe(false);
|
||||
});
|
||||
|
||||
test("forwards the vocabulary hint as `prompt`, and omits an empty one (D-WS2a)", () => {
|
||||
const worker = new FakeWorker();
|
||||
const engine = new BrowserEngine(worker);
|
||||
void engine.transcribe(new Float32Array(4), { prompt: "Maxwell\nVoid" });
|
||||
void engine.transcribe(new Float32Array(4), { prompt: "" });
|
||||
void engine.transcribe(new Float32Array(4));
|
||||
expect(worker.posted[0]!.message.prompt).toBe("Maxwell\nVoid");
|
||||
expect("prompt" in worker.posted[1]!.message).toBe(false);
|
||||
expect("prompt" in worker.posted[2]!.message).toBe(false);
|
||||
});
|
||||
|
||||
test("asks the worker to stream only when a consumer listens", () => {
|
||||
const worker = new FakeWorker();
|
||||
const engine = new BrowserEngine(worker);
|
||||
@@ -241,6 +252,23 @@ describe("ServerEngine", () => {
|
||||
expect(url).not.toContain("language");
|
||||
});
|
||||
|
||||
test("sends the vocabulary hint as the `prompt` query param, only when non-empty", async () => {
|
||||
const urls: string[] = [];
|
||||
const engine = new ServerEngine(async (input) => {
|
||||
urls.push(input);
|
||||
return ndjsonResponse([{ type: "result", text: "", chunks: [] }]);
|
||||
});
|
||||
await engine.transcribe(new Float32Array(4), {
|
||||
modelId: "onnx-community/whisper-tiny",
|
||||
prompt: "Århus, Voices of the Void",
|
||||
});
|
||||
await engine.transcribe(new Float32Array(4), { modelId: "onnx-community/whisper-tiny", prompt: "" });
|
||||
expect(new URL(urls[0]!, "http://localhost").searchParams.get("prompt")).toBe(
|
||||
"Århus, Voices of the Void",
|
||||
);
|
||||
expect(urls[1]).not.toContain("prompt");
|
||||
});
|
||||
|
||||
test("requires a modelId", () => {
|
||||
const engine = new ServerEngine(async () => ndjsonResponse([]));
|
||||
expect(engine.transcribe(new Float32Array(4))).rejects.toThrow(/model id/);
|
||||
|
||||
Vendored
+9
-4
@@ -5,7 +5,8 @@ Used by tests/transcribe.test.ts to exercise the real Bun-side supervisor
|
||||
requested model name:
|
||||
|
||||
"ok" two segments (the second reports the audio byte count,
|
||||
language, task and mode for pass-through assertions), then done
|
||||
language, task, mode and - when sent - prompt for
|
||||
pass-through assertions), then done
|
||||
"boom" an error line
|
||||
"die" exit(1) mid-request (crash simulation)
|
||||
"wait-for-cancel" one segment, then block until the next stdin line
|
||||
@@ -52,8 +53,12 @@ for line in sys.stdin:
|
||||
size = os.path.getsize(message["audio_path"])
|
||||
emit({"type": "segment", "id": request_id, "start": 0.0, "end": 1.5,
|
||||
"text": " Hello"})
|
||||
prompt = message.get("prompt")
|
||||
echo = "model=%s bytes=%d lang=%s task=%s mode=%s" % (
|
||||
model, size, message.get("language"), message.get("task"),
|
||||
message.get("mode"))
|
||||
if prompt is not None:
|
||||
echo += " prompt=%s" % prompt
|
||||
emit({"type": "segment", "id": request_id, "start": 1.5, "end": 3.0,
|
||||
"text": " world. [model=%s bytes=%d lang=%s task=%s mode=%s]"
|
||||
% (model, size, message.get("language"), message.get("task"),
|
||||
message.get("mode"))})
|
||||
"text": " world. [%s]" % echo})
|
||||
emit({"type": "done", "id": request_id, "cancelled": False})
|
||||
|
||||
@@ -44,10 +44,26 @@ def _segments(kind, model, kwargs):
|
||||
yield _Segment(0.0, 1.0, _describe(kind, model, kwargs))
|
||||
|
||||
|
||||
class _Encoding:
|
||||
def __init__(self, ids):
|
||||
self.ids = ids
|
||||
|
||||
|
||||
class _Tokenizer:
|
||||
"""Stand-in for tokenizers.Tokenizer: one id per character."""
|
||||
|
||||
def encode(self, text, add_special_tokens=True):
|
||||
return _Encoding([ord(char) for char in text])
|
||||
|
||||
def decode(self, ids):
|
||||
return "".join(chr(i) for i in ids)
|
||||
|
||||
|
||||
class WhisperModel:
|
||||
def __init__(self, name, device=None, compute_type=None, download_root=None):
|
||||
_loads["count"] += 1
|
||||
self.name = name
|
||||
self.hf_tokenizer = _Tokenizer()
|
||||
|
||||
def transcribe(self, audio, **kwargs):
|
||||
return _segments("seq", self, kwargs), None
|
||||
@@ -58,4 +74,9 @@ class BatchedInferencePipeline:
|
||||
self.model = model
|
||||
|
||||
def transcribe(self, audio, **kwargs):
|
||||
# Like faster-whisper 1.2: the batched pipeline re-encodes
|
||||
# initial_prompt itself, so it must be text, not token ids.
|
||||
prompt = kwargs.get("initial_prompt")
|
||||
if prompt is not None and not isinstance(prompt, str):
|
||||
raise TypeError("TextInputSequence must be str")
|
||||
return _segments("batched", self.model, kwargs), None
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import { join } from "node:path";
|
||||
import { fitHintToBudget, normalizeHint, PROMPT_TOKEN_BUDGET } from "../src/lib/hint";
|
||||
|
||||
// D-WS2a: the server engine truncates the hint in Python (server-py/sidecar.py,
|
||||
// with faster-whisper's tokenizer) and the browser engine in TypeScript
|
||||
// (src/lib/hint.ts, with the Transformers.js tokenizer). Both must apply the
|
||||
// same rule; with the same deterministic fake tokenizer they must agree
|
||||
// exactly. sidecar.py imports faster-whisper only inside main(), so the
|
||||
// helpers import without it.
|
||||
|
||||
const SIDECAR = join(import.meta.dir, "../server-py/sidecar.py");
|
||||
|
||||
const CASES: { hint: string; budget: number }[] = [
|
||||
{ hint: "", budget: 223 },
|
||||
{ hint: " \n\t ", budget: 223 },
|
||||
{ hint: "Maxwell\nVoices of the Void", budget: 223 },
|
||||
{ hint: " Århus\r\n\r\nÆrø, Festuge \n", budget: 223 },
|
||||
{ hint: "Max\u0000well\u0007", budget: 223 },
|
||||
{ hint: "alpha, beta gamma", budget: 10 },
|
||||
{ hint: "alpha; beta; gamma; delta", budget: 14 },
|
||||
{ hint: "x".repeat(50), budget: 10 },
|
||||
{ hint: "Maxwell, ".repeat(300), budget: 223 },
|
||||
{ hint: Array.from({ length: 400 }, (_, i) => `term${i}`).join("\n"), budget: 223 },
|
||||
];
|
||||
|
||||
/** One token per UTF-16 code unit — the same function on both sides. */
|
||||
const charEncode = (text: string) => [...text].map((c) => c.charCodeAt(0));
|
||||
|
||||
const PYTHON = `
|
||||
import importlib.util, json, sys
|
||||
spec = importlib.util.spec_from_file_location("sidecar", sys.argv[1])
|
||||
sidecar = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(sidecar)
|
||||
encode = lambda text: [ord(c) for c in text]
|
||||
cases = json.load(sys.stdin)
|
||||
print(json.dumps({"budget": sidecar.PROMPT_TOKEN_BUDGET, "results": [
|
||||
{"normalized": sidecar.normalize_hint(c["hint"]),
|
||||
"ids": sidecar.fit_prompt_ids(c["hint"], encode, c["budget"])}
|
||||
for c in cases
|
||||
]}))
|
||||
`;
|
||||
|
||||
describe("hint truncation parity: TypeScript vs sidecar.py", () => {
|
||||
test("normalize_hint / fit_prompt_ids match normalizeHint / fitHintToBudget", () => {
|
||||
const proc = Bun.spawnSync(["python3", "-I", "-c", PYTHON, SIDECAR], {
|
||||
stdin: new TextEncoder().encode(JSON.stringify(CASES)),
|
||||
stdout: "pipe",
|
||||
stderr: "pipe",
|
||||
});
|
||||
expect(proc.stderr.toString()).toBe("");
|
||||
const python = JSON.parse(proc.stdout.toString()) as {
|
||||
budget: number;
|
||||
results: { normalized: string; ids: number[] }[];
|
||||
};
|
||||
expect(python.budget).toBe(PROMPT_TOKEN_BUDGET);
|
||||
expect(python.results).toEqual(
|
||||
CASES.map(({ hint, budget }) => ({
|
||||
normalized: normalizeHint(hint),
|
||||
ids: fitHintToBudget(hint, charEncode, budget),
|
||||
})),
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,118 @@
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import {
|
||||
HINT_STORAGE_KEY,
|
||||
PROMPT_TOKEN_BUDGET,
|
||||
fitHintToBudget,
|
||||
loadHint,
|
||||
normalizeHint,
|
||||
saveHint,
|
||||
} from "../src/lib/hint";
|
||||
|
||||
/** One "token" per UTF-16 unit: deterministic, monotone, easy to reason about. */
|
||||
const charEncode = (text: string) => [...text].map((c) => c.charCodeAt(0));
|
||||
const charDecode = (ids: number[]) => String.fromCharCode(...ids);
|
||||
|
||||
function mapStorage(initial: Record<string, string> = {}) {
|
||||
const map = new Map(Object.entries(initial));
|
||||
return {
|
||||
getItem: (key: string) => map.get(key) ?? null,
|
||||
setItem: (key: string, value: string) => void map.set(key, value),
|
||||
map,
|
||||
};
|
||||
}
|
||||
|
||||
const throwingStorage = {
|
||||
getItem(): string | null {
|
||||
throw new Error("denied");
|
||||
},
|
||||
setItem(): void {
|
||||
throw new Error("denied");
|
||||
},
|
||||
};
|
||||
|
||||
describe("normalizeHint", () => {
|
||||
test("one term per line becomes a comma-separated list", () => {
|
||||
expect(normalizeHint(" Maxwell\n\n Voices of the Void \n")).toBe(
|
||||
"Maxwell, Voices of the Void",
|
||||
);
|
||||
});
|
||||
|
||||
test("a comma-separated list is unchanged", () => {
|
||||
expect(normalizeHint("Maxwell, Voices of the Void")).toBe("Maxwell, Voices of the Void");
|
||||
});
|
||||
|
||||
test("control characters are removed and whitespace-only means no hint", () => {
|
||||
expect(normalizeHint("Max\u0000well\u0007")).toBe("Maxwell");
|
||||
expect(normalizeHint(" \n\t \r\n")).toBe("");
|
||||
});
|
||||
|
||||
test("Windows line endings and tabs are handled", () => {
|
||||
expect(normalizeHint("Århus\r\nÆrø\tFestuge")).toBe("Århus, Ærø Festuge");
|
||||
});
|
||||
});
|
||||
|
||||
describe("hint storage", () => {
|
||||
test("round-trips through storage", () => {
|
||||
const storage = mapStorage();
|
||||
expect(loadHint(storage)).toBe("");
|
||||
saveHint("Maxwell\nVoid", storage);
|
||||
expect(storage.map.get(HINT_STORAGE_KEY)).toBe("Maxwell\nVoid");
|
||||
expect(loadHint(storage)).toBe("Maxwell\nVoid");
|
||||
});
|
||||
|
||||
test("unavailable storage never throws", () => {
|
||||
expect(loadHint(throwingStorage)).toBe("");
|
||||
expect(() => saveHint("x", throwingStorage)).not.toThrow();
|
||||
expect(loadHint(null)).toBe("");
|
||||
});
|
||||
});
|
||||
|
||||
describe("fitHintToBudget (D-WS2a truncation rule)", () => {
|
||||
test("empty hint means no prompt", () => {
|
||||
expect(fitHintToBudget(" \n", charEncode)).toEqual([]);
|
||||
});
|
||||
|
||||
test("a hint within budget is encoded as a space-led sentence ending in a full stop", () => {
|
||||
// Measured (D-WS2a): an unterminated list prompt suppresses punctuation
|
||||
// and triggers repetition loops on smaller models; a full stop does not.
|
||||
expect(fitHintToBudget("Maxwell\nVoices of the Void", charEncode)).toEqual(
|
||||
charEncode(" Maxwell, Voices of the Void."),
|
||||
);
|
||||
});
|
||||
|
||||
test("a hint already ending in . ? or ! gets no extra full stop", () => {
|
||||
expect(charDecode(fitHintToBudget("Is it Maxwell?", charEncode))).toBe(" Is it Maxwell?");
|
||||
expect(charDecode(fitHintToBudget("Dr. Bao.", charEncode))).toBe(" Dr. Bao.");
|
||||
});
|
||||
|
||||
test("over budget: keeps whole leading words, drops a dangling comma, ends in a full stop", () => {
|
||||
expect(charDecode(fitHintToBudget("alpha, beta gamma", charEncode, 10))).toBe(" alpha.");
|
||||
expect(charDecode(fitHintToBudget("alpha, beta gamma", charEncode, 13))).toBe(" alpha, beta.");
|
||||
});
|
||||
|
||||
test("a single word longer than the budget is cut at the budget", () => {
|
||||
const ids = fitHintToBudget("x".repeat(50), charEncode, 10);
|
||||
expect(ids).toEqual(charEncode(" xxxxxxxxx"));
|
||||
expect(ids.length).toBe(10);
|
||||
});
|
||||
|
||||
test("a very long hint fits the 120-token budget and is a word prefix of the hint", () => {
|
||||
const words = Array.from({ length: 5000 }, (_, i) => `term${i}`);
|
||||
const hint = words.join(", ");
|
||||
const ids = fitHintToBudget(hint, charEncode);
|
||||
expect(PROMPT_TOKEN_BUDGET).toBe(120);
|
||||
expect(ids.length).toBeLessThanOrEqual(120);
|
||||
expect(ids.length).toBeGreaterThan(100);
|
||||
const kept = charDecode(ids);
|
||||
// Ends on a whole term plus the full stop: the hint continues with a separator there.
|
||||
expect(kept.endsWith(".")).toBe(true);
|
||||
const body = kept.slice(0, -1);
|
||||
expect(` ${hint}`.startsWith(body)).toBe(true);
|
||||
expect([",", " "]).toContain(` ${hint}`.charAt(body.length));
|
||||
});
|
||||
|
||||
test("is deterministic", () => {
|
||||
const hint = "Maxwell, ".repeat(200);
|
||||
expect(fitHintToBudget(hint, charEncode)).toEqual(fitHintToBudget(hint, charEncode));
|
||||
});
|
||||
});
|
||||
@@ -10,6 +10,13 @@ import {
|
||||
type LiveTranscribeFn,
|
||||
} from "../src/lib/live";
|
||||
import { resolveDefaultDtype } from "../src/lib/models";
|
||||
import {
|
||||
buildPromptPrefix,
|
||||
installPromptSupport,
|
||||
PROMPT_PREFIX_KWARG,
|
||||
type PromptableModel,
|
||||
type PromptTokenizer,
|
||||
} from "../src/worker/prompt";
|
||||
import { wordErrorRate } from "./dsp";
|
||||
|
||||
/**
|
||||
@@ -266,6 +273,10 @@ describe("live pipeline accuracy (real model, real recordings)", () => {
|
||||
normalize((await transcribe(whole)).text),
|
||||
);
|
||||
|
||||
// File-mode accuracy gate (spec 001): the upload flow is held to the
|
||||
// same per-clip bar as live, so a file-mode regression fails here too.
|
||||
expect(fileWer).toBeLessThanOrEqual(MAX_WER_BY_CLIP[clip.base] ?? MAX_WER);
|
||||
|
||||
// The live session, with the injected clock advancing with the audio
|
||||
// fed so pass scheduling behaves as in real time.
|
||||
let fedSamples = 0;
|
||||
@@ -310,3 +321,103 @@ describe("live pipeline accuracy (real model, real recordings)", () => {
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Vocabulary hint (D-WS2a): the worker's real prompt code on real weights
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const HINT = "Maxwell, Voices of the Void, Slenderman, RuneScape, UFO";
|
||||
|
||||
/** Decode the first `seconds` of a clip straight to 16 kHz (no mic path needed). */
|
||||
async function decode16k(file: string, seconds?: number): Promise<Float32Array> {
|
||||
const proc = Bun.spawn(
|
||||
[
|
||||
ffmpegPath as unknown as string,
|
||||
...["-hide_banner", "-loglevel", "error", "-i", join(AUDIO_DIR, file)],
|
||||
...(seconds ? ["-t", String(seconds)] : []),
|
||||
...["-ac", "1", "-ar", String(RATE), "-f", "f32le", "-"],
|
||||
],
|
||||
{ stdout: "pipe", stderr: "pipe" },
|
||||
);
|
||||
const [bytes, exitCode] = await Promise.all([new Response(proc.stdout).arrayBuffer(), proc.exited]);
|
||||
if (exitCode !== 0) throw new Error(`ffmpeg failed on ${file}`);
|
||||
return new Float32Array(bytes);
|
||||
}
|
||||
|
||||
/** One pass with the inference worker's options and (optionally) its prompt. */
|
||||
async function transcribeWithHint(
|
||||
audio: Float32Array,
|
||||
hint: string,
|
||||
language = "en",
|
||||
): Promise<string> {
|
||||
const asr = await loadAsr();
|
||||
installPromptSupport(asr.model as unknown as PromptableModel);
|
||||
const prefix = buildPromptPrefix(asr.tokenizer as unknown as PromptTokenizer, hint);
|
||||
const output = await asr(audio, {
|
||||
chunk_length_s: 30,
|
||||
stride_length_s: 5,
|
||||
return_timestamps: true,
|
||||
max_new_tokens: 440,
|
||||
language,
|
||||
task: "transcribe",
|
||||
...(prefix.length > 0 && { [PROMPT_PREFIX_KWARG]: prefix }),
|
||||
} as Record<string, unknown>);
|
||||
const single = (Array.isArray(output) ? output[0] : output) as { text?: string };
|
||||
return single.text ?? "";
|
||||
}
|
||||
|
||||
describe("vocabulary hint (real model, D-WS2a)", () => {
|
||||
test.skipIf(!available)(
|
||||
"a hint keeps the transcript intact, keeps the hinted names, and never leaks into the text",
|
||||
async () => {
|
||||
// The first 2 min of en_alien: four windows, "Voices of the Void" and
|
||||
// several "Maxwell"s.
|
||||
const audio = await decode16k("en_alien.opus", 120);
|
||||
const plain = await transcribeWithHint(audio, "");
|
||||
const hinted = await transcribeWithHint(audio, HINT);
|
||||
const count = (text: string, term: string) => normalize(text).split(term).length - 1;
|
||||
|
||||
expect(count(hinted, "maxwell")).toBeGreaterThanOrEqual(count(plain, "maxwell"));
|
||||
expect(count(hinted, "voices of the void")).toBeGreaterThanOrEqual(1);
|
||||
// Not garbled: no repetition loop, no dropped windows (measured
|
||||
// difference between the two runs ≈ punctuation-level).
|
||||
expect(wordErrorRate(normalize(plain), normalize(hinted))).toBeLessThanOrEqual(0.15);
|
||||
// The prompt itself is never part of the transcript.
|
||||
expect(normalize(hinted).startsWith(normalize(HINT))).toBe(false);
|
||||
expect(normalize(hinted)).not.toContain("slenderman runescape");
|
||||
},
|
||||
DOWNLOAD_BUDGET_MS + 120 * 2 * MS_PER_AUDIO_SECOND,
|
||||
);
|
||||
|
||||
test.skipIf(!available)(
|
||||
"a hint longer than Whisper's prompt budget is cut, not an error or a garbled window",
|
||||
async () => {
|
||||
const longHint = Array.from({ length: 200 }, (_, i) => `Project${i} Codename${i}`)
|
||||
.join(", ")
|
||||
.slice(0, 1000);
|
||||
const reference = await Bun.file(join(AUDIO_DIR, "en_aiproblem.txt")).text();
|
||||
const text = await transcribeWithHint(await decode16k("en_aiproblem.m4a"), longHint);
|
||||
expect(wordErrorRate(normalize(reference), normalize(text))).toBeLessThanOrEqual(MAX_WER);
|
||||
expect(text).not.toContain("Codename");
|
||||
},
|
||||
DOWNLOAD_BUDGET_MS + 60 * MS_PER_AUDIO_SECOND,
|
||||
);
|
||||
|
||||
test.skipIf(!available)(
|
||||
"a short, near-silent live window with a hint does not transcribe the hint",
|
||||
async () => {
|
||||
// 3 s of faint noise — what a live pass sees in a pause that the
|
||||
// segmenter did not drop (its near-silence gate is far lower).
|
||||
let seed = 1;
|
||||
const noise = new Float32Array(3 * RATE).map(() => {
|
||||
seed = (seed * 1103515245 + 12345) % 2 ** 31;
|
||||
return (seed / 2 ** 31 - 0.5) * 0.01;
|
||||
});
|
||||
const text = normalize(await transcribeWithHint(noise, HINT));
|
||||
for (const term of ["maxwell", "voices of the void", "slenderman", "runescape"]) {
|
||||
expect(text).not.toContain(term);
|
||||
}
|
||||
},
|
||||
DOWNLOAD_BUDGET_MS + 30 * MS_PER_AUDIO_SECOND,
|
||||
);
|
||||
});
|
||||
|
||||
@@ -0,0 +1,124 @@
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import { Tensor } from "@huggingface/transformers";
|
||||
import {
|
||||
PROMPT_PREFIX_KWARG,
|
||||
WHISPER_DECODER_POSITIONS,
|
||||
buildPromptPrefix,
|
||||
installPromptSupport,
|
||||
type PromptableModel,
|
||||
} from "../src/worker/prompt";
|
||||
|
||||
const PREV = 50362;
|
||||
|
||||
/**
|
||||
* Stand-in for a Transformers.js Whisper model: records the kwargs each
|
||||
* generate() call received and, like the real one, returns the decoder input
|
||||
* followed by the "generated" tokens as an int64 [1, n] tensor.
|
||||
*/
|
||||
function fakeModel(initTokens: number[], generated: number[] = [50365, 400, 401, 50400]) {
|
||||
const calls: Record<string, unknown>[] = [];
|
||||
const model: PromptableModel & { calls: typeof calls } = {
|
||||
calls,
|
||||
_prepare_generation_config: (_config, kwargs) => ({ fromKwargs: kwargs }),
|
||||
_retrieve_init_tokens: () => [...initTokens],
|
||||
async generate(kwargs) {
|
||||
calls.push(kwargs);
|
||||
const input = (kwargs.decoder_input_ids as number[] | undefined) ?? initTokens;
|
||||
const sequence = [...input, ...generated];
|
||||
return new Tensor("int64", BigInt64Array.from(sequence.map(BigInt)), [1, sequence.length]);
|
||||
},
|
||||
};
|
||||
return model;
|
||||
}
|
||||
|
||||
function tokens(output: unknown): number[] {
|
||||
return (output as Tensor).tolist()[0]!.map(Number);
|
||||
}
|
||||
|
||||
describe("installPromptSupport (D-WS2a)", () => {
|
||||
test("without a prompt kwarg, generate is called exactly as before", async () => {
|
||||
const model = fakeModel([50258, 50259, 50360]);
|
||||
installPromptSupport(model);
|
||||
const kwargs = { max_new_tokens: 440, return_timestamps: true };
|
||||
const output = await model.generate({ ...kwargs });
|
||||
expect(model.calls).toEqual([kwargs]);
|
||||
expect(tokens(output)).toEqual([50258, 50259, 50360, 50365, 400, 401, 50400]);
|
||||
});
|
||||
|
||||
test("an empty prefix is the same as no prompt", async () => {
|
||||
const model = fakeModel([50258]);
|
||||
installPromptSupport(model);
|
||||
await model.generate({ max_new_tokens: 440, [PROMPT_PREFIX_KWARG]: [] });
|
||||
expect(model.calls).toEqual([{ max_new_tokens: 440 }]);
|
||||
});
|
||||
|
||||
test("a prefix is prepended to the init tokens and stripped from the output", async () => {
|
||||
const model = fakeModel([50258, 50259, 50360]);
|
||||
installPromptSupport(model);
|
||||
const output = await model.generate({
|
||||
max_new_tokens: 440,
|
||||
language: "en",
|
||||
[PROMPT_PREFIX_KWARG]: [PREV, 1, 2],
|
||||
});
|
||||
const call = model.calls[0]!;
|
||||
expect(call.decoder_input_ids).toEqual([PREV, 1, 2, 50258, 50259, 50360]);
|
||||
expect(call).not.toHaveProperty(PROMPT_PREFIX_KWARG);
|
||||
expect(call.language).toBe("en");
|
||||
// Prompt tokens would otherwise reach _decode_asr as transcript text.
|
||||
expect(tokens(output)).toEqual([50258, 50259, 50360, 50365, 400, 401, 50400]);
|
||||
});
|
||||
|
||||
test("works for English-only models (init tokens are just <|startoftranscript|>)", async () => {
|
||||
const model = fakeModel([50257]);
|
||||
installPromptSupport(model);
|
||||
const output = await model.generate({ max_new_tokens: 440, [PROMPT_PREFIX_KWARG]: [50361, 7] });
|
||||
expect(model.calls[0]!.decoder_input_ids).toEqual([50361, 7, 50257]);
|
||||
expect(tokens(output)[0]).toBe(50257);
|
||||
});
|
||||
|
||||
test("caps max_new_tokens so prompt + init + new tokens fit Whisper's 448 positions", async () => {
|
||||
const model = fakeModel([50258, 50259, 50360]);
|
||||
installPromptSupport(model);
|
||||
const prefix = [PREV, ...Array.from({ length: 223 }, (_, i) => 1000 + i)];
|
||||
await model.generate({ max_new_tokens: 440, [PROMPT_PREFIX_KWARG]: prefix });
|
||||
expect(WHISPER_DECODER_POSITIONS).toBe(448);
|
||||
expect(model.calls[0]!.max_new_tokens).toBe(448 - 227);
|
||||
// A short prompt keeps the D-021 value.
|
||||
await model.generate({ max_new_tokens: 440, [PROMPT_PREFIX_KWARG]: [PREV, 5] });
|
||||
expect(model.calls[1]!.max_new_tokens).toBe(440);
|
||||
});
|
||||
|
||||
test("installing twice wraps once", async () => {
|
||||
const model = fakeModel([50258]);
|
||||
installPromptSupport(model);
|
||||
installPromptSupport(model);
|
||||
await model.generate({ max_new_tokens: 440, [PROMPT_PREFIX_KWARG]: [PREV, 5] });
|
||||
expect(model.calls).toHaveLength(1);
|
||||
expect(model.calls[0]!.decoder_input_ids).toEqual([PREV, 5, 50258]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("buildPromptPrefix", () => {
|
||||
const tokenizer = {
|
||||
encode: (text: string, _options: { add_special_tokens: false }) =>
|
||||
[...text].map((c) => c.charCodeAt(0)),
|
||||
convert_tokens_to_ids: (token: string) => (token === "<|startofprev|>" ? PREV : -1),
|
||||
};
|
||||
|
||||
test("no hint → no prefix", () => {
|
||||
expect(buildPromptPrefix(tokenizer, "")).toEqual([]);
|
||||
expect(buildPromptPrefix(tokenizer, " \n ")).toEqual([]);
|
||||
});
|
||||
|
||||
test("a hint → <|startofprev|> then the space-led, full-stopped hint tokens", () => {
|
||||
expect(buildPromptPrefix(tokenizer, "Max\nVoid")).toEqual([
|
||||
PREV,
|
||||
...[..." Max, Void."].map((c) => c.charCodeAt(0)),
|
||||
]);
|
||||
});
|
||||
|
||||
test("a huge hint stays within the 120-token budget (D-WS2a)", () => {
|
||||
const prefix = buildPromptPrefix(tokenizer, "Maxwell, ".repeat(500));
|
||||
expect(prefix.length).toBeLessThanOrEqual(121);
|
||||
});
|
||||
});
|
||||
@@ -35,6 +35,7 @@ describe("isUiToWorkerMessage", () => {
|
||||
{ type: "transcribe", audio: new Float32Array(8), requestId: "live-3" },
|
||||
{ type: "transcribe", audio: new Float32Array(8), requestId: "engine-1-1", stream: true },
|
||||
{ type: "transcribe", audio: new Float32Array(8), stream: false },
|
||||
{ type: "transcribe", audio: new Float32Array(8), prompt: "Maxwell, Voices of the Void" },
|
||||
];
|
||||
for (const message of [...loads, ...transcribes]) {
|
||||
expect(isUiToWorkerMessage(message)).toBe(true);
|
||||
@@ -65,6 +66,9 @@ describe("isUiToWorkerMessage", () => {
|
||||
expect(
|
||||
isUiToWorkerMessage({ type: "transcribe", audio: new Float32Array(4), stream: "yes" }),
|
||||
).toBe(false);
|
||||
expect(
|
||||
isUiToWorkerMessage({ type: "transcribe", audio: new Float32Array(4), prompt: 5 }),
|
||||
).toBe(false);
|
||||
expect(isUiToWorkerMessage({ type: "result", text: "", chunks: [] })).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -121,7 +121,9 @@ describe("createLiveWebSocket", () => {
|
||||
handler.message(ws, toBuffer(speech(1)));
|
||||
handler.message(ws, toBuffer(silence(1.5))); // trailing silence commits the segment
|
||||
|
||||
await waitFor(() => sidecar.requests.length > 0);
|
||||
// The fake records the request before it finishes reading the PCM file —
|
||||
// wait for the payload itself, or a loaded machine races past it.
|
||||
await waitFor(() => sidecar.audioPayloads.length > 0);
|
||||
expect(sidecar.requests[0]).toEqual(
|
||||
// Live passes are short: sequential decoding, no batching/VAD (D-WS5a).
|
||||
expect.objectContaining({ model: "base", language: "da", task: "translate", mode: "live" }),
|
||||
@@ -140,6 +142,21 @@ describe("createLiveWebSocket", () => {
|
||||
await waitFor(async () => (await readdir(tempDir)).length === 0);
|
||||
});
|
||||
|
||||
test("every pass of a session carries the session's vocabulary hint (D-WS2a)", async () => {
|
||||
const tempDir = await makeTempDir();
|
||||
const sidecar = fakeSidecar();
|
||||
const handler = createLiveWebSocket({ sidecar });
|
||||
const ws = fakeSocket(sessionData({ prompt: "Maxwell, Voices of the Void", tempDir }));
|
||||
handler.open(ws);
|
||||
handler.message(ws, toBuffer(speech(1)));
|
||||
handler.message(ws, toBuffer(silence(1.5)));
|
||||
await waitFor(() => sidecar.requests.length > 0);
|
||||
for (const request of sidecar.requests) {
|
||||
expect(request.prompt).toBe("Maxwell, Voices of the Void");
|
||||
}
|
||||
await waitFor(async () => (await readdir(tempDir)).length === 0);
|
||||
});
|
||||
|
||||
test("a sidecar error surfaces on the update stream instead of crashing the session", async () => {
|
||||
const sidecar = fakeSidecar({ error: "model load failed" });
|
||||
const handler = createLiveWebSocket({ sidecar });
|
||||
|
||||
@@ -168,6 +168,36 @@ describe("parseLanguageAndTask", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("parseLanguageAndTask: vocabulary hint (D-WS2a)", () => {
|
||||
const parse = (query: string) => parseLanguageAndTask(new URL(`http://localhost/x${query}`));
|
||||
|
||||
test("normalizes the hint like the worker does", () => {
|
||||
expect(parse("?prompt=Maxwell%0A%0AVoices%20of%20the%20Void%0A")).toEqual({
|
||||
language: undefined,
|
||||
task: "transcribe",
|
||||
prompt: "Maxwell, Voices of the Void",
|
||||
});
|
||||
});
|
||||
|
||||
test("absent, empty, or whitespace-only means no prompt", () => {
|
||||
for (const query of ["", "?prompt=", "?prompt=%20%0A%09"]) {
|
||||
const parsed = parse(query);
|
||||
expect(typeof parsed).toBe("object");
|
||||
expect("prompt" in (parsed as object)).toBe(false);
|
||||
}
|
||||
});
|
||||
|
||||
test("keeps non-ASCII names intact", () => {
|
||||
const params = new URLSearchParams({ prompt: "Århus, Ærø, Festuge" });
|
||||
expect(parse(`?${params}`)).toMatchObject({ prompt: "Århus, Ærø, Festuge" });
|
||||
});
|
||||
|
||||
test("rejects a hint longer than the UI allows", () => {
|
||||
expect(parse(`?prompt=${"a".repeat(1000)}`)).toMatchObject({ prompt: "a".repeat(1000) });
|
||||
expect(parse(`?prompt=${"a".repeat(1001)}`)).toBe("prompt too long (max 1000 characters)");
|
||||
});
|
||||
});
|
||||
|
||||
describe("GET /api/transcribe/status", () => {
|
||||
test("401 without a session", async () => {
|
||||
const handlers = createTranscribeHandlers({
|
||||
@@ -345,6 +375,35 @@ describe("POST /api/transcribe", () => {
|
||||
expect(text).toContain('"type":"result"');
|
||||
});
|
||||
|
||||
test("passes the vocabulary hint to the sidecar; 400 for an over-long one (D-WS2a)", async () => {
|
||||
const sidecar = fakeSidecar();
|
||||
const handlers = createTranscribeHandlers({
|
||||
sidecar,
|
||||
getSessionUser: sessionFromCookie,
|
||||
tempDir: await makeTempDir(),
|
||||
});
|
||||
const ok = new URLSearchParams({
|
||||
model: "onnx-community/whisper-base",
|
||||
prompt: "Århus\nÆrø",
|
||||
});
|
||||
const response = await handlers.post(
|
||||
postRequest(`?${ok}`, pcmBody([0.1, 0.2]), { cookie: "sid=ok" }),
|
||||
);
|
||||
expect(response.status).toBe(200);
|
||||
await readStreamLines(response);
|
||||
expect(sidecar.requests[0]!.prompt).toBe("Århus, Ærø");
|
||||
|
||||
const tooLong = await handlers.post(
|
||||
postRequest(
|
||||
`?model=onnx-community/whisper-base&prompt=${"x".repeat(1001)}`,
|
||||
pcmBody([0.1]),
|
||||
{ cookie: "sid=ok" },
|
||||
),
|
||||
);
|
||||
expect(tooLong.status).toBe(400);
|
||||
expect(sidecar.requests).toHaveLength(1);
|
||||
});
|
||||
|
||||
test("sidecar errors travel in-stream after the 200 header", async () => {
|
||||
const tempDir = await makeTempDir();
|
||||
const handlers = createTranscribeHandlers({
|
||||
@@ -413,6 +472,18 @@ describe("createSidecar (fake python sidecar)", () => {
|
||||
const live: SidecarSegment[] = [];
|
||||
await sidecar.transcribe({ model: "ok", audioPath, mode: "live" }, (s) => live.push(s));
|
||||
expect(live[1]!.text).toContain("mode=live");
|
||||
|
||||
// The fixture echoes what it received (audio bytes, language, task).
|
||||
expect(segments[1]!.text).toContain("model=ok bytes=12 lang=da task=translate");
|
||||
expect(segments[1]!.text).not.toContain("prompt=");
|
||||
|
||||
// The vocabulary hint crosses the JSON-lines protocol intact (D-WS2a).
|
||||
const prompted: SidecarSegment[] = [];
|
||||
await sidecar.transcribe(
|
||||
{ model: "ok", audioPath, prompt: "Århus, Voices of the Void" },
|
||||
(segment) => prompted.push(segment),
|
||||
);
|
||||
expect(prompted[1]!.text).toContain("prompt=Århus, Voices of the Void]");
|
||||
});
|
||||
|
||||
test("sidecar 'error' lines reject that request only", async () => {
|
||||
@@ -528,6 +599,18 @@ describe("server-py/sidecar.py (stubbed faster-whisper, D-WS5a)", () => {
|
||||
expect(implicit.texts[0]).toContain("batched model=m1");
|
||||
});
|
||||
|
||||
test("the vocabulary hint reaches the batched pipeline as text, and live passes as ids (D-WS2a, D-WS5a)", async () => {
|
||||
const sidecar = stubbedSidecar();
|
||||
const file = await run(sidecar, { model: "m1", mode: "file", prompt: "Maxwell, Voices of the Void" });
|
||||
expect(file.texts).toHaveLength(1);
|
||||
expect(file.texts[0]).toContain("batched model=m1");
|
||||
expect(file.texts[0]).toContain("initial_prompt= Maxwell, Voices of the Void.");
|
||||
|
||||
const live = await run(sidecar, { model: "m1", mode: "live", prompt: "Maxwell" });
|
||||
expect(live.texts[0]).toContain("seq model=m1");
|
||||
expect(live.texts[0]).toContain("initial_prompt=[32, 77, 97, 120, 119, 101, 108, 108, 46]");
|
||||
});
|
||||
|
||||
test("live mode stays sequential without VAD (short passes, D-WS5a)", async () => {
|
||||
const sidecar = stubbedSidecar();
|
||||
const { texts } = await run(sidecar, { model: "m1", mode: "live", task: "translate" });
|
||||
|
||||
Reference in New Issue
Block a user