diff --git a/src/lib/hint.ts b/src/lib/hint.ts new file mode 100644 index 0000000..f66a6fe --- /dev/null +++ b/src/lib/hint.ts @@ -0,0 +1,108 @@ +/** + * The optional "Names and terms" vocabulary hint (D-WS2a): free text the user + * types once, remembered in this browser, and fed to Whisper as a decoder + * prompt (`<|startofprev|> hint …`) so names and jargon come out spelled the + * way the user spells them. + * + * Pure and dependency-free so the UI, the worker (src/worker/prompt.ts) and + * the server (src/server/transcribe.ts) share one normalization rule, and the + * truncation rule below has a Python twin in server-py/sidecar.py + * (`fit_prompt_ids`, parity-tested in tests/hint-parity.test.ts). + */ + +import type { StorageLike } from "./engine"; + +export const HINT_STORAGE_KEY = "whisper-vocabulary-hint"; + +/** + * Longest raw hint accepted (textarea `maxLength`; the server answers 400 + * above it). The hint travels as a URL query parameter on the server engine, + * so it must stay well below URL limits even fully percent-encoded. + */ +export const HINT_MAX_CHARS = 1000; + +/** + * Whisper's prompt budget: half its 448 decoder positions, minus one for + * `<|startofprev|>` — the cap openai-whisper and faster-whisper use too. + */ +export const PROMPT_TOKEN_BUDGET = 223; + +/** C0 controls + DEL, except \n and \r (line separators, handled below). Tab stays whitespace. */ +const CONTROL_CHARS = /[\u0000-\u0008\u000b\u000c\u000e-\u001f\u007f]/g; + +/** + * Canonical prompt text: one term per line or comma-separated, both become + * "a, b, c"; control characters dropped; whitespace collapsed. "" = no hint. + */ +export function normalizeHint(raw: string): string { + return raw + .replace(CONTROL_CHARS, "") + .split(/\r?\n|\r/) + .map((line) => line.replace(/\s+/g, " ").trim()) + .filter((line) => line.length > 0) + .join(", "); +} + +/** Saved hint (raw, as typed); "" when nothing is stored or storage is unavailable. */ +export function loadHint(storage: StorageLike | null = defaultStorage()): string { + try { + return storage?.getItem(HINT_STORAGE_KEY) ?? ""; + } catch { + return ""; + } +} + +export function saveHint(hint: string, storage: StorageLike | null = defaultStorage()): void { + try { + storage?.setItem(HINT_STORAGE_KEY, hint); + } catch { + // Not persistable (private mode / quota) — the hint just won't stick. + } +} + +function defaultStorage(): StorageLike | null { + try { + return (globalThis as { localStorage?: StorageLike }).localStorage ?? null; + } catch { + return null; + } +} + +/** + * Token ids for the hint as a Whisper prompt (without `<|startofprev|>`), + * truncated predictably to `budget` tokens (D-WS2a): + * + * 1. Encode `" " + normalizeHint(hint)` (Whisper prompts start with a space). + * 2. Over budget → keep the longest prefix of whole space-separated words that + * fits, with a dangling "," / ";" removed: the start of the list is kept, + * the end dropped, never mid-word. + * 3. Not even the first word fits → its first `budget` tokens. + * + * `encode` must add no special tokens. Mirrored by `fit_prompt_ids` in + * server-py/sidecar.py — change both together. + */ +export function fitHintToBudget( + hint: string, + encode: (text: string) => number[], + budget: number = PROMPT_TOKEN_BUDGET, +): number[] { + const text = normalizeHint(hint); + if (text.length === 0) return []; + const full = encode(` ${text}`); + if (full.length <= budget) return full; + + const words = text.split(" "); + const encodePrefix = (count: number) => + encode(` ${words.slice(0, count).join(" ").replace(/[,;]+$/, "")}`); + // Largest word count whose encoding fits; token count grows with the + // prefix, so binary search needs O(log n) encodes even for huge pastes. + let low = 0; + let high = words.length - 1; // the full text is already known not to fit + while (low < high) { + const mid = Math.ceil((low + high) / 2); + if (encodePrefix(mid).length <= budget) low = mid; + else high = mid - 1; + } + if (low === 0) return encode(` ${words[0]}`).slice(0, budget); + return encodePrefix(low); +} diff --git a/tests/hint.test.ts b/tests/hint.test.ts new file mode 100644 index 0000000..fdfb5ed --- /dev/null +++ b/tests/hint.test.ts @@ -0,0 +1,109 @@ +import { describe, expect, test } from "bun:test"; +import { + HINT_STORAGE_KEY, + PROMPT_TOKEN_BUDGET, + fitHintToBudget, + loadHint, + normalizeHint, + saveHint, +} from "../src/lib/hint"; + +/** One "token" per UTF-16 unit: deterministic, monotone, easy to reason about. */ +const charEncode = (text: string) => [...text].map((c) => c.charCodeAt(0)); +const charDecode = (ids: number[]) => String.fromCharCode(...ids); + +function mapStorage(initial: Record = {}) { + const map = new Map(Object.entries(initial)); + return { + getItem: (key: string) => map.get(key) ?? null, + setItem: (key: string, value: string) => void map.set(key, value), + map, + }; +} + +const throwingStorage = { + getItem(): string | null { + throw new Error("denied"); + }, + setItem(): void { + throw new Error("denied"); + }, +}; + +describe("normalizeHint", () => { + test("one term per line becomes a comma-separated list", () => { + expect(normalizeHint(" Maxwell\n\n Voices of the Void \n")).toBe( + "Maxwell, Voices of the Void", + ); + }); + + test("a comma-separated list is unchanged", () => { + expect(normalizeHint("Maxwell, Voices of the Void")).toBe("Maxwell, Voices of the Void"); + }); + + test("control characters are removed and whitespace-only means no hint", () => { + expect(normalizeHint("Max\u0000well\u0007")).toBe("Maxwell"); + expect(normalizeHint(" \n\t \r\n")).toBe(""); + }); + + test("Windows line endings and tabs are handled", () => { + expect(normalizeHint("Århus\r\nÆrø\tFestuge")).toBe("Århus, Ærø Festuge"); + }); +}); + +describe("hint storage", () => { + test("round-trips through storage", () => { + const storage = mapStorage(); + expect(loadHint(storage)).toBe(""); + saveHint("Maxwell\nVoid", storage); + expect(storage.map.get(HINT_STORAGE_KEY)).toBe("Maxwell\nVoid"); + expect(loadHint(storage)).toBe("Maxwell\nVoid"); + }); + + test("unavailable storage never throws", () => { + expect(loadHint(throwingStorage)).toBe(""); + expect(() => saveHint("x", throwingStorage)).not.toThrow(); + expect(loadHint(null)).toBe(""); + }); +}); + +describe("fitHintToBudget (D-WS2a truncation rule)", () => { + test("empty hint means no prompt", () => { + expect(fitHintToBudget(" \n", charEncode)).toEqual([]); + }); + + test("a hint within budget is encoded with Whisper's leading space", () => { + expect(fitHintToBudget("Maxwell\nVoices of the Void", charEncode)).toEqual( + charEncode(" Maxwell, Voices of the Void"), + ); + }); + + test("over budget: keeps whole leading words and drops a dangling comma", () => { + expect(charDecode(fitHintToBudget("alpha, beta gamma", charEncode, 10))).toBe(" alpha"); + }); + + test("a single word longer than the budget is cut at the budget", () => { + const ids = fitHintToBudget("x".repeat(50), charEncode, 10); + expect(ids).toEqual(charEncode(" xxxxxxxxx")); + expect(ids.length).toBe(10); + }); + + test("a very long hint fits the 223-token budget and is a word prefix of the hint", () => { + const words = Array.from({ length: 5000 }, (_, i) => `term${i}`); + const hint = words.join(", "); + const ids = fitHintToBudget(hint, charEncode); + expect(PROMPT_TOKEN_BUDGET).toBe(223); + expect(ids.length).toBeLessThanOrEqual(223); + expect(ids.length).toBeGreaterThan(200); + const kept = charDecode(ids); + expect(` ${hint}`.startsWith(kept)).toBe(true); + // Ends on a whole term: the next character of the hint is a separator. + expect(kept.endsWith(",")).toBe(false); + expect([",", " "]).toContain(` ${hint}`[kept.length]); + }); + + test("is deterministic", () => { + const hint = "Maxwell, ".repeat(200); + expect(fitHintToBudget(hint, charEncode)).toEqual(fitHintToBudget(hint, charEncode)); + }); +});