Add the vocabulary hint's normalization, storage and prompt truncation rule

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

src/lib/hint.ts

@@ -0,0 +1,108 @@
+/**
+ * The optional "Names and terms" vocabulary hint (D-WS2a): free text the user
+ * types once, remembered in this browser, and fed to Whisper as a decoder
+ * prompt (`<|startofprev|> hint …`) so names and jargon come out spelled the
+ * way the user spells them.
+ *
+ * Pure and dependency-free so the UI, the worker (src/worker/prompt.ts) and
+ * the server (src/server/transcribe.ts) share one normalization rule, and the
+ * truncation rule below has a Python twin in server-py/sidecar.py
+ * (`fit_prompt_ids`, parity-tested in tests/hint-parity.test.ts).
+ */
+
+import type { StorageLike } from "./engine";
+
+export const HINT_STORAGE_KEY = "whisper-vocabulary-hint";
+
+/**
+ * Longest raw hint accepted (textarea `maxLength`; the server answers 400
+ * above it). The hint travels as a URL query parameter on the server engine,
+ * so it must stay well below URL limits even fully percent-encoded.
+ */
+export const HINT_MAX_CHARS = 1000;
+
+/**
+ * Whisper's prompt budget: half its 448 decoder positions, minus one for
+ * `<|startofprev|>` — the cap openai-whisper and faster-whisper use too.
+ */
+export const PROMPT_TOKEN_BUDGET = 223;
+
+/** C0 controls + DEL, except \n and \r (line separators, handled below). Tab stays whitespace. */
+const CONTROL_CHARS = /[\u0000-\u0008\u000b\u000c\u000e-\u001f\u007f]/g;
+
+/**
+ * Canonical prompt text: one term per line or comma-separated, both become
+ * "a, b, c"; control characters dropped; whitespace collapsed. "" = no hint.
+ */
+export function normalizeHint(raw: string): string {
+  return raw
+    .replace(CONTROL_CHARS, "")
+    .split(/\r?\n|\r/)
+    .map((line) => line.replace(/\s+/g, " ").trim())
+    .filter((line) => line.length > 0)
+    .join(", ");
+}
+
+/** Saved hint (raw, as typed); "" when nothing is stored or storage is unavailable. */
+export function loadHint(storage: StorageLike | null = defaultStorage()): string {
+  try {
+    return storage?.getItem(HINT_STORAGE_KEY) ?? "";
+  } catch {
+    return "";
+  }
+}
+
+export function saveHint(hint: string, storage: StorageLike | null = defaultStorage()): void {
+  try {
+    storage?.setItem(HINT_STORAGE_KEY, hint);
+  } catch {
+    // Not persistable (private mode / quota) — the hint just won't stick.
+  }
+}
+
+function defaultStorage(): StorageLike | null {
+  try {
+    return (globalThis as { localStorage?: StorageLike }).localStorage ?? null;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Token ids for the hint as a Whisper prompt (without `<|startofprev|>`),
+ * truncated predictably to `budget` tokens (D-WS2a):
+ *
+ * 1. Encode `" " + normalizeHint(hint)` (Whisper prompts start with a space).
+ * 2. Over budget → keep the longest prefix of whole space-separated words that
+ *    fits, with a dangling "," / ";" removed: the start of the list is kept,
+ *    the end dropped, never mid-word.
+ * 3. Not even the first word fits → its first `budget` tokens.
+ *
+ * `encode` must add no special tokens. Mirrored by `fit_prompt_ids` in
+ * server-py/sidecar.py — change both together.
+ */
+export function fitHintToBudget(
+  hint: string,
+  encode: (text: string) => number[],
+  budget: number = PROMPT_TOKEN_BUDGET,
+): number[] {
+  const text = normalizeHint(hint);
+  if (text.length === 0) return [];
+  const full = encode(` ${text}`);
+  if (full.length <= budget) return full;
+
+  const words = text.split(" ");
+  const encodePrefix = (count: number) =>
+    encode(` ${words.slice(0, count).join(" ").replace(/[,;]+$/, "")}`);
+  // Largest word count whose encoding fits; token count grows with the
+  // prefix, so binary search needs O(log n) encodes even for huge pastes.
+  let low = 0;
+  let high = words.length - 1; // the full text is already known not to fit
+  while (low < high) {
+    const mid = Math.ceil((low + high) / 2);
+    if (encodePrefix(mid).length <= budget) low = mid;
+    else high = mid - 1;
+  }
+  if (low === 0) return encode(` ${words[0]}`).slice(0, budget);
+  return encodePrefix(low);
+}

tests/hint.test.ts

@@ -0,0 +1,109 @@
+import { describe, expect, test } from "bun:test";
+import {
+  HINT_STORAGE_KEY,
+  PROMPT_TOKEN_BUDGET,
+  fitHintToBudget,
+  loadHint,
+  normalizeHint,
+  saveHint,
+} from "../src/lib/hint";
+
+/** One "token" per UTF-16 unit: deterministic, monotone, easy to reason about. */
+const charEncode = (text: string) => [...text].map((c) => c.charCodeAt(0));
+const charDecode = (ids: number[]) => String.fromCharCode(...ids);
+
+function mapStorage(initial: Record<string, string> = {}) {
+  const map = new Map(Object.entries(initial));
+  return {
+    getItem: (key: string) => map.get(key) ?? null,
+    setItem: (key: string, value: string) => void map.set(key, value),
+    map,
+  };
+}
+
+const throwingStorage = {
+  getItem(): string | null {
+    throw new Error("denied");
+  },
+  setItem(): void {
+    throw new Error("denied");
+  },
+};
+
+describe("normalizeHint", () => {
+  test("one term per line becomes a comma-separated list", () => {
+    expect(normalizeHint("  Maxwell\n\n Voices  of the Void \n")).toBe(
+      "Maxwell, Voices of the Void",
+    );
+  });
+
+  test("a comma-separated list is unchanged", () => {
+    expect(normalizeHint("Maxwell, Voices of the Void")).toBe("Maxwell, Voices of the Void");
+  });
+
+  test("control characters are removed and whitespace-only means no hint", () => {
+    expect(normalizeHint("Max\u0000well\u0007")).toBe("Maxwell");
+    expect(normalizeHint(" \n\t \r\n")).toBe("");
+  });
+
+  test("Windows line endings and tabs are handled", () => {
+    expect(normalizeHint("Århus\r\nÆrø\tFestuge")).toBe("Århus, Ærø Festuge");
+  });
+});
+
+describe("hint storage", () => {
+  test("round-trips through storage", () => {
+    const storage = mapStorage();
+    expect(loadHint(storage)).toBe("");
+    saveHint("Maxwell\nVoid", storage);
+    expect(storage.map.get(HINT_STORAGE_KEY)).toBe("Maxwell\nVoid");
+    expect(loadHint(storage)).toBe("Maxwell\nVoid");
+  });
+
+  test("unavailable storage never throws", () => {
+    expect(loadHint(throwingStorage)).toBe("");
+    expect(() => saveHint("x", throwingStorage)).not.toThrow();
+    expect(loadHint(null)).toBe("");
+  });
+});
+
+describe("fitHintToBudget (D-WS2a truncation rule)", () => {
+  test("empty hint means no prompt", () => {
+    expect(fitHintToBudget("  \n", charEncode)).toEqual([]);
+  });
+
+  test("a hint within budget is encoded with Whisper's leading space", () => {
+    expect(fitHintToBudget("Maxwell\nVoices of the Void", charEncode)).toEqual(
+      charEncode(" Maxwell, Voices of the Void"),
+    );
+  });
+
+  test("over budget: keeps whole leading words and drops a dangling comma", () => {
+    expect(charDecode(fitHintToBudget("alpha, beta gamma", charEncode, 10))).toBe(" alpha");
+  });
+
+  test("a single word longer than the budget is cut at the budget", () => {
+    const ids = fitHintToBudget("x".repeat(50), charEncode, 10);
+    expect(ids).toEqual(charEncode(" xxxxxxxxx"));
+    expect(ids.length).toBe(10);
+  });
+
+  test("a very long hint fits the 223-token budget and is a word prefix of the hint", () => {
+    const words = Array.from({ length: 5000 }, (_, i) => `term${i}`);
+    const hint = words.join(", ");
+    const ids = fitHintToBudget(hint, charEncode);
+    expect(PROMPT_TOKEN_BUDGET).toBe(223);
+    expect(ids.length).toBeLessThanOrEqual(223);
+    expect(ids.length).toBeGreaterThan(200);
+    const kept = charDecode(ids);
+    expect(` ${hint}`.startsWith(kept)).toBe(true);
+    // Ends on a whole term: the next character of the hint is a separator.
+    expect(kept.endsWith(",")).toBe(false);
+    expect([",", " "]).toContain(` ${hint}`[kept.length]);
+  });
+
+  test("is deterministic", () => {
+    const hint = "Maxwell, ".repeat(200);
+    expect(fitHintToBudget(hint, charEncode)).toEqual(fitHintToBudget(hint, charEncode));
+  });
+});