Cap the hint prompt at 120 tokens: a full 223-token prompt garbles small models

Co-Authored-By: Claude Opus 5.5 <[email protected]>

docs/Architecture.md

@@ -149,7 +149,7 @@ environment). See `WorkBreakdown.md` for the itemized checklist.)
 5. Worker runs the pipeline (chunked, with timestamps), streaming partial results
    back via `{type:"partial"}` messages; final result as `{type:"result"}`.
    A hint becomes `<|startofprev|> hint.` once per pass (`worker/prompt.ts`,
-   cut to Whisper's 223-token prompt budget at a whole word) and is prepended
+   cut to a 120-token prompt budget at a whole word) and is prepended
    to the decoder input of every 30 s window by a wrapper around the model's
    `generate()`, which also strips it from the output again (D-WS2a).
    Live passes (4.1b) carry the same `prompt`.

server-py/sidecar.py

@@ -52,9 +52,9 @@ SAMPLE_RATE = 16000
 
 # "Names and terms" vocabulary hint (D-WS2a). Twins of normalizeHint /
 # fitHintToBudget in src/lib/hint.ts — change both together; parity is tested
-# in tests/hint-parity.test.ts. Whisper's prompt budget is half of its 448
-# decoder positions minus one (<|startofprev|>).
-PROMPT_TOKEN_BUDGET = 223
+# in tests/hint-parity.test.ts. Whisper would allow 223 tokens; 120 is the
+# measured cap at which small models are not garbled (D-WS2a).
+PROMPT_TOKEN_BUDGET = 120
 _CONTROL_CHARS = re.compile("[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]")
 
 

src/components/VocabularyHint.tsx

@@ -8,11 +8,11 @@ interface VocabularyHintProps {
 }
 
 /**
- * Past this many characters the hint likely exceeds Whisper's prompt budget
- * (223 tokens ≈ 150 English words) and gets cut at the end (D-WS2a) — say so
- * rather than silently ignoring the tail.
+ * Past this many characters the hint likely exceeds the 120-token prompt
+ * budget (a list of names runs ≈ 2.8 characters per token, measured) and gets
+ * cut at the end (D-WS2a) — say so rather than silently ignoring the tail.
  */
-const LONG_HINT_CHARS = 600;
+const LONG_HINT_CHARS = 300;
 
 /**
  * Optional "Names and terms" vocabulary hint (D-WS2a). Shown for every model;
@@ -41,7 +41,7 @@ export function VocabularyHint({ value, onChange, disabled = false }: Vocabulary
       />
       <p id={`${id}-help`} className="vocabulary-hint-help">
         {long
-          ? "That's a long list — only the first part (about 150 words) will be used."
+          ? "That's a long list — only the start of it will be used, so put the most important names first."
           : "Helps spell names, places and special words. Separate them with commas or new lines. Saved in this browser."}
       </p>
     </div>

src/lib/hint.ts

@@ -22,10 +22,14 @@ export const HINT_STORAGE_KEY = "whisper-vocabulary-hint";
 export const HINT_MAX_CHARS = 1000;
 
 /**
- * Whisper's prompt budget: half its 448 decoder positions, minus one for
- * `<|startofprev|>` — the cap openai-whisper and faster-whisper use too.
+ * Prompt budget in tokens (without `<|startofprev|>`). Whisper itself allows
+ * 223 (half its 448 decoder positions, minus one — the openai-whisper and
+ * faster-whisper cap), but measured on en_alien a prompt that long garbles
+ * small models: base.en WER 0.339 → 0.476 with a realistic 218-token list and
+ * → 1.05 with a synthetic one, while ≤ 120 tokens stays at or below the
+ * no-hint WER (D-WS2a).
  */
-export const PROMPT_TOKEN_BUDGET = 223;
+export const PROMPT_TOKEN_BUDGET = 120;
 
 /** C0 controls + DEL, except \n and \r (line separators, handled below). Tab stays whitespace. */
 const CONTROL_CHARS = /[\u0000-\u0008\u000b\u000c\u000e-\u001f\u007f]/g;

tests/hint-parity.test.ts

@@ -1,6 +1,6 @@
 import { describe, expect, test } from "bun:test";
 import { join } from "node:path";
-import { fitHintToBudget, normalizeHint } from "../src/lib/hint";
+import { fitHintToBudget, normalizeHint, PROMPT_TOKEN_BUDGET } from "../src/lib/hint";
 
 // D-WS2a: the server engine truncates the hint in Python (server-py/sidecar.py,
 // with faster-whisper's tokenizer) and the browser engine in TypeScript
@@ -34,11 +34,11 @@ sidecar = importlib.util.module_from_spec(spec)
 spec.loader.exec_module(sidecar)
 encode = lambda text: [ord(c) for c in text]
 cases = json.load(sys.stdin)
-print(json.dumps([
+print(json.dumps({"budget": sidecar.PROMPT_TOKEN_BUDGET, "results": [
     {"normalized": sidecar.normalize_hint(c["hint"]),
      "ids": sidecar.fit_prompt_ids(c["hint"], encode, c["budget"])}
     for c in cases
-]))
+]}))
 `;
 
 describe("hint truncation parity: TypeScript vs sidecar.py", () => {
@@ -49,8 +49,12 @@ describe("hint truncation parity: TypeScript vs sidecar.py", () => {
       stderr: "pipe",
     });
     expect(proc.stderr.toString()).toBe("");
-    const python = JSON.parse(proc.stdout.toString()) as { normalized: string; ids: number[] }[];
-    expect(python).toEqual(
+    const python = JSON.parse(proc.stdout.toString()) as {
+      budget: number;
+      results: { normalized: string; ids: number[] }[];
+    };
+    expect(python.budget).toBe(PROMPT_TOKEN_BUDGET);
+    expect(python.results).toEqual(
       CASES.map(({ hint, budget }) => ({
         normalized: normalizeHint(hint),
         ids: fitHintToBudget(hint, charEncode, budget),

tests/hint.test.ts

@@ -96,13 +96,13 @@ describe("fitHintToBudget (D-WS2a truncation rule)", () => {
     expect(ids.length).toBe(10);
   });
 
-  test("a very long hint fits the 223-token budget and is a word prefix of the hint", () => {
+  test("a very long hint fits the 120-token budget and is a word prefix of the hint", () => {
     const words = Array.from({ length: 5000 }, (_, i) => `term${i}`);
     const hint = words.join(", ");
     const ids = fitHintToBudget(hint, charEncode);
-    expect(PROMPT_TOKEN_BUDGET).toBe(223);
-    expect(ids.length).toBeLessThanOrEqual(223);
-    expect(ids.length).toBeGreaterThan(200);
+    expect(PROMPT_TOKEN_BUDGET).toBe(120);
+    expect(ids.length).toBeLessThanOrEqual(120);
+    expect(ids.length).toBeGreaterThan(100);
     const kept = charDecode(ids);
     // Ends on a whole term plus the full stop: the hint continues with a separator there.
     expect(kept.endsWith(".")).toBe(true);

tests/prompt.test.ts

@@ -117,8 +117,8 @@ describe("buildPromptPrefix", () => {
     ]);
   });
 
-  test("a huge hint stays within the 223-token budget", () => {
+  test("a huge hint stays within the 120-token budget (D-WS2a)", () => {
     const prefix = buildPromptPrefix(tokenizer, "Maxwell, ".repeat(500));
-    expect(prefix.length).toBeLessThanOrEqual(224);
+    expect(prefix.length).toBeLessThanOrEqual(121);
   });
 });