Enhanced transcription usability.

docs/Architecture.md

@@ -147,9 +147,13 @@ are the remaining M5 work; see `WorkBreakdown.md`.)
 3. The current segment is re-transcribed through the normal inference worker
    on a rolling basis (throttled, single-flight, latest-snapshot-wins) as
    *tentative* text; a committed segment gets one final pass and freezes.
-   `requestId` on the protocol messages correlates responses.
-4. Stopping flushes the segmenter, drains pending final passes, and yields a
-   normal transcript (text + absolute-timestamped chunks) in `TranscriptView`.
+   When inference runs slower than speech, queued committed segments are
+   batched into merged final passes (one Whisper window each, D-024) so the
+   backlog stays bounded. `requestId` on the protocol messages correlates
+   responses.
+4. Stopping flushes the segmenter, drains pending final passes (merged, with
+   a "N remaining segments" countdown in the UI), and yields a normal
+   transcript (text + absolute-timestamped chunks) in `TranscriptView`.
    Everything runs on-device; works offline like the file flow. Server-engine
    live input is a recorded future want (WorkBreakdown M7, D-016).
 

docs/Decisions.md

@@ -396,3 +396,44 @@ validation failed. Buffer with '' label is invalid". The worker now sets
 fixes its thread pool at first-session init, so the first attempt in a
 worker decides; a wasm fallback after a failed accelerator attempt may run
 single-threaded until the next worker respawn — correct over fast.
+
+## D-024 — Live mode batches queued final passes into merged passes
+**Status:** Accepted (2026-07-17)
+Reported symptom: stopping a live session got stuck on "Finishing up…" while
+the machine sat at full CPU — indistinguishable from a hang, and the saturated
+cores (D-023 uses most of them) made the whole browser feel locked up.
+**Cause — a scheduling backlog, not a deadlock.** Whisper pads every inference
+pass to a full 30 s encoder window, so a pass costs nearly the same whether it
+carries 1 s or 29 s of audio (measured on the browser runtime, ort-web WASM EP
+medium q4: ~4 s per pass at 12 threads, ~9–12 s at 4 threads, for 1–9 s
+segments alike). Segments therefore commit faster than their final passes
+complete whenever the model is slower than speech (medium/large on WASM), the
+committed-segment queue grows all session, and `stop()` used to drain it **one
+pass per segment** — a 60 s session on 4 threads measured a 52 s drain; longer
+sessions scale it into minutes. All the while the UI showed only a static
+"Finishing up…".
+**Fix (two parts):**
+1. `LiveTranscriber.pump()` batches consecutive queued segments into **one
+   merged final pass**, capped at the segmenter's `maxSegmentSeconds` so a
+   merged pass stays a single Whisper window. Segment audio is concatenated
+   (segments end on trailing silence or a soft cut, so splices land in natural
+   gaps); the result's chunk timestamps are mapped piecewise back to absolute
+   session time via each segment's own start offset. The backlog — and the
+   stop drain — is now bounded by ~one pass per 29 s of speech instead of one
+   pass per segment (the same 60 s/4-thread scenario drains in ~2 passes).
+2. `App.tsx` surfaces `LiveUpdate.pendingFinalPasses` while stopping:
+   "Finishing up… transcribing N remaining segments" counts down instead of
+   freezing, on both the browser and the server-live (D-019) paths.
+**Alternatives rejected:** skipping final passes for segments that already had
+tentative text (during a backlog new segments get *no* tentative pass, so
+skipping loses their words entirely); a "stop now" escape hatch (same data
+loss, silently); merging across the 30 s window (impossible — Whisper's
+context is one window).
+**Consequence:** a merged pass attributes its whole text to the batch's first
+segment (per-segment text attribution is meaningless after joining anyway) and
+inter-segment silence is not re-inserted, so chunk timestamps inside a merged
+pass are exact per segment region but a chunk spanning a splice maps to its
+start's region. Covered by `tests/live.test.ts` (batching + cap) and the WER
+gates (`tests/live-accuracy.test.ts`, `tests/live-real-model.test.ts` — the
+real-model test's fast feed makes real inference lag, so merging is exercised
+there with real weights).

isolate_wer2.ts

@@ -0,0 +1,15 @@
+// TEMP (delete after use): which component's q4 hurts — encoder or decoder?
+import { pipeline } from "@huggingface/transformers";
+const audio = new Float32Array(await Bun.file(process.argv[2]!).arrayBuffer());
+const S = "/tmp/claude-1000/-home-user-au-whisper-web/b192bada-c438-4a43-a29d-c2969ef71f6c/scratchpad";
+async function run(label: string, dtype: any) {
+  const asr = await pipeline("automatic-speech-recognition", "onnx-community/whisper-large-v3-turbo", { dtype, progress_callback: () => {} });
+  const t0 = performance.now();
+  const out = await asr(audio, { chunk_length_s: 30, stride_length_s: 5, return_timestamps: true, max_new_tokens: 440, language: "en", task: "transcribe" });
+  const single = (Array.isArray(out) ? out[0] : out) as { text?: string };
+  console.log(`${label}: ${((performance.now() - t0) / 1000).toFixed(0)}s`);
+  await Bun.write(`${S}/out-${label}.txt`, (single.text ?? "").trim());
+  await asr.dispose();
+}
+await run("encq8-decq4", { encoder_model: "q8", decoder_model_merged: "q4" });
+await run("encq4-decq8", { encoder_model: "q4", decoder_model_merged: "q8" });

src/App.tsx

@@ -104,6 +104,9 @@ export function App() {
   const [liveStatus, setLiveStatus] = useState<LiveStatus>("idle");
   const [liveCommitted, setLiveCommitted] = useState("");
   const [liveTentative, setLiveTentative] = useState("");
+  // Committed segments still waiting for their final pass — surfaced while
+  // stopping so a long drain shows movement instead of a frozen label.
+  const [livePending, setLivePending] = useState(0);
 
   const [modelId, setModelId] = useState(DEFAULT_MODEL_ID);
   const [devicePreference, setDevicePreference] = useState<DevicePreference>(
@@ -477,6 +480,7 @@ export function App() {
     setFileName(null);
     setLiveCommitted("");
     setLiveTentative("");
+    setLivePending(0);
     setLiveStatus("starting");
 
     const useServer =
@@ -499,6 +503,7 @@ export function App() {
         if (data.type === "update") {
           setLiveCommitted(data.update.committedText);
           setLiveTentative(data.update.tentativeText);
+          setLivePending(data.update.pendingFinalPasses ?? 0);
           if (data.update.error) setError(data.update.error);
         } else if (data.type === "result") {
           setResult(data.result);
@@ -526,7 +531,15 @@ export function App() {
         try {
           const mic = await startMicCapture((chunk) => {
             if (ws.readyState === WebSocket.OPEN) {
-              ws.send(chunk.buffer);
+              // The byte view narrows the buffer typing to what send()
+              // accepts (same pattern as ServerEngine's request body).
+              ws.send(
+                new Uint8Array(
+                  chunk.buffer as ArrayBuffer,
+                  chunk.byteOffset,
+                  chunk.byteLength,
+                ),
+              );
             }
           });
           liveWsRef.current = ws;
@@ -562,6 +575,7 @@ export function App() {
       onUpdate: (update) => {
         setLiveCommitted(update.committedText);
         setLiveTentative(update.tentativeText);
+        setLivePending(update.pendingFinalPasses);
         if (update.error) setError(update.error);
       },
     });
@@ -739,7 +753,11 @@ export function App() {
         }
         activeLabel={
           liveStatus === "stopping"
-            ? "Finishing up…"
+            ? livePending > 0
+              ? `Finishing up… transcribing ${livePending} remaining segment${
+                  livePending === 1 ? "" : "s"
+                }`
+              : "Finishing up…"
             : liveActive
               ? "Listening…"
               : undefined

src/index.ts

@@ -6,6 +6,7 @@ import { auth } from "./server/auth.ts";
 import { AUTH_ENABLED } from "./server/flags.ts";
 import { MAX_UPLOAD_BYTES, transcribe } from "./server/transcribe.ts";
 import { liveWebSocket } from "./server/live.ts";
+import { getFasterWhisperModel } from "./lib/models.ts";
 
 const ROOT = join(import.meta.dir, "..");
 const IS_PRODUCTION = process.env.NODE_ENV === "production";
@@ -129,7 +130,7 @@ async function handleApiRequest(request: Request, server: any): Promise<Response
       const url = new URL(request.url);
       const modelId = url.searchParams.get("model");
       if (!modelId) return new Response("Missing model", { status: 400 });
-      const fasterWhisperModel = transcribe.getFasterWhisperModel?.(modelId) || (await import("./lib/models.ts")).getFasterWhisperModel(modelId);
+      const fasterWhisperModel = getFasterWhisperModel(modelId);
       if (!fasterWhisperModel) return new Response("Unknown model", { status: 400 });
       
       const success = server.upgrade(request, {

src/lib/live.ts

@@ -393,6 +393,11 @@ interface SegmentState {
  * then throttled tentative refreshes of the current segment), and reports
  * every text change through `onUpdate`. Scheduling is driven entirely by
  * `push()` calls and pass completions — no timers, so it is fully testable.
+ *
+ * When inference is slower than speech, committed segments queue up faster
+ * than final passes complete; queued segments are batched into one merged
+ * pass (up to the segment length cap), which keeps the backlog — and the
+ * stop() drain — bounded (see runFinalPass).
  */
 export class LiveTranscriber {
   private readonly segmenter: LiveSegmenter;
@@ -400,6 +405,7 @@ export class LiveTranscriber {
   private readonly onUpdate: (update: LiveUpdate) => void;
   private readonly tentativeIntervalMs: number;
   private readonly minTentativeSamples: number;
+  private readonly maxMergeSamples: number;
   private readonly sampleRate: number;
   private readonly now: () => number;
 
@@ -423,6 +429,11 @@ export class LiveTranscriber {
     this.minTentativeSamples = Math.round(
       (options.minTentativeSeconds ?? 1) * this.sampleRate,
     );
+    // Merged final passes stay within one Whisper window, like any single
+    // segment (the 25 mirrors LiveSegmenter's maxSegmentSeconds default).
+    this.maxMergeSamples = Math.round(
+      (options.segmenter?.maxSegmentSeconds ?? 25) * this.sampleRate,
+    );
     this.now = options.now ?? Date.now;
   }
 
@@ -491,9 +502,25 @@ export class LiveTranscriber {
   /** Single-flight scheduler: pick and start at most one inference pass. */
   private pump(): void {
     if (this.inFlight) return;
-    const pending = this.segments.find((segment) => !segment.final);
-    if (pending) {
-      void this.runFinalPass(pending);
+    const first = this.segments.findIndex((segment) => !segment.final);
+    if (first !== -1) {
+      // Batch consecutive pending segments into one pass. Whisper pads every
+      // pass to a full 30 s window, so a pass costs nearly the same whether
+      // it carries 2 s or 25 s of audio — draining a backlog one segment at
+      // a time multiplied that fixed cost per segment and made stop() sit in
+      // "Finishing up…" for minutes at full CPU on slower machines.
+      const group: SegmentState[] = [];
+      let samples = 0;
+      for (let i = first; i < this.segments.length; i++) {
+        const segment = this.segments[i]!;
+        if (segment.final || segment.audio === null) break;
+        if (group.length > 0 && samples + segment.audio.length > this.maxMergeSamples) {
+          break;
+        }
+        group.push(segment);
+        samples += segment.audio.length;
+      }
+      void this.runFinalPass(group);
       return;
     }
     if (!this.stopped) {
@@ -506,28 +533,58 @@ export class LiveTranscriber {
     for (const resolve of resolvers) resolve();
   }
 
-  private async runFinalPass(segment: SegmentState): Promise<void> {
+  /**
+   * One final pass over a batch of consecutive committed segments. Their
+   * audio is concatenated (segments end on trailing silence or a soft cut,
+   * so the splices land in natural gaps); the result's text goes on the
+   * first segment and its chunk timestamps are mapped piecewise back to
+   * absolute session time via each segment's own start offset.
+   */
+  private async runFinalPass(group: SegmentState[]): Promise<void> {
     this.inFlight = true;
-    const audio = segment.audio!;
-    segment.audio = null; // ownership moves to the transcribe fn
-    const offsetSeconds = segment.startSample / this.sampleRate;
+    const audios = group.map((segment) => segment.audio!);
+    for (const segment of group) segment.audio = null; // ownership moves on
+    const total = audios.reduce((sum, part) => sum + part.length, 0);
+    const audio = audios.length === 1 ? audios[0]! : concat(audios, total);
+    // Merged-audio second → absolute session second, per source segment.
+    const regions: { end: number; offset: number }[] = [];
+    let at = 0;
+    for (let i = 0; i < group.length; i++) {
+      regions.push({
+        end: (at + audios[i]!.length) / this.sampleRate,
+        offset: (group[i]!.startSample - at) / this.sampleRate,
+      });
+      at += audios[i]!.length;
+    }
+    const toAbsolute = (seconds: number): number => {
+      const region =
+        regions.find((r) => seconds < r.end) ?? regions[regions.length - 1]!;
+      return seconds + region.offset;
+    };
     try {
       const result = await this.transcribe(audio);
-      segment.text = result.text;
-      segment.chunks = result.chunks.map(
+      const head = group[0]!;
+      head.text = result.text;
+      head.chunks = result.chunks.map(
         (chunk): TranscriptChunk => ({
           text: chunk.text,
           timestamp: [
-            chunk.timestamp[0] + offsetSeconds,
-            chunk.timestamp[1] === null ? null : chunk.timestamp[1] + offsetSeconds,
+            toAbsolute(chunk.timestamp[0]),
+            chunk.timestamp[1] === null ? null : toAbsolute(chunk.timestamp[1]),
           ],
         }),
       );
+      // The whole batch's words live on the head segment; later members must
+      // not repeat their (now superseded) provisional text.
+      for (const segment of group.slice(1)) {
+        segment.text = "";
+        segment.chunks = [];
+      }
     } catch (error) {
-      // Keep the provisional (tentative) text rather than losing the segment.
+      // Keep the provisional (tentative) text rather than losing the batch.
       this.error = error instanceof Error ? error.message : String(error);
     }
-    segment.final = true;
+    for (const segment of group) segment.final = true;
     this.inFlight = false;
     this.emit();
     this.pump();

tests/live.md

@@ -67,7 +67,10 @@ Re-run it after any change to `src/lib/live.ts`, `src/lib/mic.ts`,
 ## 3. Stop & result
 
 1. Click **Stop** mid-sentence. The button shows "Finishing…" while the last
-   segment gets its final pass, then the normal **Transcript** panel appears:
+   segment gets its final pass — if several segments are still queued (slow
+   model), the transcript heading counts them down ("Finishing up…
+   transcribing N remaining segments", D-024) — then the normal
+   **Transcript** panel appears:
    full text, Copy button, and a Timestamps section whose segment times are
    **absolute** (a sentence spoken ~10 s in must not be timestamped 0:00).
 2. The browser's mic indicator must turn off immediately on Stop.

tests/live.test.ts

@@ -144,7 +144,13 @@ function fakeEngine() {
     advance(ms: number) {
       now += ms;
     },
-    make(options: { tentativeIntervalMs?: number; minTentativeSeconds?: number } = {}) {
+    make(
+      options: {
+        tentativeIntervalMs?: number;
+        minTentativeSeconds?: number;
+        segmenter?: ConstructorParameters<typeof LiveSegmenter>[0];
+      } = {},
+    ) {
       return new LiveTranscriber({
         transcribe: (audio) =>
           new Promise<LiveTranscribeResult>((resolve, reject) => {
@@ -293,6 +299,84 @@ describe("LiveTranscriber", () => {
     expect(result.text).toBe("provisional recovered");
   });
 
+  test("queued final passes are batched into one merged pass", async () => {
+    const engine = fakeEngine();
+    const live = engine.make({ minTentativeSeconds: 9999 }); // finals only
+
+    // Segment 1 commits and its final pass starts (single flight). While it
+    // is in flight, segments 2 and 3 commit and queue behind it.
+    live.push(speech(1));
+    live.push(silence(1.5));
+    expect(engine.calls).toHaveLength(1);
+    live.push(speech(1));
+    live.push(silence(1.5));
+    live.push(speech(1));
+    live.push(silence(1.5));
+    expect(engine.last().pendingFinalPasses).toBe(3);
+    expect(engine.calls).toHaveLength(1); // still just the in-flight pass
+
+    engine.calls[0]!.resolve({ text: " one.", chunks: [] });
+    await tick();
+
+    // The queue drains as ONE merged pass carrying both segments' audio
+    // (each ≈ pre-roll + 1 s speech + 0.75 s commit silence).
+    expect(engine.calls).toHaveLength(2);
+    const merged = engine.calls[1]!;
+    expect(merged.audio.length).toBeGreaterThanOrEqual(3.4 * RATE);
+    expect(engine.last().pendingFinalPasses).toBe(2);
+
+    // Chunk timestamps map piecewise back to absolute session time: segment 2
+    // starts after segment 1 (> 1.5 s into the session), and a chunk in the
+    // second half of the merged audio lands even later, staying monotone.
+    const secondHalf = merged.audio.length / 2 / RATE;
+    merged.resolve({
+      text: " two. three.",
+      chunks: [
+        { text: " two.", timestamp: [0, 0.8] },
+        { text: " three.", timestamp: [secondHalf, secondHalf + 0.8] },
+      ],
+    });
+    await tick();
+    expect(engine.last().committedText).toBe("one. two. three.");
+    expect(engine.last().pendingFinalPasses).toBe(0);
+    const [first, second] = engine.last().chunks;
+    expect(first!.timestamp[0]).toBeGreaterThan(1.5);
+    expect(second!.timestamp[0]).toBeGreaterThan(first!.timestamp[1]!);
+
+    const result = await live.stop();
+    expect(result.text).toBe("one. two. three.");
+  });
+
+  test("merged passes stay under the segment length cap", async () => {
+    const engine = fakeEngine();
+    // Tiny cap: segments are ~2 s each, so two of them exceed 3 s and must
+    // NOT merge into one pass.
+    const live = engine.make({
+      minTentativeSeconds: 9999,
+      segmenter: { maxSegmentSeconds: 3 },
+    });
+
+    live.push(speech(1));
+    live.push(silence(1.5)); // segment 1 commits → pass 1 in flight
+    live.push(speech(1));
+    live.push(silence(1.5)); // segment 2 queues
+    live.push(speech(1));
+    live.push(silence(1.5)); // segment 3 queues
+    expect(engine.calls).toHaveLength(1);
+
+    engine.calls[0]!.resolve({ text: " one.", chunks: [] });
+    await tick();
+    expect(engine.calls).toHaveLength(2); // segment 2 alone (cap reached)
+    expect(engine.calls[1]!.audio.length).toBeLessThanOrEqual(3 * RATE);
+
+    engine.calls[1]!.resolve({ text: " two.", chunks: [] });
+    await tick();
+    expect(engine.calls).toHaveLength(3);
+    engine.calls[2]!.resolve({ text: " three.", chunks: [] });
+    const result = await live.stop();
+    expect(result.text).toBe("one. two. three.");
+  });
+
   test("a tentative result for an already-committed segment is discarded", async () => {
     const engine = fakeEngine();
     const live = engine.make();