From 74858f63fcdb8d4400ba193f0eb0d72f9252e69d Mon Sep 17 00:00:00 2001 From: Lucas Winther Date: Fri, 17 Jul 2026 15:18:15 +0200 Subject: [PATCH] Enhanced transcription usability. --- docs/Architecture.md | 10 ++++-- docs/Decisions.md | 41 +++++++++++++++++++++ isolate_wer2.ts | 15 ++++++++ src/App.tsx | 22 ++++++++++-- src/index.ts | 3 +- src/lib/live.ts | 83 +++++++++++++++++++++++++++++++++++------- tests/live.md | 5 ++- tests/live.test.ts | 86 +++++++++++++++++++++++++++++++++++++++++++- 8 files changed, 244 insertions(+), 21 deletions(-) create mode 100644 isolate_wer2.ts diff --git a/docs/Architecture.md b/docs/Architecture.md index bfb3734..36df6c9 100644 --- a/docs/Architecture.md +++ b/docs/Architecture.md @@ -147,9 +147,13 @@ are the remaining M5 work; see `WorkBreakdown.md`.) 3. The current segment is re-transcribed through the normal inference worker on a rolling basis (throttled, single-flight, latest-snapshot-wins) as *tentative* text; a committed segment gets one final pass and freezes. - `requestId` on the protocol messages correlates responses. -4. Stopping flushes the segmenter, drains pending final passes, and yields a - normal transcript (text + absolute-timestamped chunks) in `TranscriptView`. + When inference runs slower than speech, queued committed segments are + batched into merged final passes (one Whisper window each, D-024) so the + backlog stays bounded. `requestId` on the protocol messages correlates + responses. +4. Stopping flushes the segmenter, drains pending final passes (merged, with + a "N remaining segments" countdown in the UI), and yields a normal + transcript (text + absolute-timestamped chunks) in `TranscriptView`. Everything runs on-device; works offline like the file flow. Server-engine live input is a recorded future want (WorkBreakdown M7, D-016). diff --git a/docs/Decisions.md b/docs/Decisions.md index 21100e2..81862c1 100644 --- a/docs/Decisions.md +++ b/docs/Decisions.md @@ -396,3 +396,44 @@ validation failed. Buffer with '' label is invalid". The worker now sets fixes its thread pool at first-session init, so the first attempt in a worker decides; a wasm fallback after a failed accelerator attempt may run single-threaded until the next worker respawn — correct over fast. + +## D-024 — Live mode batches queued final passes into merged passes +**Status:** Accepted (2026-07-17) +Reported symptom: stopping a live session got stuck on "Finishing up…" while +the machine sat at full CPU — indistinguishable from a hang, and the saturated +cores (D-023 uses most of them) made the whole browser feel locked up. +**Cause — a scheduling backlog, not a deadlock.** Whisper pads every inference +pass to a full 30 s encoder window, so a pass costs nearly the same whether it +carries 1 s or 29 s of audio (measured on the browser runtime, ort-web WASM EP +medium q4: ~4 s per pass at 12 threads, ~9–12 s at 4 threads, for 1–9 s +segments alike). Segments therefore commit faster than their final passes +complete whenever the model is slower than speech (medium/large on WASM), the +committed-segment queue grows all session, and `stop()` used to drain it **one +pass per segment** — a 60 s session on 4 threads measured a 52 s drain; longer +sessions scale it into minutes. All the while the UI showed only a static +"Finishing up…". +**Fix (two parts):** +1. `LiveTranscriber.pump()` batches consecutive queued segments into **one + merged final pass**, capped at the segmenter's `maxSegmentSeconds` so a + merged pass stays a single Whisper window. Segment audio is concatenated + (segments end on trailing silence or a soft cut, so splices land in natural + gaps); the result's chunk timestamps are mapped piecewise back to absolute + session time via each segment's own start offset. The backlog — and the + stop drain — is now bounded by ~one pass per 29 s of speech instead of one + pass per segment (the same 60 s/4-thread scenario drains in ~2 passes). +2. `App.tsx` surfaces `LiveUpdate.pendingFinalPasses` while stopping: + "Finishing up… transcribing N remaining segments" counts down instead of + freezing, on both the browser and the server-live (D-019) paths. +**Alternatives rejected:** skipping final passes for segments that already had +tentative text (during a backlog new segments get *no* tentative pass, so +skipping loses their words entirely); a "stop now" escape hatch (same data +loss, silently); merging across the 30 s window (impossible — Whisper's +context is one window). +**Consequence:** a merged pass attributes its whole text to the batch's first +segment (per-segment text attribution is meaningless after joining anyway) and +inter-segment silence is not re-inserted, so chunk timestamps inside a merged +pass are exact per segment region but a chunk spanning a splice maps to its +start's region. Covered by `tests/live.test.ts` (batching + cap) and the WER +gates (`tests/live-accuracy.test.ts`, `tests/live-real-model.test.ts` — the +real-model test's fast feed makes real inference lag, so merging is exercised +there with real weights). diff --git a/isolate_wer2.ts b/isolate_wer2.ts new file mode 100644 index 0000000..d3dc5a5 --- /dev/null +++ b/isolate_wer2.ts @@ -0,0 +1,15 @@ +// TEMP (delete after use): which component's q4 hurts — encoder or decoder? +import { pipeline } from "@huggingface/transformers"; +const audio = new Float32Array(await Bun.file(process.argv[2]!).arrayBuffer()); +const S = "/tmp/claude-1000/-home-user-au-whisper-web/b192bada-c438-4a43-a29d-c2969ef71f6c/scratchpad"; +async function run(label: string, dtype: any) { + const asr = await pipeline("automatic-speech-recognition", "onnx-community/whisper-large-v3-turbo", { dtype, progress_callback: () => {} }); + const t0 = performance.now(); + const out = await asr(audio, { chunk_length_s: 30, stride_length_s: 5, return_timestamps: true, max_new_tokens: 440, language: "en", task: "transcribe" }); + const single = (Array.isArray(out) ? out[0] : out) as { text?: string }; + console.log(`${label}: ${((performance.now() - t0) / 1000).toFixed(0)}s`); + await Bun.write(`${S}/out-${label}.txt`, (single.text ?? "").trim()); + await asr.dispose(); +} +await run("encq8-decq4", { encoder_model: "q8", decoder_model_merged: "q4" }); +await run("encq4-decq8", { encoder_model: "q4", decoder_model_merged: "q8" }); diff --git a/src/App.tsx b/src/App.tsx index 21371fc..ead5c6a 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -104,6 +104,9 @@ export function App() { const [liveStatus, setLiveStatus] = useState("idle"); const [liveCommitted, setLiveCommitted] = useState(""); const [liveTentative, setLiveTentative] = useState(""); + // Committed segments still waiting for their final pass — surfaced while + // stopping so a long drain shows movement instead of a frozen label. + const [livePending, setLivePending] = useState(0); const [modelId, setModelId] = useState(DEFAULT_MODEL_ID); const [devicePreference, setDevicePreference] = useState( @@ -477,6 +480,7 @@ export function App() { setFileName(null); setLiveCommitted(""); setLiveTentative(""); + setLivePending(0); setLiveStatus("starting"); const useServer = @@ -499,6 +503,7 @@ export function App() { if (data.type === "update") { setLiveCommitted(data.update.committedText); setLiveTentative(data.update.tentativeText); + setLivePending(data.update.pendingFinalPasses ?? 0); if (data.update.error) setError(data.update.error); } else if (data.type === "result") { setResult(data.result); @@ -526,7 +531,15 @@ export function App() { try { const mic = await startMicCapture((chunk) => { if (ws.readyState === WebSocket.OPEN) { - ws.send(chunk.buffer); + // The byte view narrows the buffer typing to what send() + // accepts (same pattern as ServerEngine's request body). + ws.send( + new Uint8Array( + chunk.buffer as ArrayBuffer, + chunk.byteOffset, + chunk.byteLength, + ), + ); } }); liveWsRef.current = ws; @@ -562,6 +575,7 @@ export function App() { onUpdate: (update) => { setLiveCommitted(update.committedText); setLiveTentative(update.tentativeText); + setLivePending(update.pendingFinalPasses); if (update.error) setError(update.error); }, }); @@ -739,7 +753,11 @@ export function App() { } activeLabel={ liveStatus === "stopping" - ? "Finishing up…" + ? livePending > 0 + ? `Finishing up… transcribing ${livePending} remaining segment${ + livePending === 1 ? "" : "s" + }` + : "Finishing up…" : liveActive ? "Listening…" : undefined diff --git a/src/index.ts b/src/index.ts index da09cf2..3893dfa 100644 --- a/src/index.ts +++ b/src/index.ts @@ -6,6 +6,7 @@ import { auth } from "./server/auth.ts"; import { AUTH_ENABLED } from "./server/flags.ts"; import { MAX_UPLOAD_BYTES, transcribe } from "./server/transcribe.ts"; import { liveWebSocket } from "./server/live.ts"; +import { getFasterWhisperModel } from "./lib/models.ts"; const ROOT = join(import.meta.dir, ".."); const IS_PRODUCTION = process.env.NODE_ENV === "production"; @@ -129,7 +130,7 @@ async function handleApiRequest(request: Request, server: any): Promise void; private readonly tentativeIntervalMs: number; private readonly minTentativeSamples: number; + private readonly maxMergeSamples: number; private readonly sampleRate: number; private readonly now: () => number; @@ -423,6 +429,11 @@ export class LiveTranscriber { this.minTentativeSamples = Math.round( (options.minTentativeSeconds ?? 1) * this.sampleRate, ); + // Merged final passes stay within one Whisper window, like any single + // segment (the 25 mirrors LiveSegmenter's maxSegmentSeconds default). + this.maxMergeSamples = Math.round( + (options.segmenter?.maxSegmentSeconds ?? 25) * this.sampleRate, + ); this.now = options.now ?? Date.now; } @@ -491,9 +502,25 @@ export class LiveTranscriber { /** Single-flight scheduler: pick and start at most one inference pass. */ private pump(): void { if (this.inFlight) return; - const pending = this.segments.find((segment) => !segment.final); - if (pending) { - void this.runFinalPass(pending); + const first = this.segments.findIndex((segment) => !segment.final); + if (first !== -1) { + // Batch consecutive pending segments into one pass. Whisper pads every + // pass to a full 30 s window, so a pass costs nearly the same whether + // it carries 2 s or 25 s of audio — draining a backlog one segment at + // a time multiplied that fixed cost per segment and made stop() sit in + // "Finishing up…" for minutes at full CPU on slower machines. + const group: SegmentState[] = []; + let samples = 0; + for (let i = first; i < this.segments.length; i++) { + const segment = this.segments[i]!; + if (segment.final || segment.audio === null) break; + if (group.length > 0 && samples + segment.audio.length > this.maxMergeSamples) { + break; + } + group.push(segment); + samples += segment.audio.length; + } + void this.runFinalPass(group); return; } if (!this.stopped) { @@ -506,28 +533,58 @@ export class LiveTranscriber { for (const resolve of resolvers) resolve(); } - private async runFinalPass(segment: SegmentState): Promise { + /** + * One final pass over a batch of consecutive committed segments. Their + * audio is concatenated (segments end on trailing silence or a soft cut, + * so the splices land in natural gaps); the result's text goes on the + * first segment and its chunk timestamps are mapped piecewise back to + * absolute session time via each segment's own start offset. + */ + private async runFinalPass(group: SegmentState[]): Promise { this.inFlight = true; - const audio = segment.audio!; - segment.audio = null; // ownership moves to the transcribe fn - const offsetSeconds = segment.startSample / this.sampleRate; + const audios = group.map((segment) => segment.audio!); + for (const segment of group) segment.audio = null; // ownership moves on + const total = audios.reduce((sum, part) => sum + part.length, 0); + const audio = audios.length === 1 ? audios[0]! : concat(audios, total); + // Merged-audio second → absolute session second, per source segment. + const regions: { end: number; offset: number }[] = []; + let at = 0; + for (let i = 0; i < group.length; i++) { + regions.push({ + end: (at + audios[i]!.length) / this.sampleRate, + offset: (group[i]!.startSample - at) / this.sampleRate, + }); + at += audios[i]!.length; + } + const toAbsolute = (seconds: number): number => { + const region = + regions.find((r) => seconds < r.end) ?? regions[regions.length - 1]!; + return seconds + region.offset; + }; try { const result = await this.transcribe(audio); - segment.text = result.text; - segment.chunks = result.chunks.map( + const head = group[0]!; + head.text = result.text; + head.chunks = result.chunks.map( (chunk): TranscriptChunk => ({ text: chunk.text, timestamp: [ - chunk.timestamp[0] + offsetSeconds, - chunk.timestamp[1] === null ? null : chunk.timestamp[1] + offsetSeconds, + toAbsolute(chunk.timestamp[0]), + chunk.timestamp[1] === null ? null : toAbsolute(chunk.timestamp[1]), ], }), ); + // The whole batch's words live on the head segment; later members must + // not repeat their (now superseded) provisional text. + for (const segment of group.slice(1)) { + segment.text = ""; + segment.chunks = []; + } } catch (error) { - // Keep the provisional (tentative) text rather than losing the segment. + // Keep the provisional (tentative) text rather than losing the batch. this.error = error instanceof Error ? error.message : String(error); } - segment.final = true; + for (const segment of group) segment.final = true; this.inFlight = false; this.emit(); this.pump(); diff --git a/tests/live.md b/tests/live.md index 55b2aca..5d47577 100644 --- a/tests/live.md +++ b/tests/live.md @@ -67,7 +67,10 @@ Re-run it after any change to `src/lib/live.ts`, `src/lib/mic.ts`, ## 3. Stop & result 1. Click **Stop** mid-sentence. The button shows "Finishing…" while the last - segment gets its final pass, then the normal **Transcript** panel appears: + segment gets its final pass — if several segments are still queued (slow + model), the transcript heading counts them down ("Finishing up… + transcribing N remaining segments", D-024) — then the normal + **Transcript** panel appears: full text, Copy button, and a Timestamps section whose segment times are **absolute** (a sentence spoken ~10 s in must not be timestamped 0:00). 2. The browser's mic indicator must turn off immediately on Stop. diff --git a/tests/live.test.ts b/tests/live.test.ts index c877e39..526da4e 100644 --- a/tests/live.test.ts +++ b/tests/live.test.ts @@ -144,7 +144,13 @@ function fakeEngine() { advance(ms: number) { now += ms; }, - make(options: { tentativeIntervalMs?: number; minTentativeSeconds?: number } = {}) { + make( + options: { + tentativeIntervalMs?: number; + minTentativeSeconds?: number; + segmenter?: ConstructorParameters[0]; + } = {}, + ) { return new LiveTranscriber({ transcribe: (audio) => new Promise((resolve, reject) => { @@ -293,6 +299,84 @@ describe("LiveTranscriber", () => { expect(result.text).toBe("provisional recovered"); }); + test("queued final passes are batched into one merged pass", async () => { + const engine = fakeEngine(); + const live = engine.make({ minTentativeSeconds: 9999 }); // finals only + + // Segment 1 commits and its final pass starts (single flight). While it + // is in flight, segments 2 and 3 commit and queue behind it. + live.push(speech(1)); + live.push(silence(1.5)); + expect(engine.calls).toHaveLength(1); + live.push(speech(1)); + live.push(silence(1.5)); + live.push(speech(1)); + live.push(silence(1.5)); + expect(engine.last().pendingFinalPasses).toBe(3); + expect(engine.calls).toHaveLength(1); // still just the in-flight pass + + engine.calls[0]!.resolve({ text: " one.", chunks: [] }); + await tick(); + + // The queue drains as ONE merged pass carrying both segments' audio + // (each ≈ pre-roll + 1 s speech + 0.75 s commit silence). + expect(engine.calls).toHaveLength(2); + const merged = engine.calls[1]!; + expect(merged.audio.length).toBeGreaterThanOrEqual(3.4 * RATE); + expect(engine.last().pendingFinalPasses).toBe(2); + + // Chunk timestamps map piecewise back to absolute session time: segment 2 + // starts after segment 1 (> 1.5 s into the session), and a chunk in the + // second half of the merged audio lands even later, staying monotone. + const secondHalf = merged.audio.length / 2 / RATE; + merged.resolve({ + text: " two. three.", + chunks: [ + { text: " two.", timestamp: [0, 0.8] }, + { text: " three.", timestamp: [secondHalf, secondHalf + 0.8] }, + ], + }); + await tick(); + expect(engine.last().committedText).toBe("one. two. three."); + expect(engine.last().pendingFinalPasses).toBe(0); + const [first, second] = engine.last().chunks; + expect(first!.timestamp[0]).toBeGreaterThan(1.5); + expect(second!.timestamp[0]).toBeGreaterThan(first!.timestamp[1]!); + + const result = await live.stop(); + expect(result.text).toBe("one. two. three."); + }); + + test("merged passes stay under the segment length cap", async () => { + const engine = fakeEngine(); + // Tiny cap: segments are ~2 s each, so two of them exceed 3 s and must + // NOT merge into one pass. + const live = engine.make({ + minTentativeSeconds: 9999, + segmenter: { maxSegmentSeconds: 3 }, + }); + + live.push(speech(1)); + live.push(silence(1.5)); // segment 1 commits → pass 1 in flight + live.push(speech(1)); + live.push(silence(1.5)); // segment 2 queues + live.push(speech(1)); + live.push(silence(1.5)); // segment 3 queues + expect(engine.calls).toHaveLength(1); + + engine.calls[0]!.resolve({ text: " one.", chunks: [] }); + await tick(); + expect(engine.calls).toHaveLength(2); // segment 2 alone (cap reached) + expect(engine.calls[1]!.audio.length).toBeLessThanOrEqual(3 * RATE); + + engine.calls[1]!.resolve({ text: " two.", chunks: [] }); + await tick(); + expect(engine.calls).toHaveLength(3); + engine.calls[2]!.resolve({ text: " three.", chunks: [] }); + const result = await live.stop(); + expect(result.text).toBe("one. two. three."); + }); + test("a tentative result for an already-committed segment is discarded", async () => { const engine = fakeEngine(); const live = engine.make();