From 104c1c4e4d363f0d2dcfb8257cd52f34252a61d9 Mon Sep 17 00:00:00 2001 From: Lucas Winther Date: Fri, 30 Oct 2026 00:23:55 +0100 Subject: [PATCH] Record the batched-VAD measurements and check heartbeat, bursts and cancel against the real sidecar Co-Authored-By: Claude Opus 5.5 --- docs/Architecture.md | 4 +- docs/Decisions.md | 45 +++++++++++++++++++++ docs/WorkBreakdown.md | 3 ++ tests/server-engine.md | 6 ++- tests/transcribe.test.ts | 87 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 143 insertions(+), 2 deletions(-) diff --git a/docs/Architecture.md b/docs/Architecture.md index ea5fc28..1fc6645 100644 --- a/docs/Architecture.md +++ b/docs/Architecture.md @@ -204,7 +204,9 @@ environment). See `WorkBreakdown.md` for the itemized checklist.) (`server-py/`) that Bun spawns and supervises (D-013; faster-whisper has no browser/WASM build, so it is server-only — the browser stays on Transformers.js). `src/server/transcribe.ts` bridges the sidecar's segment stream to the streamed - JSON-lines HTTP response (D-011). The shared catalog maps model ids to + JSON-lines HTTP response (D-011). File requests (`mode: "file"`) decode with + faster-whisper's batched pipeline over Silero-VAD speech chunks; server-live + passes send `mode: "live"` and stay sequential (D-WS5a). The shared catalog maps model ids to `Systran/faster-whisper-*` CTranslate2 conversions. If the sidecar is absent, the server engine reports unavailable; the browser/anonymous flow is unaffected. 4. Both engines implement the same `Engine` interface in `src/lib/engine.ts`, so the diff --git a/docs/Decisions.md b/docs/Decisions.md index 5dc8f08..2d2bc89 100644 --- a/docs/Decisions.md +++ b/docs/Decisions.md @@ -1224,3 +1224,48 @@ whole-file output is equally bad, which is exactly what the absolute bar exists to catch. **Consequence:** full `bun test` ≈ 19 min on CPU (en_alien ≈ 20 min of it on a cold run); the fast suite excludes this file. + +## D-WS5a — Server engine: batched faster-whisper + tuned Silero VAD for files; live passes stay sequential +**Status:** Accepted (2026-10-30, spec 001 Goal 5) +`server-py/sidecar.py` decoded every request with the sequential +`WhisperModel.transcribe`, no VAD. +**Decision:** the sidecar request gains `mode: "file" | "live"` (absent ⇒ +`"file"`, so an older Bun still gets the new path). `"file"` (`POST +/api/transcribe`) runs `BatchedInferencePipeline(model).transcribe(audio, +vad_filter=True, vad_parameters={threshold: 0.2, speech_pad_ms: 800}, +batch_size=8)`; the pipeline is a weightless wrapper cached and rebuilt with +the one resident model. `"live"` (server-live passes, `src/server/live.ts`) +keeps the sequential decoder. Beam size stays at the default 5. +**Measurements** (CPU int8, AMD Ryzen 9 9950X, 16 cores shared with other +jobs — load average 7–24 during runs, so wall-clock is ±20 %; WER via jiwer +on the `tests/live-real-model.test.ts` normalisation vs `tests/audio_files/*.txt`): + +| model | audio | config | wall-clock | WER | +|---|---|---|---|---| +| large-v3-turbo | en_alien (888 s) | sequential (before) | 97 s / 130 s (two runs) | 0.097 / 0.067 | +| large-v3-turbo | en_alien | sequential + VAD | 79 s | 0.136 | +| large-v3-turbo | en_alien | batched, default VAD (0.5 / 400 ms) | 47 s / 59 s | 0.117 | +| large-v3-turbo | en_alien | batched, beam 1 | 44 s | 0.114 | +| large-v3-turbo | en_alien | **batched, VAD 0.2 / 800 ms (after)** | **51 s / 58 s** | **0.086** | +| large-v3-turbo | en_alien | batched, VAD 0.2 / 800 ms, word-level timestamps | 56 s | 0.144 | +| large-v3-turbo | da_festuge (21.5 s, da) | sequential (before) | 2.9 s | 0.094 | +| large-v3-turbo | da_festuge | batched, default VAD | 2.9 s | 0.141 | +| large-v3-turbo | da_festuge | **batched, VAD 0.2 / 800 ms (after)** | **2.7 s** | **0.109** | +| medium.en | en_alien | sequential (before) | 114 s | 0.126 | +| medium.en | en_alien | **batched, VAD 0.2 / 800 ms (after)** | **67 s** | **0.111** | + +The sequential decoder is not deterministic run to run on en_alien (temperature +fallback; 0.067–0.097 over three runs), while the batched result repeated +exactly. Default-VAD errors were deletions (121 vs 64 sequential): Silero at +0.5 dropped quiet speech under music; 0.2 / 800 ms recovered most of them. +da_festuge's gap is one word of 64. Live-style 6 s passes over the first +180 s of en_alien: sequential 2.45 s/pass, batched 2.12, sequential + VAD +1.88 — within the load noise, each losing a few words, and real live +passes are already speech-gated client-side (D-016), so live stays as it was. +**Consequences:** file transcriptions are ≈ 1.7–2.5× faster on CPU at +equal-or-better WER on en_alien (one word worse on the 21 s Danish clip). Segments are coarser — one per VAD chunk (≤ 30 s) instead of one per +sentence — because word-level timestamps (`without_timestamps=False`) cost +WER here (0.144); SRT/VTT cues from the server engine are correspondingly +long. Segments arrive in bursts of 8 chunks (≈ 4 min of speech) and the first +burst can take ~15–25 s on a long file; the D-021 heartbeat covers it, and +cancel lands at the next burst boundary. No new env vars. diff --git a/docs/WorkBreakdown.md b/docs/WorkBreakdown.md index 336b0b4..ce4d042 100644 --- a/docs/WorkBreakdown.md +++ b/docs/WorkBreakdown.md @@ -145,6 +145,9 @@ Notes (M4): (CTranslate2) via a Python sidecar in `server-py/`, spawned/supervised by Bun (D-013 supersedes the Transformers.js-under-Bun plan); map the shared model catalog to faster-whisper model ids in `src/lib/models.ts` +- [x] Batched inference + tuned Silero VAD for server-engine file + transcriptions, live passes sequential (spec 001 Goal 5, D-WS5a): + ≈ 2× faster on CPU int8 at equal-or-better WER; segments ≤ 30 s chunks - [x] Degrade gracefully when Python/the sidecar is absent (server engine unavailable; browser engine untouched — anonymous path deploys with Bun alone) - [x] `POST /api/transcribe` — auth-guarded, streamed JSON-lines response (D-011), diff --git a/tests/server-engine.md b/tests/server-engine.md index 782b7f7..6ac7ba9 100644 --- a/tests/server-engine.md +++ b/tests/server-engine.md @@ -31,6 +31,9 @@ faster-whisper isn't installed for `python3` itself. Without faster-whisper at all, `bun test tests/transcribe.test.ts` still runs the real `server-py/sidecar.py` against a stub (`tests/fixtures/stub_fw`) to check the mode routing, the one-model cache and cancel between segments. +With `FW_INTEGRATION=1` it also pushes the whole 15-min `en_alien` clip +(`tiny.en`) through `POST /api/transcribe` — heartbeat lines before the first +burst, segments in bursts — and cancels a batched run after its first burst. - [ ] The `real faster-whisper sidecar` describe block passes. @@ -52,7 +55,8 @@ check the mode routing, the one-model cache and cancel between segments. batched pipeline (D-WS5a) they arrive in bursts — one burst per 8 speech chunks (≈ up to 4 min of audio) — with blank heartbeat lines every 5 s in between; on a clip under ~30 s of speech, everything - arrives as a single segment at the end. + arrives as a single segment at the end. Each segment covers one VAD + speech chunk (up to 30 s), not one sentence — expected (D-WS5a). - [ ] Cancelling a long server transcription (abort / page refresh) stops the sidecar within one burst; the next request then starts normally. - [ ] Both transcripts agree in content (minor punctuation/casing drift diff --git a/tests/transcribe.test.ts b/tests/transcribe.test.ts index 9b7a0ba..b8033b1 100644 --- a/tests/transcribe.test.ts +++ b/tests/transcribe.test.ts @@ -624,4 +624,91 @@ describe.skipIf(!process.env.FW_INTEGRATION)("real faster-whisper sidecar", () = }, { timeout: 300_000 }, ); + + // The whole 15-min en_alien clip (≈ 17 VAD chunks ⇒ 3 batches) as uploaded PCM. + async function wholeAlienPcm(): Promise { + const clip = join(import.meta.dir, "audio_files/en_alien.opus"); + const pcm = await Bun.$`ffmpeg -v error -i ${clip} -f f32le -ac 1 -ar 16000 -`.arrayBuffer(); + const audioPath = join(await makeTempDir(), "alien-full.f32"); + await Bun.write(audioPath, pcm); + return audioPath; + } + + test( + "a long file streams in bursts behind heartbeats over HTTP (D-021, D-WS5a)", + async () => { + const handlers = createTranscribeHandlers({ + sidecar: createSidecar(), + getSessionUser: sessionFromCookie, + tempDir: await makeTempDir(), + heartbeatIntervalMs: 100, + }); + const body = await Bun.file(await wholeAlienPcm()).arrayBuffer(); + const response = await handlers.post( + postRequest("?model=onnx-community/whisper-tiny.en&language=en", body, { + cookie: "sid=ok", + }), + ); + expect(response.status).toBe(200); + + // Timestamp every line as it arrives. + const started = performance.now(); + const lines: { at: number; line: string }[] = []; + const decoder = new TextDecoder(); + let buffered = ""; + for await (const chunk of response.body!) { + buffered += decoder.decode(chunk, { stream: true }); + let newline: number; + while ((newline = buffered.indexOf("\n")) >= 0) { + lines.push({ at: performance.now() - started, line: buffered.slice(0, newline) }); + buffered = buffered.slice(newline + 1); + } + } + const segmentLines = lines.filter(({ line }) => line.includes('"type":"segment"')); + const firstSegment = lines.findIndex(({ line }) => line.includes('"type":"segment"')); + // VAD over 15 min of audio + the first batch: heartbeats fill the gap. + expect(lines.slice(0, firstSegment).filter(({ line }) => line === "").length).toBeGreaterThan(0); + expect(segmentLines.length).toBeGreaterThan(3); + // Streamed, not one blob at the end: later bursts arrive measurably later. + expect(segmentLines.at(-1)!.at - segmentLines[0]!.at).toBeGreaterThan(100); + expect(lines.at(-1)!.line).toContain('"type":"result"'); + console.log( + "[D-WS5a] segment arrival (ms):", + segmentLines.map(({ at }) => Math.round(at)).join(" "), + ); + }, + { timeout: 600_000 }, + ); + + test( + "cancelling a batched file run stops it early and the next request still works", + async () => { + const sidecar = createSidecar(); + const audioPath = await wholeAlienPcm(); + const controller = new AbortController(); + const cancelledRun: SidecarSegment[] = []; + const outcome = await sidecar.transcribe( + { model: "tiny.en", audioPath, language: "en" }, + (segment) => { + cancelledRun.push(segment); + controller.abort(); + }, + controller.signal, + ); + expect(outcome.cancelled).toBe(true); + // At most the first burst (8 chunks) got out; the full clip is ≈ 17 chunks. + expect(cancelledRun.length).toBeGreaterThan(0); + expect(cancelledRun.length).toBeLessThanOrEqual(8); + expect(cancelledRun.at(-1)!.end).toBeLessThan(800); + + const again: SidecarSegment[] = []; + const next = await sidecar.transcribe( + { model: "tiny.en", audioPath, language: "en" }, + (segment) => again.push(segment), + ); + expect(next.cancelled).toBe(false); + expect(again.length).toBeGreaterThan(cancelledRun.length); + }, + { timeout: 600_000 }, + ); });