import { describe, test, expect } from "bun:test"; import { StreamingResampler } from "../src/lib/audio"; import { LIVE_TUNING, LiveTranscriber, type LiveTranscribeFn, type LiveTranscribeResult, type LiveUpdate, } from "../src/lib/live"; import { GAP_SECONDS, ORACLE_WORDS, RATE, WORD_SECONDS, buildSession, concatAudio, oracleDecode, silence, sine, wordErrorRate, wordFreq, } from "./dsp"; // End-to-end accuracy of the live pipeline (segmenter + scheduler + mic-path // resampler) measured as WER, using the deterministic tone-word oracle from // tests/dsp.ts instead of a model. Runs with the production LIVE_TUNING so // what we measure is what ships. See tests/live.md for what still needs a // real browser + real Whisper. function tick(): Promise { return new Promise((resolve) => setTimeout(resolve, 0)); } interface SessionRun { result: LiveTranscribeResult; updates: LiveUpdate[]; } /** * Stream `audio` through a LiveTranscriber in mic-sized chunks. The injected * clock advances with the audio fed, so the tentative-pass throttling behaves * exactly as it would in real time. */ async function runLiveSession( audio: Float32Array, transcribe: LiveTranscribeFn, chunkSeconds = 0.1, ): Promise { let fedSamples = 0; const updates: LiveUpdate[] = []; const live = new LiveTranscriber({ ...LIVE_TUNING, transcribe, onUpdate: (update) => updates.push(update), now: () => fedSamples / (RATE / 1000), }); const chunk = Math.round(chunkSeconds * RATE); for (let i = 0; i < audio.length; i += chunk) { live.push(audio.slice(i, Math.min(i + chunk, audio.length))); fedSamples = Math.min(i + chunk, audio.length); await tick(); // let in-flight passes resolve, as they would in real time } const result = await live.stop(); return { result, updates }; } const oracleEngine: LiveTranscribeFn = async (audio) => oracleDecode(audio); describe("live pipeline accuracy (oracle engine, production tuning)", () => { test("dictation with pauses: zero WER and accurate absolute timestamps", async () => { const session = buildSession( [ ["alpha", "bravo", "charlie"], ["delta", "echo", "foxtrot"], ["golf", "hotel", "alpha", "bravo"], ], { pauseSeconds: LIVE_TUNING.segmenter.commitSilenceSeconds + 0.6 }, ); const { result, updates } = await runLiveSession(session.audio, oracleEngine); expect(wordErrorRate(session.reference, result.text)).toBe(0); // Tentative text must have been visible at some point mid-utterance. expect(updates.some((u) => u.tentativeText.trim().length > 0)).toBe(true); // One chunk per word, timestamped in absolute session time: the oracle // reports times relative to each segment's audio and the transcriber // offsets by the segment's session position — quantization is ~20 ms // (oracle frames) + 32 ms (segmenter frames). expect(result.chunks).toHaveLength(session.truth.length); result.chunks.forEach((chunk, i) => { expect(chunk.text.trim()).toBe(session.truth[i]!.word); expect(Math.abs(chunk.timestamp[0] - session.truth[i]!.start)).toBeLessThan(0.15); }); }); test("continuous speech across the max-length cap: soft cuts keep every word intact", async () => { // ~40 s with only 80 ms inter-word gaps — silence never reaches the // 1.2 s commit threshold, so the session *must* cut at the 29 s cap at // least once. The soft cut has to land in a gap, not inside a word. // // Stretching the first word sweeps where the cap falls relative to the // word/gap cycle (segments start at speech onset, so a longer lead-in // would shift onset and cap together and test nothing). A hard cutter // can get lucky on one phase — the cap lands in a gap — but not on all // five: word+gap = 0.33 s, and the offsets cover the whole cycle. const words: string[] = []; const total = Math.ceil(40 / (WORD_SECONDS + GAP_SECONDS)); for (let i = 0; i < total; i++) words.push(ORACLE_WORDS[i % ORACLE_WORDS.length]!); for (let phase = 0; phase < 5; phase++) { const session = buildSession([words], { firstWordSeconds: WORD_SECONDS + phase * ((WORD_SECONDS + GAP_SECONDS) / 5), }); expect(session.audio.length).toBeGreaterThan( LIVE_TUNING.segmenter.maxSegmentSeconds * RATE, ); const { result } = await runLiveSession(session.audio, oracleEngine, 0.5); // A mid-word cut shows up as an insertion (two fragments) or a // deletion (both halves too short) — either way WER > 0. expect(wordErrorRate(session.reference, result.text)).toBe(0); expect(result.chunks).toHaveLength(words.length); } }, 30_000); test("48 kHz mic path: StreamingResampler preserves accuracy end to end", async () => { // Same dictation, but generated at 48 kHz and resampled in ~100 ms chunks // exactly like src/lib/mic.ts does. Aliasing, pitch shift, or dropped // samples at chunk boundaries would decode as wrong or missing words. const session = buildSession( [ ["hotel", "golf", "foxtrot"], ["echo", "delta", "charlie", "bravo"], ], { rate: 48_000, pauseSeconds: LIVE_TUNING.segmenter.commitSilenceSeconds + 0.6 }, ); const resampler = new StreamingResampler(48_000, RATE); let fedSamples = 0; const live = new LiveTranscriber({ ...LIVE_TUNING, transcribe: oracleEngine, onUpdate: () => {}, now: () => fedSamples / (RATE / 1000), }); const chunk = 4800; // 100 ms at the native rate for (let i = 0; i < session.audio.length; i += chunk) { const out = resampler.process( session.audio.slice(i, Math.min(i + chunk, session.audio.length)), ); if (out.length > 0) { live.push(out); fedSamples += out.length; } await tick(); } const tail = resampler.flush(); if (tail.length > 0) live.push(tail); const result = await live.stop(); expect(wordErrorRate(session.reference, result.text)).toBe(0); }); test("hallucinated text over a near-silent blip is dropped and cleared", async () => { // Whisper's classic failure mode: fed near-silence, it emits "Thank you." // The energy gate must keep that audio from ever reaching the committed // transcript, and the tentative flash of it must be cleared on drop. const hallucinating: LiveTranscribeFn = async () => ({ text: " Thank you.", chunks: [], }); const audio = concatAudio([ silence(0.5), sine(wordFreq(0), 0.12), // 120 ms blip < minSpeechSeconds 0.25 silence(2.5), ]); const { result, updates } = await runLiveSession(audio, hallucinating); // The blip opened a segment ≥ minTentativeSeconds, so a tentative pass // really ran and "Thank you." was briefly tentative… expect(updates.some((u) => u.tentativeText === " Thank you.")).toBe(true); // …but the drop cleared it, and nothing was ever committed. expect(updates[updates.length - 1]!.tentativeText).toBe(""); expect(result.text).toBe(""); expect(result.chunks).toHaveLength(0); }); test("stale tentative text from a dropped segment never leaks into the next commit", async () => { // Manual-resolution engine so the ordering is exact. const calls: { resolve: (result: LiveTranscribeResult) => void; }[] = []; const updates: LiveUpdate[] = []; let now = 0; const live = new LiveTranscriber({ ...LIVE_TUNING, transcribe: (audio) => new Promise((resolve) => { void audio; calls.push({ resolve }); }), onUpdate: (update) => updates.push(update), now: () => now, }); const last = () => updates[updates.length - 1]!; // A breathy blip keeps a segment open long enough for a tentative pass… live.push(silence(0.3)); live.push(sine(wordFreq(0), 0.15)); now += 450; live.push(silence(1.0)); now += 1000; expect(calls).toHaveLength(1); // tentative pass over mostly-silence audio calls[0]!.resolve({ text: " Thank you.", chunks: [] }); await tick(); expect(last().tentativeText).toBe(" Thank you."); // …then the segment is dropped as near-silence (0.15 s < 0.25 s speech). live.push(silence(0.5)); now += 500; expect(last().tentativeText).toBe(""); expect(last().committedText).toBe(""); // Real speech commits next: it must NOT adopt "Thank you." as its // provisional text while waiting for its final pass. live.push(sine(wordFreq(1), 0.6)); now += 600; live.push(silence(1.5)); now += 1500; expect(last().pendingFinalPasses).toBe(1); expect(last().committedText).toBe(""); // provisional text is empty, not stale calls[1]!.resolve({ text: " bravo", chunks: [] }); const result = await live.stop(); expect(result.text).toBe("bravo"); }); });