diff --git a/Decisions.md b/Decisions.md index 2d8c8d8..bf18d45 100644 --- a/Decisions.md +++ b/Decisions.md @@ -174,19 +174,34 @@ up-to-30 s windows), so "live" is approximated by a **segment loop**: - **Capture (main thread):** `getUserMedia` → `AudioContext` at the device's native rate → an `AudioWorklet` chunker posts ~100 ms Float32 frames to the - main thread, which resamples them to 16 kHz with the existing pure - `resampleLinear` (D-012's documented fallback; adequate for speech). - The worklet module is loaded from a **Blob URL** built from an inline code - string — no extra served file, no build/service-worker changes, and it keeps - working offline. Where `audioWorklet`/Blob modules are unavailable - (older Safari), fall back to a `ScriptProcessorNode`. The context is *not* - forced to 16 kHz because Firefox cannot connect a `MediaStreamAudioSourceNode` - across sample rates. + main thread, which resamples them to 16 kHz with the pure, stateful + `StreamingResampler` (windowed-sinc low-pass; `resampleLinear` has no + anti-aliasing filter, so 48 kHz → 16 kHz decimation folded everything above + 8 kHz into the speech band and measurably hurt accuracy — see + `tests/resampler.test.ts`). The worklet module is loaded from a **Blob URL** + built from an inline code string — no extra served file, no + build/service-worker changes, and it keeps working offline. Where + `audioWorklet`/Blob modules are unavailable (older Safari), fall back to a + `ScriptProcessorNode`. The context is *not* forced to 16 kHz because Firefox + cannot connect a `MediaStreamAudioSourceNode` across sample rates. - **Segmentation (pure, unit-tested):** a simple RMS energy gate (`src/lib/live.ts`) splits speech into segments: a segment starts at speech onset (with a short pre-roll), commits after a run of trailing silence or at - a max length (~25 s, under Whisper's 30 s window), and pure-silence audio is + a max length (under Whisper's 30 s window), and pure-silence audio is dropped entirely — Whisper hallucinates on silence, so never transcribe it. + Two accuracy refinements (2026-07-14): the gate has **hysteresis** (once a + segment is open, frames stay "speech" down to half the onset threshold, so + quiet phonemes don't fragment utterances), and a max-length commit is a + **soft cut** — it splits at the last sub-threshold frame in the trailing + window instead of mid-word at the hard cap, seeding the next segment with + the remainder (a mid-word cut corrupts the boundary word in both segments). + Dropped (near-silent) segments also clear any tentative text they produced, + so a hallucinated tentative pass can neither linger on screen nor be adopted + as the next segment's provisional text. Production tuning lives in + `LIVE_TUNING` (`src/lib/live.ts`), shared by `src/App.tsx` and the accuracy + harness (`tests/live-accuracy.test.ts` — tone-word oracle engine + WER, so + segmentation/scheduling/resampling accuracy is regression-tested under + `bun test` without a model). No ML VAD (e.g. Silero) until the energy gate proves insufficient in real use. - **Inference (existing worker, serialized):** the current segment is re-transcribed through the **existing** inference worker/pipeline on a diff --git a/src/App.tsx b/src/App.tsx index 70e45a1..1aeedef 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -9,7 +9,7 @@ import { ModelPicker } from "./components/ModelPicker"; import { TranscriptView } from "./components/TranscriptView"; import { TranscriptionOptions } from "./components/TranscriptionOptions"; import { decodeAudioFile } from "./lib/audio"; -import { LiveTranscriber, type LiveTranscribeResult } from "./lib/live"; +import { LIVE_TUNING, LiveTranscriber, type LiveTranscribeResult } from "./lib/live"; import { startMicCapture, type MicCapture } from "./lib/mic"; import { getDownloadedModels } from "./lib/modelCache"; import { registerServiceWorker } from "./pwa/register"; @@ -297,9 +297,7 @@ export function App() { // live. `.en` models take neither (same rule as the file flow). const options = (getModel(modelId)?.multilingual ?? false) ? { language, task } : {}; const transcriber = new LiveTranscriber({ - tentativeIntervalMs: 2500, - minTentativeSeconds: 2.0, - segmenter: { commitSilenceSeconds: 2.5, maxSegmentSeconds: 29 }, + ...LIVE_TUNING, transcribe: (audio) => transcribeLiveSegment(worker, audio, options), onUpdate: (update) => { setLiveCommitted(update.committedText); diff --git a/src/index.ts b/src/index.ts index 63e51e1..99d26a3 100644 --- a/src/index.ts +++ b/src/index.ts @@ -203,12 +203,12 @@ async function developmentServeOptions(): Promise { } const iconsMatch = pathname.match(/\/icons\/(icon-\d+\.png)$/); - if (iconsMatch) { + if (iconsMatch?.[1]) { return serveFile(join(ICONS_DIR, iconsMatch[1])); } const ortMatch = pathname.match(/\/ort\/(.*(?:wasm|mjs))$/); - if (ortMatch) { + if (ortMatch?.[1]) { return serveOrtFile(ORT_DIST, ortMatch[1]); } diff --git a/src/lib/audio.ts b/src/lib/audio.ts index 1f85a02..7ce173c 100644 --- a/src/lib/audio.ts +++ b/src/lib/audio.ts @@ -75,6 +75,116 @@ export function resampleLinear( return out; } +/** + * Streaming anti-aliased resampler (windowed-sinc low-pass interpolation). + * Pure — safe to call anywhere. + * + * `resampleLinear` has no low-pass filter, so decimating mic audio + * (48 kHz → 16 kHz) folds everything above 8 kHz back into the speech band — + * audible as aliasing noise and measurably bad for Whisper accuracy. This + * class filters at the correct cutoff while resampling, and it is stateful: + * feed it arbitrary chunk sizes (the mic path delivers ~100 ms blocks) and + * the output is identical to resampling the whole signal at once — no edge + * artifacts at chunk boundaries. + */ +export class StreamingResampler { + /** Input samples advanced per output sample. */ + private readonly step: number; + /** Kernel half-width, in input samples. */ + private readonly half: number; + /** Normalized low-pass cutoff, cycles per input sample. */ + private readonly cutoff: number; + private readonly passthrough: boolean; + + /** Unconsumed input tail (kernel needs `half` samples of look-ahead). */ + private pending = new Float32Array(0); + /** Fractional index into `pending` of the next output sample's center. */ + private position = 0; + + constructor(fromRate: number, toRate: number) { + if (!(fromRate > 0) || !(toRate > 0)) { + throw new Error("StreamingResampler: sample rates must be positive"); + } + this.passthrough = fromRate === toRate; + this.step = fromRate / toRate; + // 0.45×Nyquist of the slower rate: keeps the whole speech band while the + // transition band stays inside what the window can attenuate. + this.cutoff = (0.45 * Math.min(fromRate, toRate)) / fromRate; + this.half = Math.ceil(2 / this.cutoff); + } + + /** Blackman-windowed sinc, low-pass at `cutoff`, support (−half, half). */ + private kernel(distance: number): number { + const scaled = 2 * this.cutoff * distance; + const x = Math.PI * scaled; + const sinc = scaled === 0 ? 1 : Math.sin(x) / x; + const t = (distance / this.half + 1) / 2; // 0..1 across the support + const window = + 0.42 - 0.5 * Math.cos(2 * Math.PI * t) + 0.08 * Math.cos(4 * Math.PI * t); + return 2 * this.cutoff * sinc * window; + } + + /** Emit output samples for every center that has full kernel support. */ + private produce(buffer: Float32Array, limit: number): Float32Array { + const count = Math.max( + 0, + Math.floor((limit - this.position) / this.step) + 1, + ); + const out = new Float32Array(count); + for (let i = 0; i < count; i++) { + const center = this.position + i * this.step; + const from = Math.max(0, Math.ceil(center - this.half)); + const to = Math.min(buffer.length - 1, Math.floor(center + this.half)); + let acc = 0; + let norm = 0; + for (let j = from; j <= to; j++) { + const weight = this.kernel(j - center); + acc += buffer[j]! * weight; + norm += weight; + } + // Normalizing keeps unity gain exact despite kernel truncation. + out[i] = norm !== 0 ? acc / norm : 0; + } + this.position += count * this.step; + return out; + } + + /** Resample the next chunk; may return 0 samples while buffering. */ + process(chunk: Float32Array): Float32Array { + if (this.passthrough) return chunk; + let buffer: Float32Array; + if (this.pending.length > 0) { + buffer = new Float32Array(this.pending.length + chunk.length); + buffer.set(this.pending, 0); + buffer.set(chunk, this.pending.length); + } else { + buffer = chunk; + } + // Only centers whose kernel fits entirely in the buffered input. + const out = this.produce(buffer, buffer.length - 1 - this.half); + const keepFrom = Math.max(0, Math.floor(this.position) - this.half); + this.pending = buffer.slice(keepFrom); + this.position -= keepFrom; + return out; + } + + /** Drain the buffered tail (end of stream) and reset. */ + flush(): Float32Array { + if (this.passthrough || this.pending.length === 0) { + this.pending = new Float32Array(0); + this.position = 0; + return new Float32Array(0); + } + const limit = this.pending.length - 1; + const padded = new Float32Array(this.pending.length + this.half + 1); + padded.set(this.pending, 0); + const out = this.produce(padded, limit); + this.pending = new Float32Array(0); + this.position = 0; + return out; + } +} + /** * Downmix + resample a decoded `AudioBuffer` to 16 kHz mono using * `OfflineAudioContext` (D-012). Browser-only. diff --git a/src/lib/live.ts b/src/lib/live.ts index 0ff5f17..979c440 100644 --- a/src/lib/live.ts +++ b/src/lib/live.ts @@ -45,6 +45,12 @@ export interface LiveSegmenterOptions { frameSeconds?: number; /** A frame at/above this RMS counts as speech. Default 0.015. */ speechRms?: number; + /** + * Hysteresis: once a segment is open, frames stay "speech" down to + * `speechRms × sustainRmsRatio`, so quiet phonemes and word tails don't + * count as silence and fragment the utterance. Default 0.5. + */ + sustainRmsRatio?: number; /** Trailing silence that commits the current segment. Default 0.75 s. */ commitSilenceSeconds?: number; /** @@ -52,6 +58,13 @@ export interface LiveSegmenterOptions { * segment is always a single-chunk transcription. Default 25 s. */ maxSegmentSeconds?: number; + /** + * When a segment hits the max length, prefer to cut at the quietest gap + * within this trailing window instead of mid-word at the hard cap; the + * remainder after the gap seeds the next segment. Cutting mid-word corrupts + * the boundary word in *both* segments. Default 3 s. + */ + softCutWindowSeconds?: number; /** Audio kept from just before speech onset. Default 0.3 s. */ prerollSeconds?: number; /** @@ -65,6 +78,8 @@ export type CommitReason = "silence" | "max-length" | "flush"; /** One committed speech segment, ready for a final transcription pass. */ export interface CommittedSegment { + /** The segmenter's id for this segment (matches `currentSegmentId`). */ + id: number; audio: Float32Array; /** Absolute offset of `audio[0]` in samples since the session started. */ startSample: number; @@ -75,6 +90,9 @@ interface Frame { samples: Float32Array; /** Absolute offset of this frame's first sample. */ start: number; + rms: number; + /** Classified as speech when appended (onset/sustain threshold applied). */ + speech: boolean; } /** @@ -85,8 +103,10 @@ export class LiveSegmenter { private readonly sampleRate: number; private readonly frameSamples: number; private readonly speechRms: number; + private readonly sustainRms: number; private readonly commitSilenceFrames: number; private readonly maxSegmentSamples: number; + private readonly softCutWindowFrames: number; private readonly maxPrerollFrames: number; private readonly minSpeechSamples: number; @@ -109,6 +129,7 @@ export class LiveSegmenter { const frameSeconds = options.frameSeconds ?? 0.032; this.frameSamples = Math.max(1, Math.round(frameSeconds * this.sampleRate)); this.speechRms = options.speechRms ?? 0.015; + this.sustainRms = this.speechRms * (options.sustainRmsRatio ?? 0.5); this.commitSilenceFrames = Math.max( 1, Math.ceil((options.commitSilenceSeconds ?? 0.75) / frameSeconds), @@ -116,6 +137,10 @@ export class LiveSegmenter { this.maxSegmentSamples = Math.round( (options.maxSegmentSeconds ?? 25) * this.sampleRate, ); + this.softCutWindowFrames = Math.max( + 1, + Math.ceil((options.softCutWindowSeconds ?? 3) / frameSeconds), + ); this.maxPrerollFrames = Math.ceil((options.prerollSeconds ?? 0.3) / frameSeconds); this.minSpeechSamples = Math.round( (options.minSpeechSeconds ?? 0.25) * this.sampleRate, @@ -157,13 +182,16 @@ export class LiveSegmenter { let offset = 0; while (buffer.length - offset >= this.frameSamples) { // Copy the frame — the caller may reuse/transfer its buffer. + const samples = buffer.slice(offset, offset + this.frameSamples); const frame: Frame = { - samples: buffer.slice(offset, offset + this.frameSamples), + samples, start: this.absSample, + rms: rms(samples), + speech: false, // classified in processFrame (threshold is stateful) }; offset += this.frameSamples; this.absSample += this.frameSamples; - const segment = this.processFrame(frame, rms(frame.samples) >= this.speechRms); + const segment = this.processFrame(frame); if (segment) committed.push(segment); } this.carry = buffer.slice(offset); @@ -173,8 +201,13 @@ export class LiveSegmenter { /** Commit whatever is in progress (session stop). */ flush(): CommittedSegment | null { if (this.segment && this.carry.length > 0) { - // Sub-frame tail — too short to classify, but part of the speech. - this.segment.push({ samples: this.carry, start: this.absSample }); + // Sub-frame tail — too short to gate reliably, but part of the speech. + this.segment.push({ + samples: this.carry, + start: this.absSample, + rms: rms(this.carry), + speech: false, + }); this.segmentSamples += this.carry.length; this.absSample += this.carry.length; this.carry = new Float32Array(0); @@ -182,13 +215,14 @@ export class LiveSegmenter { return this.commit("flush"); } - private processFrame(frame: Frame, speech: boolean): CommittedSegment | null { + private processFrame(frame: Frame): CommittedSegment | null { if (!this.segment) { - if (!speech) { + if (frame.rms < this.speechRms) { this.preroll.push(frame); if (this.preroll.length > this.maxPrerollFrames) this.preroll.shift(); return null; } + frame.speech = true; this.segment = [...this.preroll, frame]; this.segmentStart = this.segment[0]!.start; this.segmentSamples = this.segment.length * this.frameSamples; @@ -199,25 +233,73 @@ export class LiveSegmenter { return null; } + // Hysteresis: inside a segment the bar is lower, so quiet word tails and + // soft phonemes extend the utterance instead of counting as silence. + frame.speech = frame.rms >= this.sustainRms; this.segment.push(frame); this.segmentSamples += this.frameSamples; - if (speech) { + if (frame.speech) { this.speechSamples += this.frameSamples; this.silenceFrames = 0; } else { this.silenceFrames += 1; if (this.silenceFrames >= this.commitSilenceFrames) return this.commit("silence"); } - if (this.segmentSamples >= this.maxSegmentSamples) return this.commit("max-length"); + if (this.segmentSamples >= this.maxSegmentSamples) return this.softCommitAtMaxLength(); return null; } + /** + * Max-length commit: cut at the last quiet frame in the trailing window so + * the boundary lands in a real gap, and seed the next segment with the + * remainder. Continuous loud speech (no quiet frame) hard-cuts at the cap. + */ + private softCommitAtMaxLength(): CommittedSegment | null { + const frames = this.segment!; + let cut = -1; + const windowStart = Math.max(1, frames.length - this.softCutWindowFrames); + for (let i = frames.length - 1; i >= windowStart; i--) { + if (frames[i]!.rms < this.sustainRms) { + cut = i; + break; + } + } + if (cut === -1 || cut === frames.length - 1) return this.commit("max-length"); + + const head = frames.slice(0, cut + 1); + const tail = frames.slice(cut + 1); + const headSpeech = head.filter((frame) => frame.speech).length * this.frameSamples; + const result: CommittedSegment | null = + headSpeech >= this.minSpeechSamples + ? { + id: this.segmentId!, + audio: concat( + head.map((frame) => frame.samples), + head.length * this.frameSamples, + ), + startSample: this.segmentStart, + reason: "max-length", + } + : null; // near-silence head: drop it, keep the live tail (D-016) + + this.segment = tail; + this.segmentStart = tail[0]!.start; + this.segmentSamples = tail.length * this.frameSamples; + this.speechSamples = tail.filter((frame) => frame.speech).length * this.frameSamples; + let trailingSilence = 0; + for (let i = tail.length - 1; i >= 0 && !tail[i]!.speech; i--) trailingSilence++; + this.silenceFrames = trailingSilence; + this.segmentId = this.nextSegmentId++; + return result; + } + private commit(reason: CommitReason): CommittedSegment | null { const segment = this.segment; if (!segment) return null; const result: CommittedSegment | null = this.speechSamples >= this.minSpeechSamples ? { + id: this.segmentId!, audio: concat( segment.map((frame) => frame.samples), this.segmentSamples, @@ -264,6 +346,25 @@ export interface LiveUpdate { error: string | null; } +/** + * Production tuning, shared between src/App.tsx and the accuracy tests + * (tests/live-accuracy.test.ts) so what we test is what users get. + * + * Latency rationale: the single-flight scheduler already guarantees inference + * never piles up, so the throttles only need to bound CPU, not protect + * correctness. 1.2 s commit silence keeps utterances whole (with sustain + * hysteresis covering quiet phonemes) while committing fast enough that text + * freezes shortly after a natural pause. + */ +export const LIVE_TUNING = { + tentativeIntervalMs: 2000, + minTentativeSeconds: 1, + segmenter: { + commitSilenceSeconds: 1.2, + maxSegmentSeconds: 29, + } satisfies LiveSegmenterOptions, +} as const; + export interface LiveTranscriberOptions { transcribe: LiveTranscribeFn; onUpdate: (update: LiveUpdate) => void; @@ -304,6 +405,8 @@ export class LiveTranscriber { private segments: SegmentState[] = []; private tentativeText = ""; + /** Segmenter id of the segment `tentativeText` describes (null: none). */ + private tentativeSegmentId: number | null = null; private lastTentative: { segmentId: number; samples: number; at: number } | null = null; private inFlight = false; @@ -329,6 +432,17 @@ export class LiveTranscriber { for (const committed of this.segmenter.push(samples)) { this.acceptSegment(committed); } + if ( + this.tentativeSegmentId !== null && + this.segmenter.currentSegmentId !== this.tentativeSegmentId + ) { + // The segment the tentative text described ended without committing — + // the segmenter dropped it as near-silence. Its text is very likely a + // hallucination over breath/noise; discard instead of leaving it on + // screen (or letting the next segment adopt it as provisional text). + this.clearTentative(); + this.emit(); + } this.pump(); } @@ -343,7 +457,7 @@ export class LiveTranscriber { if (flushed) this.acceptSegment(flushed); // Anything tentative either moved into the flushed segment or belonged // to audio the segmenter dropped as near-silence. - this.tentativeText = ""; + this.clearTentative(); this.pump(); } await this.whenDrained(); @@ -351,18 +465,26 @@ export class LiveTranscriber { return { text: this.committedText(), chunks: this.allChunks() }; } + private clearTentative(): void { + this.tentativeText = ""; + this.tentativeSegmentId = null; + this.lastTentative = null; + } + private acceptSegment(committed: CommittedSegment): void { - // The committing segment is the one the tentative text described; carry - // it as the provisional text until the final pass replaces it. + // Carry the tentative text as this segment's provisional text until the + // final pass replaces it — but only when it actually described this + // segment (a dropped predecessor's leftover text must not leak in). + const provisional = + committed.id === this.tentativeSegmentId ? this.tentativeText : ""; this.segments.push({ startSample: committed.startSample, audio: committed.audio, - text: this.tentativeText, + text: provisional, chunks: [], final: false, }); - this.tentativeText = ""; - this.lastTentative = null; + this.clearTentative(); this.emit(); } @@ -431,6 +553,7 @@ export class LiveTranscriber { // pass supersedes this) or been dropped while we were busy. if (this.segmenter.currentSegmentId === segmentId) { this.tentativeText = result.text; + this.tentativeSegmentId = segmentId; } }, (error: unknown) => { diff --git a/src/lib/mic.ts b/src/lib/mic.ts index 8f807bf..147cf2e 100644 --- a/src/lib/mic.ts +++ b/src/lib/mic.ts @@ -5,8 +5,10 @@ * cannot connect a MediaStreamAudioSourceNode across sample rates, so the * context is never forced to 16 kHz) → an AudioWorklet chunker posts ~100 ms * Float32 blocks to the main thread, which resamples them to 16 kHz with the - * pure `resampleLinear` before handing them to the live segment loop - * (src/lib/live.ts). + * pure `StreamingResampler` (anti-aliased, chunk-boundary safe — plain + * `resampleLinear` aliases everything above 8 kHz into the speech band and + * measurably hurts Whisper accuracy) before handing them to the live segment + * loop (src/lib/live.ts). * * The worklet module is created from an inline source string via a Blob URL: * no extra served file, no build or service-worker changes, and capture keeps @@ -17,7 +19,7 @@ * (Architecture.md §5). */ -import { TARGET_SAMPLE_RATE, resampleLinear } from "./audio"; +import { StreamingResampler, TARGET_SAMPLE_RATE } from "./audio"; export interface MicCapture { /** Native rate of the capture context (chunks are resampled to 16 kHz). */ @@ -131,18 +133,18 @@ export async function startMicCapture( audio: { echoCancellation: true, noiseSuppression: true, autoGainControl: true }, }); - const context = new AudioContext({ sampleRate: TARGET_SAMPLE_RATE }); + // Native rate, never forced to 16 kHz: Firefox cannot connect a + // MediaStreamAudioSourceNode across sample rates (D-016). + const context = new AudioContext(); // Contexts created outside a user gesture start suspended; the start button // click satisfies the gesture requirement, so this resolves immediately. await context.resume().catch(() => {}); const source = context.createMediaStreamSource(stream); + const resampler = new StreamingResampler(context.sampleRate, TARGET_SAMPLE_RATE); const deliver = (samples: Float32Array) => { - onChunk( - context.sampleRate === TARGET_SAMPLE_RATE - ? samples - : resampleLinear(samples, context.sampleRate, TARGET_SAMPLE_RATE), - ); + const resampled = resampler.process(samples); + if (resampled.length > 0) onChunk(resampled); }; let detach: () => void; @@ -162,6 +164,10 @@ export async function startMicCapture( if (stopped) return; stopped = true; detach(); + // Drain the resampler's look-ahead tail so the last spoken samples + // reach the segment loop before the session flushes. + const tail = resampler.flush(); + if (tail.length > 0) onChunk(tail); for (const track of stream.getTracks()) track.stop(); await context.close().catch(() => {}); }, diff --git a/tests/audio_files/Festuge.48k.wav b/tests/audio_files/Festuge.48k.wav new file mode 100644 index 0000000..e693d89 Binary files /dev/null and b/tests/audio_files/Festuge.48k.wav differ diff --git a/tests/audio_files/Festuge.m4a b/tests/audio_files/Festuge.m4a new file mode 100644 index 0000000..adb8668 Binary files /dev/null and b/tests/audio_files/Festuge.m4a differ diff --git a/tests/audio_files/Festuge.txt b/tests/audio_files/Festuge.txt new file mode 100644 index 0000000..94442a9 --- /dev/null +++ b/tests/audio_files/Festuge.txt @@ -0,0 +1,3 @@ +På Aarhus Festuges sekretariat har vi løbet rigtig stærkt de sidste måneder. +Selvom du måske lige nu mest tænker på sol og sommerferie, så er Festugen lige rundt om hjørnet for os. +Men nu trækker vi stikket, går på sommerferie og lader op til det sidste store ryk frem mod Festugen, så vi er helt klar, når det hele begynder fredag den 28. august. \ No newline at end of file diff --git a/tests/dsp.ts b/tests/dsp.ts new file mode 100644 index 0000000..311b7bf --- /dev/null +++ b/tests/dsp.ts @@ -0,0 +1,250 @@ +/** + * Shared DSP helpers for the live-accuracy and resampler tests. + * + * The accuracy harness encodes "words" as pure tones at word-specific + * frequencies and decodes them with a Goertzel filter bank ("oracle engine"). + * That makes transcript accuracy of the live *pipeline* measurable under + * `bun test` with no model: if the segment loop cuts a word in half, drops + * audio, duplicates a boundary, leaks stale text, or the resampler shifts / + * aliases frequencies, the decoded word sequence changes and WER rises above + * zero. + */ + +import type { LiveTranscribeResult } from "../src/lib/live"; + +export const RATE = 16_000; + +// --------------------------------------------------------------------------- +// Signal generation +// --------------------------------------------------------------------------- + +export function sine( + freq: number, + seconds: number, + rate: number = RATE, + amplitude = 0.2, +): Float32Array { + const out = new Float32Array(Math.round(seconds * rate)); + for (let i = 0; i < out.length; i++) { + out[i] = amplitude * Math.sin((2 * Math.PI * freq * i) / rate); + } + return out; +} + +export function silence(seconds: number, rate: number = RATE): Float32Array { + return new Float32Array(Math.round(seconds * rate)); +} + +export function concatAudio(parts: readonly Float32Array[]): Float32Array { + const out = new Float32Array(parts.reduce((n, p) => n + p.length, 0)); + let offset = 0; + for (const part of parts) { + out.set(part, offset); + offset += part.length; + } + return out; +} + +// --------------------------------------------------------------------------- +// Tone-word vocabulary and session builder +// --------------------------------------------------------------------------- + +export const ORACLE_WORDS = [ + "alpha", + "bravo", + "charlie", + "delta", + "echo", + "foxtrot", + "golf", + "hotel", +] as const; + +/** Word k ↔ tone at 500 + 250·k Hz (500–2250 Hz, well inside speech band). */ +export function wordFreq(index: number): number { + return 500 + 250 * index; +} + +export const WORD_SECONDS = 0.25; +export const GAP_SECONDS = 0.08; + +export interface BuiltSession { + audio: Float32Array; + /** Ground truth: every word with its absolute onset time in seconds. */ + truth: { word: string; start: number }[]; + /** The reference transcript (all words, in order, space-joined). */ + reference: string; +} + +/** + * Lay out utterances (word lists) separated by `pauseSeconds` of silence, + * with `GAP_SECONDS` between words inside an utterance. + */ +export function buildSession( + utterances: readonly (readonly string[])[], + options: { + rate?: number; + leadSeconds?: number; + pauseSeconds?: number; + /** + * Duration of the very first word. Segments start relative to speech + * onset, so stretching the first word is how a test shifts where a + * max-length cap lands within the word/gap cycle (a longer lead-in + * silence shifts onset and cap together and changes nothing). + */ + firstWordSeconds?: number; + } = {}, +): BuiltSession { + const rate = options.rate ?? RATE; + const lead = options.leadSeconds ?? 0.5; + const pause = options.pauseSeconds ?? 2.0; + + const parts: Float32Array[] = [silence(lead, rate)]; + const truth: { word: string; start: number }[] = []; + let t = lead; + let first = true; + utterances.forEach((words, u) => { + words.forEach((word, w) => { + const index = ORACLE_WORDS.indexOf(word as (typeof ORACLE_WORDS)[number]); + if (index === -1) throw new Error(`unknown oracle word: ${word}`); + const duration = first ? (options.firstWordSeconds ?? WORD_SECONDS) : WORD_SECONDS; + first = false; + truth.push({ word, start: t }); + parts.push(sine(wordFreq(index), duration, rate)); + t += duration; + if (w < words.length - 1) { + parts.push(silence(GAP_SECONDS, rate)); + t += GAP_SECONDS; + } + }); + if (u < utterances.length - 1) { + parts.push(silence(pause, rate)); + t += pause; + } + }); + return { + audio: concatAudio(parts), + truth, + reference: truth.map((entry) => entry.word).join(" "), + }; +} + +// --------------------------------------------------------------------------- +// Goertzel filter bank + oracle decoder +// --------------------------------------------------------------------------- + +/** Normalized power of `freq` over samples[from, to). */ +export function goertzelPower( + samples: Float32Array, + from: number, + to: number, + freq: number, + rate: number, +): number { + const coeff = 2 * Math.cos((2 * Math.PI * freq) / rate); + let s1 = 0; + let s2 = 0; + for (let i = from; i < to; i++) { + const s0 = samples[i]! + coeff * s1 - s2; + s2 = s1; + s1 = s0; + } + const n = to - from; + return (s1 * s1 + s2 * s2 - coeff * s1 * s2) / (n * n); +} + +const ORACLE_FRAME_SECONDS = 0.02; +/** Runs shorter than this are boundary noise and ignored. */ +const ORACLE_FRAGMENT_MIN_SECONDS = 0.06; +/** Runs at least this long are a complete word (words are 0.25 s). */ +const ORACLE_WORD_MIN_SECONDS = 0.22; +/** What a truncated word decodes as — a substitution, like real Whisper. */ +export const ORACLE_FRAGMENT = "▯"; + +/** + * Decode tone-words from 16 kHz audio. Deterministic stand-in for Whisper: + * frames are labeled by dominant vocabulary frequency; runs ≥ 220 ms decode + * as the word, and truncated runs (60–220 ms — a word cut by a segment + * boundary) decode as the garbage token `ORACLE_FRAGMENT`, because a real + * model mis-transcribes a cut word rather than silently skipping it. Either + * fragment of a mid-word cut therefore raises WER. + * Output mimics Whisper: leading-space text, per-word timestamped chunks. + */ +export function oracleDecode(audio: Float32Array): LiveTranscribeResult { + const hop = Math.round(RATE * ORACLE_FRAME_SECONDS); + const fragmentRun = Math.round(ORACLE_FRAGMENT_MIN_SECONDS / ORACLE_FRAME_SECONDS); + const wordRun = Math.round(ORACLE_WORD_MIN_SECONDS / ORACLE_FRAME_SECONDS); + + const labels: number[] = []; + for (let start = 0; start + hop <= audio.length; start += hop) { + let energy = 0; + for (let i = start; i < start + hop; i++) energy += audio[i]! * audio[i]!; + if (Math.sqrt(energy / hop) < 0.05) { + labels.push(-1); + continue; + } + let best = -1; + let bestPower = 0; + for (let k = 0; k < ORACLE_WORDS.length; k++) { + const power = goertzelPower(audio, start, start + hop, wordFreq(k), RATE); + if (power > bestPower) { + bestPower = power; + best = k; + } + } + labels.push(best); + } + + const words: string[] = []; + const chunks: LiveTranscribeResult["chunks"] = []; + let runLabel = -1; + let runStart = 0; + const endRun = (endFrame: number) => { + const length = endFrame - runStart; + if (runLabel < 0 || length < fragmentRun) return; + const word = length >= wordRun ? ORACLE_WORDS[runLabel]! : ORACLE_FRAGMENT; + words.push(word); + chunks.push({ + text: ` ${word}`, + timestamp: [runStart * ORACLE_FRAME_SECONDS, endFrame * ORACLE_FRAME_SECONDS], + }); + }; + labels.forEach((label, frame) => { + if (label !== runLabel) { + endRun(frame); + runLabel = label; + runStart = frame; + } + }); + endRun(labels.length); + + return { text: words.length > 0 ? ` ${words.join(" ")}` : "", chunks }; +} + +// --------------------------------------------------------------------------- +// Word error rate +// --------------------------------------------------------------------------- + +/** + * Standard WER: word-level Levenshtein distance over the reference length. + * 0 = perfect; insertions (duplicated boundary fragments), deletions + * (dropped audio), and substitutions all count. + */ +export function wordErrorRate(reference: string, hypothesis: string): number { + const ref = reference.trim().split(/\s+/).filter(Boolean); + const hyp = hypothesis.trim().split(/\s+/).filter(Boolean); + let previous = Array.from({ length: hyp.length + 1 }, (_, j) => j); + for (let i = 1; i <= ref.length; i++) { + const current = new Array(hyp.length + 1); + current[0] = i; + for (let j = 1; j <= hyp.length; j++) { + current[j] = Math.min( + previous[j]! + 1, + current[j - 1]! + 1, + previous[j - 1]! + (ref[i - 1] === hyp[j - 1] ? 0 : 1), + ); + } + previous = current; + } + return previous[hyp.length]! / Math.max(1, ref.length); +} diff --git a/tests/live-accuracy.test.ts b/tests/live-accuracy.test.ts new file mode 100644 index 0000000..52c0d5c --- /dev/null +++ b/tests/live-accuracy.test.ts @@ -0,0 +1,242 @@ +import { describe, test, expect } from "bun:test"; +import { StreamingResampler } from "../src/lib/audio"; +import { + LIVE_TUNING, + LiveTranscriber, + type LiveTranscribeFn, + type LiveTranscribeResult, + type LiveUpdate, +} from "../src/lib/live"; +import { + GAP_SECONDS, + ORACLE_WORDS, + RATE, + WORD_SECONDS, + buildSession, + concatAudio, + oracleDecode, + silence, + sine, + wordErrorRate, + wordFreq, +} from "./dsp"; + +// End-to-end accuracy of the live pipeline (segmenter + scheduler + mic-path +// resampler) measured as WER, using the deterministic tone-word oracle from +// tests/dsp.ts instead of a model. Runs with the production LIVE_TUNING so +// what we measure is what ships. See tests/live.md for what still needs a +// real browser + real Whisper. + +function tick(): Promise { + return new Promise((resolve) => setTimeout(resolve, 0)); +} + +interface SessionRun { + result: LiveTranscribeResult; + updates: LiveUpdate[]; +} + +/** + * Stream `audio` through a LiveTranscriber in mic-sized chunks. The injected + * clock advances with the audio fed, so the tentative-pass throttling behaves + * exactly as it would in real time. + */ +async function runLiveSession( + audio: Float32Array, + transcribe: LiveTranscribeFn, + chunkSeconds = 0.1, +): Promise { + let fedSamples = 0; + const updates: LiveUpdate[] = []; + const live = new LiveTranscriber({ + ...LIVE_TUNING, + transcribe, + onUpdate: (update) => updates.push(update), + now: () => fedSamples / (RATE / 1000), + }); + const chunk = Math.round(chunkSeconds * RATE); + for (let i = 0; i < audio.length; i += chunk) { + live.push(audio.slice(i, Math.min(i + chunk, audio.length))); + fedSamples = Math.min(i + chunk, audio.length); + await tick(); // let in-flight passes resolve, as they would in real time + } + const result = await live.stop(); + return { result, updates }; +} + +const oracleEngine: LiveTranscribeFn = async (audio) => oracleDecode(audio); + +describe("live pipeline accuracy (oracle engine, production tuning)", () => { + test("dictation with pauses: zero WER and accurate absolute timestamps", async () => { + const session = buildSession( + [ + ["alpha", "bravo", "charlie"], + ["delta", "echo", "foxtrot"], + ["golf", "hotel", "alpha", "bravo"], + ], + { pauseSeconds: LIVE_TUNING.segmenter.commitSilenceSeconds + 0.6 }, + ); + + const { result, updates } = await runLiveSession(session.audio, oracleEngine); + + expect(wordErrorRate(session.reference, result.text)).toBe(0); + + // Tentative text must have been visible at some point mid-utterance. + expect(updates.some((u) => u.tentativeText.trim().length > 0)).toBe(true); + + // One chunk per word, timestamped in absolute session time: the oracle + // reports times relative to each segment's audio and the transcriber + // offsets by the segment's session position — quantization is ~20 ms + // (oracle frames) + 32 ms (segmenter frames). + expect(result.chunks).toHaveLength(session.truth.length); + result.chunks.forEach((chunk, i) => { + expect(chunk.text.trim()).toBe(session.truth[i]!.word); + expect(Math.abs(chunk.timestamp[0] - session.truth[i]!.start)).toBeLessThan(0.15); + }); + }); + + test("continuous speech across the max-length cap: soft cuts keep every word intact", async () => { + // ~40 s with only 80 ms inter-word gaps — silence never reaches the + // 1.2 s commit threshold, so the session *must* cut at the 29 s cap at + // least once. The soft cut has to land in a gap, not inside a word. + // + // Stretching the first word sweeps where the cap falls relative to the + // word/gap cycle (segments start at speech onset, so a longer lead-in + // would shift onset and cap together and test nothing). A hard cutter + // can get lucky on one phase — the cap lands in a gap — but not on all + // five: word+gap = 0.33 s, and the offsets cover the whole cycle. + const words: string[] = []; + const total = Math.ceil(40 / (WORD_SECONDS + GAP_SECONDS)); + for (let i = 0; i < total; i++) words.push(ORACLE_WORDS[i % ORACLE_WORDS.length]!); + + for (let phase = 0; phase < 5; phase++) { + const session = buildSession([words], { + firstWordSeconds: + WORD_SECONDS + phase * ((WORD_SECONDS + GAP_SECONDS) / 5), + }); + expect(session.audio.length).toBeGreaterThan( + LIVE_TUNING.segmenter.maxSegmentSeconds * RATE, + ); + + const { result } = await runLiveSession(session.audio, oracleEngine, 0.5); + + // A mid-word cut shows up as an insertion (two fragments) or a + // deletion (both halves too short) — either way WER > 0. + expect(wordErrorRate(session.reference, result.text)).toBe(0); + expect(result.chunks).toHaveLength(words.length); + } + }, 30_000); + + test("48 kHz mic path: StreamingResampler preserves accuracy end to end", async () => { + // Same dictation, but generated at 48 kHz and resampled in ~100 ms chunks + // exactly like src/lib/mic.ts does. Aliasing, pitch shift, or dropped + // samples at chunk boundaries would decode as wrong or missing words. + const session = buildSession( + [ + ["hotel", "golf", "foxtrot"], + ["echo", "delta", "charlie", "bravo"], + ], + { rate: 48_000, pauseSeconds: LIVE_TUNING.segmenter.commitSilenceSeconds + 0.6 }, + ); + + const resampler = new StreamingResampler(48_000, RATE); + let fedSamples = 0; + const live = new LiveTranscriber({ + ...LIVE_TUNING, + transcribe: oracleEngine, + onUpdate: () => {}, + now: () => fedSamples / (RATE / 1000), + }); + const chunk = 4800; // 100 ms at the native rate + for (let i = 0; i < session.audio.length; i += chunk) { + const out = resampler.process( + session.audio.slice(i, Math.min(i + chunk, session.audio.length)), + ); + if (out.length > 0) { + live.push(out); + fedSamples += out.length; + } + await tick(); + } + const tail = resampler.flush(); + if (tail.length > 0) live.push(tail); + const result = await live.stop(); + + expect(wordErrorRate(session.reference, result.text)).toBe(0); + }); + + test("hallucinated text over a near-silent blip is dropped and cleared", async () => { + // Whisper's classic failure mode: fed near-silence, it emits "Thank you." + // The energy gate must keep that audio from ever reaching the committed + // transcript, and the tentative flash of it must be cleared on drop. + const hallucinating: LiveTranscribeFn = async () => ({ + text: " Thank you.", + chunks: [], + }); + const audio = concatAudio([ + silence(0.5), + sine(wordFreq(0), 0.12), // 120 ms blip < minSpeechSeconds 0.25 + silence(2.5), + ]); + + const { result, updates } = await runLiveSession(audio, hallucinating); + + // The blip opened a segment ≥ minTentativeSeconds, so a tentative pass + // really ran and "Thank you." was briefly tentative… + expect(updates.some((u) => u.tentativeText === " Thank you.")).toBe(true); + // …but the drop cleared it, and nothing was ever committed. + expect(updates[updates.length - 1]!.tentativeText).toBe(""); + expect(result.text).toBe(""); + expect(result.chunks).toHaveLength(0); + }); + + test("stale tentative text from a dropped segment never leaks into the next commit", async () => { + // Manual-resolution engine so the ordering is exact. + const calls: { + resolve: (result: LiveTranscribeResult) => void; + }[] = []; + const updates: LiveUpdate[] = []; + let now = 0; + const live = new LiveTranscriber({ + ...LIVE_TUNING, + transcribe: (audio) => + new Promise((resolve) => { + void audio; + calls.push({ resolve }); + }), + onUpdate: (update) => updates.push(update), + now: () => now, + }); + const last = () => updates[updates.length - 1]!; + + // A breathy blip keeps a segment open long enough for a tentative pass… + live.push(silence(0.3)); + live.push(sine(wordFreq(0), 0.15)); + now += 450; + live.push(silence(1.0)); + now += 1000; + expect(calls).toHaveLength(1); // tentative pass over mostly-silence audio + calls[0]!.resolve({ text: " Thank you.", chunks: [] }); + await tick(); + expect(last().tentativeText).toBe(" Thank you."); + + // …then the segment is dropped as near-silence (0.15 s < 0.25 s speech). + live.push(silence(0.5)); + now += 500; + expect(last().tentativeText).toBe(""); + expect(last().committedText).toBe(""); + + // Real speech commits next: it must NOT adopt "Thank you." as its + // provisional text while waiting for its final pass. + live.push(sine(wordFreq(1), 0.6)); + now += 600; + live.push(silence(1.5)); + now += 1500; + expect(last().pendingFinalPasses).toBe(1); + expect(last().committedText).toBe(""); // provisional text is empty, not stale + + calls[1]!.resolve({ text: " bravo", chunks: [] }); + const result = await live.stop(); + expect(result.text).toBe("bravo"); + }); +}); diff --git a/tests/live-real-model.test.ts b/tests/live-real-model.test.ts new file mode 100644 index 0000000..c6dd033 --- /dev/null +++ b/tests/live-real-model.test.ts @@ -0,0 +1,217 @@ +import { describe, test, expect } from "bun:test"; +import { existsSync } from "node:fs"; +import { join } from "node:path"; +import { env, pipeline } from "@huggingface/transformers"; +import { StreamingResampler } from "../src/lib/audio"; +import { + LIVE_TUNING, + LiveTranscriber, + type LiveTranscribeFn, +} from "../src/lib/live"; +import { resolveDefaultDtype } from "../src/lib/models"; +import { wordErrorRate } from "./dsp"; + +/** + * Real-model, real-speech accuracy of the live pipeline, measured as WER over + * a real Danish recording (tests/audio_files/Festuge.m4a, reference + * transcript Festuge.txt). tests/live-accuracy.test.ts proves the pipeline + * moves audio losslessly with a deterministic oracle; this test closes the + * remaining gap by running actual Whisper weights end to end over the exact + * mic path: 48 kHz PCM → StreamingResampler (as src/lib/mic.ts) → + * LiveTranscriber with production LIVE_TUNING → the same Transformers.js + * pipeline options the inference worker sends (src/worker/inference.worker.ts). + * + * Model: the catalog's Whisper Large v3 Turbo at its WASM-default dtype (q4, + * D-006) — the model a Danish user should pick. Under bun test it runs on + * onnxruntime-node (CPU), i.e. the same weights and quantization as the + * browser WASM engine, just a different execution provider. `language` is + * pinned to "da" exactly as the worker does when the user selects Danish in + * the UI; "auto" misdetects this clip as English (whole-file mode too) and + * comes back as a rough English translation — see tests/live.md. + * + * The first run downloads ~600 MB into @huggingface/transformers' cache + * (node_modules/@huggingface/transformers/.cache/). When the model is neither + * cached nor reachable on the Hub, the test skips so offline development + * stays green (this repo is offline-first). + * + * Fixture: Festuge.48k.wav is Festuge.m4a decoded once, so the test needs no + * audio decoder: ffmpeg -i Festuge.m4a -ac 1 -ar 48000 -sample_fmt s16 \ + * Festuge.48k.wav + */ + +const MODEL_ID = "onnx-community/whisper-large-v3-turbo"; +const MIC_RATE = 48_000; +const RATE = 16_000; + +/** + * WER normalization: casing and punctuation are not transcription accuracy. + * "Århus"/"Aarhus" are the same word in Danish orthography, so they unify. + */ +function normalize(text: string): string { + return text + .toLowerCase() + .replaceAll("århus", "aarhus") + .replace(/[.,!?;:"“”‘’«»()\-–—…]/g, " ") + .replace(/\s+/g, " ") + .trim(); +} + +/** Minimal RIFF/WAVE reader for the fixture (16-bit PCM mono). */ +function readWav(buffer: ArrayBuffer): { samples: Float32Array; rate: number } { + const view = new DataView(buffer); + const tag = (offset: number) => + String.fromCharCode( + view.getUint8(offset), + view.getUint8(offset + 1), + view.getUint8(offset + 2), + view.getUint8(offset + 3), + ); + if (tag(0) !== "RIFF" || tag(8) !== "WAVE") throw new Error("not a WAV file"); + + let rate = 0; + let offset = 12; + while (offset + 8 <= view.byteLength) { + const id = tag(offset); + const size = view.getUint32(offset + 4, true); + if (id === "fmt ") { + const format = view.getUint16(offset + 8, true); + const channels = view.getUint16(offset + 10, true); + const bits = view.getUint16(offset + 22, true); + if (format !== 1 || channels !== 1 || bits !== 16) { + throw new Error("fixture must be 16-bit PCM mono"); + } + rate = view.getUint32(offset + 12, true); + } else if (id === "data") { + const pcm = new Int16Array(buffer, offset + 8, size / 2); + const samples = new Float32Array(pcm.length); + for (let i = 0; i < pcm.length; i++) samples[i] = pcm[i]! / 32768; + return { samples, rate }; + } + offset += 8 + size + (size % 2); + } + throw new Error("WAV data chunk not found"); +} + +/** Cached locally, or downloadable right now — otherwise the test skips. */ +async function modelAvailable(id: string): Promise { + if (existsSync(join(env.cacheDir, id))) return true; + try { + const response = await fetch( + `https://huggingface.co/${id}/resolve/main/config.json`, + { method: "HEAD", signal: AbortSignal.timeout(5_000) }, + ); + return response.ok; + } catch { + return false; + } +} + +const available = await modelAvailable(MODEL_ID); + +describe("live pipeline accuracy (real model, real Danish speech)", () => { + test.skipIf(!available)( + "Festuge.m4a over the mic path stays close to the reference transcript", + async () => { + const dir = join(import.meta.dir, "audio_files"); + const reference = await Bun.file(join(dir, "Festuge.txt")).text(); + const wav = readWav(await Bun.file(join(dir, "Festuge.48k.wav")).arrayBuffer()); + expect(wav.rate).toBe(MIC_RATE); + + const asr = await pipeline("automatic-speech-recognition", MODEL_ID, { + dtype: resolveDefaultDtype(MODEL_ID, "wasm"), + }); + try { + const transcribe: LiveTranscribeFn = async (audio) => { + // Mirrors src/worker/inference.worker.ts `transcribe` options with + // the UI set to Danish. + const output = await asr(audio, { + chunk_length_s: 30, + stride_length_s: 5, + return_timestamps: true, + language: "da", + task: "transcribe", + }); + const single = (Array.isArray(output) ? output[0] : output) as { + text?: string; + chunks?: { text: string; timestamp: [number, number | null] }[]; + }; + return { text: single.text ?? "", chunks: single.chunks ?? [] }; + }; + + // Resample in ~100 ms mic-sized blocks exactly like src/lib/mic.ts. + const resampler = new StreamingResampler(MIC_RATE, RATE); + const blocks: Float32Array[] = []; + const chunk = MIC_RATE / 10; + for (let i = 0; i < wav.samples.length; i += chunk) { + const out = resampler.process( + wav.samples.slice(i, Math.min(i + chunk, wav.samples.length)), + ); + if (out.length > 0) blocks.push(out); + } + const tail = resampler.flush(); + if (tail.length > 0) blocks.push(tail); + const totalSamples = blocks.reduce((n, b) => n + b.length, 0); + const durationSeconds = totalSamples / RATE; + + // Whole-clip baseline: what the file-upload flow would produce from + // the same audio and model. The live pipeline is then held to it. + const whole = new Float32Array(totalSamples); + let at = 0; + for (const block of blocks) { + whole.set(block, at); + at += block.length; + } + const fileWer = wordErrorRate( + normalize(reference), + normalize((await transcribe(whole)).text), + ); + + // The live session, with the injected clock advancing with the audio + // fed so pass scheduling behaves as in real time. + let fedSamples = 0; + let sawTentative = false; + const live = new LiveTranscriber({ + ...LIVE_TUNING, + transcribe, + onUpdate: (update) => { + if (update.tentativeText.trim().length > 0) sawTentative = true; + }, + now: () => fedSamples / (RATE / 1000), + }); + for (const block of blocks) { + live.push(block); + fedSamples += block.length; + await new Promise((resolve) => setTimeout(resolve, 0)); + } + const result = await live.stop(); + const liveWer = wordErrorRate(normalize(reference), normalize(result.text)); + + // "Close enough": measured 0.11 (live) / 0.14 (whole-file) on this + // clip; the margin absorbs onnxruntime version-to-version drift, not + // real regressions — a single mid-word cut or dropped segment on a + // 64-word reference moves WER by far more than the slack here. + expect(liveWer).toBeLessThanOrEqual(0.25); + // Segmentation must not cost accuracy relative to whole-file mode. + expect(liveWer).toBeLessThanOrEqual(fileWer + 0.1); + + // Live UX invariants, with a real model: tentative text streamed, and + // the final chunks carry absolute session timestamps spanning the clip. + expect(sawTentative).toBe(true); + expect(result.chunks.length).toBeGreaterThanOrEqual(3); + const first = result.chunks[0]!; + const last = result.chunks[result.chunks.length - 1]!; + expect(first.timestamp[0]).toBeGreaterThan(0.5); // speech starts ~1.8 s in + expect(first.timestamp[0]).toBeLessThan(5); + expect(last.timestamp[1] ?? last.timestamp[0]).toBeGreaterThan( + durationSeconds - 5, + ); + expect(last.timestamp[1] ?? last.timestamp[0]).toBeLessThanOrEqual( + durationSeconds + 1, + ); + } finally { + await asr.dispose(); + } + }, + 900_000, // first run downloads ~600 MB of weights + ); +}); diff --git a/tests/live.md b/tests/live.md index 348dc39..f1a89a9 100644 --- a/tests/live.md +++ b/tests/live.md @@ -1,8 +1,23 @@ # Manual live-transcription verification (M7, D-016) The segment loop (`src/lib/live.ts`) and the protocol changes are covered by -unit tests; microphone capture, the AudioWorklet path, and end-to-end live -quality can only be exercised in a real browser. This is that procedure. +unit tests, and transcript *accuracy* of the pipeline (segmentation, pass +scheduling, mic-path resampling) is regression-tested as WER by +`tests/live-accuracy.test.ts` + `tests/resampler.test.ts`. End-to-end quality +with real Whisper weights and real speech is regression-tested too: +`tests/live-real-model.test.ts` runs the Danish clip +`tests/audio_files/Festuge.m4a` through the full mic path with +`whisper-large-v3-turbo` (q4) and holds live WER against the reference +transcript (≤ 0.25; measured ≈ 0.11) and against whole-file mode (it skips +itself when the model is neither cached nor downloadable). Microphone capture, +the AudioWorklet path, and WebGPU/WASM execution in an actual browser can +only be exercised manually. This is that procedure. + +Known model behavior (not a pipeline bug): with language **auto**, Whisper +misdetects the Festuge clip as English and returns a rough English +translation — in whole-file mode as well as live. Danish (and likely other +non-English) users should pick their language explicitly; the worker forwards +it and accuracy is then as tested above. Re-run it after any change to `src/lib/live.ts`, `src/lib/mic.ts`, `src/components/LiveControls.tsx`, or the live wiring in `src/App.tsx`. @@ -57,11 +72,15 @@ Re-run it after any change to `src/lib/live.ts`, `src/lib/mic.ts`, ## 4. Quality/latency knobs (only if needed) -If commits are too eager/lazy in real rooms, tune `LiveSegmenterOptions` -(`speechRms` gate 0.015, `commitSilenceSeconds` 0.75, `maxSegmentSeconds` 25, -`minSpeechSeconds` 0.25) and the `LiveTranscriber` throttle -(`tentativeIntervalMs` 1500) — all injected from `src/App.tsx` defaults, all -unit-tested in `tests/live.test.ts`. Record any tuned values in D-016. +Production tuning lives in `LIVE_TUNING` (`src/lib/live.ts`) and is shared by +`src/App.tsx` and the accuracy harness (`tests/live-accuracy.test.ts`), so a +knob change is automatically re-validated for WER under `bun test`. If commits +are too eager/lazy in real rooms, tune `LIVE_TUNING.segmenter` +(`commitSilenceSeconds` 1.2, `maxSegmentSeconds` 29) or the +`LiveSegmenterOptions` defaults (`speechRms` gate 0.015, `sustainRmsRatio` +0.5, `softCutWindowSeconds` 3, `minSpeechSeconds` 0.25) and the throttle +(`tentativeIntervalMs` 2000, `minTentativeSeconds` 1). Record any tuned +values in D-016. ## 5. Offline live transcription diff --git a/tests/resampler.test.ts b/tests/resampler.test.ts new file mode 100644 index 0000000..82c984e --- /dev/null +++ b/tests/resampler.test.ts @@ -0,0 +1,110 @@ +import { describe, test, expect } from "bun:test"; +import { StreamingResampler, resampleLinear } from "../src/lib/audio"; +import { goertzelPower, sine } from "./dsp"; + +// Signal-quality tests for the mic-path resampler (D-016). Whisper accuracy +// depends on the 48 kHz → 16 kHz decimation NOT folding ultrasonic/fricative +// energy into the speech band, which is exactly what a plain linear resampler +// does — these tests pin the anti-aliasing behavior down numerically. + +function rmsOf(samples: Float32Array, trim = 0): number { + let sum = 0; + let count = 0; + for (let i = trim; i < samples.length - trim; i++) { + sum += samples[i]! * samples[i]!; + count++; + } + return Math.sqrt(sum / Math.max(1, count)); +} + +/** Run a whole signal through in one call + flush. */ +function resampleWhole(input: Float32Array, from: number, to: number): Float32Array { + const resampler = new StreamingResampler(from, to); + const head = resampler.process(input); + const tail = resampler.flush(); + const out = new Float32Array(head.length + tail.length); + out.set(head, 0); + out.set(tail, head.length); + return out; +} + +describe("StreamingResampler", () => { + test("passthrough at equal rates returns the input unchanged", () => { + const input = sine(440, 0.1, 16_000); + const resampler = new StreamingResampler(16_000, 16_000); + expect(resampler.process(input)).toBe(input); + expect(resampler.flush().length).toBe(0); + }); + + test("output length matches the rate ratio", () => { + const out = resampleWhole(sine(1000, 1, 48_000), 48_000, 16_000); + expect(Math.abs(out.length - 16_000)).toBeLessThanOrEqual(2); + }); + + test("passband: a 1 kHz tone survives 48 kHz → 16 kHz at unity gain and pitch", () => { + const input = sine(1000, 1, 48_000); + const out = resampleWhole(input, 48_000, 16_000); + // Amplitude preserved (edges trimmed: half a kernel of warm-up). + expect(rmsOf(out, 100)).toBeCloseTo(rmsOf(input, 300), 2); + // Pitch preserved: energy sits at 1 kHz, not at any shifted frequency. + const at1k = goertzelPower(out, 200, out.length - 200, 1000, 16_000); + const at1k2 = goertzelPower(out, 200, out.length - 200, 1200, 16_000); + expect(at1k).toBeGreaterThan(100 * at1k2); + }); + + test("stopband: energy above the target Nyquist is rejected, not aliased", () => { + // 11 kHz is inaudible to a 16 kHz stream — it must vanish. Linear + // resampling folds it to 5 kHz (16 − 11) at nearly full amplitude, + // polluting the band Whisper reads. Require > 34 dB rejection. + const input = sine(11_000, 0.5, 48_000); + const out = resampleWhole(input, 48_000, 16_000); + expect(rmsOf(out, 100)).toBeLessThan(0.02 * rmsOf(input)); + + // Document the motivation: the old path really did alias. + const linear = resampleLinear(input, 48_000, 16_000); + expect(rmsOf(linear)).toBeGreaterThan(0.1 * rmsOf(input)); + }); + + test("chunked processing is equivalent to whole-signal processing", () => { + // Mixed deterministic signal, awkward chunk size, non-integer ratio. + const input = new Float32Array(44_100); + for (let i = 0; i < input.length; i++) { + input[i] = + 0.3 * Math.sin((2 * Math.PI * 700 * i) / 44_100) + + 0.2 * Math.sin((2 * Math.PI * 3100 * i) / 44_100 + 1.3); + } + const whole = resampleWhole(input, 44_100, 16_000); + + const resampler = new StreamingResampler(44_100, 16_000); + const pieces: Float32Array[] = []; + for (let i = 0; i < input.length; i += 333) { + pieces.push(resampler.process(input.slice(i, Math.min(i + 333, input.length)))); + } + pieces.push(resampler.flush()); + const chunked = new Float32Array(pieces.reduce((n, p) => n + p.length, 0)); + let offset = 0; + for (const piece of pieces) { + chunked.set(piece, offset); + offset += piece.length; + } + + expect(chunked.length).toBe(whole.length); + for (let i = 0; i < whole.length; i++) { + if (Math.abs(chunked[i]! - whole[i]!) > 1e-6) { + throw new Error( + `chunked/whole mismatch at ${i}: ${chunked[i]} vs ${whole[i]}`, + ); + } + } + }); + + test("upsampling 8 kHz → 16 kHz preserves the signal", () => { + const input = sine(1000, 0.5, 8_000); + const out = resampleWhole(input, 8_000, 16_000); + expect(Math.abs(out.length - 8_000)).toBeLessThanOrEqual(2); + expect(rmsOf(out, 100)).toBeCloseTo(rmsOf(input, 50), 2); + const at1k = goertzelPower(out, 100, out.length - 100, 1000, 16_000); + const at1k5 = goertzelPower(out, 100, out.length - 100, 1500, 16_000); + expect(at1k).toBeGreaterThan(100 * at1k5); + }); +});