whisper_web/tests/live-accuracy.test.ts

import { describe, test, expect } from "bun:test";
import { StreamingResampler } from "../src/lib/audio";
import {
  LIVE_TUNING,
  LiveTranscriber,
  type LiveTranscribeFn,
  type LiveTranscribeResult,
  type LiveUpdate,
} from "../src/lib/live";
import {
  GAP_SECONDS,
  ORACLE_WORDS,
  RATE,
  WORD_SECONDS,
  buildSession,
  concatAudio,
  oracleDecode,
  silence,
  sine,
  wordErrorRate,
  wordFreq,
} from "./dsp";

// End-to-end accuracy of the live pipeline (segmenter + scheduler + mic-path
// resampler) measured as WER, using the deterministic tone-word oracle from
// tests/dsp.ts instead of a model. Runs with the production LIVE_TUNING so
// what we measure is what ships. See tests/live.md for what still needs a
// real browser + real Whisper.

function tick(): Promise<void> {
  return new Promise((resolve) => setTimeout(resolve, 0));
}

interface SessionRun {
  result: LiveTranscribeResult;
  updates: LiveUpdate[];
}

/**
 * Stream `audio` through a LiveTranscriber in mic-sized chunks. The injected
 * clock advances with the audio fed, so the tentative-pass throttling behaves
 * exactly as it would in real time.
 */
async function runLiveSession(
  audio: Float32Array,
  transcribe: LiveTranscribeFn,
  chunkSeconds = 0.1,
): Promise<SessionRun> {
  let fedSamples = 0;
  const updates: LiveUpdate[] = [];
  const live = new LiveTranscriber({
    ...LIVE_TUNING,
    transcribe,
    onUpdate: (update) => updates.push(update),
    now: () => fedSamples / (RATE / 1000),
  });
  const chunk = Math.round(chunkSeconds * RATE);
  for (let i = 0; i < audio.length; i += chunk) {
    live.push(audio.slice(i, Math.min(i + chunk, audio.length)));
    fedSamples = Math.min(i + chunk, audio.length);
    await tick(); // let in-flight passes resolve, as they would in real time
  }
  const result = await live.stop();
  return { result, updates };
}

const oracleEngine: LiveTranscribeFn = async (audio) => oracleDecode(audio);

describe("live pipeline accuracy (oracle engine, production tuning)", () => {
  test("dictation with pauses: zero WER and accurate absolute timestamps", async () => {
    const session = buildSession(
      [
        ["alpha", "bravo", "charlie"],
        ["delta", "echo", "foxtrot"],
        ["golf", "hotel", "alpha", "bravo"],
      ],
      { pauseSeconds: LIVE_TUNING.segmenter.commitSilenceSeconds + 0.6 },
    );

    const { result, updates } = await runLiveSession(session.audio, oracleEngine);

    expect(wordErrorRate(session.reference, result.text)).toBe(0);

    // Tentative text must have been visible at some point mid-utterance.
    expect(updates.some((u) => u.tentativeText.trim().length > 0)).toBe(true);

    // One chunk per word, timestamped in absolute session time: the oracle
    // reports times relative to each segment's audio and the transcriber
    // offsets by the segment's session position — quantization is ~20 ms
    // (oracle frames) + 32 ms (segmenter frames).
    expect(result.chunks).toHaveLength(session.truth.length);
    result.chunks.forEach((chunk, i) => {
      expect(chunk.text.trim()).toBe(session.truth[i]!.word);
      expect(Math.abs(chunk.timestamp[0] - session.truth[i]!.start)).toBeLessThan(0.15);
    });
  });

  test("continuous speech across the max-length cap: soft cuts keep every word intact", async () => {
    // ~40 s with only 80 ms inter-word gaps — silence never reaches the
    // 1.2 s commit threshold, so the session *must* cut at the 29 s cap at
    // least once. The soft cut has to land in a gap, not inside a word.
    //
    // Stretching the first word sweeps where the cap falls relative to the
    // word/gap cycle (segments start at speech onset, so a longer lead-in
    // would shift onset and cap together and test nothing). A hard cutter
    // can get lucky on one phase — the cap lands in a gap — but not on all
    // five: word+gap = 0.33 s, and the offsets cover the whole cycle.
    const words: string[] = [];
    const total = Math.ceil(40 / (WORD_SECONDS + GAP_SECONDS));
    for (let i = 0; i < total; i++) words.push(ORACLE_WORDS[i % ORACLE_WORDS.length]!);

    for (let phase = 0; phase < 5; phase++) {
      const session = buildSession([words], {
        firstWordSeconds:
          WORD_SECONDS + phase * ((WORD_SECONDS + GAP_SECONDS) / 5),
      });
      expect(session.audio.length).toBeGreaterThan(
        LIVE_TUNING.segmenter.maxSegmentSeconds * RATE,
      );

      const { result } = await runLiveSession(session.audio, oracleEngine, 0.5);

      // A mid-word cut shows up as an insertion (two fragments) or a
      // deletion (both halves too short) — either way WER > 0.
      expect(wordErrorRate(session.reference, result.text)).toBe(0);
      expect(result.chunks).toHaveLength(words.length);
    }
  }, 30_000);

  test("48 kHz mic path: StreamingResampler preserves accuracy end to end", async () => {
    // Same dictation, but generated at 48 kHz and resampled in ~100 ms chunks
    // exactly like src/lib/mic.ts does. Aliasing, pitch shift, or dropped
    // samples at chunk boundaries would decode as wrong or missing words.
    const session = buildSession(
      [
        ["hotel", "golf", "foxtrot"],
        ["echo", "delta", "charlie", "bravo"],
      ],
      { rate: 48_000, pauseSeconds: LIVE_TUNING.segmenter.commitSilenceSeconds + 0.6 },
    );

    const resampler = new StreamingResampler(48_000, RATE);
    let fedSamples = 0;
    const live = new LiveTranscriber({
      ...LIVE_TUNING,
      transcribe: oracleEngine,
      onUpdate: () => {},
      now: () => fedSamples / (RATE / 1000),
    });
    const chunk = 4800; // 100 ms at the native rate
    for (let i = 0; i < session.audio.length; i += chunk) {
      const out = resampler.process(
        session.audio.slice(i, Math.min(i + chunk, session.audio.length)),
      );
      if (out.length > 0) {
        live.push(out);
        fedSamples += out.length;
      }
      await tick();
    }
    const tail = resampler.flush();
    if (tail.length > 0) live.push(tail);
    const result = await live.stop();

    expect(wordErrorRate(session.reference, result.text)).toBe(0);
  });

  test("hallucinated text over a near-silent blip is dropped and cleared", async () => {
    // Whisper's classic failure mode: fed near-silence, it emits "Thank you."
    // The energy gate must keep that audio from ever reaching the committed
    // transcript, and the tentative flash of it must be cleared on drop.
    const hallucinating: LiveTranscribeFn = async () => ({
      text: " Thank you.",
      chunks: [],
    });
    const audio = concatAudio([
      silence(0.5),
      sine(wordFreq(0), 0.12), // 120 ms blip < minSpeechSeconds 0.25
      silence(2.5),
    ]);

    const { result, updates } = await runLiveSession(audio, hallucinating);

    // The blip opened a segment ≥ minTentativeSeconds, so a tentative pass
    // really ran and "Thank you." was briefly tentative…
    expect(updates.some((u) => u.tentativeText === " Thank you.")).toBe(true);
    // …but the drop cleared it, and nothing was ever committed.
    expect(updates[updates.length - 1]!.tentativeText).toBe("");
    expect(result.text).toBe("");
    expect(result.chunks).toHaveLength(0);
  });

  test("stale tentative text from a dropped segment never leaks into the next commit", async () => {
    // Manual-resolution engine so the ordering is exact.
    const calls: {
      resolve: (result: LiveTranscribeResult) => void;
    }[] = [];
    const updates: LiveUpdate[] = [];
    let now = 0;
    const live = new LiveTranscriber({
      ...LIVE_TUNING,
      transcribe: (audio) =>
        new Promise<LiveTranscribeResult>((resolve) => {
          void audio;
          calls.push({ resolve });
        }),
      onUpdate: (update) => updates.push(update),
      now: () => now,
    });
    const last = () => updates[updates.length - 1]!;

    // A breathy blip keeps a segment open long enough for a tentative pass…
    live.push(silence(0.3));
    live.push(sine(wordFreq(0), 0.15));
    now += 450;
    live.push(silence(1.0));
    now += 1000;
    expect(calls).toHaveLength(1); // tentative pass over mostly-silence audio
    calls[0]!.resolve({ text: " Thank you.", chunks: [] });
    await tick();
    expect(last().tentativeText).toBe(" Thank you.");

    // …then the segment is dropped as near-silence (0.15 s < 0.25 s speech).
    live.push(silence(0.5));
    now += 500;
    expect(last().tentativeText).toBe("");
    expect(last().committedText).toBe("");

    // Real speech commits next: it must NOT adopt "Thank you." as its
    // provisional text while waiting for its final pass.
    live.push(sine(wordFreq(1), 0.6));
    now += 600;
    live.push(silence(1.5));
    now += 1500;
    expect(last().pendingFinalPasses).toBe(1);
    expect(last().committedText).toBe(""); // provisional text is empty, not stale

    calls[1]!.resolve({ text: " bravo", chunks: [] });
    const result = await live.stop();
    expect(result.text).toBe("bravo");
  });
});