Improvements and tests for live transcription.

src/App.tsx

@@ -9,7 +9,7 @@ import { ModelPicker } from "./components/ModelPicker";
 import { TranscriptView } from "./components/TranscriptView";
 import { TranscriptionOptions } from "./components/TranscriptionOptions";
 import { decodeAudioFile } from "./lib/audio";
-import { LiveTranscriber, type LiveTranscribeResult } from "./lib/live";
+import { LIVE_TUNING, LiveTranscriber, type LiveTranscribeResult } from "./lib/live";
 import { startMicCapture, type MicCapture } from "./lib/mic";
 import { getDownloadedModels } from "./lib/modelCache";
 import { registerServiceWorker } from "./pwa/register";
@@ -297,9 +297,7 @@ export function App() {
     // live. `.en` models take neither (same rule as the file flow).
     const options = (getModel(modelId)?.multilingual ?? false) ? { language, task } : {};
     const transcriber = new LiveTranscriber({
-      tentativeIntervalMs: 2500,
-      minTentativeSeconds: 2.0,
-      segmenter: { commitSilenceSeconds: 2.5, maxSegmentSeconds: 29 },
+      ...LIVE_TUNING,
       transcribe: (audio) => transcribeLiveSegment(worker, audio, options),
       onUpdate: (update) => {
         setLiveCommitted(update.committedText);

src/index.ts

@@ -203,12 +203,12 @@ async function developmentServeOptions(): Promise<ServeOptions> {
       }
 
       const iconsMatch = pathname.match(/\/icons\/(icon-\d+\.png)$/);
-      if (iconsMatch) {
+      if (iconsMatch?.[1]) {
         return serveFile(join(ICONS_DIR, iconsMatch[1]));
       }
 
       const ortMatch = pathname.match(/\/ort\/(.*(?:wasm|mjs))$/);
-      if (ortMatch) {
+      if (ortMatch?.[1]) {
         return serveOrtFile(ORT_DIST, ortMatch[1]);
       }
 

src/lib/audio.ts

@@ -75,6 +75,116 @@ export function resampleLinear(
   return out;
 }
 
+/**
+ * Streaming anti-aliased resampler (windowed-sinc low-pass interpolation).
+ * Pure — safe to call anywhere.
+ *
+ * `resampleLinear` has no low-pass filter, so decimating mic audio
+ * (48 kHz → 16 kHz) folds everything above 8 kHz back into the speech band —
+ * audible as aliasing noise and measurably bad for Whisper accuracy. This
+ * class filters at the correct cutoff while resampling, and it is stateful:
+ * feed it arbitrary chunk sizes (the mic path delivers ~100 ms blocks) and
+ * the output is identical to resampling the whole signal at once — no edge
+ * artifacts at chunk boundaries.
+ */
+export class StreamingResampler {
+  /** Input samples advanced per output sample. */
+  private readonly step: number;
+  /** Kernel half-width, in input samples. */
+  private readonly half: number;
+  /** Normalized low-pass cutoff, cycles per input sample. */
+  private readonly cutoff: number;
+  private readonly passthrough: boolean;
+
+  /** Unconsumed input tail (kernel needs `half` samples of look-ahead). */
+  private pending = new Float32Array(0);
+  /** Fractional index into `pending` of the next output sample's center. */
+  private position = 0;
+
+  constructor(fromRate: number, toRate: number) {
+    if (!(fromRate > 0) || !(toRate > 0)) {
+      throw new Error("StreamingResampler: sample rates must be positive");
+    }
+    this.passthrough = fromRate === toRate;
+    this.step = fromRate / toRate;
+    // 0.45×Nyquist of the slower rate: keeps the whole speech band while the
+    // transition band stays inside what the window can attenuate.
+    this.cutoff = (0.45 * Math.min(fromRate, toRate)) / fromRate;
+    this.half = Math.ceil(2 / this.cutoff);
+  }
+
+  /** Blackman-windowed sinc, low-pass at `cutoff`, support (−half, half). */
+  private kernel(distance: number): number {
+    const scaled = 2 * this.cutoff * distance;
+    const x = Math.PI * scaled;
+    const sinc = scaled === 0 ? 1 : Math.sin(x) / x;
+    const t = (distance / this.half + 1) / 2; // 0..1 across the support
+    const window =
+      0.42 - 0.5 * Math.cos(2 * Math.PI * t) + 0.08 * Math.cos(4 * Math.PI * t);
+    return 2 * this.cutoff * sinc * window;
+  }
+
+  /** Emit output samples for every center that has full kernel support. */
+  private produce(buffer: Float32Array, limit: number): Float32Array {
+    const count = Math.max(
+      0,
+      Math.floor((limit - this.position) / this.step) + 1,
+    );
+    const out = new Float32Array(count);
+    for (let i = 0; i < count; i++) {
+      const center = this.position + i * this.step;
+      const from = Math.max(0, Math.ceil(center - this.half));
+      const to = Math.min(buffer.length - 1, Math.floor(center + this.half));
+      let acc = 0;
+      let norm = 0;
+      for (let j = from; j <= to; j++) {
+        const weight = this.kernel(j - center);
+        acc += buffer[j]! * weight;
+        norm += weight;
+      }
+      // Normalizing keeps unity gain exact despite kernel truncation.
+      out[i] = norm !== 0 ? acc / norm : 0;
+    }
+    this.position += count * this.step;
+    return out;
+  }
+
+  /** Resample the next chunk; may return 0 samples while buffering. */
+  process(chunk: Float32Array): Float32Array {
+    if (this.passthrough) return chunk;
+    let buffer: Float32Array;
+    if (this.pending.length > 0) {
+      buffer = new Float32Array(this.pending.length + chunk.length);
+      buffer.set(this.pending, 0);
+      buffer.set(chunk, this.pending.length);
+    } else {
+      buffer = chunk;
+    }
+    // Only centers whose kernel fits entirely in the buffered input.
+    const out = this.produce(buffer, buffer.length - 1 - this.half);
+    const keepFrom = Math.max(0, Math.floor(this.position) - this.half);
+    this.pending = buffer.slice(keepFrom);
+    this.position -= keepFrom;
+    return out;
+  }
+
+  /** Drain the buffered tail (end of stream) and reset. */
+  flush(): Float32Array {
+    if (this.passthrough || this.pending.length === 0) {
+      this.pending = new Float32Array(0);
+      this.position = 0;
+      return new Float32Array(0);
+    }
+    const limit = this.pending.length - 1;
+    const padded = new Float32Array(this.pending.length + this.half + 1);
+    padded.set(this.pending, 0);
+    const out = this.produce(padded, limit);
+    this.pending = new Float32Array(0);
+    this.position = 0;
+    return out;
+  }
+}
+
 /**
  * Downmix + resample a decoded `AudioBuffer` to 16 kHz mono using
  * `OfflineAudioContext` (D-012). Browser-only.

src/lib/live.ts

@@ -45,6 +45,12 @@ export interface LiveSegmenterOptions {
   frameSeconds?: number;
   /** A frame at/above this RMS counts as speech. Default 0.015. */
   speechRms?: number;
+  /**
+   * Hysteresis: once a segment is open, frames stay "speech" down to
+   * `speechRms × sustainRmsRatio`, so quiet phonemes and word tails don't
+   * count as silence and fragment the utterance. Default 0.5.
+   */
+  sustainRmsRatio?: number;
   /** Trailing silence that commits the current segment. Default 0.75 s. */
   commitSilenceSeconds?: number;
   /**
@@ -52,6 +58,13 @@ export interface LiveSegmenterOptions {
    * segment is always a single-chunk transcription. Default 25 s.
    */
   maxSegmentSeconds?: number;
+  /**
+   * When a segment hits the max length, prefer to cut at the quietest gap
+   * within this trailing window instead of mid-word at the hard cap; the
+   * remainder after the gap seeds the next segment. Cutting mid-word corrupts
+   * the boundary word in *both* segments. Default 3 s.
+   */
+  softCutWindowSeconds?: number;
   /** Audio kept from just before speech onset. Default 0.3 s. */
   prerollSeconds?: number;
   /**
@@ -65,6 +78,8 @@ export type CommitReason = "silence" | "max-length" | "flush";
 
 /** One committed speech segment, ready for a final transcription pass. */
 export interface CommittedSegment {
+  /** The segmenter's id for this segment (matches `currentSegmentId`). */
+  id: number;
   audio: Float32Array;
   /** Absolute offset of `audio[0]` in samples since the session started. */
   startSample: number;
@@ -75,6 +90,9 @@ interface Frame {
   samples: Float32Array;
   /** Absolute offset of this frame's first sample. */
   start: number;
+  rms: number;
+  /** Classified as speech when appended (onset/sustain threshold applied). */
+  speech: boolean;
 }
 
 /**
@@ -85,8 +103,10 @@ export class LiveSegmenter {
   private readonly sampleRate: number;
   private readonly frameSamples: number;
   private readonly speechRms: number;
+  private readonly sustainRms: number;
   private readonly commitSilenceFrames: number;
   private readonly maxSegmentSamples: number;
+  private readonly softCutWindowFrames: number;
   private readonly maxPrerollFrames: number;
   private readonly minSpeechSamples: number;
 
@@ -109,6 +129,7 @@ export class LiveSegmenter {
     const frameSeconds = options.frameSeconds ?? 0.032;
     this.frameSamples = Math.max(1, Math.round(frameSeconds * this.sampleRate));
     this.speechRms = options.speechRms ?? 0.015;
+    this.sustainRms = this.speechRms * (options.sustainRmsRatio ?? 0.5);
     this.commitSilenceFrames = Math.max(
       1,
       Math.ceil((options.commitSilenceSeconds ?? 0.75) / frameSeconds),
@@ -116,6 +137,10 @@ export class LiveSegmenter {
     this.maxSegmentSamples = Math.round(
       (options.maxSegmentSeconds ?? 25) * this.sampleRate,
     );
+    this.softCutWindowFrames = Math.max(
+      1,
+      Math.ceil((options.softCutWindowSeconds ?? 3) / frameSeconds),
+    );
     this.maxPrerollFrames = Math.ceil((options.prerollSeconds ?? 0.3) / frameSeconds);
     this.minSpeechSamples = Math.round(
       (options.minSpeechSeconds ?? 0.25) * this.sampleRate,
@@ -157,13 +182,16 @@ export class LiveSegmenter {
     let offset = 0;
     while (buffer.length - offset >= this.frameSamples) {
       // Copy the frame — the caller may reuse/transfer its buffer.
+      const samples = buffer.slice(offset, offset + this.frameSamples);
       const frame: Frame = {
-        samples: buffer.slice(offset, offset + this.frameSamples),
+        samples,
         start: this.absSample,
+        rms: rms(samples),
+        speech: false, // classified in processFrame (threshold is stateful)
       };
       offset += this.frameSamples;
       this.absSample += this.frameSamples;
-      const segment = this.processFrame(frame, rms(frame.samples) >= this.speechRms);
+      const segment = this.processFrame(frame);
       if (segment) committed.push(segment);
     }
     this.carry = buffer.slice(offset);
@@ -173,8 +201,13 @@ export class LiveSegmenter {
   /** Commit whatever is in progress (session stop). */
   flush(): CommittedSegment | null {
     if (this.segment && this.carry.length > 0) {
-      // Sub-frame tail — too short to classify, but part of the speech.
-      this.segment.push({ samples: this.carry, start: this.absSample });
+      // Sub-frame tail — too short to gate reliably, but part of the speech.
+      this.segment.push({
+        samples: this.carry,
+        start: this.absSample,
+        rms: rms(this.carry),
+        speech: false,
+      });
       this.segmentSamples += this.carry.length;
       this.absSample += this.carry.length;
       this.carry = new Float32Array(0);
@@ -182,13 +215,14 @@ export class LiveSegmenter {
     return this.commit("flush");
   }
 
-  private processFrame(frame: Frame, speech: boolean): CommittedSegment | null {
+  private processFrame(frame: Frame): CommittedSegment | null {
     if (!this.segment) {
-      if (!speech) {
+      if (frame.rms < this.speechRms) {
         this.preroll.push(frame);
         if (this.preroll.length > this.maxPrerollFrames) this.preroll.shift();
         return null;
       }
+      frame.speech = true;
       this.segment = [...this.preroll, frame];
       this.segmentStart = this.segment[0]!.start;
       this.segmentSamples = this.segment.length * this.frameSamples;
@@ -199,25 +233,73 @@ export class LiveSegmenter {
       return null;
     }
 
+    // Hysteresis: inside a segment the bar is lower, so quiet word tails and
+    // soft phonemes extend the utterance instead of counting as silence.
+    frame.speech = frame.rms >= this.sustainRms;
     this.segment.push(frame);
     this.segmentSamples += this.frameSamples;
-    if (speech) {
+    if (frame.speech) {
       this.speechSamples += this.frameSamples;
       this.silenceFrames = 0;
     } else {
       this.silenceFrames += 1;
       if (this.silenceFrames >= this.commitSilenceFrames) return this.commit("silence");
     }
-    if (this.segmentSamples >= this.maxSegmentSamples) return this.commit("max-length");
+    if (this.segmentSamples >= this.maxSegmentSamples) return this.softCommitAtMaxLength();
     return null;
   }
 
+  /**
+   * Max-length commit: cut at the last quiet frame in the trailing window so
+   * the boundary lands in a real gap, and seed the next segment with the
+   * remainder. Continuous loud speech (no quiet frame) hard-cuts at the cap.
+   */
+  private softCommitAtMaxLength(): CommittedSegment | null {
+    const frames = this.segment!;
+    let cut = -1;
+    const windowStart = Math.max(1, frames.length - this.softCutWindowFrames);
+    for (let i = frames.length - 1; i >= windowStart; i--) {
+      if (frames[i]!.rms < this.sustainRms) {
+        cut = i;
+        break;
+      }
+    }
+    if (cut === -1 || cut === frames.length - 1) return this.commit("max-length");
+
+    const head = frames.slice(0, cut + 1);
+    const tail = frames.slice(cut + 1);
+    const headSpeech = head.filter((frame) => frame.speech).length * this.frameSamples;
+    const result: CommittedSegment | null =
+      headSpeech >= this.minSpeechSamples
+        ? {
+            id: this.segmentId!,
+            audio: concat(
+              head.map((frame) => frame.samples),
+              head.length * this.frameSamples,
+            ),
+            startSample: this.segmentStart,
+            reason: "max-length",
+          }
+        : null; // near-silence head: drop it, keep the live tail (D-016)
+
+    this.segment = tail;
+    this.segmentStart = tail[0]!.start;
+    this.segmentSamples = tail.length * this.frameSamples;
+    this.speechSamples = tail.filter((frame) => frame.speech).length * this.frameSamples;
+    let trailingSilence = 0;
+    for (let i = tail.length - 1; i >= 0 && !tail[i]!.speech; i--) trailingSilence++;
+    this.silenceFrames = trailingSilence;
+    this.segmentId = this.nextSegmentId++;
+    return result;
+  }
+
   private commit(reason: CommitReason): CommittedSegment | null {
     const segment = this.segment;
     if (!segment) return null;
     const result: CommittedSegment | null =
       this.speechSamples >= this.minSpeechSamples
         ? {
+            id: this.segmentId!,
             audio: concat(
               segment.map((frame) => frame.samples),
               this.segmentSamples,
@@ -264,6 +346,25 @@ export interface LiveUpdate {
   error: string | null;
 }
 
+/**
+ * Production tuning, shared between src/App.tsx and the accuracy tests
+ * (tests/live-accuracy.test.ts) so what we test is what users get.
+ *
+ * Latency rationale: the single-flight scheduler already guarantees inference
+ * never piles up, so the throttles only need to bound CPU, not protect
+ * correctness. 1.2 s commit silence keeps utterances whole (with sustain
+ * hysteresis covering quiet phonemes) while committing fast enough that text
+ * freezes shortly after a natural pause.
+ */
+export const LIVE_TUNING = {
+  tentativeIntervalMs: 2000,
+  minTentativeSeconds: 1,
+  segmenter: {
+    commitSilenceSeconds: 1.2,
+    maxSegmentSeconds: 29,
+  } satisfies LiveSegmenterOptions,
+} as const;
+
 export interface LiveTranscriberOptions {
   transcribe: LiveTranscribeFn;
   onUpdate: (update: LiveUpdate) => void;
@@ -304,6 +405,8 @@ export class LiveTranscriber {
 
   private segments: SegmentState[] = [];
   private tentativeText = "";
+  /** Segmenter id of the segment `tentativeText` describes (null: none). */
+  private tentativeSegmentId: number | null = null;
   private lastTentative: { segmentId: number; samples: number; at: number } | null =
     null;
   private inFlight = false;
@@ -329,6 +432,17 @@ export class LiveTranscriber {
     for (const committed of this.segmenter.push(samples)) {
       this.acceptSegment(committed);
     }
+    if (
+      this.tentativeSegmentId !== null &&
+      this.segmenter.currentSegmentId !== this.tentativeSegmentId
+    ) {
+      // The segment the tentative text described ended without committing —
+      // the segmenter dropped it as near-silence. Its text is very likely a
+      // hallucination over breath/noise; discard instead of leaving it on
+      // screen (or letting the next segment adopt it as provisional text).
+      this.clearTentative();
+      this.emit();
+    }
     this.pump();
   }
 
@@ -343,7 +457,7 @@ export class LiveTranscriber {
       if (flushed) this.acceptSegment(flushed);
       // Anything tentative either moved into the flushed segment or belonged
       // to audio the segmenter dropped as near-silence.
-      this.tentativeText = "";
+      this.clearTentative();
       this.pump();
     }
     await this.whenDrained();
@@ -351,18 +465,26 @@ export class LiveTranscriber {
     return { text: this.committedText(), chunks: this.allChunks() };
   }
 
+  private clearTentative(): void {
+    this.tentativeText = "";
+    this.tentativeSegmentId = null;
+    this.lastTentative = null;
+  }
+
   private acceptSegment(committed: CommittedSegment): void {
-    // The committing segment is the one the tentative text described; carry
-    // it as the provisional text until the final pass replaces it.
+    // Carry the tentative text as this segment's provisional text until the
+    // final pass replaces it — but only when it actually described this
+    // segment (a dropped predecessor's leftover text must not leak in).
+    const provisional =
+      committed.id === this.tentativeSegmentId ? this.tentativeText : "";
     this.segments.push({
       startSample: committed.startSample,
       audio: committed.audio,
-      text: this.tentativeText,
+      text: provisional,
       chunks: [],
       final: false,
     });
-    this.tentativeText = "";
-    this.lastTentative = null;
+    this.clearTentative();
     this.emit();
   }
 
@@ -431,6 +553,7 @@ export class LiveTranscriber {
           // pass supersedes this) or been dropped while we were busy.
           if (this.segmenter.currentSegmentId === segmentId) {
             this.tentativeText = result.text;
+            this.tentativeSegmentId = segmentId;
           }
         },
         (error: unknown) => {

src/lib/mic.ts

@@ -5,8 +5,10 @@
  * cannot connect a MediaStreamAudioSourceNode across sample rates, so the
  * context is never forced to 16 kHz) → an AudioWorklet chunker posts ~100 ms
  * Float32 blocks to the main thread, which resamples them to 16 kHz with the
- * pure `resampleLinear` before handing them to the live segment loop
- * (src/lib/live.ts).
+ * pure `StreamingResampler` (anti-aliased, chunk-boundary safe — plain
+ * `resampleLinear` aliases everything above 8 kHz into the speech band and
+ * measurably hurts Whisper accuracy) before handing them to the live segment
+ * loop (src/lib/live.ts).
  *
  * The worklet module is created from an inline source string via a Blob URL:
  * no extra served file, no build or service-worker changes, and capture keeps
@@ -17,7 +19,7 @@
  * (Architecture.md §5).
  */
 
-import { TARGET_SAMPLE_RATE, resampleLinear } from "./audio";
+import { StreamingResampler, TARGET_SAMPLE_RATE } from "./audio";
 
 export interface MicCapture {
   /** Native rate of the capture context (chunks are resampled to 16 kHz). */
@@ -131,18 +133,18 @@ export async function startMicCapture(
     audio: { echoCancellation: true, noiseSuppression: true, autoGainControl: true },
   });
 
-  const context = new AudioContext({ sampleRate: TARGET_SAMPLE_RATE });
+  // Native rate, never forced to 16 kHz: Firefox cannot connect a
+  // MediaStreamAudioSourceNode across sample rates (D-016).
+  const context = new AudioContext();
   // Contexts created outside a user gesture start suspended; the start button
   // click satisfies the gesture requirement, so this resolves immediately.
   await context.resume().catch(() => {});
   const source = context.createMediaStreamSource(stream);
 
+  const resampler = new StreamingResampler(context.sampleRate, TARGET_SAMPLE_RATE);
   const deliver = (samples: Float32Array) => {
-    onChunk(
-      context.sampleRate === TARGET_SAMPLE_RATE
-        ? samples
-        : resampleLinear(samples, context.sampleRate, TARGET_SAMPLE_RATE),
-    );
+    const resampled = resampler.process(samples);
+    if (resampled.length > 0) onChunk(resampled);
   };
 
   let detach: () => void;
@@ -162,6 +164,10 @@ export async function startMicCapture(
       if (stopped) return;
       stopped = true;
       detach();
+      // Drain the resampler's look-ahead tail so the last spoken samples
+      // reach the segment loop before the session flushes.
+      const tail = resampler.flush();
+      if (tail.length > 0) onChunk(tail);
       for (const track of stream.getTracks()) track.stop();
       await context.close().catch(() => {});
     },