au_whisper_web/src/lib/mic.ts

/**
 * Microphone capture for live transcription (Decisions.md D-016). Browser-only.
 *
 * `getUserMedia` → `AudioContext` at the device's **native** rate (Firefox
 * cannot connect a MediaStreamAudioSourceNode across sample rates, so the
 * context is never forced to 16 kHz) → an AudioWorklet chunker posts ~100 ms
 * Float32 blocks to the main thread, which resamples them to 16 kHz with the
 * pure `StreamingResampler` (anti-aliased, chunk-boundary safe — plain
 * `resampleLinear` aliases everything above 8 kHz into the speech band and
 * measurably hurts Whisper accuracy) before handing them to the live segment
 * loop (src/lib/live.ts).
 *
 * The worklet module is created from an inline source string via a Blob URL:
 * no extra served file, no build or service-worker changes, and capture keeps
 * working offline. Browsers without AudioWorklet (or that reject Blob-URL
 * modules) fall back to a ScriptProcessorNode.
 *
 * Mic audio is processed entirely on-device, like every browser-engine path
 * (Architecture.md §5).
 */

import { StreamingResampler, TARGET_SAMPLE_RATE } from "./audio";

export interface MicCapture {
  /** Native rate of the capture context (chunks are resampled to 16 kHz). */
  sampleRate: number;
  /** True when the AudioWorklet path failed and ScriptProcessorNode is used. */
  usingFallback: boolean;
  /** Release the mic (stops the permission indicator) and the audio graph. */
  stop(): Promise<void>;
}

const WORKLET_NAME = "live-mic-chunker";

// Runs in the AudioWorklet global scope — plain JS, kept dependency-free.
// Accumulates the 128-sample render quanta into ~100 ms chunks so the main
// thread handles ~10 messages/s instead of ~375.
const WORKLET_SOURCE = `
registerProcessor("${WORKLET_NAME}", class extends AudioWorkletProcessor {
  constructor() {
    super();
    this.blocks = [];
    this.length = 0;
    this.target = Math.max(128, Math.round(sampleRate / 10));
  }
  process(inputs) {
    const channel = inputs[0] && inputs[0][0];
    if (channel && channel.length > 0) {
      this.blocks.push(new Float32Array(channel));
      this.length += channel.length;
      if (this.length >= this.target) {
        const out = new Float32Array(this.length);
        let offset = 0;
        for (const block of this.blocks) {
          out.set(block, offset);
          offset += block.length;
        }
        this.blocks = [];
        this.length = 0;
        this.port.postMessage(out, [out.buffer]);
      }
    }
    return true;
  }
});
`;

async function attachWorklet(
  context: AudioContext,
  source: MediaStreamAudioSourceNode,
  deliver: (samples: Float32Array) => void,
): Promise<() => void> {
  const url = URL.createObjectURL(
    new Blob([WORKLET_SOURCE], { type: "text/javascript" }),
  );
  try {
    await context.audioWorklet.addModule(url);
  } finally {
    URL.revokeObjectURL(url);
  }
  const node = new AudioWorkletNode(context, WORKLET_NAME, {
    numberOfInputs: 1,
    numberOfOutputs: 0,
    channelCount: 1,
  });
  node.port.onmessage = (event: MessageEvent) => deliver(event.data as Float32Array);
  source.connect(node);
  return () => {
    node.port.onmessage = null;
    source.disconnect(node);
  };
}

function attachScriptProcessor(
  context: AudioContext,
  source: MediaStreamAudioSourceNode,
  deliver: (samples: Float32Array) => void,
): () => void {
  const node = context.createScriptProcessor(4096, 1, 1);
  node.onaudioprocess = (event) => {
    // The input buffer is reused between callbacks — copy before handing off.
    deliver(new Float32Array(event.inputBuffer.getChannelData(0)));
  };
  // ScriptProcessorNode only fires when connected to the destination; route
  // it through a muted gain so the mic is never played back.
  const mute = context.createGain();
  mute.gain.value = 0;
  source.connect(node);
  node.connect(mute);
  mute.connect(context.destination);
  return () => {
    node.onaudioprocess = null;
    source.disconnect(node);
    node.disconnect();
    mute.disconnect();
  };
}

/**
 * Open the microphone and stream 16 kHz mono Float32 chunks to `onChunk`
 * until `stop()`. Rejects if the user denies mic permission (surface the
 * error; do not retry without a user gesture).
 */
export async function startMicCapture(
  onChunk: (samples: Float32Array) => void,
): Promise<MicCapture> {
  if (!navigator.mediaDevices?.getUserMedia) {
    throw new Error(
      "Microphone capture is not supported in this browser (requires a secure context).",
    );
  }
  const stream = await navigator.mediaDevices.getUserMedia({
    audio: { echoCancellation: true, noiseSuppression: true, autoGainControl: true },
  });

  // Native rate, never forced to 16 kHz: Firefox cannot connect a
  // MediaStreamAudioSourceNode across sample rates (D-016).
  const context = new AudioContext();
  // Contexts created outside a user gesture start suspended; the start button
  // click satisfies the gesture requirement, so this resolves immediately.
  await context.resume().catch(() => {});
  const source = context.createMediaStreamSource(stream);

  const resampler = new StreamingResampler(context.sampleRate, TARGET_SAMPLE_RATE);
  const deliver = (samples: Float32Array) => {
    const resampled = resampler.process(samples);
    if (resampled.length > 0) onChunk(resampled);
  };

  let detach: () => void;
  let usingFallback = false;
  try {
    detach = await attachWorklet(context, source, deliver);
  } catch {
    detach = attachScriptProcessor(context, source, deliver);
    usingFallback = true;
  }

  let stopped = false;
  return {
    sampleRate: context.sampleRate,
    usingFallback,
    async stop() {
      if (stopped) return;
      stopped = true;
      detach();
      // Drain the resampler's look-ahead tail so the last spoken samples
      // reach the segment loop before the session flushes.
      const tail = resampler.flush();
      if (tail.length > 0) onChunk(tail);
      for (const track of stream.getTracks()) track.stop();
      await context.close().catch(() => {});
    },
  };
}