/** * Audio decoding for the browser engine (Decisions.md D-012). * * Uploaded audio/video File → `AudioContext.decodeAudioData` → mono downmix → * resample to 16 kHz via `OfflineAudioContext` → `Float32Array` (transferred * to the inference worker, zero-copy). * * Browser-engine audio never touches the network (Architecture.md §5): all * work happens locally with Web Audio APIs. * * `downmixToMono` / `resampleLinear` are pure and unit-testable under * `bun test`; the browser path uses `OfflineAudioContext` (higher-quality * resampler) and only needs the pure helpers as a documented fallback. * * No ffmpeg.wasm so far — `decodeAudioData` has covered every container we * have tried. Note here if we hit a real gap (per D-012). */ /** Whisper models consume 16 kHz mono PCM. */ export const TARGET_SAMPLE_RATE = 16_000; /** * Average N channels into one. Pure — safe to call anywhere. * Channels of unequal length are truncated to the shortest. */ export function downmixToMono(channels: readonly Float32Array[]): Float32Array { if (channels.length === 0) { throw new Error("downmixToMono: need at least one channel"); } if (channels.length === 1) { return Float32Array.from(channels[0]!); } const length = Math.min(...channels.map((c) => c.length)); const out = new Float32Array(length); for (let i = 0; i < length; i++) { let sum = 0; for (const channel of channels) { sum += channel[i]!; } out[i] = sum / channels.length; } return out; } /** * Linear-interpolation resampler. Pure — safe to call anywhere. * The browser path prefers `OfflineAudioContext` (D-012); this exists as a * testable reference implementation and non-browser fallback. */ export function resampleLinear( input: Float32Array, fromRate: number, toRate: number, ): Float32Array { if (!(fromRate > 0) || !(toRate > 0)) { throw new Error("resampleLinear: sample rates must be positive"); } if (fromRate === toRate) { return Float32Array.from(input); } if (input.length === 0) { return new Float32Array(0); } const outLength = Math.max(1, Math.round((input.length * toRate) / fromRate)); const out = new Float32Array(outLength); const step = fromRate / toRate; const last = input.length - 1; for (let i = 0; i < outLength; i++) { const position = Math.min(i * step, last); const index = Math.floor(position); const nextIndex = Math.min(index + 1, last); const fraction = position - index; out[i] = input[index]! * (1 - fraction) + input[nextIndex]! * fraction; } return out; } /** * Streaming anti-aliased resampler (windowed-sinc low-pass interpolation). * Pure — safe to call anywhere. * * `resampleLinear` has no low-pass filter, so decimating mic audio * (48 kHz → 16 kHz) folds everything above 8 kHz back into the speech band — * audible as aliasing noise and measurably bad for Whisper accuracy. This * class filters at the correct cutoff while resampling, and it is stateful: * feed it arbitrary chunk sizes (the mic path delivers ~100 ms blocks) and * the output is identical to resampling the whole signal at once — no edge * artifacts at chunk boundaries. */ export class StreamingResampler { /** Input samples advanced per output sample. */ private readonly step: number; /** Kernel half-width, in input samples. */ private readonly half: number; /** Normalized low-pass cutoff, cycles per input sample. */ private readonly cutoff: number; private readonly passthrough: boolean; /** Unconsumed input tail (kernel needs `half` samples of look-ahead). */ private pending = new Float32Array(0); /** Fractional index into `pending` of the next output sample's center. */ private position = 0; constructor(fromRate: number, toRate: number) { if (!(fromRate > 0) || !(toRate > 0)) { throw new Error("StreamingResampler: sample rates must be positive"); } this.passthrough = fromRate === toRate; this.step = fromRate / toRate; // 0.45×Nyquist of the slower rate: keeps the whole speech band while the // transition band stays inside what the window can attenuate. this.cutoff = (0.45 * Math.min(fromRate, toRate)) / fromRate; this.half = Math.ceil(2 / this.cutoff); } /** Blackman-windowed sinc, low-pass at `cutoff`, support (−half, half). */ private kernel(distance: number): number { const scaled = 2 * this.cutoff * distance; const x = Math.PI * scaled; const sinc = scaled === 0 ? 1 : Math.sin(x) / x; const t = (distance / this.half + 1) / 2; // 0..1 across the support const window = 0.42 - 0.5 * Math.cos(2 * Math.PI * t) + 0.08 * Math.cos(4 * Math.PI * t); return 2 * this.cutoff * sinc * window; } /** Emit output samples for every center that has full kernel support. */ private produce(buffer: Float32Array, limit: number): Float32Array { const count = Math.max( 0, Math.floor((limit - this.position) / this.step) + 1, ); const out = new Float32Array(count); for (let i = 0; i < count; i++) { const center = this.position + i * this.step; const from = Math.max(0, Math.ceil(center - this.half)); const to = Math.min(buffer.length - 1, Math.floor(center + this.half)); let acc = 0; let norm = 0; for (let j = from; j <= to; j++) { const weight = this.kernel(j - center); acc += buffer[j]! * weight; norm += weight; } // Normalizing keeps unity gain exact despite kernel truncation. out[i] = norm !== 0 ? acc / norm : 0; } this.position += count * this.step; return out; } /** Resample the next chunk; may return 0 samples while buffering. */ process(chunk: Float32Array): Float32Array { if (this.passthrough) return chunk; let buffer: Float32Array; if (this.pending.length > 0) { buffer = new Float32Array(this.pending.length + chunk.length); buffer.set(this.pending, 0); buffer.set(chunk, this.pending.length); } else { buffer = chunk; } // Only centers whose kernel fits entirely in the buffered input. const out = this.produce(buffer, buffer.length - 1 - this.half); const keepFrom = Math.max(0, Math.floor(this.position) - this.half); this.pending = buffer.slice(keepFrom); this.position -= keepFrom; return out; } /** Drain the buffered tail (end of stream) and reset. */ flush(): Float32Array { if (this.passthrough || this.pending.length === 0) { this.pending = new Float32Array(0); this.position = 0; return new Float32Array(0); } const limit = this.pending.length - 1; const padded = new Float32Array(this.pending.length + this.half + 1); padded.set(this.pending, 0); const out = this.produce(padded, limit); this.pending = new Float32Array(0); this.position = 0; return out; } } /** * Downmix + resample a decoded `AudioBuffer` to 16 kHz mono using * `OfflineAudioContext` (D-012). Browser-only. */ export async function audioBufferToModelInput( buffer: AudioBuffer, ): Promise { if (buffer.sampleRate === TARGET_SAMPLE_RATE && buffer.numberOfChannels === 1) { return buffer.getChannelData(0); } const length = Math.max(1, Math.ceil(buffer.duration * TARGET_SAMPLE_RATE)); // Single-channel destination: the OfflineAudioContext performs the mono // downmix and the (windowed-sinc quality) resample in one render pass. const offline = new OfflineAudioContext(1, length, TARGET_SAMPLE_RATE); const source = offline.createBufferSource(); source.buffer = buffer; source.connect(offline.destination); source.start(0); const rendered = await offline.startRendering(); return rendered.getChannelData(0); } /** * Decode an uploaded audio or video file to 16 kHz mono `Float32Array`, * ready to transfer to the inference worker. Browser-only (Web Audio API). */ export async function decodeAudioFile(file: Blob): Promise { const bytes = await file.arrayBuffer(); // decodeAudioData handles most audio containers and audio-in-video (D-012). // A suspended AudioContext (no user gesture) can still decode. const context = new AudioContext(); try { const decoded = await context.decodeAudioData(bytes); return await audioBufferToModelInput(decoded); } catch (error) { throw new Error( "Could not decode this file. Make sure it is an audio or video file " + `in a format your browser supports. (${error instanceof Error ? error.message : String(error)})`, ); } finally { void context.close(); } }