/** * Audio decoding for the browser engine (Decisions.md D-012). * * Uploaded audio/video File → `AudioContext.decodeAudioData` → mono downmix → * resample to 16 kHz via `OfflineAudioContext` → `Float32Array` (transferred * to the inference worker, zero-copy). * * Browser-engine audio never touches the network (Architecture.md §5): all * work happens locally with Web Audio APIs. * * `downmixToMono` / `resampleLinear` are pure and unit-testable under * `bun test`; the browser path uses `OfflineAudioContext` (higher-quality * resampler) and only needs the pure helpers as a documented fallback. * * No ffmpeg.wasm so far — `decodeAudioData` has covered every container we * have tried. Note here if we hit a real gap (per D-012). */ /** Whisper models consume 16 kHz mono PCM. */ export const TARGET_SAMPLE_RATE = 16_000; /** * Average N channels into one. Pure — safe to call anywhere. * Channels of unequal length are truncated to the shortest. */ export function downmixToMono(channels: readonly Float32Array[]): Float32Array { if (channels.length === 0) { throw new Error("downmixToMono: need at least one channel"); } if (channels.length === 1) { return Float32Array.from(channels[0]!); } const length = Math.min(...channels.map((c) => c.length)); const out = new Float32Array(length); for (let i = 0; i < length; i++) { let sum = 0; for (const channel of channels) { sum += channel[i]!; } out[i] = sum / channels.length; } return out; } /** * Linear-interpolation resampler. Pure — safe to call anywhere. * The browser path prefers `OfflineAudioContext` (D-012); this exists as a * testable reference implementation and non-browser fallback. */ export function resampleLinear( input: Float32Array, fromRate: number, toRate: number, ): Float32Array { if (!(fromRate > 0) || !(toRate > 0)) { throw new Error("resampleLinear: sample rates must be positive"); } if (fromRate === toRate) { return Float32Array.from(input); } if (input.length === 0) { return new Float32Array(0); } const outLength = Math.max(1, Math.round((input.length * toRate) / fromRate)); const out = new Float32Array(outLength); const step = fromRate / toRate; const last = input.length - 1; for (let i = 0; i < outLength; i++) { const position = Math.min(i * step, last); const index = Math.floor(position); const nextIndex = Math.min(index + 1, last); const fraction = position - index; out[i] = input[index]! * (1 - fraction) + input[nextIndex]! * fraction; } return out; } /** * Downmix + resample a decoded `AudioBuffer` to 16 kHz mono using * `OfflineAudioContext` (D-012). Browser-only. */ export async function audioBufferToModelInput( buffer: AudioBuffer, ): Promise { if (buffer.sampleRate === TARGET_SAMPLE_RATE && buffer.numberOfChannels === 1) { return buffer.getChannelData(0); } const length = Math.max(1, Math.ceil(buffer.duration * TARGET_SAMPLE_RATE)); // Single-channel destination: the OfflineAudioContext performs the mono // downmix and the (windowed-sinc quality) resample in one render pass. const offline = new OfflineAudioContext(1, length, TARGET_SAMPLE_RATE); const source = offline.createBufferSource(); source.buffer = buffer; source.connect(offline.destination); source.start(0); const rendered = await offline.startRendering(); return rendered.getChannelData(0); } /** * Decode an uploaded audio or video file to 16 kHz mono `Float32Array`, * ready to transfer to the inference worker. Browser-only (Web Audio API). */ export async function decodeAudioFile(file: Blob): Promise { const bytes = await file.arrayBuffer(); // decodeAudioData handles most audio containers and audio-in-video (D-012). // A suspended AudioContext (no user gesture) can still decode. const context = new AudioContext(); try { const decoded = await context.decodeAudioData(bytes); return await audioBufferToModelInput(decoded); } catch (error) { throw new Error( "Could not decode this file. Make sure it is an audio or video file " + `in a format your browser supports. (${error instanceof Error ? error.message : String(error)})`, ); } finally { void context.close(); } }