121 lines
4.2 KiB
TypeScript
121 lines
4.2 KiB
TypeScript
/**
|
|
* Audio decoding for the browser engine (Decisions.md D-012).
|
|
*
|
|
* Uploaded audio/video File → `AudioContext.decodeAudioData` → mono downmix →
|
|
* resample to 16 kHz via `OfflineAudioContext` → `Float32Array` (transferred
|
|
* to the inference worker, zero-copy).
|
|
*
|
|
* Browser-engine audio never touches the network (Architecture.md §5): all
|
|
* work happens locally with Web Audio APIs.
|
|
*
|
|
* `downmixToMono` / `resampleLinear` are pure and unit-testable under
|
|
* `bun test`; the browser path uses `OfflineAudioContext` (higher-quality
|
|
* resampler) and only needs the pure helpers as a documented fallback.
|
|
*
|
|
* No ffmpeg.wasm so far — `decodeAudioData` has covered every container we
|
|
* have tried. Note here if we hit a real gap (per D-012).
|
|
*/
|
|
|
|
/** Whisper models consume 16 kHz mono PCM. */
|
|
export const TARGET_SAMPLE_RATE = 16_000;
|
|
|
|
/**
|
|
* Average N channels into one. Pure — safe to call anywhere.
|
|
* Channels of unequal length are truncated to the shortest.
|
|
*/
|
|
export function downmixToMono(channels: readonly Float32Array[]): Float32Array {
|
|
if (channels.length === 0) {
|
|
throw new Error("downmixToMono: need at least one channel");
|
|
}
|
|
if (channels.length === 1) {
|
|
return Float32Array.from(channels[0]!);
|
|
}
|
|
const length = Math.min(...channels.map((c) => c.length));
|
|
const out = new Float32Array(length);
|
|
for (let i = 0; i < length; i++) {
|
|
let sum = 0;
|
|
for (const channel of channels) {
|
|
sum += channel[i]!;
|
|
}
|
|
out[i] = sum / channels.length;
|
|
}
|
|
return out;
|
|
}
|
|
|
|
/**
|
|
* Linear-interpolation resampler. Pure — safe to call anywhere.
|
|
* The browser path prefers `OfflineAudioContext` (D-012); this exists as a
|
|
* testable reference implementation and non-browser fallback.
|
|
*/
|
|
export function resampleLinear(
|
|
input: Float32Array,
|
|
fromRate: number,
|
|
toRate: number,
|
|
): Float32Array {
|
|
if (!(fromRate > 0) || !(toRate > 0)) {
|
|
throw new Error("resampleLinear: sample rates must be positive");
|
|
}
|
|
if (fromRate === toRate) {
|
|
return Float32Array.from(input);
|
|
}
|
|
if (input.length === 0) {
|
|
return new Float32Array(0);
|
|
}
|
|
const outLength = Math.max(1, Math.round((input.length * toRate) / fromRate));
|
|
const out = new Float32Array(outLength);
|
|
const step = fromRate / toRate;
|
|
const last = input.length - 1;
|
|
for (let i = 0; i < outLength; i++) {
|
|
const position = Math.min(i * step, last);
|
|
const index = Math.floor(position);
|
|
const nextIndex = Math.min(index + 1, last);
|
|
const fraction = position - index;
|
|
out[i] = input[index]! * (1 - fraction) + input[nextIndex]! * fraction;
|
|
}
|
|
return out;
|
|
}
|
|
|
|
/**
|
|
* Downmix + resample a decoded `AudioBuffer` to 16 kHz mono using
|
|
* `OfflineAudioContext` (D-012). Browser-only.
|
|
*/
|
|
export async function audioBufferToModelInput(
|
|
buffer: AudioBuffer,
|
|
): Promise<Float32Array> {
|
|
if (buffer.sampleRate === TARGET_SAMPLE_RATE && buffer.numberOfChannels === 1) {
|
|
return buffer.getChannelData(0);
|
|
}
|
|
const length = Math.max(1, Math.ceil(buffer.duration * TARGET_SAMPLE_RATE));
|
|
// Single-channel destination: the OfflineAudioContext performs the mono
|
|
// downmix and the (windowed-sinc quality) resample in one render pass.
|
|
const offline = new OfflineAudioContext(1, length, TARGET_SAMPLE_RATE);
|
|
const source = offline.createBufferSource();
|
|
source.buffer = buffer;
|
|
source.connect(offline.destination);
|
|
source.start(0);
|
|
const rendered = await offline.startRendering();
|
|
return rendered.getChannelData(0);
|
|
}
|
|
|
|
/**
|
|
* Decode an uploaded audio or video file to 16 kHz mono `Float32Array`,
|
|
* ready to transfer to the inference worker. Browser-only (Web Audio API).
|
|
*/
|
|
export async function decodeAudioFile(file: Blob): Promise<Float32Array> {
|
|
const bytes = await file.arrayBuffer();
|
|
// decodeAudioData handles most audio containers and audio-in-video (D-012).
|
|
// A suspended AudioContext (no user gesture) can still decode.
|
|
const context = new AudioContext();
|
|
try {
|
|
const decoded = await context.decodeAudioData(bytes);
|
|
return await audioBufferToModelInput(decoded);
|
|
} catch (error) {
|
|
throw new Error(
|
|
"Could not decode this file. Make sure it is an audio or video file " +
|
|
`in a format your browser supports. (${error instanceof Error ? error.message : String(error)})`,
|
|
);
|
|
} finally {
|
|
void context.close();
|
|
}
|
|
}
|