231 lines
8.4 KiB
TypeScript
231 lines
8.4 KiB
TypeScript
/**
|
||
* Audio decoding for the browser engine (Decisions.md D-012).
|
||
*
|
||
* Uploaded audio/video File → `AudioContext.decodeAudioData` → mono downmix →
|
||
* resample to 16 kHz via `OfflineAudioContext` → `Float32Array` (transferred
|
||
* to the inference worker, zero-copy).
|
||
*
|
||
* Browser-engine audio never touches the network (Architecture.md §5): all
|
||
* work happens locally with Web Audio APIs.
|
||
*
|
||
* `downmixToMono` / `resampleLinear` are pure and unit-testable under
|
||
* `bun test`; the browser path uses `OfflineAudioContext` (higher-quality
|
||
* resampler) and only needs the pure helpers as a documented fallback.
|
||
*
|
||
* No ffmpeg.wasm so far — `decodeAudioData` has covered every container we
|
||
* have tried. Note here if we hit a real gap (per D-012).
|
||
*/
|
||
|
||
/** Whisper models consume 16 kHz mono PCM. */
|
||
export const TARGET_SAMPLE_RATE = 16_000;
|
||
|
||
/**
|
||
* Average N channels into one. Pure — safe to call anywhere.
|
||
* Channels of unequal length are truncated to the shortest.
|
||
*/
|
||
export function downmixToMono(channels: readonly Float32Array[]): Float32Array {
|
||
if (channels.length === 0) {
|
||
throw new Error("downmixToMono: need at least one channel");
|
||
}
|
||
if (channels.length === 1) {
|
||
return Float32Array.from(channels[0]!);
|
||
}
|
||
const length = Math.min(...channels.map((c) => c.length));
|
||
const out = new Float32Array(length);
|
||
for (let i = 0; i < length; i++) {
|
||
let sum = 0;
|
||
for (const channel of channels) {
|
||
sum += channel[i]!;
|
||
}
|
||
out[i] = sum / channels.length;
|
||
}
|
||
return out;
|
||
}
|
||
|
||
/**
|
||
* Linear-interpolation resampler. Pure — safe to call anywhere.
|
||
* The browser path prefers `OfflineAudioContext` (D-012); this exists as a
|
||
* testable reference implementation and non-browser fallback.
|
||
*/
|
||
export function resampleLinear(
|
||
input: Float32Array,
|
||
fromRate: number,
|
||
toRate: number,
|
||
): Float32Array {
|
||
if (!(fromRate > 0) || !(toRate > 0)) {
|
||
throw new Error("resampleLinear: sample rates must be positive");
|
||
}
|
||
if (fromRate === toRate) {
|
||
return Float32Array.from(input);
|
||
}
|
||
if (input.length === 0) {
|
||
return new Float32Array(0);
|
||
}
|
||
const outLength = Math.max(1, Math.round((input.length * toRate) / fromRate));
|
||
const out = new Float32Array(outLength);
|
||
const step = fromRate / toRate;
|
||
const last = input.length - 1;
|
||
for (let i = 0; i < outLength; i++) {
|
||
const position = Math.min(i * step, last);
|
||
const index = Math.floor(position);
|
||
const nextIndex = Math.min(index + 1, last);
|
||
const fraction = position - index;
|
||
out[i] = input[index]! * (1 - fraction) + input[nextIndex]! * fraction;
|
||
}
|
||
return out;
|
||
}
|
||
|
||
/**
|
||
* Streaming anti-aliased resampler (windowed-sinc low-pass interpolation).
|
||
* Pure — safe to call anywhere.
|
||
*
|
||
* `resampleLinear` has no low-pass filter, so decimating mic audio
|
||
* (48 kHz → 16 kHz) folds everything above 8 kHz back into the speech band —
|
||
* audible as aliasing noise and measurably bad for Whisper accuracy. This
|
||
* class filters at the correct cutoff while resampling, and it is stateful:
|
||
* feed it arbitrary chunk sizes (the mic path delivers ~100 ms blocks) and
|
||
* the output is identical to resampling the whole signal at once — no edge
|
||
* artifacts at chunk boundaries.
|
||
*/
|
||
export class StreamingResampler {
|
||
/** Input samples advanced per output sample. */
|
||
private readonly step: number;
|
||
/** Kernel half-width, in input samples. */
|
||
private readonly half: number;
|
||
/** Normalized low-pass cutoff, cycles per input sample. */
|
||
private readonly cutoff: number;
|
||
private readonly passthrough: boolean;
|
||
|
||
/** Unconsumed input tail (kernel needs `half` samples of look-ahead). */
|
||
private pending = new Float32Array(0);
|
||
/** Fractional index into `pending` of the next output sample's center. */
|
||
private position = 0;
|
||
|
||
constructor(fromRate: number, toRate: number) {
|
||
if (!(fromRate > 0) || !(toRate > 0)) {
|
||
throw new Error("StreamingResampler: sample rates must be positive");
|
||
}
|
||
this.passthrough = fromRate === toRate;
|
||
this.step = fromRate / toRate;
|
||
// 0.45×Nyquist of the slower rate: keeps the whole speech band while the
|
||
// transition band stays inside what the window can attenuate.
|
||
this.cutoff = (0.45 * Math.min(fromRate, toRate)) / fromRate;
|
||
this.half = Math.ceil(2 / this.cutoff);
|
||
}
|
||
|
||
/** Blackman-windowed sinc, low-pass at `cutoff`, support (−half, half). */
|
||
private kernel(distance: number): number {
|
||
const scaled = 2 * this.cutoff * distance;
|
||
const x = Math.PI * scaled;
|
||
const sinc = scaled === 0 ? 1 : Math.sin(x) / x;
|
||
const t = (distance / this.half + 1) / 2; // 0..1 across the support
|
||
const window =
|
||
0.42 - 0.5 * Math.cos(2 * Math.PI * t) + 0.08 * Math.cos(4 * Math.PI * t);
|
||
return 2 * this.cutoff * sinc * window;
|
||
}
|
||
|
||
/** Emit output samples for every center that has full kernel support. */
|
||
private produce(buffer: Float32Array, limit: number): Float32Array {
|
||
const count = Math.max(
|
||
0,
|
||
Math.floor((limit - this.position) / this.step) + 1,
|
||
);
|
||
const out = new Float32Array(count);
|
||
for (let i = 0; i < count; i++) {
|
||
const center = this.position + i * this.step;
|
||
const from = Math.max(0, Math.ceil(center - this.half));
|
||
const to = Math.min(buffer.length - 1, Math.floor(center + this.half));
|
||
let acc = 0;
|
||
let norm = 0;
|
||
for (let j = from; j <= to; j++) {
|
||
const weight = this.kernel(j - center);
|
||
acc += buffer[j]! * weight;
|
||
norm += weight;
|
||
}
|
||
// Normalizing keeps unity gain exact despite kernel truncation.
|
||
out[i] = norm !== 0 ? acc / norm : 0;
|
||
}
|
||
this.position += count * this.step;
|
||
return out;
|
||
}
|
||
|
||
/** Resample the next chunk; may return 0 samples while buffering. */
|
||
process(chunk: Float32Array): Float32Array {
|
||
if (this.passthrough) return chunk;
|
||
let buffer: Float32Array;
|
||
if (this.pending.length > 0) {
|
||
buffer = new Float32Array(this.pending.length + chunk.length);
|
||
buffer.set(this.pending, 0);
|
||
buffer.set(chunk, this.pending.length);
|
||
} else {
|
||
buffer = chunk;
|
||
}
|
||
// Only centers whose kernel fits entirely in the buffered input.
|
||
const out = this.produce(buffer, buffer.length - 1 - this.half);
|
||
const keepFrom = Math.max(0, Math.floor(this.position) - this.half);
|
||
this.pending = buffer.slice(keepFrom);
|
||
this.position -= keepFrom;
|
||
return out;
|
||
}
|
||
|
||
/** Drain the buffered tail (end of stream) and reset. */
|
||
flush(): Float32Array {
|
||
if (this.passthrough || this.pending.length === 0) {
|
||
this.pending = new Float32Array(0);
|
||
this.position = 0;
|
||
return new Float32Array(0);
|
||
}
|
||
const limit = this.pending.length - 1;
|
||
const padded = new Float32Array(this.pending.length + this.half + 1);
|
||
padded.set(this.pending, 0);
|
||
const out = this.produce(padded, limit);
|
||
this.pending = new Float32Array(0);
|
||
this.position = 0;
|
||
return out;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Downmix + resample a decoded `AudioBuffer` to 16 kHz mono using
|
||
* `OfflineAudioContext` (D-012). Browser-only.
|
||
*/
|
||
export async function audioBufferToModelInput(
|
||
buffer: AudioBuffer,
|
||
): Promise<Float32Array> {
|
||
if (buffer.sampleRate === TARGET_SAMPLE_RATE && buffer.numberOfChannels === 1) {
|
||
return buffer.getChannelData(0);
|
||
}
|
||
const length = Math.max(1, Math.ceil(buffer.duration * TARGET_SAMPLE_RATE));
|
||
// Single-channel destination: the OfflineAudioContext performs the mono
|
||
// downmix and the (windowed-sinc quality) resample in one render pass.
|
||
const offline = new OfflineAudioContext(1, length, TARGET_SAMPLE_RATE);
|
||
const source = offline.createBufferSource();
|
||
source.buffer = buffer;
|
||
source.connect(offline.destination);
|
||
source.start(0);
|
||
const rendered = await offline.startRendering();
|
||
return rendered.getChannelData(0);
|
||
}
|
||
|
||
/**
|
||
* Decode an uploaded audio or video file to 16 kHz mono `Float32Array`,
|
||
* ready to transfer to the inference worker. Browser-only (Web Audio API).
|
||
*/
|
||
export async function decodeAudioFile(file: Blob): Promise<Float32Array> {
|
||
const bytes = await file.arrayBuffer();
|
||
// decodeAudioData handles most audio containers and audio-in-video (D-012).
|
||
// A suspended AudioContext (no user gesture) can still decode.
|
||
const context = new AudioContext();
|
||
try {
|
||
const decoded = await context.decodeAudioData(bytes);
|
||
return await audioBufferToModelInput(decoded);
|
||
} catch (error) {
|
||
throw new Error(
|
||
"Could not decode this file. Make sure it is an audio or video file " +
|
||
`in a format your browser supports. (${error instanceof Error ? error.message : String(error)})`,
|
||
);
|
||
} finally {
|
||
void context.close();
|
||
}
|
||
}
|