feat: stem rendering, fx and mixing

Co-Authored-By: Claude Sonnet 5.5 <[email protected]>

src/audio/fx.ts

@@ -0,0 +1,121 @@
+import type { Audio, FxPreset, Role } from "../core/types.ts";
+
+const map = (a: Audio, f: (ch: Float32Array, idx: number) => Float32Array): Audio => ({
+  sampleRate: a.sampleRate,
+  channels: a.channels.map(f),
+});
+
+/** Constant-power pan; p in [-1 (left), 1 (right)]. Mono input is widened to stereo; stereo is balanced. */
+export function pan(a: Audio, p: number): Audio {
+  const th = ((Math.max(-1, Math.min(1, p)) + 1) * Math.PI) / 4;
+  const gl = Math.cos(th), gr = Math.sin(th);
+  const l = a.channels[0]!, r = a.channels[1] ?? l;
+  const n = l.length;
+  const L = new Float32Array(n), R = new Float32Array(n);
+  // At centre gl=gr=sqrt(1/2); scale by sqrt(2) so a centred pan is unity gain.
+  for (let i = 0; i < n; i++) { L[i] = l[i]! * gl * Math.SQRT2; R[i] = r[i]! * gr * Math.SQRT2; }
+  return { sampleRate: a.sampleRate, channels: [L, R] };
+}
+
+function onePoleLp(x: Float32Array, fc: number, sr: number): Float32Array {
+  const a = 1 - Math.exp((-2 * Math.PI * fc) / sr);
+  const y = new Float32Array(x.length);
+  let s = 0;
+  for (let i = 0; i < x.length; i++) { s += a * (x[i]! - s); y[i] = s; }
+  return y;
+}
+
+function drive(ch: Float32Array, sr: number): Float32Array {
+  const y = new Float32Array(ch.length);
+  const norm = 1 / Math.tanh(4); // keep full-scale input at full-scale output
+  for (let i = 0; i < ch.length; i++) y[i] = Math.tanh(4 * ch[i]!) * norm;
+  return onePoleLp(y, 5000, sr);
+}
+
+function chorus(ch: Float32Array, sr: number, phase: number): Float32Array {
+  const base = 0.015 * sr, depth = 0.003 * sr, rate = 0.8 / sr;
+  const n = ch.length;
+  const y = new Float32Array(n);
+  // Two modulated delay lines in quadrature, mixed 50/50 with the dry signal.
+  const tap = (i: number, d: number) => {
+    const pos = i - d;
+    if (pos < 0) return 0;
+    const i0 = Math.floor(pos), f = pos - i0;
+    return ch[i0]! * (1 - f) + (i0 + 1 < n ? ch[i0 + 1]! : 0) * f;
+  };
+  for (let i = 0; i < n; i++) {
+    const d1 = base + depth * Math.sin(2 * Math.PI * (rate * i + phase));
+    const d2 = base + depth * Math.sin(2 * Math.PI * (rate * i + phase + 0.5));
+    y[i] = 0.5 * ch[i]! + 0.25 * tap(i, d1) + 0.25 * tap(i, d2);
+  }
+  return y;
+}
+
+/** Deterministic per-preset effect. Output length equals input length. */
+export function applyFx(a: Audio, preset: FxPreset, sampleRate: number): Audio {
+  switch (preset) {
+    case "none":
+    case "clean":
+      return { sampleRate: a.sampleRate, channels: a.channels.map((c) => c.slice()) };
+    case "light-drive":
+      return map(a, (c) => drive(c, sampleRate));
+    case "clean-chorus":
+      return map(a, (c, i) => chorus(c, sampleRate, i === 0 ? 0 : 0.25));
+  }
+}
+
+// Freeverb tunings (samples at 44.1 kHz).
+const COMBS = [1116, 1188, 1277, 1356, 1422, 1491, 1557, 1617];
+const ALLPASS = [556, 441, 341, 225];
+const STEREO_SPREAD = 23;
+
+function freeverbChannel(x: Float32Array, sr: number, spread: number): Float32Array {
+  const k = sr / 44100;
+  const feedback = 0.84, damp = 0.2;
+  const n = x.length;
+  const out = new Float32Array(n);
+  for (const len0 of COMBS) {
+    const len = Math.round((len0 + spread) * k);
+    const buf = new Float32Array(len);
+    let idx = 0, store = 0;
+    for (let i = 0; i < n; i++) {
+      const o = buf[idx]!;
+      store = o * (1 - damp) + store * damp;
+      buf[idx] = x[i]! * 0.015 + store * feedback;
+      if (++idx === len) idx = 0;
+      out[i]! += o;
+    }
+  }
+  for (const len0 of ALLPASS) {
+    const len = Math.round((len0 + spread) * k);
+    const buf = new Float32Array(len);
+    let idx = 0;
+    for (let i = 0; i < n; i++) {
+      const b = buf[idx]!;
+      const v = out[i]!;
+      buf[idx] = v + b * 0.5;
+      out[i] = b - v;
+      if (++idx === len) idx = 0;
+    }
+  }
+  return out;
+}
+
+/** Light Freeverb-style reverb (8 combs + 4 allpasses per channel). Length preserved (tail is truncated). */
+export function applyReverb(a: Audio, wet = 0.15): Audio {
+  const sr = a.sampleRate;
+  const channels = a.channels.map((c, i) => {
+    const rv = freeverbChannel(c, sr, i === 0 ? 0 : STEREO_SPREAD);
+    const y = new Float32Array(c.length);
+    // Freeverb input gain is 0.015 per comb x 8 combs; rescale so wet≈ level of dry.
+    for (let j = 0; j < c.length; j++) y[j] = c[j]! * (1 - wet) + rv[j]! * wet * 3;
+    return y;
+  });
+  return { sampleRate: sr, channels };
+}
+
+/** Full per-track chain used for oracle stems: preset fx, then the shared reverb on every pitched (non-drum) track. */
+export function applyTrackFx(a: Audio, role: Role, preset: FxPreset, sampleRate: number): Audio {
+  const dry = applyFx(a, preset, sampleRate);
+  return role === "drums" ? dry : applyReverb(dry, 0.15);
+}

src/audio/mix.ts

@@ -0,0 +1,28 @@
+import type { Audio } from "../core/types.ts";
+
+const PEAK_TARGET = Math.pow(10, -1 / 20);
+
+/** Sum stems (with per-stem gain in dB), then peak-normalise to -1 dBFS. */
+export function mixStems(stems: { audio: Audio; gainDb: number }[]): Audio {
+  if (stems.length === 0) throw new Error("mixStems: no stems");
+  const sr = stems[0]!.audio.sampleRate;
+  const nch = Math.max(...stems.map((s) => s.audio.channels.length));
+  const n = Math.max(...stems.map((s) => s.audio.channels[0]?.length ?? 0));
+  const out = Array.from({ length: nch }, () => new Float32Array(n));
+  for (const { audio, gainDb } of stems) {
+    if (audio.sampleRate !== sr) throw new Error("mixStems: sample rate mismatch");
+    const g = Math.pow(10, gainDb / 20);
+    for (let c = 0; c < nch; c++) {
+      const src = audio.channels[c] ?? audio.channels[0]!;
+      const dst = out[c]!;
+      for (let i = 0; i < src.length; i++) dst[i]! += src[i]! * g;
+    }
+  }
+  let peak = 0;
+  for (const ch of out) for (let i = 0; i < n; i++) peak = Math.max(peak, Math.abs(ch[i]!));
+  if (peak > 0) {
+    const k = PEAK_TARGET / peak;
+    for (const ch of out) for (let i = 0; i < n; i++) ch[i]! *= k;
+  }
+  return { sampleRate: sr, channels: out };
+}

src/audio/mp3.ts

@@ -0,0 +1,4 @@
+/** Encode a WAV to MP3 (192 kbps) with ffmpeg. */
+export async function encodeMp3(wavPath: string, mp3Path: string): Promise<void> {
+  await Bun.$`ffmpeg -y -loglevel error -i ${wavPath} -b:a 192k ${mp3Path}`.quiet();
+}

src/audio/render.ts

@@ -0,0 +1,62 @@
+import * as at from "@coderline/alphatab";
+import type { Audio, Song } from "../core/types.ts";
+import { songToScore } from "../score/gp.ts";
+
+const TAIL_SECONDS = 2;
+
+/** Offline-render sink: AlphaSynth requires an ISynthOutput but exportAudio never uses it. */
+function dummyOutput(sampleRate: number): at.synth.ISynthOutput {
+  const ev = { on() {}, off() {} };
+  return {
+    sampleRate, open() {}, play() {}, destroy() {}, pause() {}, addSamples() {}, resetSamples() {}, activate() {},
+    ready: ev, samplesPlayed: ev, sampleRequest: ev,
+    enumerateOutputDevices: async () => [], setOutputDevice: async () => {}, getOutputDevice: async () => null,
+  } as unknown as at.synth.ISynthOutput;
+}
+
+function songSeconds(song: Song): number {
+  return (song.bars * song.timeSig[0] * (4 / song.timeSig[1]) * 60) / song.bpm;
+}
+
+/**
+ * Render each track in isolation with alphaSynth. Isolation: `AudioExportOptions.trackVolume` is keyed by MIDI
+ * channel (despite the doc's "track index"), so every channel not owned by the target track is muted (volume 0).
+ * Stems are stereo, centred (alphaSynth output is not panned), and all exactly song length + 2 s.
+ * alphaSynth adds no pre-roll or latency (verified in tests: onset at 1.000 s for a note at tick 960, bpm 60).
+ */
+export function renderStems(song: Song, sf2: Uint8Array, sampleRate = 44100): Map<string, Audio> {
+  const score = songToScore(song);
+  const midi = new at.midi.MidiFile();
+  new at.midi.MidiFileGenerator(score, new at.Settings(), new at.midi.AlphaSynthMidiFileHandler(midi, true)).generate();
+
+  const total = Math.ceil((songSeconds(song) + TAIL_SECONDS) * sampleRate);
+  const stems = new Map<string, Audio>();
+  for (const track of score.tracks) {
+    const own = new Set([track.playbackInfo.primaryChannel, track.playbackInfo.secondaryChannel]);
+    const trackVolume = new Map<number, number>();
+    for (let ch = 0; ch < 16; ch++) if (!own.has(ch)) trackVolume.set(ch, 0);
+
+    const synth = new at.synth.AlphaSynth(dummyOutput(sampleRate), 500);
+    const opts = new at.synth.AudioExportOptions();
+    opts.soundFonts = [sf2];
+    opts.sampleRate = sampleRate;
+    opts.useSyncPoints = false;
+    opts.trackVolume = trackVolume;
+    const exporter = synth.exportAudio(opts, midi, [], new Map());
+
+    const L = new Float32Array(total), R = new Float32Array(total);
+    let pos = 0;
+    for (;;) {
+      const chunk = exporter.render(500);
+      if (!chunk) break;
+      const s = chunk.samples;
+      const frames = Math.min(s.length >> 1, total - pos);
+      for (let i = 0; i < frames; i++) { L[pos + i] = s[2 * i]!; R[pos + i] = s[2 * i + 1]!; }
+      pos += frames;
+      if (pos >= total) break;
+    }
+    synth.destroy();
+    stems.set(track.name, { sampleRate, channels: [L, R] });
+  }
+  return stems;
+}

src/audio/wav.ts

@@ -0,0 +1,106 @@
+import type { Audio } from "../core/types.ts";
+
+/** Read a PCM16 / PCM24 / float32 WAV file into float channels. */
+export async function readWav(path: string): Promise<Audio> {
+  const buf = new Uint8Array(await Bun.file(path).arrayBuffer());
+  const dv = new DataView(buf.buffer, buf.byteOffset, buf.byteLength);
+  const tag = (o: number) => String.fromCharCode(buf[o]!, buf[o + 1]!, buf[o + 2]!, buf[o + 3]!);
+  if (tag(0) !== "RIFF" || tag(8) !== "WAVE") throw new Error(`${path}: not a RIFF/WAVE file`);
+  let fmt = 0, nch = 0, sampleRate = 0, bits = 0;
+  let dataOff = -1, dataLen = 0;
+  for (let o = 12; o + 8 <= buf.length; ) {
+    const id = tag(o);
+    const len = dv.getUint32(o + 4, true);
+    if (id === "fmt ") {
+      fmt = dv.getUint16(o + 8, true);
+      nch = dv.getUint16(o + 10, true);
+      sampleRate = dv.getUint32(o + 12, true);
+      bits = dv.getUint16(o + 22, true);
+      if (fmt === 0xfffe) fmt = dv.getUint16(o + 32, true); // WAVE_FORMAT_EXTENSIBLE sub-format
+    } else if (id === "data") {
+      dataOff = o + 8;
+      dataLen = Math.min(len, buf.length - dataOff);
+      break;
+    }
+    o += 8 + len + (len & 1);
+  }
+  if (dataOff < 0 || nch < 1) throw new Error(`${path}: missing fmt/data chunk`);
+  const bytes = bits / 8;
+  const frames = Math.floor(dataLen / (bytes * nch));
+  const channels = Array.from({ length: nch }, () => new Float32Array(frames));
+  for (let i = 0; i < frames; i++) {
+    for (let c = 0; c < nch; c++) {
+      const o = dataOff + (i * nch + c) * bytes;
+      let v: number;
+      if (fmt === 3 && bits === 32) v = dv.getFloat32(o, true);
+      else if (fmt === 1 && bits === 16) v = dv.getInt16(o, true) / 32768;
+      else if (fmt === 1 && bits === 24) v = ((dv.getUint8(o) | (dv.getUint8(o + 1) << 8) | (dv.getInt8(o + 2) << 16)) as number) / 8388608;
+      else if (fmt === 1 && bits === 32) v = dv.getInt32(o, true) / 2147483648;
+      else throw new Error(`${path}: unsupported WAV format ${fmt}/${bits}-bit`);
+      channels[c]![i] = v;
+    }
+  }
+  return { sampleRate, channels };
+}
+
+/** Write 16-bit PCM WAV (clipped to [-1, 1]). */
+export async function writeWav(path: string, a: Audio): Promise<void> {
+  const nch = a.channels.length;
+  const frames = a.channels[0]?.length ?? 0;
+  const dataLen = frames * nch * 2;
+  const out = new Uint8Array(44 + dataLen);
+  const dv = new DataView(out.buffer);
+  const put = (o: number, s: string) => { for (let i = 0; i < 4; i++) out[o + i] = s.charCodeAt(i); };
+  put(0, "RIFF"); dv.setUint32(4, 36 + dataLen, true); put(8, "WAVE");
+  put(12, "fmt "); dv.setUint32(16, 16, true); dv.setUint16(20, 1, true); dv.setUint16(22, nch, true);
+  dv.setUint32(24, a.sampleRate, true); dv.setUint32(28, a.sampleRate * nch * 2, true);
+  dv.setUint16(32, nch * 2, true); dv.setUint16(34, 16, true);
+  put(36, "data"); dv.setUint32(40, dataLen, true);
+  let o = 44;
+  for (let i = 0; i < frames; i++) {
+    for (let c = 0; c < nch; c++) {
+      const s = Math.max(-1, Math.min(1, a.channels[c]![i]!));
+      dv.setInt16(o, Math.round(s < 0 ? s * 32768 : s * 32767), true);
+      o += 2;
+    }
+  }
+  await Bun.write(path, out);
+}
+
+/** Average all channels into one mono signal. */
+export function toMono(a: Audio): Float32Array {
+  const n = a.channels[0]?.length ?? 0;
+  if (a.channels.length === 1) return a.channels[0]!.slice();
+  const out = new Float32Array(n);
+  for (const ch of a.channels) for (let i = 0; i < n; i++) out[i]! += ch[i]!;
+  const k = 1 / a.channels.length;
+  for (let i = 0; i < n; i++) out[i]! *= k;
+  return out;
+}
+
+const sinc = (x: number) => (x === 0 ? 1 : Math.sin(Math.PI * x) / (Math.PI * x));
+
+/** Windowed-sinc (Blackman) resampler; low-passes at the lower Nyquist when downsampling. */
+export function resample(x: Float32Array, from: number, to: number): Float32Array {
+  if (from === to) return x.slice();
+  const ratio = to / from;
+  const outLen = Math.round(x.length * ratio);
+  const out = new Float32Array(outLen);
+  const cutoff = Math.min(1, ratio); // in units of input Nyquist
+  const half = 16 / cutoff; // kernel half-width in input samples (16 zero-crossings)
+  for (let j = 0; j < outLen; j++) {
+    const t = j / ratio;
+    const lo = Math.max(0, Math.ceil(t - half)), hi = Math.min(x.length - 1, Math.floor(t + half));
+    let acc = 0, wsum = 0;
+    for (let i = lo; i <= hi; i++) {
+      const d = i - t;
+      const w = 0.42 + 0.5 * Math.cos((Math.PI * d) / half) + 0.08 * Math.cos((2 * Math.PI * d) / half);
+      const k = cutoff * sinc(cutoff * d) * w;
+      acc += x[i]! * k;
+      wsum += k;
+    }
+    // Normalise to unity DC gain so edges and non-integer ratios stay calibrated.
+    out[j] = wsum !== 0 ? acc / wsum : 0;
+  }
+  return out;
+}