feat: stem rendering, fx and mixing

Co-Authored-By: Claude Sonnet 5.5 <[email protected]>

src/audio/fx.ts

@@ -0,0 +1,121 @@
+import type { Audio, FxPreset, Role } from "../core/types.ts";
+
+const map = (a: Audio, f: (ch: Float32Array, idx: number) => Float32Array): Audio => ({
+  sampleRate: a.sampleRate,
+  channels: a.channels.map(f),
+});
+
+/** Constant-power pan; p in [-1 (left), 1 (right)]. Mono input is widened to stereo; stereo is balanced. */
+export function pan(a: Audio, p: number): Audio {
+  const th = ((Math.max(-1, Math.min(1, p)) + 1) * Math.PI) / 4;
+  const gl = Math.cos(th), gr = Math.sin(th);
+  const l = a.channels[0]!, r = a.channels[1] ?? l;
+  const n = l.length;
+  const L = new Float32Array(n), R = new Float32Array(n);
+  // At centre gl=gr=sqrt(1/2); scale by sqrt(2) so a centred pan is unity gain.
+  for (let i = 0; i < n; i++) { L[i] = l[i]! * gl * Math.SQRT2; R[i] = r[i]! * gr * Math.SQRT2; }
+  return { sampleRate: a.sampleRate, channels: [L, R] };
+}
+
+function onePoleLp(x: Float32Array, fc: number, sr: number): Float32Array {
+  const a = 1 - Math.exp((-2 * Math.PI * fc) / sr);
+  const y = new Float32Array(x.length);
+  let s = 0;
+  for (let i = 0; i < x.length; i++) { s += a * (x[i]! - s); y[i] = s; }
+  return y;
+}
+
+function drive(ch: Float32Array, sr: number): Float32Array {
+  const y = new Float32Array(ch.length);
+  const norm = 1 / Math.tanh(4); // keep full-scale input at full-scale output
+  for (let i = 0; i < ch.length; i++) y[i] = Math.tanh(4 * ch[i]!) * norm;
+  return onePoleLp(y, 5000, sr);
+}
+
+function chorus(ch: Float32Array, sr: number, phase: number): Float32Array {
+  const base = 0.015 * sr, depth = 0.003 * sr, rate = 0.8 / sr;
+  const n = ch.length;
+  const y = new Float32Array(n);
+  // Two modulated delay lines in quadrature, mixed 50/50 with the dry signal.
+  const tap = (i: number, d: number) => {
+    const pos = i - d;
+    if (pos < 0) return 0;
+    const i0 = Math.floor(pos), f = pos - i0;
+    return ch[i0]! * (1 - f) + (i0 + 1 < n ? ch[i0 + 1]! : 0) * f;
+  };
+  for (let i = 0; i < n; i++) {
+    const d1 = base + depth * Math.sin(2 * Math.PI * (rate * i + phase));
+    const d2 = base + depth * Math.sin(2 * Math.PI * (rate * i + phase + 0.5));
+    y[i] = 0.5 * ch[i]! + 0.25 * tap(i, d1) + 0.25 * tap(i, d2);
+  }
+  return y;
+}
+
+/** Deterministic per-preset effect. Output length equals input length. */
+export function applyFx(a: Audio, preset: FxPreset, sampleRate: number): Audio {
+  switch (preset) {
+    case "none":
+    case "clean":
+      return { sampleRate: a.sampleRate, channels: a.channels.map((c) => c.slice()) };
+    case "light-drive":
+      return map(a, (c) => drive(c, sampleRate));
+    case "clean-chorus":
+      return map(a, (c, i) => chorus(c, sampleRate, i === 0 ? 0 : 0.25));
+  }
+}
+
+// Freeverb tunings (samples at 44.1 kHz).
+const COMBS = [1116, 1188, 1277, 1356, 1422, 1491, 1557, 1617];
+const ALLPASS = [556, 441, 341, 225];
+const STEREO_SPREAD = 23;
+
+function freeverbChannel(x: Float32Array, sr: number, spread: number): Float32Array {
+  const k = sr / 44100;
+  const feedback = 0.84, damp = 0.2;
+  const n = x.length;
+  const out = new Float32Array(n);
+  for (const len0 of COMBS) {
+    const len = Math.round((len0 + spread) * k);
+    const buf = new Float32Array(len);
+    let idx = 0, store = 0;
+    for (let i = 0; i < n; i++) {
+      const o = buf[idx]!;
+      store = o * (1 - damp) + store * damp;
+      buf[idx] = x[i]! * 0.015 + store * feedback;
+      if (++idx === len) idx = 0;
+      out[i]! += o;
+    }
+  }
+  for (const len0 of ALLPASS) {
+    const len = Math.round((len0 + spread) * k);
+    const buf = new Float32Array(len);
+    let idx = 0;
+    for (let i = 0; i < n; i++) {
+      const b = buf[idx]!;
+      const v = out[i]!;
+      buf[idx] = v + b * 0.5;
+      out[i] = b - v;
+      if (++idx === len) idx = 0;
+    }
+  }
+  return out;
+}
+
+/** Light Freeverb-style reverb (8 combs + 4 allpasses per channel). Length preserved (tail is truncated). */
+export function applyReverb(a: Audio, wet = 0.15): Audio {
+  const sr = a.sampleRate;
+  const channels = a.channels.map((c, i) => {
+    const rv = freeverbChannel(c, sr, i === 0 ? 0 : STEREO_SPREAD);
+    const y = new Float32Array(c.length);
+    // Freeverb input gain is 0.015 per comb x 8 combs; rescale so wet≈ level of dry.
+    for (let j = 0; j < c.length; j++) y[j] = c[j]! * (1 - wet) + rv[j]! * wet * 3;
+    return y;
+  });
+  return { sampleRate: sr, channels };
+}
+
+/** Full per-track chain used for oracle stems: preset fx, then the shared reverb on every pitched (non-drum) track. */
+export function applyTrackFx(a: Audio, role: Role, preset: FxPreset, sampleRate: number): Audio {
+  const dry = applyFx(a, preset, sampleRate);
+  return role === "drums" ? dry : applyReverb(dry, 0.15);
+}

src/audio/mix.ts

@@ -0,0 +1,28 @@
+import type { Audio } from "../core/types.ts";
+
+const PEAK_TARGET = Math.pow(10, -1 / 20);
+
+/** Sum stems (with per-stem gain in dB), then peak-normalise to -1 dBFS. */
+export function mixStems(stems: { audio: Audio; gainDb: number }[]): Audio {
+  if (stems.length === 0) throw new Error("mixStems: no stems");
+  const sr = stems[0]!.audio.sampleRate;
+  const nch = Math.max(...stems.map((s) => s.audio.channels.length));
+  const n = Math.max(...stems.map((s) => s.audio.channels[0]?.length ?? 0));
+  const out = Array.from({ length: nch }, () => new Float32Array(n));
+  for (const { audio, gainDb } of stems) {
+    if (audio.sampleRate !== sr) throw new Error("mixStems: sample rate mismatch");
+    const g = Math.pow(10, gainDb / 20);
+    for (let c = 0; c < nch; c++) {
+      const src = audio.channels[c] ?? audio.channels[0]!;
+      const dst = out[c]!;
+      for (let i = 0; i < src.length; i++) dst[i]! += src[i]! * g;
+    }
+  }
+  let peak = 0;
+  for (const ch of out) for (let i = 0; i < n; i++) peak = Math.max(peak, Math.abs(ch[i]!));
+  if (peak > 0) {
+    const k = PEAK_TARGET / peak;
+    for (const ch of out) for (let i = 0; i < n; i++) ch[i]! *= k;
+  }
+  return { sampleRate: sr, channels: out };
+}

src/audio/mp3.ts

@@ -0,0 +1,4 @@
+/** Encode a WAV to MP3 (192 kbps) with ffmpeg. */
+export async function encodeMp3(wavPath: string, mp3Path: string): Promise<void> {
+  await Bun.$`ffmpeg -y -loglevel error -i ${wavPath} -b:a 192k ${mp3Path}`.quiet();
+}

src/audio/render.ts

@@ -0,0 +1,62 @@
+import * as at from "@coderline/alphatab";
+import type { Audio, Song } from "../core/types.ts";
+import { songToScore } from "../score/gp.ts";
+
+const TAIL_SECONDS = 2;
+
+/** Offline-render sink: AlphaSynth requires an ISynthOutput but exportAudio never uses it. */
+function dummyOutput(sampleRate: number): at.synth.ISynthOutput {
+  const ev = { on() {}, off() {} };
+  return {
+    sampleRate, open() {}, play() {}, destroy() {}, pause() {}, addSamples() {}, resetSamples() {}, activate() {},
+    ready: ev, samplesPlayed: ev, sampleRequest: ev,
+    enumerateOutputDevices: async () => [], setOutputDevice: async () => {}, getOutputDevice: async () => null,
+  } as unknown as at.synth.ISynthOutput;
+}
+
+function songSeconds(song: Song): number {
+  return (song.bars * song.timeSig[0] * (4 / song.timeSig[1]) * 60) / song.bpm;
+}
+
+/**
+ * Render each track in isolation with alphaSynth. Isolation: `AudioExportOptions.trackVolume` is keyed by MIDI
+ * channel (despite the doc's "track index"), so every channel not owned by the target track is muted (volume 0).
+ * Stems are stereo, centred (alphaSynth output is not panned), and all exactly song length + 2 s.
+ * alphaSynth adds no pre-roll or latency (verified in tests: onset at 1.000 s for a note at tick 960, bpm 60).
+ */
+export function renderStems(song: Song, sf2: Uint8Array, sampleRate = 44100): Map<string, Audio> {
+  const score = songToScore(song);
+  const midi = new at.midi.MidiFile();
+  new at.midi.MidiFileGenerator(score, new at.Settings(), new at.midi.AlphaSynthMidiFileHandler(midi, true)).generate();
+
+  const total = Math.ceil((songSeconds(song) + TAIL_SECONDS) * sampleRate);
+  const stems = new Map<string, Audio>();
+  for (const track of score.tracks) {
+    const own = new Set([track.playbackInfo.primaryChannel, track.playbackInfo.secondaryChannel]);
+    const trackVolume = new Map<number, number>();
+    for (let ch = 0; ch < 16; ch++) if (!own.has(ch)) trackVolume.set(ch, 0);
+
+    const synth = new at.synth.AlphaSynth(dummyOutput(sampleRate), 500);
+    const opts = new at.synth.AudioExportOptions();
+    opts.soundFonts = [sf2];
+    opts.sampleRate = sampleRate;
+    opts.useSyncPoints = false;
+    opts.trackVolume = trackVolume;
+    const exporter = synth.exportAudio(opts, midi, [], new Map());
+
+    const L = new Float32Array(total), R = new Float32Array(total);
+    let pos = 0;
+    for (;;) {
+      const chunk = exporter.render(500);
+      if (!chunk) break;
+      const s = chunk.samples;
+      const frames = Math.min(s.length >> 1, total - pos);
+      for (let i = 0; i < frames; i++) { L[pos + i] = s[2 * i]!; R[pos + i] = s[2 * i + 1]!; }
+      pos += frames;
+      if (pos >= total) break;
+    }
+    synth.destroy();
+    stems.set(track.name, { sampleRate, channels: [L, R] });
+  }
+  return stems;
+}

src/audio/wav.ts

@@ -0,0 +1,106 @@
+import type { Audio } from "../core/types.ts";
+
+/** Read a PCM16 / PCM24 / float32 WAV file into float channels. */
+export async function readWav(path: string): Promise<Audio> {
+  const buf = new Uint8Array(await Bun.file(path).arrayBuffer());
+  const dv = new DataView(buf.buffer, buf.byteOffset, buf.byteLength);
+  const tag = (o: number) => String.fromCharCode(buf[o]!, buf[o + 1]!, buf[o + 2]!, buf[o + 3]!);
+  if (tag(0) !== "RIFF" || tag(8) !== "WAVE") throw new Error(`${path}: not a RIFF/WAVE file`);
+  let fmt = 0, nch = 0, sampleRate = 0, bits = 0;
+  let dataOff = -1, dataLen = 0;
+  for (let o = 12; o + 8 <= buf.length; ) {
+    const id = tag(o);
+    const len = dv.getUint32(o + 4, true);
+    if (id === "fmt ") {
+      fmt = dv.getUint16(o + 8, true);
+      nch = dv.getUint16(o + 10, true);
+      sampleRate = dv.getUint32(o + 12, true);
+      bits = dv.getUint16(o + 22, true);
+      if (fmt === 0xfffe) fmt = dv.getUint16(o + 32, true); // WAVE_FORMAT_EXTENSIBLE sub-format
+    } else if (id === "data") {
+      dataOff = o + 8;
+      dataLen = Math.min(len, buf.length - dataOff);
+      break;
+    }
+    o += 8 + len + (len & 1);
+  }
+  if (dataOff < 0 || nch < 1) throw new Error(`${path}: missing fmt/data chunk`);
+  const bytes = bits / 8;
+  const frames = Math.floor(dataLen / (bytes * nch));
+  const channels = Array.from({ length: nch }, () => new Float32Array(frames));
+  for (let i = 0; i < frames; i++) {
+    for (let c = 0; c < nch; c++) {
+      const o = dataOff + (i * nch + c) * bytes;
+      let v: number;
+      if (fmt === 3 && bits === 32) v = dv.getFloat32(o, true);
+      else if (fmt === 1 && bits === 16) v = dv.getInt16(o, true) / 32768;
+      else if (fmt === 1 && bits === 24) v = ((dv.getUint8(o) | (dv.getUint8(o + 1) << 8) | (dv.getInt8(o + 2) << 16)) as number) / 8388608;
+      else if (fmt === 1 && bits === 32) v = dv.getInt32(o, true) / 2147483648;
+      else throw new Error(`${path}: unsupported WAV format ${fmt}/${bits}-bit`);
+      channels[c]![i] = v;
+    }
+  }
+  return { sampleRate, channels };
+}
+
+/** Write 16-bit PCM WAV (clipped to [-1, 1]). */
+export async function writeWav(path: string, a: Audio): Promise<void> {
+  const nch = a.channels.length;
+  const frames = a.channels[0]?.length ?? 0;
+  const dataLen = frames * nch * 2;
+  const out = new Uint8Array(44 + dataLen);
+  const dv = new DataView(out.buffer);
+  const put = (o: number, s: string) => { for (let i = 0; i < 4; i++) out[o + i] = s.charCodeAt(i); };
+  put(0, "RIFF"); dv.setUint32(4, 36 + dataLen, true); put(8, "WAVE");
+  put(12, "fmt "); dv.setUint32(16, 16, true); dv.setUint16(20, 1, true); dv.setUint16(22, nch, true);
+  dv.setUint32(24, a.sampleRate, true); dv.setUint32(28, a.sampleRate * nch * 2, true);
+  dv.setUint16(32, nch * 2, true); dv.setUint16(34, 16, true);
+  put(36, "data"); dv.setUint32(40, dataLen, true);
+  let o = 44;
+  for (let i = 0; i < frames; i++) {
+    for (let c = 0; c < nch; c++) {
+      const s = Math.max(-1, Math.min(1, a.channels[c]![i]!));
+      dv.setInt16(o, Math.round(s < 0 ? s * 32768 : s * 32767), true);
+      o += 2;
+    }
+  }
+  await Bun.write(path, out);
+}
+
+/** Average all channels into one mono signal. */
+export function toMono(a: Audio): Float32Array {
+  const n = a.channels[0]?.length ?? 0;
+  if (a.channels.length === 1) return a.channels[0]!.slice();
+  const out = new Float32Array(n);
+  for (const ch of a.channels) for (let i = 0; i < n; i++) out[i]! += ch[i]!;
+  const k = 1 / a.channels.length;
+  for (let i = 0; i < n; i++) out[i]! *= k;
+  return out;
+}
+
+const sinc = (x: number) => (x === 0 ? 1 : Math.sin(Math.PI * x) / (Math.PI * x));
+
+/** Windowed-sinc (Blackman) resampler; low-passes at the lower Nyquist when downsampling. */
+export function resample(x: Float32Array, from: number, to: number): Float32Array {
+  if (from === to) return x.slice();
+  const ratio = to / from;
+  const outLen = Math.round(x.length * ratio);
+  const out = new Float32Array(outLen);
+  const cutoff = Math.min(1, ratio); // in units of input Nyquist
+  const half = 16 / cutoff; // kernel half-width in input samples (16 zero-crossings)
+  for (let j = 0; j < outLen; j++) {
+    const t = j / ratio;
+    const lo = Math.max(0, Math.ceil(t - half)), hi = Math.min(x.length - 1, Math.floor(t + half));
+    let acc = 0, wsum = 0;
+    for (let i = lo; i <= hi; i++) {
+      const d = i - t;
+      const w = 0.42 + 0.5 * Math.cos((Math.PI * d) / half) + 0.08 * Math.cos((2 * Math.PI * d) / half);
+      const k = cutoff * sinc(cutoff * d) * w;
+      acc += x[i]! * k;
+      wsum += k;
+    }
+    // Normalise to unity DC gain so edges and non-integer ratios stay calibrated.
+    out[j] = wsum !== 0 ? acc / wsum : 0;
+  }
+  return out;
+}

tests/audio/audio.test.ts

@@ -0,0 +1,127 @@
+import { test, expect } from "bun:test";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { readWav, writeWav, toMono, resample } from "../../src/audio/wav.ts";
+import { pan, applyFx, applyReverb } from "../../src/audio/fx.ts";
+import { mixStems } from "../../src/audio/mix.ts";
+import { renderStems } from "../../src/audio/render.ts";
+import { generateSong } from "../../src/gen/generate.ts";
+import { GUITAR_STANDARD, type Audio, type Song } from "../../src/core/types.ts";
+
+const SF2 = new Uint8Array(await Bun.file("node_modules/@coderline/alphatab/dist/soundfont/sonivox.sf2").arrayBuffer());
+const rms = (x: Float32Array) => Math.sqrt(x.reduce((s, v) => s + v * v, 0) / Math.max(1, x.length));
+
+test("WAV round trip (PCM16)", async () => {
+  const sr = 44100;
+  const ch = () => Float32Array.from({ length: sr }, () => Math.random() * 2 - 1);
+  const a: Audio = { sampleRate: sr, channels: [ch(), ch()] };
+  const p = join(tmpdir(), `stsheets-rt-${process.pid}.wav`);
+  await writeWav(p, a);
+  const b = await readWav(p);
+  expect(b.sampleRate).toBe(sr);
+  expect(b.channels.length).toBe(2);
+  expect(b.channels[0]!.length).toBe(sr);
+  let max = 0;
+  for (let c = 0; c < 2; c++) for (let i = 0; i < sr; i++) max = Math.max(max, Math.abs(a.channels[c]![i]! - b.channels[c]![i]!));
+  expect(max).toBeLessThan(1e-4);
+});
+
+test("toMono averages; resample preserves a tone's frequency/level", () => {
+  expect(Array.from(toMono({ sampleRate: 1, channels: [Float32Array.of(1, 0), Float32Array.of(0, 1)] }))).toEqual([0.5, 0.5]);
+  const sr = 44100, f = 440;
+  const x = Float32Array.from({ length: sr }, (_, i) => Math.sin((2 * Math.PI * f * i) / sr));
+  const y = resample(x, sr, 22050);
+  expect(y.length).toBe(22050);
+  const ref = Float32Array.from({ length: 22050 }, (_, i) => Math.sin((2 * Math.PI * f * i) / 22050));
+  let err = 0;
+  for (let i = 100; i < 22050 - 100; i++) err = Math.max(err, Math.abs(y[i]! - ref[i]!));
+  expect(err).toBeLessThan(5e-3);
+});
+
+test("pan(-1) silences the right channel; centre is unity", () => {
+  const n = 4410;
+  const m = Float32Array.from({ length: n }, (_, i) => Math.sin(i / 10));
+  const a: Audio = { sampleRate: 44100, channels: [m, m.slice()] };
+  const p = pan(a, -1);
+  expect(rms(p.channels[1]!)).toBeLessThan(1e-3 * rms(p.channels[0]!));
+  const c = pan(a, 0);
+  expect(rms(c.channels[0]!) / rms(m)).toBeCloseTo(1, 3);
+});
+
+test("fx keep length and are deterministic", () => {
+  const n = 22050;
+  const m = Float32Array.from({ length: n }, (_, i) => 0.5 * Math.sin(i / 7));
+  const a: Audio = { sampleRate: 44100, channels: [m, m.slice()] };
+  for (const p of ["none", "clean", "clean-chorus", "light-drive"] as const) {
+    const o1 = applyFx(a, p, 44100), o2 = applyFx(a, p, 44100);
+    expect(o1.channels[0]!.length).toBe(n);
+    expect(o1.channels[0]).toEqual(o2.channels[0]!);
+    expect(rms(o1.channels[0]!)).toBeGreaterThan(0.05);
+  }
+  const rv = applyReverb(a);
+  expect(rv.channels[0]!.length).toBe(n);
+  expect(rv.channels[0]).not.toEqual(m);
+});
+
+test("mixStems peaks at -1 dBFS", () => {
+  const x = Float32Array.from({ length: 1000 }, (_, i) => Math.sin(i / 5));
+  const mk = (g: number): Audio => ({ sampleRate: 44100, channels: [x.map((v) => v * g), x.map((v) => v * g)] });
+  const mix = mixStems([{ audio: mk(0.3), gainDb: 0 }, { audio: mk(0.2), gainDb: -6 }]);
+  let peak = 0;
+  for (const ch of mix.channels) for (const v of ch) peak = Math.max(peak, Math.abs(v));
+  expect(Math.abs(peak - Math.pow(10, -1 / 20))).toBeLessThan(1e-3);
+});
+
+test("renderStems on t1b: one stem per track, equal length, audible", () => {
+  const song = generateSong("t1b", 1);
+  const stems = renderStems(song, SF2);
+  expect(stems.size).toBe(3);
+  const secs = (song.bars * song.timeSig[0] * 60) / song.bpm;
+  const lens = new Set<number>();
+  for (const [name, a] of stems) {
+    lens.add(a.channels[0]!.length);
+    expect(a.channels.length).toBe(2);
+    expect(a.channels[0]!.length / a.sampleRate).toBeGreaterThanOrEqual(secs);
+    expect(rms(a.channels[0]!)).toBeGreaterThan(1e-3);
+    expect(name.length).toBeGreaterThan(0);
+  }
+  expect(lens.size).toBe(1);
+}, 60000);
+
+test("stems are isolated: stem energies sum to roughly the full-mix energy", () => {
+  const song = generateSong("t1b", 1);
+  const stems = renderStems(song, SF2);
+  // Each track alone must differ from the others (muting works) and have limited overlap.
+  const arr = [...stems.values()].map((a) => a.channels[0]!);
+  const e = arr.map((x) => x.reduce((s, v) => s + v * v, 0));
+  const dot = (a: Float32Array, b: Float32Array) => { let s = 0; for (let i = 0; i < a.length; i++) s += a[i]! * b[i]!; return s; };
+  for (let i = 0; i < arr.length; i++) for (let j = i + 1; j < arr.length; j++) {
+    expect(Math.abs(dot(arr[i]!, arr[j]!)) / Math.sqrt(e[i]! * e[j]!)).toBeLessThan(0.3);
+  }
+}, 60000);
+
+test("onset latency: guitar note at tick 960 (bpm 60) starts at 1.0 s +-20 ms", () => {
+  const song: Song = {
+    title: "lat", tier: "t", seed: 1, bpm: 60, timeSig: [4, 4], bars: 2, key: "C",
+    tracks: [{
+      name: "Lead", role: "guitar", program: 25, tuning: GUITAR_STANDARD, pan: 0, fx: "none", gainDb: 0,
+      notes: [{ tick: 960, dur: 960, pitch: 64, string: 1, fret: 0 }],
+    }],
+  };
+  const a = renderStems(song, SF2).get("Lead")!;
+  const x = a.channels[0]!;
+  const win = Math.round(0.005 * a.sampleRate);
+  const env: number[] = [];
+  for (let i = 0; i + win <= x.length; i += 1) {
+    if (i % 8 !== 0) { continue; }
+    let s = 0;
+    for (let k = 0; k < win; k++) s += x[i + k]! * x[i + k]!;
+    env.push(Math.sqrt(s / win));
+  }
+  const max = Math.max(...env);
+  const first = env.findIndex((v) => v > 0.1 * max);
+  const t = (first * 8) / a.sampleRate;
+  console.log(`measured onset: ${t.toFixed(4)} s`);
+  expect(t).toBeGreaterThanOrEqual(0.98);
+  expect(t).toBeLessThanOrEqual(1.02);
+});