From a1e9ce86a0465e5854b7d61fc063e98b3bce98d1 Mon Sep 17 00:00:00 2001 From: Lucas Winther Date: Fri, 30 Oct 2026 02:48:59 +0100 Subject: [PATCH] feat: stem rendering, fx and mixing Co-Authored-By: Claude Sonnet 5.5 --- src/audio/fx.ts | 121 ++++++++++++++++++++++++++++++++++++ src/audio/mix.ts | 28 +++++++++ src/audio/mp3.ts | 4 ++ src/audio/render.ts | 62 +++++++++++++++++++ src/audio/wav.ts | 106 +++++++++++++++++++++++++++++++ tests/audio/audio.test.ts | 127 ++++++++++++++++++++++++++++++++++++++ 6 files changed, 448 insertions(+) create mode 100644 src/audio/fx.ts create mode 100644 src/audio/mix.ts create mode 100644 src/audio/mp3.ts create mode 100644 src/audio/render.ts create mode 100644 src/audio/wav.ts create mode 100644 tests/audio/audio.test.ts diff --git a/src/audio/fx.ts b/src/audio/fx.ts new file mode 100644 index 0000000..16fab28 --- /dev/null +++ b/src/audio/fx.ts @@ -0,0 +1,121 @@ +import type { Audio, FxPreset, Role } from "../core/types.ts"; + +const map = (a: Audio, f: (ch: Float32Array, idx: number) => Float32Array): Audio => ({ + sampleRate: a.sampleRate, + channels: a.channels.map(f), +}); + +/** Constant-power pan; p in [-1 (left), 1 (right)]. Mono input is widened to stereo; stereo is balanced. */ +export function pan(a: Audio, p: number): Audio { + const th = ((Math.max(-1, Math.min(1, p)) + 1) * Math.PI) / 4; + const gl = Math.cos(th), gr = Math.sin(th); + const l = a.channels[0]!, r = a.channels[1] ?? l; + const n = l.length; + const L = new Float32Array(n), R = new Float32Array(n); + // At centre gl=gr=sqrt(1/2); scale by sqrt(2) so a centred pan is unity gain. + for (let i = 0; i < n; i++) { L[i] = l[i]! * gl * Math.SQRT2; R[i] = r[i]! * gr * Math.SQRT2; } + return { sampleRate: a.sampleRate, channels: [L, R] }; +} + +function onePoleLp(x: Float32Array, fc: number, sr: number): Float32Array { + const a = 1 - Math.exp((-2 * Math.PI * fc) / sr); + const y = new Float32Array(x.length); + let s = 0; + for (let i = 0; i < x.length; i++) { s += a * (x[i]! - s); y[i] = s; } + return y; +} + +function drive(ch: Float32Array, sr: number): Float32Array { + const y = new Float32Array(ch.length); + const norm = 1 / Math.tanh(4); // keep full-scale input at full-scale output + for (let i = 0; i < ch.length; i++) y[i] = Math.tanh(4 * ch[i]!) * norm; + return onePoleLp(y, 5000, sr); +} + +function chorus(ch: Float32Array, sr: number, phase: number): Float32Array { + const base = 0.015 * sr, depth = 0.003 * sr, rate = 0.8 / sr; + const n = ch.length; + const y = new Float32Array(n); + // Two modulated delay lines in quadrature, mixed 50/50 with the dry signal. + const tap = (i: number, d: number) => { + const pos = i - d; + if (pos < 0) return 0; + const i0 = Math.floor(pos), f = pos - i0; + return ch[i0]! * (1 - f) + (i0 + 1 < n ? ch[i0 + 1]! : 0) * f; + }; + for (let i = 0; i < n; i++) { + const d1 = base + depth * Math.sin(2 * Math.PI * (rate * i + phase)); + const d2 = base + depth * Math.sin(2 * Math.PI * (rate * i + phase + 0.5)); + y[i] = 0.5 * ch[i]! + 0.25 * tap(i, d1) + 0.25 * tap(i, d2); + } + return y; +} + +/** Deterministic per-preset effect. Output length equals input length. */ +export function applyFx(a: Audio, preset: FxPreset, sampleRate: number): Audio { + switch (preset) { + case "none": + case "clean": + return { sampleRate: a.sampleRate, channels: a.channels.map((c) => c.slice()) }; + case "light-drive": + return map(a, (c) => drive(c, sampleRate)); + case "clean-chorus": + return map(a, (c, i) => chorus(c, sampleRate, i === 0 ? 0 : 0.25)); + } +} + +// Freeverb tunings (samples at 44.1 kHz). +const COMBS = [1116, 1188, 1277, 1356, 1422, 1491, 1557, 1617]; +const ALLPASS = [556, 441, 341, 225]; +const STEREO_SPREAD = 23; + +function freeverbChannel(x: Float32Array, sr: number, spread: number): Float32Array { + const k = sr / 44100; + const feedback = 0.84, damp = 0.2; + const n = x.length; + const out = new Float32Array(n); + for (const len0 of COMBS) { + const len = Math.round((len0 + spread) * k); + const buf = new Float32Array(len); + let idx = 0, store = 0; + for (let i = 0; i < n; i++) { + const o = buf[idx]!; + store = o * (1 - damp) + store * damp; + buf[idx] = x[i]! * 0.015 + store * feedback; + if (++idx === len) idx = 0; + out[i]! += o; + } + } + for (const len0 of ALLPASS) { + const len = Math.round((len0 + spread) * k); + const buf = new Float32Array(len); + let idx = 0; + for (let i = 0; i < n; i++) { + const b = buf[idx]!; + const v = out[i]!; + buf[idx] = v + b * 0.5; + out[i] = b - v; + if (++idx === len) idx = 0; + } + } + return out; +} + +/** Light Freeverb-style reverb (8 combs + 4 allpasses per channel). Length preserved (tail is truncated). */ +export function applyReverb(a: Audio, wet = 0.15): Audio { + const sr = a.sampleRate; + const channels = a.channels.map((c, i) => { + const rv = freeverbChannel(c, sr, i === 0 ? 0 : STEREO_SPREAD); + const y = new Float32Array(c.length); + // Freeverb input gain is 0.015 per comb x 8 combs; rescale so wet≈ level of dry. + for (let j = 0; j < c.length; j++) y[j] = c[j]! * (1 - wet) + rv[j]! * wet * 3; + return y; + }); + return { sampleRate: sr, channels }; +} + +/** Full per-track chain used for oracle stems: preset fx, then the shared reverb on every pitched (non-drum) track. */ +export function applyTrackFx(a: Audio, role: Role, preset: FxPreset, sampleRate: number): Audio { + const dry = applyFx(a, preset, sampleRate); + return role === "drums" ? dry : applyReverb(dry, 0.15); +} diff --git a/src/audio/mix.ts b/src/audio/mix.ts new file mode 100644 index 0000000..e6fae3a --- /dev/null +++ b/src/audio/mix.ts @@ -0,0 +1,28 @@ +import type { Audio } from "../core/types.ts"; + +const PEAK_TARGET = Math.pow(10, -1 / 20); + +/** Sum stems (with per-stem gain in dB), then peak-normalise to -1 dBFS. */ +export function mixStems(stems: { audio: Audio; gainDb: number }[]): Audio { + if (stems.length === 0) throw new Error("mixStems: no stems"); + const sr = stems[0]!.audio.sampleRate; + const nch = Math.max(...stems.map((s) => s.audio.channels.length)); + const n = Math.max(...stems.map((s) => s.audio.channels[0]?.length ?? 0)); + const out = Array.from({ length: nch }, () => new Float32Array(n)); + for (const { audio, gainDb } of stems) { + if (audio.sampleRate !== sr) throw new Error("mixStems: sample rate mismatch"); + const g = Math.pow(10, gainDb / 20); + for (let c = 0; c < nch; c++) { + const src = audio.channels[c] ?? audio.channels[0]!; + const dst = out[c]!; + for (let i = 0; i < src.length; i++) dst[i]! += src[i]! * g; + } + } + let peak = 0; + for (const ch of out) for (let i = 0; i < n; i++) peak = Math.max(peak, Math.abs(ch[i]!)); + if (peak > 0) { + const k = PEAK_TARGET / peak; + for (const ch of out) for (let i = 0; i < n; i++) ch[i]! *= k; + } + return { sampleRate: sr, channels: out }; +} diff --git a/src/audio/mp3.ts b/src/audio/mp3.ts new file mode 100644 index 0000000..519f9e6 --- /dev/null +++ b/src/audio/mp3.ts @@ -0,0 +1,4 @@ +/** Encode a WAV to MP3 (192 kbps) with ffmpeg. */ +export async function encodeMp3(wavPath: string, mp3Path: string): Promise { + await Bun.$`ffmpeg -y -loglevel error -i ${wavPath} -b:a 192k ${mp3Path}`.quiet(); +} diff --git a/src/audio/render.ts b/src/audio/render.ts new file mode 100644 index 0000000..a008026 --- /dev/null +++ b/src/audio/render.ts @@ -0,0 +1,62 @@ +import * as at from "@coderline/alphatab"; +import type { Audio, Song } from "../core/types.ts"; +import { songToScore } from "../score/gp.ts"; + +const TAIL_SECONDS = 2; + +/** Offline-render sink: AlphaSynth requires an ISynthOutput but exportAudio never uses it. */ +function dummyOutput(sampleRate: number): at.synth.ISynthOutput { + const ev = { on() {}, off() {} }; + return { + sampleRate, open() {}, play() {}, destroy() {}, pause() {}, addSamples() {}, resetSamples() {}, activate() {}, + ready: ev, samplesPlayed: ev, sampleRequest: ev, + enumerateOutputDevices: async () => [], setOutputDevice: async () => {}, getOutputDevice: async () => null, + } as unknown as at.synth.ISynthOutput; +} + +function songSeconds(song: Song): number { + return (song.bars * song.timeSig[0] * (4 / song.timeSig[1]) * 60) / song.bpm; +} + +/** + * Render each track in isolation with alphaSynth. Isolation: `AudioExportOptions.trackVolume` is keyed by MIDI + * channel (despite the doc's "track index"), so every channel not owned by the target track is muted (volume 0). + * Stems are stereo, centred (alphaSynth output is not panned), and all exactly song length + 2 s. + * alphaSynth adds no pre-roll or latency (verified in tests: onset at 1.000 s for a note at tick 960, bpm 60). + */ +export function renderStems(song: Song, sf2: Uint8Array, sampleRate = 44100): Map { + const score = songToScore(song); + const midi = new at.midi.MidiFile(); + new at.midi.MidiFileGenerator(score, new at.Settings(), new at.midi.AlphaSynthMidiFileHandler(midi, true)).generate(); + + const total = Math.ceil((songSeconds(song) + TAIL_SECONDS) * sampleRate); + const stems = new Map(); + for (const track of score.tracks) { + const own = new Set([track.playbackInfo.primaryChannel, track.playbackInfo.secondaryChannel]); + const trackVolume = new Map(); + for (let ch = 0; ch < 16; ch++) if (!own.has(ch)) trackVolume.set(ch, 0); + + const synth = new at.synth.AlphaSynth(dummyOutput(sampleRate), 500); + const opts = new at.synth.AudioExportOptions(); + opts.soundFonts = [sf2]; + opts.sampleRate = sampleRate; + opts.useSyncPoints = false; + opts.trackVolume = trackVolume; + const exporter = synth.exportAudio(opts, midi, [], new Map()); + + const L = new Float32Array(total), R = new Float32Array(total); + let pos = 0; + for (;;) { + const chunk = exporter.render(500); + if (!chunk) break; + const s = chunk.samples; + const frames = Math.min(s.length >> 1, total - pos); + for (let i = 0; i < frames; i++) { L[pos + i] = s[2 * i]!; R[pos + i] = s[2 * i + 1]!; } + pos += frames; + if (pos >= total) break; + } + synth.destroy(); + stems.set(track.name, { sampleRate, channels: [L, R] }); + } + return stems; +} diff --git a/src/audio/wav.ts b/src/audio/wav.ts new file mode 100644 index 0000000..9c68599 --- /dev/null +++ b/src/audio/wav.ts @@ -0,0 +1,106 @@ +import type { Audio } from "../core/types.ts"; + +/** Read a PCM16 / PCM24 / float32 WAV file into float channels. */ +export async function readWav(path: string): Promise