From 63b32a488db0a719ba473a813c3ba1a5e2dce6a9 Mon Sep 17 00:00:00 2001 From: Lucas Winther Date: Fri, 30 Oct 2026 06:34:15 +0100 Subject: [PATCH] feat(align): warped MIDI rendering and peaks Co-Authored-By: Claude Sonnet 5.5 --- src/align/peaks.ts | 14 +++++++++ src/align/render.ts | 48 +++++++++++++++++++++++++++++++ src/audio/render.ts | 58 +++++++++++++++++++++++--------------- tests/align/render.test.ts | 51 +++++++++++++++++++++++++++++++++ 4 files changed, 149 insertions(+), 22 deletions(-) create mode 100644 src/align/peaks.ts create mode 100644 src/align/render.ts create mode 100644 tests/align/render.test.ts diff --git a/src/align/peaks.ts b/src/align/peaks.ts new file mode 100644 index 0000000..afec5ae --- /dev/null +++ b/src/align/peaks.ts @@ -0,0 +1,14 @@ +/** Per-bucket min/max of a waveform, for drawing. Pure. */ +export function peaks(x: Float32Array, buckets: number): { min: Float32Array; max: Float32Array } { + const min = new Float32Array(buckets), max = new Float32Array(buckets); + const n = x.length; + for (let b = 0; b < buckets; b++) { + const from = Math.floor((b * n) / buckets); + const to = Math.max(from + 1, Math.floor(((b + 1) * n) / buckets)); + let lo = Infinity, hi = -Infinity; + for (let i = from; i < to && i < n; i++) { const v = x[i]!; if (v < lo) lo = v; if (v > hi) hi = v; } + min[b] = lo === Infinity ? 0 : lo; + max[b] = hi === -Infinity ? 0 : hi; + } + return { min, max }; +} diff --git a/src/align/render.ts b/src/align/render.ts new file mode 100644 index 0000000..af093ac --- /dev/null +++ b/src/align/render.ts @@ -0,0 +1,48 @@ +import * as at from "@coderline/alphatab"; +import type { Audio } from "../core/types.ts"; +import { renderMidiFile } from "../audio/render.ts"; +import type { RefTrack } from "./smf.ts"; +import type { Warp } from "./warp.ts"; + +const TPS = 1920; // division 960 at a fixed 120 bpm +const PEAK = Math.pow(10, -1 / 20); +const MELODIC_CHANNELS = [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13, 14, 15]; + +/** + * Render reference tracks to audio with `warp` applied (MIDI time -> audio time). Notes are placed on a fixed-tempo + * tick grid so alphaSynth output time equals audio time. Peak-normalised to -1 dBFS; `durationSec` pads/trims. + * Pure: no I/O. Notes landing before audio t=0 are clamped to 0 (dropped if they end before it). + */ +export function renderMidi( + tracks: RefTrack[], warp: Warp, sf2: Uint8Array, + opts: { sampleRate?: number; mono?: boolean; durationSec?: number } = {}, +): Audio { + const sampleRate = opts.sampleRate ?? 44100; + const midi = new at.midi.MidiFile(); + midi.division = 960; + midi.addEvent(new at.midi.TempoChangeEvent(0, 500000)); + const seen = new Set(); + let next = 0; + for (const t of tracks) { + const ch = t.isDrums ? 9 : MELODIC_CHANNELS[next++ % MELODIC_CHANNELS.length]!; + if (!seen.has(ch)) { seen.add(ch); midi.addEvent(new at.midi.ProgramChangeEvent(0, 0, ch, t.isDrums ? 0 : t.program)); } + for (const n of t.notes) { + const on = Math.round(Math.max(0, warp.toAudio(n.onset)) * TPS); + const off = Math.max(on + 1, Math.round(warp.toAudio(n.offset) * TPS)); + if (warp.toAudio(n.offset) <= 0) continue; + midi.addEvent(new at.midi.NoteOnEvent(0, on, ch, n.pitch, n.velocity)); + midi.addEvent(new at.midi.NoteOffEvent(0, off, ch, n.pitch, 0)); + } + } + const frames = opts.durationSec === undefined ? undefined : Math.round(opts.durationSec * sampleRate); + const out = renderMidiFile(midi, sf2, sampleRate, { frames }); + let chans = out.channels; + if (opts.mono) { + const [L, R] = chans as [Float32Array, Float32Array]; + chans = [L.map((v, i) => (v + R[i]!) / 2)]; + } + let peak = 0; + for (const c of chans) for (let i = 0; i < c.length; i++) { const a = Math.abs(c[i]!); if (a > peak) peak = a; } + if (peak > 0) { const g = PEAK / peak; for (const c of chans) for (let i = 0; i < c.length; i++) c[i]! *= g; } + return { sampleRate, channels: chans }; +} diff --git a/src/audio/render.ts b/src/audio/render.ts index a008026..4e4dd34 100644 --- a/src/audio/render.ts +++ b/src/audio/render.ts @@ -5,7 +5,7 @@ import { songToScore } from "../score/gp.ts"; const TAIL_SECONDS = 2; /** Offline-render sink: AlphaSynth requires an ISynthOutput but exportAudio never uses it. */ -function dummyOutput(sampleRate: number): at.synth.ISynthOutput { +export function dummyOutput(sampleRate: number): at.synth.ISynthOutput { const ev = { on() {}, off() {} }; return { sampleRate, open() {}, play() {}, destroy() {}, pause() {}, addSamples() {}, resetSamples() {}, activate() {}, @@ -18,6 +18,39 @@ function songSeconds(song: Song): number { return (song.bars * song.timeSig[0] * (4 / song.timeSig[1]) * 60) / song.bpm; } +/** Render a MidiFile offline. With `frames`, output is exactly that long (zero-padded or truncated); else full length. */ +export function renderMidiFile( + midi: at.midi.MidiFile, sf2: Uint8Array, sampleRate: number, + opts: { trackVolume?: Map; frames?: number } = {}, +): Audio { + const synth = new at.synth.AlphaSynth(dummyOutput(sampleRate), 500); + const eo = new at.synth.AudioExportOptions(); + eo.soundFonts = [sf2]; + eo.sampleRate = sampleRate; + eo.useSyncPoints = false; + if (opts.trackVolume) eo.trackVolume = opts.trackVolume; + const exporter = synth.exportAudio(eo, midi, [], new Map()); + const cap = opts.frames ?? Infinity; + const Ls: Float32Array[] = [], Rs: Float32Array[] = []; + let pos = 0; + while (pos < cap) { + const chunk = exporter.render(500); + if (!chunk) break; + const s = chunk.samples; + const frames = Math.min(s.length >> 1, cap - pos); + const l = new Float32Array(frames), r = new Float32Array(frames); + for (let i = 0; i < frames; i++) { l[i] = s[2 * i]!; r[i] = s[2 * i + 1]!; } + Ls.push(l); Rs.push(r); + pos += frames; + } + synth.destroy(); + const n = opts.frames ?? pos; + const L = new Float32Array(n), R = new Float32Array(n); + let o = 0; + for (let k = 0; k < Ls.length; k++) { L.set(Ls[k]!, o); R.set(Rs[k]!, o); o += Ls[k]!.length; } + return { sampleRate, channels: [L, R] }; +} + /** * Render each track in isolation with alphaSynth. Isolation: `AudioExportOptions.trackVolume` is keyed by MIDI * channel (despite the doc's "track index"), so every channel not owned by the target track is muted (volume 0). @@ -36,27 +69,8 @@ export function renderStems(song: Song, sf2: Uint8Array, sampleRate = 44100): Ma const trackVolume = new Map(); for (let ch = 0; ch < 16; ch++) if (!own.has(ch)) trackVolume.set(ch, 0); - const synth = new at.synth.AlphaSynth(dummyOutput(sampleRate), 500); - const opts = new at.synth.AudioExportOptions(); - opts.soundFonts = [sf2]; - opts.sampleRate = sampleRate; - opts.useSyncPoints = false; - opts.trackVolume = trackVolume; - const exporter = synth.exportAudio(opts, midi, [], new Map()); - - const L = new Float32Array(total), R = new Float32Array(total); - let pos = 0; - for (;;) { - const chunk = exporter.render(500); - if (!chunk) break; - const s = chunk.samples; - const frames = Math.min(s.length >> 1, total - pos); - for (let i = 0; i < frames; i++) { L[pos + i] = s[2 * i]!; R[pos + i] = s[2 * i + 1]!; } - pos += frames; - if (pos >= total) break; - } - synth.destroy(); - stems.set(track.name, { sampleRate, channels: [L, R] }); + const audio = renderMidiFile(midi, sf2, sampleRate, { trackVolume, frames: total }); + stems.set(track.name, audio); } return stems; } diff --git a/tests/align/render.test.ts b/tests/align/render.test.ts new file mode 100644 index 0000000..51be879 --- /dev/null +++ b/tests/align/render.test.ts @@ -0,0 +1,51 @@ +import { test, expect } from "bun:test"; +import { renderMidi } from "../../src/align/render.ts"; +import { peaks } from "../../src/align/peaks.ts"; +import { makeWarp } from "../../src/align/warp.ts"; +import type { RefTrack, RefNote } from "../../src/align/smf.ts"; + +const SF2 = new Uint8Array(await Bun.file("node_modules/@coderline/alphatab/dist/soundfont/sonivox.sf2").arrayBuffer()); +const note = (onset: number, offset: number, pitch: number): RefNote => ({ onset, offset, pitch, velocity: 100, channel: 0 }); +const track = (notes: RefNote[], isDrums = false): RefTrack => ({ index: 0, name: "t", program: 25, isDrums, notes }); +const SR = 44100; // alphaSynth yields NaN at 22050 Hz + +function firstAbove(x: Float32Array, thr: number): number { + for (let i = 0; i < x.length; i++) if (Math.abs(x[i]!) > thr) return i; + return -1; +} + +test("warp offset moves the note onset", () => { + const a = renderMidi([track([note(1, 2, 60)])], makeWarp([{ midiSec: 0, audioSec: 0.5, kind: "manual" }]), SF2, { sampleRate: SR, mono: true, durationSec: 4 }); + expect(a.channels.length).toBe(1); + expect(a.channels[0]!.length).toBe(4 * SR); + const t = firstAbove(a.channels[0]!, 0.02) / SR; + expect(t).toBeGreaterThan(1.48); + expect(t).toBeLessThan(1.52); +}); + +test("dense chord is peak-normalised to -1 dBFS", () => { + const notes = Array.from({ length: 20 }, (_, i) => note(0, 1, 40 + i * 2)); + const a = renderMidi([track(notes)], makeWarp([]), SF2, { sampleRate: SR }); + let p = 0; + for (const c of a.channels) for (const v of c) p = Math.max(p, Math.abs(v)); + expect(p).toBeLessThanOrEqual(Math.pow(10, -1 / 20) + 1e-3); + expect(p).toBeGreaterThan(0.85); +}); + +test("drums track is audible", () => { + const a = renderMidi([track([note(0.2, 0.3, 38)], true)], makeWarp([]), SF2, { sampleRate: SR, durationSec: 1 }); + expect(a.channels[0]!.some((v) => Math.abs(v) > 0.1)).toBe(true); +}); + +test("peaks of a known sine", () => { + const x = new Float32Array(1000).map((_, i) => Math.sin((2 * Math.PI * i) / 100)); + const p = peaks(x, 10); + expect(p.max.length).toBe(10); + for (let b = 0; b < 10; b++) { + expect(p.max[b]!).toBeCloseTo(1, 1); + expect(p.min[b]!).toBeCloseTo(-1, 1); + } + const q = peaks(new Float32Array([0.5, 0.25, -0.75, 0]), 2); + expect([...q.max]).toEqual([0.5, 0]); + expect([...q.min]).toEqual([0.25, -0.75]); +});