feat(align): warped MIDI rendering and peaks
Co-Authored-By: Claude Sonnet 5.5 <[email protected]>
This commit is contained in:
Lucas Winther
co-authored by
Claude Sonnet 5.5
Claude Sonnet 5.5
parent
ca5942956a
commit
63b32a488d
@@ -0,0 +1,14 @@
|
||||
/** Per-bucket min/max of a waveform, for drawing. Pure. */
|
||||
export function peaks(x: Float32Array, buckets: number): { min: Float32Array; max: Float32Array } {
|
||||
const min = new Float32Array(buckets), max = new Float32Array(buckets);
|
||||
const n = x.length;
|
||||
for (let b = 0; b < buckets; b++) {
|
||||
const from = Math.floor((b * n) / buckets);
|
||||
const to = Math.max(from + 1, Math.floor(((b + 1) * n) / buckets));
|
||||
let lo = Infinity, hi = -Infinity;
|
||||
for (let i = from; i < to && i < n; i++) { const v = x[i]!; if (v < lo) lo = v; if (v > hi) hi = v; }
|
||||
min[b] = lo === Infinity ? 0 : lo;
|
||||
max[b] = hi === -Infinity ? 0 : hi;
|
||||
}
|
||||
return { min, max };
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
import * as at from "@coderline/alphatab";
|
||||
import type { Audio } from "../core/types.ts";
|
||||
import { renderMidiFile } from "../audio/render.ts";
|
||||
import type { RefTrack } from "./smf.ts";
|
||||
import type { Warp } from "./warp.ts";
|
||||
|
||||
const TPS = 1920; // division 960 at a fixed 120 bpm
|
||||
const PEAK = Math.pow(10, -1 / 20);
|
||||
const MELODIC_CHANNELS = [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13, 14, 15];
|
||||
|
||||
/**
|
||||
* Render reference tracks to audio with `warp` applied (MIDI time -> audio time). Notes are placed on a fixed-tempo
|
||||
* tick grid so alphaSynth output time equals audio time. Peak-normalised to -1 dBFS; `durationSec` pads/trims.
|
||||
* Pure: no I/O. Notes landing before audio t=0 are clamped to 0 (dropped if they end before it).
|
||||
*/
|
||||
export function renderMidi(
|
||||
tracks: RefTrack[], warp: Warp, sf2: Uint8Array,
|
||||
opts: { sampleRate?: number; mono?: boolean; durationSec?: number } = {},
|
||||
): Audio {
|
||||
const sampleRate = opts.sampleRate ?? 44100;
|
||||
const midi = new at.midi.MidiFile();
|
||||
midi.division = 960;
|
||||
midi.addEvent(new at.midi.TempoChangeEvent(0, 500000));
|
||||
const seen = new Set<number>();
|
||||
let next = 0;
|
||||
for (const t of tracks) {
|
||||
const ch = t.isDrums ? 9 : MELODIC_CHANNELS[next++ % MELODIC_CHANNELS.length]!;
|
||||
if (!seen.has(ch)) { seen.add(ch); midi.addEvent(new at.midi.ProgramChangeEvent(0, 0, ch, t.isDrums ? 0 : t.program)); }
|
||||
for (const n of t.notes) {
|
||||
const on = Math.round(Math.max(0, warp.toAudio(n.onset)) * TPS);
|
||||
const off = Math.max(on + 1, Math.round(warp.toAudio(n.offset) * TPS));
|
||||
if (warp.toAudio(n.offset) <= 0) continue;
|
||||
midi.addEvent(new at.midi.NoteOnEvent(0, on, ch, n.pitch, n.velocity));
|
||||
midi.addEvent(new at.midi.NoteOffEvent(0, off, ch, n.pitch, 0));
|
||||
}
|
||||
}
|
||||
const frames = opts.durationSec === undefined ? undefined : Math.round(opts.durationSec * sampleRate);
|
||||
const out = renderMidiFile(midi, sf2, sampleRate, { frames });
|
||||
let chans = out.channels;
|
||||
if (opts.mono) {
|
||||
const [L, R] = chans as [Float32Array, Float32Array];
|
||||
chans = [L.map((v, i) => (v + R[i]!) / 2)];
|
||||
}
|
||||
let peak = 0;
|
||||
for (const c of chans) for (let i = 0; i < c.length; i++) { const a = Math.abs(c[i]!); if (a > peak) peak = a; }
|
||||
if (peak > 0) { const g = PEAK / peak; for (const c of chans) for (let i = 0; i < c.length; i++) c[i]! *= g; }
|
||||
return { sampleRate, channels: chans };
|
||||
}
|
||||
+36
-22
@@ -5,7 +5,7 @@ import { songToScore } from "../score/gp.ts";
|
||||
const TAIL_SECONDS = 2;
|
||||
|
||||
/** Offline-render sink: AlphaSynth requires an ISynthOutput but exportAudio never uses it. */
|
||||
function dummyOutput(sampleRate: number): at.synth.ISynthOutput {
|
||||
export function dummyOutput(sampleRate: number): at.synth.ISynthOutput {
|
||||
const ev = { on() {}, off() {} };
|
||||
return {
|
||||
sampleRate, open() {}, play() {}, destroy() {}, pause() {}, addSamples() {}, resetSamples() {}, activate() {},
|
||||
@@ -18,6 +18,39 @@ function songSeconds(song: Song): number {
|
||||
return (song.bars * song.timeSig[0] * (4 / song.timeSig[1]) * 60) / song.bpm;
|
||||
}
|
||||
|
||||
/** Render a MidiFile offline. With `frames`, output is exactly that long (zero-padded or truncated); else full length. */
|
||||
export function renderMidiFile(
|
||||
midi: at.midi.MidiFile, sf2: Uint8Array, sampleRate: number,
|
||||
opts: { trackVolume?: Map<number, number>; frames?: number } = {},
|
||||
): Audio {
|
||||
const synth = new at.synth.AlphaSynth(dummyOutput(sampleRate), 500);
|
||||
const eo = new at.synth.AudioExportOptions();
|
||||
eo.soundFonts = [sf2];
|
||||
eo.sampleRate = sampleRate;
|
||||
eo.useSyncPoints = false;
|
||||
if (opts.trackVolume) eo.trackVolume = opts.trackVolume;
|
||||
const exporter = synth.exportAudio(eo, midi, [], new Map());
|
||||
const cap = opts.frames ?? Infinity;
|
||||
const Ls: Float32Array[] = [], Rs: Float32Array[] = [];
|
||||
let pos = 0;
|
||||
while (pos < cap) {
|
||||
const chunk = exporter.render(500);
|
||||
if (!chunk) break;
|
||||
const s = chunk.samples;
|
||||
const frames = Math.min(s.length >> 1, cap - pos);
|
||||
const l = new Float32Array(frames), r = new Float32Array(frames);
|
||||
for (let i = 0; i < frames; i++) { l[i] = s[2 * i]!; r[i] = s[2 * i + 1]!; }
|
||||
Ls.push(l); Rs.push(r);
|
||||
pos += frames;
|
||||
}
|
||||
synth.destroy();
|
||||
const n = opts.frames ?? pos;
|
||||
const L = new Float32Array(n), R = new Float32Array(n);
|
||||
let o = 0;
|
||||
for (let k = 0; k < Ls.length; k++) { L.set(Ls[k]!, o); R.set(Rs[k]!, o); o += Ls[k]!.length; }
|
||||
return { sampleRate, channels: [L, R] };
|
||||
}
|
||||
|
||||
/**
|
||||
* Render each track in isolation with alphaSynth. Isolation: `AudioExportOptions.trackVolume` is keyed by MIDI
|
||||
* channel (despite the doc's "track index"), so every channel not owned by the target track is muted (volume 0).
|
||||
@@ -36,27 +69,8 @@ export function renderStems(song: Song, sf2: Uint8Array, sampleRate = 44100): Ma
|
||||
const trackVolume = new Map<number, number>();
|
||||
for (let ch = 0; ch < 16; ch++) if (!own.has(ch)) trackVolume.set(ch, 0);
|
||||
|
||||
const synth = new at.synth.AlphaSynth(dummyOutput(sampleRate), 500);
|
||||
const opts = new at.synth.AudioExportOptions();
|
||||
opts.soundFonts = [sf2];
|
||||
opts.sampleRate = sampleRate;
|
||||
opts.useSyncPoints = false;
|
||||
opts.trackVolume = trackVolume;
|
||||
const exporter = synth.exportAudio(opts, midi, [], new Map());
|
||||
|
||||
const L = new Float32Array(total), R = new Float32Array(total);
|
||||
let pos = 0;
|
||||
for (;;) {
|
||||
const chunk = exporter.render(500);
|
||||
if (!chunk) break;
|
||||
const s = chunk.samples;
|
||||
const frames = Math.min(s.length >> 1, total - pos);
|
||||
for (let i = 0; i < frames; i++) { L[pos + i] = s[2 * i]!; R[pos + i] = s[2 * i + 1]!; }
|
||||
pos += frames;
|
||||
if (pos >= total) break;
|
||||
}
|
||||
synth.destroy();
|
||||
stems.set(track.name, { sampleRate, channels: [L, R] });
|
||||
const audio = renderMidiFile(midi, sf2, sampleRate, { trackVolume, frames: total });
|
||||
stems.set(track.name, audio);
|
||||
}
|
||||
return stems;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
import { test, expect } from "bun:test";
|
||||
import { renderMidi } from "../../src/align/render.ts";
|
||||
import { peaks } from "../../src/align/peaks.ts";
|
||||
import { makeWarp } from "../../src/align/warp.ts";
|
||||
import type { RefTrack, RefNote } from "../../src/align/smf.ts";
|
||||
|
||||
const SF2 = new Uint8Array(await Bun.file("node_modules/@coderline/alphatab/dist/soundfont/sonivox.sf2").arrayBuffer());
|
||||
const note = (onset: number, offset: number, pitch: number): RefNote => ({ onset, offset, pitch, velocity: 100, channel: 0 });
|
||||
const track = (notes: RefNote[], isDrums = false): RefTrack => ({ index: 0, name: "t", program: 25, isDrums, notes });
|
||||
const SR = 44100; // alphaSynth yields NaN at 22050 Hz
|
||||
|
||||
function firstAbove(x: Float32Array, thr: number): number {
|
||||
for (let i = 0; i < x.length; i++) if (Math.abs(x[i]!) > thr) return i;
|
||||
return -1;
|
||||
}
|
||||
|
||||
test("warp offset moves the note onset", () => {
|
||||
const a = renderMidi([track([note(1, 2, 60)])], makeWarp([{ midiSec: 0, audioSec: 0.5, kind: "manual" }]), SF2, { sampleRate: SR, mono: true, durationSec: 4 });
|
||||
expect(a.channels.length).toBe(1);
|
||||
expect(a.channels[0]!.length).toBe(4 * SR);
|
||||
const t = firstAbove(a.channels[0]!, 0.02) / SR;
|
||||
expect(t).toBeGreaterThan(1.48);
|
||||
expect(t).toBeLessThan(1.52);
|
||||
});
|
||||
|
||||
test("dense chord is peak-normalised to -1 dBFS", () => {
|
||||
const notes = Array.from({ length: 20 }, (_, i) => note(0, 1, 40 + i * 2));
|
||||
const a = renderMidi([track(notes)], makeWarp([]), SF2, { sampleRate: SR });
|
||||
let p = 0;
|
||||
for (const c of a.channels) for (const v of c) p = Math.max(p, Math.abs(v));
|
||||
expect(p).toBeLessThanOrEqual(Math.pow(10, -1 / 20) + 1e-3);
|
||||
expect(p).toBeGreaterThan(0.85);
|
||||
});
|
||||
|
||||
test("drums track is audible", () => {
|
||||
const a = renderMidi([track([note(0.2, 0.3, 38)], true)], makeWarp([]), SF2, { sampleRate: SR, durationSec: 1 });
|
||||
expect(a.channels[0]!.some((v) => Math.abs(v) > 0.1)).toBe(true);
|
||||
});
|
||||
|
||||
test("peaks of a known sine", () => {
|
||||
const x = new Float32Array(1000).map((_, i) => Math.sin((2 * Math.PI * i) / 100));
|
||||
const p = peaks(x, 10);
|
||||
expect(p.max.length).toBe(10);
|
||||
for (let b = 0; b < 10; b++) {
|
||||
expect(p.max[b]!).toBeCloseTo(1, 1);
|
||||
expect(p.min[b]!).toBeCloseTo(-1, 1);
|
||||
}
|
||||
const q = peaks(new Float32Array([0.5, 0.25, -0.75, 0]), 2);
|
||||
expect([...q.max]).toEqual([0.5, 0]);
|
||||
expect([...q.min]).toEqual([0.25, -0.75]);
|
||||
});
|
||||
Reference in New Issue
Block a user