feat(bench): real-song benchmark
Co-Authored-By: Claude Sonnet 5.5 <[email protected]>
This commit is contained in:
Lucas Winther
co-authored by
Claude Sonnet 5.5
Claude Sonnet 5.5
parent
f0249995cf
commit
d623026f6d
+8
-1
@@ -1,4 +1,5 @@
|
||||
import { runBench } from "./runner.ts";
|
||||
import { runReal } from "./real.ts";
|
||||
import { buildSong } from "./dataset.ts";
|
||||
import type { TierId } from "../gen/tiers.ts";
|
||||
import { TIERS } from "../gen/tiers.ts";
|
||||
@@ -47,8 +48,14 @@ async function main() {
|
||||
console.log(`results/${runId}.md`);
|
||||
break;
|
||||
}
|
||||
case "real": {
|
||||
const runId = arg(rest, "run-id", new Date().toISOString().replace(/[:.]/g, "-"));
|
||||
await runReal({ runId, auto: rest.includes("--auto"), root: arg(rest, "root", "reference"), out: arg(rest, "out", "runs") });
|
||||
console.log(`results/real-${runId}.md`);
|
||||
break;
|
||||
}
|
||||
default:
|
||||
console.error("usage: bun run bench gen [--tiers t1a,t1b] [--seeds 1-5] [--root data]\n bun run bench run [--tiers ..] [--seeds ..] [--modes full,oracle-stems,oracle-tempo,oracle-notes] [--run-id id] [--root data] [--out runs]");
|
||||
console.error("usage: bun run bench gen [--tiers t1a,t1b] [--seeds 1-5] [--root data]\n bun run bench run [--tiers ..] [--seeds ..] [--modes full,oracle-stems,oracle-tempo,oracle-notes] [--run-id id] [--root data] [--out runs]\n bun run bench real [--auto] [--run-id id] [--root reference] [--out runs]");
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,133 @@
|
||||
// Real-song benchmark: full pipeline on a team recording, scored per role against the GP-exported MIDI warped through the saved alignment.
|
||||
import { join } from "node:path";
|
||||
import { mkdir } from "node:fs/promises";
|
||||
import type { Role, TimedNote } from "../core/types.ts";
|
||||
import { createPipeline } from "../pipeline/run.ts";
|
||||
import { matchNotes, type MatchResult } from "../eval/notes.ts";
|
||||
import { DRUM_CLASSES, ROLE_ORDER, drumClass, toClassPitch, type DrumClass } from "../eval/score.ts";
|
||||
import { readManifest, readAlignment, loadSong, trackKeys } from "../ref/manifest.ts";
|
||||
import { makeWarp, type Anchor } from "../align/warp.ts";
|
||||
import { autoAlign, alignmentFit } from "../align/auto.ts";
|
||||
import { fitSummary, type FitReport } from "../align/fit.ts";
|
||||
|
||||
export interface RealSongResult {
|
||||
id: string; title: string;
|
||||
alignment: { source: "manual" | "auto"; manual: number; auto: number; savedAt: string | null };
|
||||
fit: FitReport;
|
||||
perRole: Partial<Record<Role, MatchResult>>;
|
||||
/** F1 at the lenient +-100 ms tolerance, per role (null when the role has no notes on either side). */
|
||||
lenient: Partial<Record<Role, number | null>>;
|
||||
drumsPerClass: Record<DrumClass, MatchResult> | null;
|
||||
excludedTracks: string[];
|
||||
/** Capo per track from the manifest; informational only (not applied to scoring). */
|
||||
capo: Record<string, number>;
|
||||
seconds: number;
|
||||
}
|
||||
|
||||
const MODELS = { demucs: "models/demucs/htdemucs_6s.onnx", basicPitch: "models/basic-pitch/nmp.onnx" };
|
||||
const SF2 = join(import.meta.dir, "../../node_modules/@coderline/alphatab/dist/soundfont/sonivox.sf2");
|
||||
const TOL = 0.05, LENIENT_TOL = 0.1;
|
||||
|
||||
export async function runReal(opts: { runId: string; auto: boolean; root?: string; out?: string; resultsDir?: string }): Promise<RealSongResult[]> {
|
||||
const root = opts.root ?? "reference", out = opts.out ?? "runs", rdir = opts.resultsDir ?? "results";
|
||||
const songs = await readManifest(root);
|
||||
const pipeline = await createPipeline({
|
||||
demucs: new Uint8Array(await Bun.file(MODELS.demucs).arrayBuffer()),
|
||||
basicPitch: new Uint8Array(await Bun.file(MODELS.basicPitch).arrayBuffer()),
|
||||
});
|
||||
const sf2 = opts.auto ? new Uint8Array(await Bun.file(SF2).arrayBuffer()) : null;
|
||||
const results: RealSongResult[] = [];
|
||||
const skipped: string[] = [];
|
||||
for (const entry of songs) {
|
||||
const saved = await readAlignment(entry.id, root);
|
||||
if (!saved && !opts.auto) { skipped.push(entry.id); console.log(`${entry.id}: skipped (no alignment.json; pass --auto to compute one)`); continue; }
|
||||
const t0 = performance.now();
|
||||
const { smf, audio } = await loadSong(entry.id, root);
|
||||
const keys = trackKeys(smf.tracks);
|
||||
const roleOf = (i: number): Role | null => entry.roles[keys[i]!] ?? null;
|
||||
|
||||
let anchors: Anchor[], savedAt: string | null, source: "manual" | "auto";
|
||||
if (saved) { anchors = saved.anchors; savedAt = saved.savedAt ?? null; source = "manual"; }
|
||||
else { anchors = autoAlign(audio, smf, entry.roles, [], sf2!).anchors; savedAt = null; source = "auto"; }
|
||||
const fit = alignmentFit(audio, smf, entry.roles, anchors);
|
||||
const warp = makeWarp(anchors);
|
||||
|
||||
const res = await pipeline({ mix: audio }, { oracleStems: false, oracleTempo: false, oracleNotes: false });
|
||||
const refOf = (role: Role): TimedNote[] => smf.tracks.flatMap((t, i) => roleOf(i) === role
|
||||
? t.notes.map((n) => ({ onset: warp.toAudio(n.onset), offset: warp.toAudio(n.offset), pitch: n.pitch })) : []);
|
||||
const estOf = (role: Role): TimedNote[] => res.output.tracks.filter((t) => t.role === role).flatMap((t) => t.notes);
|
||||
|
||||
const perRole: RealSongResult["perRole"] = {}, lenient: RealSongResult["lenient"] = {};
|
||||
let drumsPerClass: RealSongResult["drumsPerClass"] = null;
|
||||
for (const role of ROLE_ORDER) {
|
||||
let r = refOf(role), e = estOf(role);
|
||||
if (role === "drums") {
|
||||
const rawR = r, rawE = e;
|
||||
r = r.map(toClassPitch); e = e.map(toClassPitch);
|
||||
if (rawR.length + rawE.length > 0) {
|
||||
drumsPerClass = {} as Record<DrumClass, MatchResult>;
|
||||
for (const c of Object.keys(DRUM_CLASSES) as DrumClass[]) {
|
||||
drumsPerClass[c] = matchNotes(rawR.filter((n) => drumClass(n.pitch) === c).map(toClassPitch), rawE.filter((n) => drumClass(n.pitch) === c).map(toClassPitch), { onsetTol: TOL });
|
||||
}
|
||||
}
|
||||
}
|
||||
perRole[role] = matchNotes(r, e, { onsetTol: TOL });
|
||||
lenient[role] = matchNotes(r, e, { onsetTol: LENIENT_TOL }).f1;
|
||||
}
|
||||
const seconds = (performance.now() - t0) / 1000;
|
||||
const result: RealSongResult = {
|
||||
id: entry.id, title: entry.title,
|
||||
alignment: { source, manual: anchors.filter((a) => a.kind === "manual").length, auto: anchors.filter((a) => a.kind === "auto").length, savedAt },
|
||||
fit, perRole, lenient, drumsPerClass,
|
||||
excludedTracks: smf.tracks.map((t, i) => (roleOf(i) ? null : keys[i]!)).filter((k): k is string => k !== null),
|
||||
capo: entry.capo ?? {}, seconds,
|
||||
};
|
||||
const d = join(out, `real-${opts.runId}`, entry.id);
|
||||
await mkdir(d, { recursive: true });
|
||||
await Bun.write(join(d, "notes.json"), JSON.stringify({ grid: res.output.grid, tracks: res.output.tracks, warnings: res.warnings, anchors, seconds }));
|
||||
await Bun.write(join(d, "out.gp"), res.gp);
|
||||
await Bun.write(join(d, "score.json"), JSON.stringify(result, null, 1));
|
||||
results.push(result);
|
||||
console.log(`${entry.id}: guitar F1 ${fmt(perRole.guitar?.f1)} (${seconds.toFixed(1)}s)`);
|
||||
}
|
||||
await mkdir(rdir, { recursive: true });
|
||||
await Bun.write(join(rdir, `real-${opts.runId}.json`), JSON.stringify(results, (k, v) => (k === "matches" ? undefined : v), 1));
|
||||
await Bun.write(join(rdir, `real-${opts.runId}.md`), renderRealReport(opts.runId, results, skipped));
|
||||
return results;
|
||||
}
|
||||
|
||||
const NA = "n/a";
|
||||
const fmt = (x: number | null | undefined) => (x == null ? NA : x.toFixed(2));
|
||||
const row = (cells: string[]) => `| ${cells.join(" | ")} |`;
|
||||
|
||||
export function renderRealReport(runId: string, results: RealSongResult[], skipped: string[] = []): string {
|
||||
const o: string[] = [`# STSheets real-song benchmark: ${runId}`, ""];
|
||||
o.push("> Reference = the team's Guitar Pro-exported MIDI warped onto the recording by the saved alignment; scores are note-onset matches (pitch must equal; drums compared by class). " +
|
||||
"The fit line reports the *consistency of the alignment* (warped MIDI onsets vs onset-flux peaks of the recording), not an independent validation of it: " +
|
||||
"a poor alignment lowers every score below. Capo is informational and not applied.", "");
|
||||
for (const r of results) {
|
||||
const a = r.alignment;
|
||||
const src = a.source === "auto" ? `auto (${a.manual} manual / ${a.auto} auto anchors, computed this run)` : `saved (${a.manual} manual / ${a.auto} auto anchors, saved ${a.savedAt ?? "unknown"})`;
|
||||
o.push(`## ${r.title} (${r.id})`, "");
|
||||
o.push(`- Alignment: ${src}`, `- Consistency of the alignment: ${fitSummary(r.fit)}`);
|
||||
const capos = Object.entries(r.capo);
|
||||
o.push(`- Capo: ${capos.length ? capos.map(([k, v]) => `${k}: ${v}`).join(", ") : "none listed"}`);
|
||||
o.push(`- Excluded tracks (no role): ${r.excludedTracks.length ? r.excludedTracks.join(", ") : "none"}`, `- Pipeline + scoring: ${r.seconds.toFixed(1)} s`, "");
|
||||
const head = ["Role", "Ref notes", "Est notes", "P ±50 ms", "R ±50 ms", "F1 ±50 ms", "F1 ±100 ms"];
|
||||
o.push(row(head), row(head.map(() => "---")));
|
||||
for (const role of ROLE_ORDER) {
|
||||
const m = r.perRole[role];
|
||||
if (!m) continue;
|
||||
o.push(row([role, String(m.tp + m.fn), String(m.tp + m.fp), fmt(m.precision), fmt(m.recall), fmt(m.f1), fmt(r.lenient[role])]));
|
||||
}
|
||||
o.push("");
|
||||
if (r.drumsPerClass) {
|
||||
const dh = ["Drum class", "Ref", "Est", "F1 ±50 ms"];
|
||||
o.push(row(dh), row(dh.map(() => "---")));
|
||||
for (const [c, m] of Object.entries(r.drumsPerClass)) o.push(row([c, String(m.tp + m.fn), String(m.tp + m.fp), fmt(m.f1)]));
|
||||
o.push("");
|
||||
}
|
||||
}
|
||||
if (skipped.length) o.push(`Skipped (no alignment.json): ${skipped.join(", ")}`, "");
|
||||
return o.join("\n");
|
||||
}
|
||||
+2
-2
@@ -30,12 +30,12 @@ export const DRUM_CLASSES: Record<DrumClass, number[]> = {
|
||||
};
|
||||
const PASS_F1 = 0.85;
|
||||
|
||||
function drumClass(pitch: number): DrumClass | null {
|
||||
export function drumClass(pitch: number): DrumClass | null {
|
||||
for (const [c, ps] of Object.entries(DRUM_CLASSES)) if (ps.includes(pitch)) return c as DrumClass;
|
||||
return null;
|
||||
}
|
||||
// Drum pitches are compared by class: map to the class's first GM number; unknown pitches stay distinct.
|
||||
const toClassPitch = (n: TimedNote): TimedNote => {
|
||||
export const toClassPitch = (n: TimedNote): TimedNote => {
|
||||
const c = drumClass(n.pitch);
|
||||
return { ...n, pitch: c ? DRUM_CLASSES[c][0]! : 1000 + n.pitch };
|
||||
};
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import { mkdtemp, rm } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { renderRealReport, runReal, type RealSongResult } from "../../src/bench/real.ts";
|
||||
import { buildSong, loadSongDir } from "../../src/bench/dataset.ts";
|
||||
import { writeSmf } from "../../src/align/smf.ts";
|
||||
import { writeAlignment, type SongEntry } from "../../src/ref/manifest.ts";
|
||||
import { createPipeline } from "../../src/pipeline/run.ts";
|
||||
import { scoreSong } from "../../src/eval/score.ts";
|
||||
import type { Role } from "../../src/core/types.ts";
|
||||
|
||||
const hasReference = await Bun.file("reference/songs.json").exists();
|
||||
const mr = (f1: number | null, tp = 8, fp = 2, fn = 2) => ({ tp, fp, fn, precision: f1, recall: f1, f1, octaveErrors: 0, matches: [] as [number, number][] });
|
||||
const fit = (rate: number) => ({ matchRate: rate, medianResidualMs: 12.3, windows: [{ startSec: 0, matchRate: rate, medianResidualMs: 12.3, flagged: false }, { startSec: 30, matchRate: 0.2, medianResidualMs: 80, flagged: true }] });
|
||||
const fixture = (): RealSongResult[] => [
|
||||
{
|
||||
id: "a", title: "Song A", alignment: { source: "auto", manual: 0, auto: 40, savedAt: null }, fit: fit(0.8),
|
||||
perRole: { bass: mr(0.5), guitar: mr(0.9), keys: mr(null, 0, 0, 0) }, lenient: { bass: 0.6, guitar: 0.95, keys: null },
|
||||
drumsPerClass: null, excludedTracks: ["Click", "Pad"], capo: { Lead: 2 }, seconds: 3,
|
||||
},
|
||||
{
|
||||
id: "b", title: "Song B", alignment: { source: "manual", manual: 5, auto: 30, savedAt: "2026-10-29T10:00:00Z" }, fit: fit(0.6),
|
||||
perRole: { guitar: mr(0.3) }, lenient: { guitar: 0.4 }, drumsPerClass: null, excludedTracks: [], capo: {}, seconds: 4,
|
||||
},
|
||||
];
|
||||
|
||||
describe("renderRealReport", () => {
|
||||
const md = renderRealReport("fx", fixture());
|
||||
test("guitar row precedes the other roles; auto source and fit shown", () => {
|
||||
expect(md).toContain("Song A");
|
||||
expect(md).toContain("computed this run");
|
||||
expect(md).toContain("2026-10-29T10:00:00Z");
|
||||
expect(md).toContain("0 manual / 40 auto");
|
||||
expect(md).toContain("80.0% of MIDI onsets matched");
|
||||
expect(md).toContain("consistency of the alignment");
|
||||
const a = md.slice(md.indexOf("Song A"), md.indexOf("Song B"));
|
||||
expect(a.indexOf("| guitar")).toBeGreaterThan(-1);
|
||||
expect(a.indexOf("| guitar")).toBeLessThan(a.indexOf("| bass"));
|
||||
expect(a.indexOf("| bass")).toBeLessThan(a.indexOf("| keys"));
|
||||
expect(a).toContain("0.90");
|
||||
expect(a).toContain("0.95");
|
||||
expect(a).toContain("Click, Pad");
|
||||
expect(a).toContain("Lead: 2");
|
||||
});
|
||||
});
|
||||
|
||||
describe("runReal", () => {
|
||||
test("t1a fixture with identity alignment matches the synthetic full-mode score", async () => {
|
||||
const root = await mkdtemp(join(tmpdir(), "stsheets-real-"));
|
||||
try {
|
||||
const dir = await buildSong("t1a", 1, join(root, "data"));
|
||||
const { truth } = await loadSongDir(dir);
|
||||
const prog: Record<Role, number> = { guitar: 25, bass: 33, keys: 0, vocals: 53, drums: 0 };
|
||||
const tracks = truth.tracks.map((t) => ({ name: t.name, program: prog[t.role], isDrums: t.role === "drums", notes: t.notes }));
|
||||
const ref = join(root, "ref");
|
||||
await Bun.write(join(ref, "t1a-1", "song.mid"), writeSmf(tracks, truth.bpm));
|
||||
const entry: SongEntry = {
|
||||
id: "t1a-1", title: "Synthetic t1a/1", audio: join(dir, "mix.wav"), midi: join(ref, "t1a-1", "song.mid"),
|
||||
roles: Object.fromEntries(truth.tracks.map((t) => [t.name, t.role])), capo: {},
|
||||
};
|
||||
await Bun.write(join(ref, "songs.json"), JSON.stringify([entry, { ...entry, id: "noalign", title: "No alignment" }]));
|
||||
const dur = 600;
|
||||
await writeAlignment("t1a-1", { anchors: [{ midiSec: 0, audioSec: 0, kind: "manual" }, { midiSec: dur, audioSec: dur, kind: "manual" }], savedAt: "2026-10-30T00:00:00Z", fit: null }, ref);
|
||||
|
||||
const results = await runReal({ runId: "t", auto: false, root: ref, out: join(root, "runs"), resultsDir: join(root, "results") });
|
||||
expect(results.map((r) => r.id)).toEqual(["t1a-1"]);
|
||||
const r = results[0]!;
|
||||
expect(r.alignment).toEqual({ source: "manual", manual: 2, auto: 0, savedAt: "2026-10-30T00:00:00Z" });
|
||||
expect(r.fit.matchRate).not.toBeNull();
|
||||
|
||||
const pipe = await createPipeline({
|
||||
demucs: new Uint8Array(await Bun.file("models/demucs/htdemucs_6s.onnx").arrayBuffer()),
|
||||
basicPitch: new Uint8Array(await Bun.file("models/basic-pitch/nmp.onnx").arrayBuffer()),
|
||||
});
|
||||
const flags = { oracleStems: false, oracleTempo: false, oracleNotes: false };
|
||||
const loaded = await loadSongDir(dir);
|
||||
const synth = scoreSong(truth, (await pipe({ mix: loaded.mix }, flags)).output, flags);
|
||||
expect(Math.abs((r.perRole.guitar!.f1 ?? 0) - (synth.guitarMerged.f1 ?? 0))).toBeLessThan(0.02);
|
||||
|
||||
for (const f of ["notes.json", "out.gp", "score.json"]) expect(await Bun.file(join(root, "runs", "real-t", "t1a-1", f)).exists()).toBe(true);
|
||||
const json = await Bun.file(join(root, "results", "real-t.json")).text();
|
||||
expect(json).not.toContain('"matches"');
|
||||
expect(await Bun.file(join(root, "results", "real-t.md")).text()).toContain("Synthetic t1a/1");
|
||||
} finally { await rm(root, { recursive: true, force: true }); }
|
||||
}, 600_000);
|
||||
|
||||
test.skipIf(!hasReference)("real reference manifest loads", async () => {
|
||||
const { readManifest } = await import("../../src/ref/manifest.ts");
|
||||
expect((await readManifest()).length).toBeGreaterThan(0);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user