feat(bench): real-song benchmark
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
Lucas Winther
co-authored by
Claude Sonnet 5.5
Claude Sonnet 5.5
parent
f0249995cf
commit
d623026f6d
+8
-1
@@ -1,4 +1,5 @@
|
||||
import { runBench } from "./runner.ts";
|
||||
import { runReal } from "./real.ts";
|
||||
import { buildSong } from "./dataset.ts";
|
||||
import type { TierId } from "../gen/tiers.ts";
|
||||
import { TIERS } from "../gen/tiers.ts";
|
||||
@@ -47,8 +48,14 @@ async function main() {
|
||||
console.log(`results/${runId}.md`);
|
||||
break;
|
||||
}
|
||||
case "real": {
|
||||
const runId = arg(rest, "run-id", new Date().toISOString().replace(/[:.]/g, "-"));
|
||||
await runReal({ runId, auto: rest.includes("--auto"), root: arg(rest, "root", "reference"), out: arg(rest, "out", "runs") });
|
||||
console.log(`results/real-${runId}.md`);
|
||||
break;
|
||||
}
|
||||
default:
|
||||
console.error("usage: bun run bench gen [--tiers t1a,t1b] [--seeds 1-5] [--root data]\n bun run bench run [--tiers ..] [--seeds ..] [--modes full,oracle-stems,oracle-tempo,oracle-notes] [--run-id id] [--root data] [--out runs]");
|
||||
console.error("usage: bun run bench gen [--tiers t1a,t1b] [--seeds 1-5] [--root data]\n bun run bench run [--tiers ..] [--seeds ..] [--modes full,oracle-stems,oracle-tempo,oracle-notes] [--run-id id] [--root data] [--out runs]\n bun run bench real [--auto] [--run-id id] [--root reference] [--out runs]");
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,133 @@
|
||||
// Real-song benchmark: full pipeline on a team recording, scored per role against the GP-exported MIDI warped through the saved alignment.
|
||||
import { join } from "node:path";
|
||||
import { mkdir } from "node:fs/promises";
|
||||
import type { Role, TimedNote } from "../core/types.ts";
|
||||
import { createPipeline } from "../pipeline/run.ts";
|
||||
import { matchNotes, type MatchResult } from "../eval/notes.ts";
|
||||
import { DRUM_CLASSES, ROLE_ORDER, drumClass, toClassPitch, type DrumClass } from "../eval/score.ts";
|
||||
import { readManifest, readAlignment, loadSong, trackKeys } from "../ref/manifest.ts";
|
||||
import { makeWarp, type Anchor } from "../align/warp.ts";
|
||||
import { autoAlign, alignmentFit } from "../align/auto.ts";
|
||||
import { fitSummary, type FitReport } from "../align/fit.ts";
|
||||
|
||||
export interface RealSongResult {
|
||||
id: string; title: string;
|
||||
alignment: { source: "manual" | "auto"; manual: number; auto: number; savedAt: string | null };
|
||||
fit: FitReport;
|
||||
perRole: Partial<Record<Role, MatchResult>>;
|
||||
/** F1 at the lenient +-100 ms tolerance, per role (null when the role has no notes on either side). */
|
||||
lenient: Partial<Record<Role, number | null>>;
|
||||
drumsPerClass: Record<DrumClass, MatchResult> | null;
|
||||
excludedTracks: string[];
|
||||
/** Capo per track from the manifest; informational only (not applied to scoring). */
|
||||
capo: Record<string, number>;
|
||||
seconds: number;
|
||||
}
|
||||
|
||||
const MODELS = { demucs: "models/demucs/htdemucs_6s.onnx", basicPitch: "models/basic-pitch/nmp.onnx" };
|
||||
const SF2 = join(import.meta.dir, "../../node_modules/@coderline/alphatab/dist/soundfont/sonivox.sf2");
|
||||
const TOL = 0.05, LENIENT_TOL = 0.1;
|
||||
|
||||
export async function runReal(opts: { runId: string; auto: boolean; root?: string; out?: string; resultsDir?: string }): Promise<RealSongResult[]> {
|
||||
const root = opts.root ?? "reference", out = opts.out ?? "runs", rdir = opts.resultsDir ?? "results";
|
||||
const songs = await readManifest(root);
|
||||
const pipeline = await createPipeline({
|
||||
demucs: new Uint8Array(await Bun.file(MODELS.demucs).arrayBuffer()),
|
||||
basicPitch: new Uint8Array(await Bun.file(MODELS.basicPitch).arrayBuffer()),
|
||||
});
|
||||
const sf2 = opts.auto ? new Uint8Array(await Bun.file(SF2).arrayBuffer()) : null;
|
||||
const results: RealSongResult[] = [];
|
||||
const skipped: string[] = [];
|
||||
for (const entry of songs) {
|
||||
const saved = await readAlignment(entry.id, root);
|
||||
if (!saved && !opts.auto) { skipped.push(entry.id); console.log(`${entry.id}: skipped (no alignment.json; pass --auto to compute one)`); continue; }
|
||||
const t0 = performance.now();
|
||||
const { smf, audio } = await loadSong(entry.id, root);
|
||||
const keys = trackKeys(smf.tracks);
|
||||
const roleOf = (i: number): Role | null => entry.roles[keys[i]!] ?? null;
|
||||
|
||||
let anchors: Anchor[], savedAt: string | null, source: "manual" | "auto";
|
||||
if (saved) { anchors = saved.anchors; savedAt = saved.savedAt ?? null; source = "manual"; }
|
||||
else { anchors = autoAlign(audio, smf, entry.roles, [], sf2!).anchors; savedAt = null; source = "auto"; }
|
||||
const fit = alignmentFit(audio, smf, entry.roles, anchors);
|
||||
const warp = makeWarp(anchors);
|
||||
|
||||
const res = await pipeline({ mix: audio }, { oracleStems: false, oracleTempo: false, oracleNotes: false });
|
||||
const refOf = (role: Role): TimedNote[] => smf.tracks.flatMap((t, i) => roleOf(i) === role
|
||||
? t.notes.map((n) => ({ onset: warp.toAudio(n.onset), offset: warp.toAudio(n.offset), pitch: n.pitch })) : []);
|
||||
const estOf = (role: Role): TimedNote[] => res.output.tracks.filter((t) => t.role === role).flatMap((t) => t.notes);
|
||||
|
||||
const perRole: RealSongResult["perRole"] = {}, lenient: RealSongResult["lenient"] = {};
|
||||
let drumsPerClass: RealSongResult["drumsPerClass"] = null;
|
||||
for (const role of ROLE_ORDER) {
|
||||
let r = refOf(role), e = estOf(role);
|
||||
if (role === "drums") {
|
||||
const rawR = r, rawE = e;
|
||||
r = r.map(toClassPitch); e = e.map(toClassPitch);
|
||||
if (rawR.length + rawE.length > 0) {
|
||||
drumsPerClass = {} as Record<DrumClass, MatchResult>;
|
||||
for (const c of Object.keys(DRUM_CLASSES) as DrumClass[]) {
|
||||
drumsPerClass[c] = matchNotes(rawR.filter((n) => drumClass(n.pitch) === c).map(toClassPitch), rawE.filter((n) => drumClass(n.pitch) === c).map(toClassPitch), { onsetTol: TOL });
|
||||
}
|
||||
}
|
||||
}
|
||||
perRole[role] = matchNotes(r, e, { onsetTol: TOL });
|
||||
lenient[role] = matchNotes(r, e, { onsetTol: LENIENT_TOL }).f1;
|
||||
}
|
||||
const seconds = (performance.now() - t0) / 1000;
|
||||
const result: RealSongResult = {
|
||||
id: entry.id, title: entry.title,
|
||||
alignment: { source, manual: anchors.filter((a) => a.kind === "manual").length, auto: anchors.filter((a) => a.kind === "auto").length, savedAt },
|
||||
fit, perRole, lenient, drumsPerClass,
|
||||
excludedTracks: smf.tracks.map((t, i) => (roleOf(i) ? null : keys[i]!)).filter((k): k is string => k !== null),
|
||||
capo: entry.capo ?? {}, seconds,
|
||||
};
|
||||
const d = join(out, `real-${opts.runId}`, entry.id);
|
||||
await mkdir(d, { recursive: true });
|
||||
await Bun.write(join(d, "notes.json"), JSON.stringify({ grid: res.output.grid, tracks: res.output.tracks, warnings: res.warnings, anchors, seconds }));
|
||||
await Bun.write(join(d, "out.gp"), res.gp);
|
||||
await Bun.write(join(d, "score.json"), JSON.stringify(result, null, 1));
|
||||
results.push(result);
|
||||
console.log(`${entry.id}: guitar F1 ${fmt(perRole.guitar?.f1)} (${seconds.toFixed(1)}s)`);
|
||||
}
|
||||
await mkdir(rdir, { recursive: true });
|
||||
await Bun.write(join(rdir, `real-${opts.runId}.json`), JSON.stringify(results, (k, v) => (k === "matches" ? undefined : v), 1));
|
||||
await Bun.write(join(rdir, `real-${opts.runId}.md`), renderRealReport(opts.runId, results, skipped));
|
||||
return results;
|
||||
}
|
||||
|
||||
const NA = "n/a";
|
||||
const fmt = (x: number | null | undefined) => (x == null ? NA : x.toFixed(2));
|
||||
const row = (cells: string[]) => `| ${cells.join(" | ")} |`;
|
||||
|
||||
export function renderRealReport(runId: string, results: RealSongResult[], skipped: string[] = []): string {
|
||||
const o: string[] = [`# STSheets real-song benchmark: ${runId}`, ""];
|
||||
o.push("> Reference = the team's Guitar Pro-exported MIDI warped onto the recording by the saved alignment; scores are note-onset matches (pitch must equal; drums compared by class). " +
|
||||
"The fit line reports the *consistency of the alignment* (warped MIDI onsets vs onset-flux peaks of the recording), not an independent validation of it: " +
|
||||
"a poor alignment lowers every score below. Capo is informational and not applied.", "");
|
||||
for (const r of results) {
|
||||
const a = r.alignment;
|
||||
const src = a.source === "auto" ? `auto (${a.manual} manual / ${a.auto} auto anchors, computed this run)` : `saved (${a.manual} manual / ${a.auto} auto anchors, saved ${a.savedAt ?? "unknown"})`;
|
||||
o.push(`## ${r.title} (${r.id})`, "");
|
||||
o.push(`- Alignment: ${src}`, `- Consistency of the alignment: ${fitSummary(r.fit)}`);
|
||||
const capos = Object.entries(r.capo);
|
||||
o.push(`- Capo: ${capos.length ? capos.map(([k, v]) => `${k}: ${v}`).join(", ") : "none listed"}`);
|
||||
o.push(`- Excluded tracks (no role): ${r.excludedTracks.length ? r.excludedTracks.join(", ") : "none"}`, `- Pipeline + scoring: ${r.seconds.toFixed(1)} s`, "");
|
||||
const head = ["Role", "Ref notes", "Est notes", "P ±50 ms", "R ±50 ms", "F1 ±50 ms", "F1 ±100 ms"];
|
||||
o.push(row(head), row(head.map(() => "---")));
|
||||
for (const role of ROLE_ORDER) {
|
||||
const m = r.perRole[role];
|
||||
if (!m) continue;
|
||||
o.push(row([role, String(m.tp + m.fn), String(m.tp + m.fp), fmt(m.precision), fmt(m.recall), fmt(m.f1), fmt(r.lenient[role])]));
|
||||
}
|
||||
o.push("");
|
||||
if (r.drumsPerClass) {
|
||||
const dh = ["Drum class", "Ref", "Est", "F1 ±50 ms"];
|
||||
o.push(row(dh), row(dh.map(() => "---")));
|
||||
for (const [c, m] of Object.entries(r.drumsPerClass)) o.push(row([c, String(m.tp + m.fn), String(m.tp + m.fp), fmt(m.f1)]));
|
||||
o.push("");
|
||||
}
|
||||
}
|
||||
if (skipped.length) o.push(`Skipped (no alignment.json): ${skipped.join(", ")}`, "");
|
||||
return o.join("\n");
|
||||
}
|
||||
+2
-2
@@ -30,12 +30,12 @@ export const DRUM_CLASSES: Record<DrumClass, number[]> = {
|
||||
};
|
||||
const PASS_F1 = 0.85;
|
||||
|
||||
function drumClass(pitch: number): DrumClass | null {
|
||||
export function drumClass(pitch: number): DrumClass | null {
|
||||
for (const [c, ps] of Object.entries(DRUM_CLASSES)) if (ps.includes(pitch)) return c as DrumClass;
|
||||
return null;
|
||||
}
|
||||
// Drum pitches are compared by class: map to the class's first GM number; unknown pitches stay distinct.
|
||||
const toClassPitch = (n: TimedNote): TimedNote => {
|
||||
export const toClassPitch = (n: TimedNote): TimedNote => {
|
||||
const c = drumClass(n.pitch);
|
||||
return { ...n, pitch: c ? DRUM_CLASSES[c][0]! : 1000 + n.pitch };
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user