feat: benchmark runner and report
Co-Authored-By: Claude Sonnet 5.5 <[email protected]>
This commit is contained in:
Lucas Winther
co-authored by
Claude Sonnet 5.5
Claude Sonnet 5.5
parent
4c89f0d664
commit
42d88a2401
+12
-1
@@ -1,3 +1,4 @@
|
||||
import { runBench } from "./runner.ts";
|
||||
import { buildSong } from "./dataset.ts";
|
||||
import type { TierId } from "../gen/tiers.ts";
|
||||
import { TIERS } from "../gen/tiers.ts";
|
||||
@@ -36,8 +37,18 @@ async function main() {
|
||||
}
|
||||
break;
|
||||
}
|
||||
case "run": {
|
||||
const tiers = arg(rest, "tiers", "t1a,t1b,t1c,t1d").split(",") as TierId[];
|
||||
for (const t of tiers) if (!Object.hasOwn(TIERS, t)) throw new Error(`unknown tier: ${t}`);
|
||||
const seeds = parseSeeds(arg(rest, "seeds", "1-5"));
|
||||
const modes = arg(rest, "modes", "full,oracle-stems,oracle-tempo,oracle-notes").split(",");
|
||||
const runId = arg(rest, "run-id", new Date().toISOString().replace(/[:.]/g, "-"));
|
||||
await runBench({ tiers, seeds, modes, runId, root: arg(rest, "root", "data"), out: arg(rest, "out", "runs") });
|
||||
console.log(`results/${runId}.md`);
|
||||
break;
|
||||
}
|
||||
default:
|
||||
console.error("usage: bun run bench gen [--tiers t1a,t1b] [--seeds 1-5] [--root data]");
|
||||
console.error("usage: bun run bench gen [--tiers t1a,t1b] [--seeds 1-5] [--root data]\n bun run bench run [--tiers ..] [--seeds ..] [--modes full,oracle-stems,oracle-tempo,oracle-notes] [--run-id id] [--root data] [--out runs]");
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
// Markdown tier report. Pure: no I/O.
|
||||
import type { TierId } from "../gen/tiers.ts";
|
||||
import type { OracleFlags, SongScore } from "../eval/score.ts";
|
||||
|
||||
export interface BenchResult { tier: TierId; seed: number; mode: string; score: SongScore; seconds: number }
|
||||
export interface BenchRun {
|
||||
runId: string;
|
||||
date: string;
|
||||
modes: { name: string; flags: OracleFlags }[];
|
||||
results: BenchResult[];
|
||||
meta?: { machine?: string; models?: Record<string, string>; soundFont?: string };
|
||||
}
|
||||
|
||||
const PASS_F1 = 0.85;
|
||||
const NA = "n/a";
|
||||
|
||||
export function median(xs: (number | null | undefined)[]): number | null {
|
||||
const v = xs.filter((x): x is number => x != null && !Number.isNaN(x)).sort((a, b) => a - b);
|
||||
if (!v.length) return null;
|
||||
const m = v.length >> 1;
|
||||
return v.length % 2 ? v[m]! : (v[m - 1]! + v[m]!) / 2;
|
||||
}
|
||||
const fmt = (x: number | null | undefined, d = 2) => (x == null ? NA : x.toFixed(d));
|
||||
|
||||
/** "median (min–max)" over non-null values; n/a when all null. */
|
||||
function stat(xs: (number | null | undefined)[], d = 2): string {
|
||||
const v = xs.filter((x): x is number => x != null && !Number.isNaN(x));
|
||||
if (!v.length) return NA;
|
||||
return `${fmt(median(v), d)} (${fmt(Math.min(...v), d)}–${fmt(Math.max(...v), d)})`;
|
||||
}
|
||||
|
||||
type Col = { head: string; get: (s: SongScore) => number | null; d?: number };
|
||||
const drum = (c: "kick" | "snare" | "hihat") => (s: SongScore) => s.drumsPerClass?.[c].f1 ?? null;
|
||||
const COLS: Col[] = [
|
||||
{ head: "Guitar merged F1", get: (s) => s.guitarMerged.f1 },
|
||||
{ head: "Guitar octave errors", get: (s) => s.guitarMerged.octaveErrors, d: 0 },
|
||||
{ head: "Spurious rate", get: (s) => s.spuriousRate },
|
||||
{ head: "Part-split F1 (t1d)", get: (s) => s.partSplit?.f1 ?? null },
|
||||
{ head: "Vocals F1", get: (s) => s.perRole.vocals?.f1 ?? null },
|
||||
{ head: "Bass F1", get: (s) => s.perRole.bass?.f1 ?? null },
|
||||
{ head: "Drums kick F1", get: drum("kick") },
|
||||
{ head: "Drums snare F1", get: drum("snare") },
|
||||
{ head: "Drums hihat F1", get: drum("hihat") },
|
||||
{ head: "Keys F1", get: (s) => s.perRole.keys?.f1 ?? null },
|
||||
{ head: "Rhythm onBeat", get: (s) => s.rhythm?.onBeat ?? null },
|
||||
{ head: "Rhythm durationOk", get: (s) => s.rhythm?.durationOk ?? null },
|
||||
{ head: "Tempo factor", get: (s) => s.rhythm?.factor ?? null },
|
||||
{ head: "Beat offset", get: (s) => s.rhythm?.beatOffset ?? null, d: 0 },
|
||||
{ head: "String agreement", get: (s) => s.stringAgreement },
|
||||
];
|
||||
|
||||
const row = (cells: string[]) => `| ${cells.join(" | ")} |`;
|
||||
const table = (head: string[], rows: string[][]) =>
|
||||
[row(head), row(head.map(() => "---")), ...rows.map(row)].join("\n");
|
||||
|
||||
export function renderReport(run: BenchRun): string {
|
||||
const out: string[] = [];
|
||||
out.push(`# STSheets benchmark report: ${run.runId}`, "");
|
||||
out.push(`- Run id: ${run.runId}`, `- Date: ${run.date}`);
|
||||
out.push(`- Machine: ${run.meta?.machine ?? NA}`);
|
||||
const models = Object.entries(run.meta?.models ?? {});
|
||||
out.push(`- Models (sha256): ${models.length ? models.map(([k, v]) => `${k} ${v}`).join("; ") : NA}`);
|
||||
out.push(`- SoundFont: ${run.meta?.soundFont ?? "alphaTab bundled sonivox.sf2 (Apache-2.0)"}`);
|
||||
out.push(`- Modes: ${run.modes.map((m) => `${m.name} (stems=${m.flags.oracleStems}, tempo=${m.flags.oracleTempo}, notes=${m.flags.oracleNotes})`).join("; ")}`);
|
||||
out.push("", "> Caveats: vocals are rendered with a GM \"Voice Oohs\" patch, not real singing, so separation/transcription numbers for vocals are indicative only. " +
|
||||
"Drums use a heuristic baseline transcriber. The SONiVOX guitar is partly routed to Demucs' piano stem (known synthetic-timbre gap). " +
|
||||
"Pitch is pass/fail (song passes at merged-guitar pitch F1 >= 0.85); rhythm is secondary; fingering (string agreement) is informational. Cells are median (min–max).", "");
|
||||
|
||||
const tiers = [...new Set(run.results.map((r) => r.tier))].sort();
|
||||
for (const tier of tiers) {
|
||||
const rs = run.results.filter((r) => r.tier === tier);
|
||||
out.push(`## Tier ${tier}`, "");
|
||||
const rows = run.modes.filter((m) => rs.some((r) => r.mode === m.name)).map((m) => {
|
||||
const ms = rs.filter((r) => r.mode === m.name);
|
||||
const guitarCell = `${stat(ms.map((r) => r.score.guitarMerged.f1))} ${ms.every((r) => r.score.guitarMerged.pass) ? "✓" : "✗"}`;
|
||||
return [m.name, String(ms.length), guitarCell,
|
||||
...COLS.slice(1).map((c) => stat(ms.map((r) => c.get(r.score)), c.d ?? 2))];
|
||||
});
|
||||
out.push(table(["Mode", "Songs", ...COLS.map((c) => c.head)], rows), "");
|
||||
const full = rs.filter((r) => r.mode === "full");
|
||||
const med = median(full.map((r) => r.score.guitarMerged.f1));
|
||||
out.push(med == null
|
||||
? `**Tier ${tier}: n/a** (no \`full\` mode results)`
|
||||
: `**Tier ${tier}: ${med >= PASS_F1 ? "PASS" : "FAIL"}** (median merged-guitar F1 ${fmt(med)} in \`full\` mode, threshold ${PASS_F1})`, "");
|
||||
}
|
||||
|
||||
out.push("## Per-song appendix", "");
|
||||
const sorted = [...run.results].sort((a, b) => a.tier.localeCompare(b.tier) || a.seed - b.seed || a.mode.localeCompare(b.mode));
|
||||
out.push(table(["Tier", "Seed", "Mode", "Guitar pass", ...COLS.map((c) => c.head), "Seconds"],
|
||||
sorted.map((r) => [r.tier, String(r.seed), r.mode, r.score.guitarMerged.pass ? "✓" : "✗",
|
||||
...COLS.map((c) => fmt(c.get(r.score), c.d ?? 2)), r.seconds.toFixed(1)])), "");
|
||||
return out.join("\n");
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
// Benchmark orchestration: builds songs, runs the pipeline per oracle mode, scores, writes artefacts.
|
||||
import { join } from "node:path";
|
||||
import { mkdir } from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import type { TierId } from "../gen/tiers.ts";
|
||||
import { buildSong, loadSongDir } from "./dataset.ts";
|
||||
import { createPipeline } from "../pipeline/run.ts";
|
||||
import { scoreSong, type OracleFlags } from "../eval/score.ts";
|
||||
import { renderReport, type BenchRun } from "./report.ts";
|
||||
|
||||
export const MODES: Record<string, OracleFlags> = {
|
||||
"full": { oracleStems: false, oracleTempo: false, oracleNotes: false },
|
||||
"oracle-stems": { oracleStems: true, oracleTempo: false, oracleNotes: false },
|
||||
"oracle-tempo": { oracleStems: true, oracleTempo: true, oracleNotes: false },
|
||||
"oracle-notes": { oracleStems: true, oracleTempo: true, oracleNotes: true },
|
||||
};
|
||||
|
||||
const MODEL_PATHS = { demucs: "models/demucs/htdemucs_6s.onnx", basicPitch: "models/basic-pitch/nmp.onnx" };
|
||||
|
||||
async function sha256(bytes: Uint8Array): Promise<string> {
|
||||
const h = new Bun.CryptoHasher("sha256");
|
||||
h.update(bytes);
|
||||
return h.digest("hex");
|
||||
}
|
||||
|
||||
export async function runBench(opts: {
|
||||
tiers: TierId[]; seeds: number[]; modes: string[]; runId: string; root: string; out: string; resultsDir?: string;
|
||||
}): Promise<BenchRun> {
|
||||
for (const m of opts.modes) if (!Object.hasOwn(MODES, m)) throw new Error(`unknown mode: ${m}`);
|
||||
const demucs = new Uint8Array(await Bun.file(MODEL_PATHS.demucs).arrayBuffer());
|
||||
const basicPitch = new Uint8Array(await Bun.file(MODEL_PATHS.basicPitch).arrayBuffer());
|
||||
const pipeline = await createPipeline({ demucs, basicPitch }); // model load once per run
|
||||
const run: BenchRun = {
|
||||
runId: opts.runId,
|
||||
date: new Date().toISOString(),
|
||||
modes: opts.modes.map((name) => ({ name, flags: MODES[name]! })),
|
||||
results: [],
|
||||
meta: {
|
||||
machine: `${os.cpus()[0]?.model ?? "unknown cpu"} x${os.cpus().length}, ${os.platform()} ${os.arch()}, bun ${Bun.version}`,
|
||||
models: { "htdemucs_6s.onnx": await sha256(demucs), "nmp.onnx": await sha256(basicPitch) },
|
||||
soundFont: "alphaTab bundled sonivox.sf2 (Apache-2.0)",
|
||||
},
|
||||
};
|
||||
for (const tier of opts.tiers) for (const seed of opts.seeds) {
|
||||
const dir = await buildSong(tier, seed, opts.root);
|
||||
const { truth, mix, stems } = await loadSongDir(dir);
|
||||
const outDir = join(opts.out, opts.runId, tier, String(seed));
|
||||
for (const mode of run.modes) {
|
||||
const t0 = performance.now();
|
||||
const res = await pipeline({ mix, oracle: { stems, truth } }, mode.flags);
|
||||
const seconds = (performance.now() - t0) / 1000;
|
||||
const score = scoreSong(truth, res.output, mode.flags);
|
||||
const d = join(outDir, mode.name);
|
||||
await mkdir(d, { recursive: true });
|
||||
await Bun.write(join(d, "notes.json"), JSON.stringify({
|
||||
grid: res.output.grid, tracks: res.output.tracks, warnings: res.warnings, flags: mode.flags, seconds,
|
||||
}));
|
||||
await Bun.write(join(d, "out.gp"), res.gp);
|
||||
await Bun.write(join(d, "score.json"), JSON.stringify(score, null, 1));
|
||||
run.results.push({ tier, seed, mode: mode.name, score, seconds });
|
||||
console.log(`${tier}/${seed} ${mode.name}: guitar F1 ${score.guitarMerged.f1?.toFixed(3) ?? "n/a"} (${seconds.toFixed(1)}s)`);
|
||||
}
|
||||
}
|
||||
const rdir = opts.resultsDir ?? "results";
|
||||
await mkdir(rdir, { recursive: true });
|
||||
await Bun.write(join(rdir, `${opts.runId}.json`), JSON.stringify(run, null, 1));
|
||||
await Bun.write(join(rdir, `${opts.runId}.md`), renderReport(run));
|
||||
return run;
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import { renderReport, type BenchRun } from "../../src/bench/report.ts";
|
||||
import type { SongScore } from "../../src/eval/score.ts";
|
||||
|
||||
const mr = (f1: number | null, oct = 0) => ({ tp: 0, fp: 0, fn: 0, precision: f1, recall: f1, f1, octaveErrors: oct, matches: [] as [number, number][] });
|
||||
function score(gf1: number, vocals: number | null, flags = { oracleStems: false, oracleTempo: false, oracleNotes: false }): SongScore {
|
||||
return {
|
||||
flags,
|
||||
guitarMerged: { ...mr(gf1, 1), pass: gf1 >= 0.85 },
|
||||
partSplit: null,
|
||||
perRole: { guitar: mr(gf1), vocals: mr(vocals), bass: mr(0.5) },
|
||||
drumsPerClass: null,
|
||||
rhythm: { factor: 1, beatOffset: 0, onBeat: 0.9, durationOk: null },
|
||||
stringAgreement: null,
|
||||
spuriousRate: 0.1,
|
||||
};
|
||||
}
|
||||
const modes = [
|
||||
{ name: "full", flags: { oracleStems: false, oracleTempo: false, oracleNotes: false } },
|
||||
{ name: "oracle-notes", flags: { oracleStems: true, oracleTempo: true, oracleNotes: true } },
|
||||
];
|
||||
const mk = (g: number): BenchRun => ({
|
||||
runId: "fx", date: "2026-10-30", modes,
|
||||
results: [
|
||||
{ tier: "t1a", seed: 1, mode: "full", score: score(g, null), seconds: 1 },
|
||||
{ tier: "t1a", seed: 2, mode: "full", score: score(g, 0.4), seconds: 2 },
|
||||
{ tier: "t1a", seed: 1, mode: "oracle-notes", score: score(1, 1, modes[1]!.flags), seconds: 1 },
|
||||
],
|
||||
});
|
||||
|
||||
describe("renderReport", () => {
|
||||
test("column order, n/a, caveat", () => {
|
||||
const md = renderReport(mk(0.9));
|
||||
expect(md.indexOf("Guitar")).toBeLessThan(md.indexOf("Vocals"));
|
||||
expect(md.indexOf("Vocals")).toBeLessThan(md.indexOf("Bass"));
|
||||
expect(md).toContain("n/a");
|
||||
expect(md).toContain("Voice Oohs");
|
||||
expect(md).toContain("fx");
|
||||
});
|
||||
test("tier pass line", () => {
|
||||
expect(renderReport(mk(0.9))).toContain("PASS");
|
||||
const bad = renderReport(mk(0.5));
|
||||
expect(bad).toContain("FAIL");
|
||||
expect(bad).not.toContain("Tier t1a: PASS");
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user