From 42d88a2401ac51320a5a0f1506c1c878b7a94fdc Mon Sep 17 00:00:00 2001 From: Lucas Winther Date: Fri, 30 Oct 2026 05:14:41 +0100 Subject: [PATCH] feat: benchmark runner and report Co-Authored-By: Claude Sonnet 5.5 --- src/bench/cli.ts | 13 +++++- src/bench/report.ts | 93 ++++++++++++++++++++++++++++++++++++++ src/bench/runner.ts | 69 ++++++++++++++++++++++++++++ tests/bench/report.test.ts | 46 +++++++++++++++++++ 4 files changed, 220 insertions(+), 1 deletion(-) create mode 100644 src/bench/report.ts create mode 100644 src/bench/runner.ts create mode 100644 tests/bench/report.test.ts diff --git a/src/bench/cli.ts b/src/bench/cli.ts index 7b55adf..3157f34 100644 --- a/src/bench/cli.ts +++ b/src/bench/cli.ts @@ -1,3 +1,4 @@ +import { runBench } from "./runner.ts"; import { buildSong } from "./dataset.ts"; import type { TierId } from "../gen/tiers.ts"; import { TIERS } from "../gen/tiers.ts"; @@ -36,8 +37,18 @@ async function main() { } break; } + case "run": { + const tiers = arg(rest, "tiers", "t1a,t1b,t1c,t1d").split(",") as TierId[]; + for (const t of tiers) if (!Object.hasOwn(TIERS, t)) throw new Error(`unknown tier: ${t}`); + const seeds = parseSeeds(arg(rest, "seeds", "1-5")); + const modes = arg(rest, "modes", "full,oracle-stems,oracle-tempo,oracle-notes").split(","); + const runId = arg(rest, "run-id", new Date().toISOString().replace(/[:.]/g, "-")); + await runBench({ tiers, seeds, modes, runId, root: arg(rest, "root", "data"), out: arg(rest, "out", "runs") }); + console.log(`results/${runId}.md`); + break; + } default: - console.error("usage: bun run bench gen [--tiers t1a,t1b] [--seeds 1-5] [--root data]"); + console.error("usage: bun run bench gen [--tiers t1a,t1b] [--seeds 1-5] [--root data]\n bun run bench run [--tiers ..] [--seeds ..] [--modes full,oracle-stems,oracle-tempo,oracle-notes] [--run-id id] [--root data] [--out runs]"); process.exit(1); } } diff --git a/src/bench/report.ts b/src/bench/report.ts new file mode 100644 index 0000000..c21819a --- /dev/null +++ b/src/bench/report.ts @@ -0,0 +1,93 @@ +// Markdown tier report. Pure: no I/O. +import type { TierId } from "../gen/tiers.ts"; +import type { OracleFlags, SongScore } from "../eval/score.ts"; + +export interface BenchResult { tier: TierId; seed: number; mode: string; score: SongScore; seconds: number } +export interface BenchRun { + runId: string; + date: string; + modes: { name: string; flags: OracleFlags }[]; + results: BenchResult[]; + meta?: { machine?: string; models?: Record; soundFont?: string }; +} + +const PASS_F1 = 0.85; +const NA = "n/a"; + +export function median(xs: (number | null | undefined)[]): number | null { + const v = xs.filter((x): x is number => x != null && !Number.isNaN(x)).sort((a, b) => a - b); + if (!v.length) return null; + const m = v.length >> 1; + return v.length % 2 ? v[m]! : (v[m - 1]! + v[m]!) / 2; +} +const fmt = (x: number | null | undefined, d = 2) => (x == null ? NA : x.toFixed(d)); + +/** "median (min–max)" over non-null values; n/a when all null. */ +function stat(xs: (number | null | undefined)[], d = 2): string { + const v = xs.filter((x): x is number => x != null && !Number.isNaN(x)); + if (!v.length) return NA; + return `${fmt(median(v), d)} (${fmt(Math.min(...v), d)}–${fmt(Math.max(...v), d)})`; +} + +type Col = { head: string; get: (s: SongScore) => number | null; d?: number }; +const drum = (c: "kick" | "snare" | "hihat") => (s: SongScore) => s.drumsPerClass?.[c].f1 ?? null; +const COLS: Col[] = [ + { head: "Guitar merged F1", get: (s) => s.guitarMerged.f1 }, + { head: "Guitar octave errors", get: (s) => s.guitarMerged.octaveErrors, d: 0 }, + { head: "Spurious rate", get: (s) => s.spuriousRate }, + { head: "Part-split F1 (t1d)", get: (s) => s.partSplit?.f1 ?? null }, + { head: "Vocals F1", get: (s) => s.perRole.vocals?.f1 ?? null }, + { head: "Bass F1", get: (s) => s.perRole.bass?.f1 ?? null }, + { head: "Drums kick F1", get: drum("kick") }, + { head: "Drums snare F1", get: drum("snare") }, + { head: "Drums hihat F1", get: drum("hihat") }, + { head: "Keys F1", get: (s) => s.perRole.keys?.f1 ?? null }, + { head: "Rhythm onBeat", get: (s) => s.rhythm?.onBeat ?? null }, + { head: "Rhythm durationOk", get: (s) => s.rhythm?.durationOk ?? null }, + { head: "Tempo factor", get: (s) => s.rhythm?.factor ?? null }, + { head: "Beat offset", get: (s) => s.rhythm?.beatOffset ?? null, d: 0 }, + { head: "String agreement", get: (s) => s.stringAgreement }, +]; + +const row = (cells: string[]) => `| ${cells.join(" | ")} |`; +const table = (head: string[], rows: string[][]) => + [row(head), row(head.map(() => "---")), ...rows.map(row)].join("\n"); + +export function renderReport(run: BenchRun): string { + const out: string[] = []; + out.push(`# STSheets benchmark report: ${run.runId}`, ""); + out.push(`- Run id: ${run.runId}`, `- Date: ${run.date}`); + out.push(`- Machine: ${run.meta?.machine ?? NA}`); + const models = Object.entries(run.meta?.models ?? {}); + out.push(`- Models (sha256): ${models.length ? models.map(([k, v]) => `${k} ${v}`).join("; ") : NA}`); + out.push(`- SoundFont: ${run.meta?.soundFont ?? "alphaTab bundled sonivox.sf2 (Apache-2.0)"}`); + out.push(`- Modes: ${run.modes.map((m) => `${m.name} (stems=${m.flags.oracleStems}, tempo=${m.flags.oracleTempo}, notes=${m.flags.oracleNotes})`).join("; ")}`); + out.push("", "> Caveats: vocals are rendered with a GM \"Voice Oohs\" patch, not real singing, so separation/transcription numbers for vocals are indicative only. " + + "Drums use a heuristic baseline transcriber. The SONiVOX guitar is partly routed to Demucs' piano stem (known synthetic-timbre gap). " + + "Pitch is pass/fail (song passes at merged-guitar pitch F1 >= 0.85); rhythm is secondary; fingering (string agreement) is informational. Cells are median (min–max).", ""); + + const tiers = [...new Set(run.results.map((r) => r.tier))].sort(); + for (const tier of tiers) { + const rs = run.results.filter((r) => r.tier === tier); + out.push(`## Tier ${tier}`, ""); + const rows = run.modes.filter((m) => rs.some((r) => r.mode === m.name)).map((m) => { + const ms = rs.filter((r) => r.mode === m.name); + const guitarCell = `${stat(ms.map((r) => r.score.guitarMerged.f1))} ${ms.every((r) => r.score.guitarMerged.pass) ? "✓" : "✗"}`; + return [m.name, String(ms.length), guitarCell, + ...COLS.slice(1).map((c) => stat(ms.map((r) => c.get(r.score)), c.d ?? 2))]; + }); + out.push(table(["Mode", "Songs", ...COLS.map((c) => c.head)], rows), ""); + const full = rs.filter((r) => r.mode === "full"); + const med = median(full.map((r) => r.score.guitarMerged.f1)); + out.push(med == null + ? `**Tier ${tier}: n/a** (no \`full\` mode results)` + : `**Tier ${tier}: ${med >= PASS_F1 ? "PASS" : "FAIL"}** (median merged-guitar F1 ${fmt(med)} in \`full\` mode, threshold ${PASS_F1})`, ""); + } + + out.push("## Per-song appendix", ""); + const sorted = [...run.results].sort((a, b) => a.tier.localeCompare(b.tier) || a.seed - b.seed || a.mode.localeCompare(b.mode)); + out.push(table(["Tier", "Seed", "Mode", "Guitar pass", ...COLS.map((c) => c.head), "Seconds"], + sorted.map((r) => [r.tier, String(r.seed), r.mode, r.score.guitarMerged.pass ? "✓" : "✗", + ...COLS.map((c) => fmt(c.get(r.score), c.d ?? 2)), r.seconds.toFixed(1)])), ""); + return out.join("\n"); +} diff --git a/src/bench/runner.ts b/src/bench/runner.ts new file mode 100644 index 0000000..439c69b --- /dev/null +++ b/src/bench/runner.ts @@ -0,0 +1,69 @@ +// Benchmark orchestration: builds songs, runs the pipeline per oracle mode, scores, writes artefacts. +import { join } from "node:path"; +import { mkdir } from "node:fs/promises"; +import os from "node:os"; +import type { TierId } from "../gen/tiers.ts"; +import { buildSong, loadSongDir } from "./dataset.ts"; +import { createPipeline } from "../pipeline/run.ts"; +import { scoreSong, type OracleFlags } from "../eval/score.ts"; +import { renderReport, type BenchRun } from "./report.ts"; + +export const MODES: Record = { + "full": { oracleStems: false, oracleTempo: false, oracleNotes: false }, + "oracle-stems": { oracleStems: true, oracleTempo: false, oracleNotes: false }, + "oracle-tempo": { oracleStems: true, oracleTempo: true, oracleNotes: false }, + "oracle-notes": { oracleStems: true, oracleTempo: true, oracleNotes: true }, +}; + +const MODEL_PATHS = { demucs: "models/demucs/htdemucs_6s.onnx", basicPitch: "models/basic-pitch/nmp.onnx" }; + +async function sha256(bytes: Uint8Array): Promise { + const h = new Bun.CryptoHasher("sha256"); + h.update(bytes); + return h.digest("hex"); +} + +export async function runBench(opts: { + tiers: TierId[]; seeds: number[]; modes: string[]; runId: string; root: string; out: string; resultsDir?: string; +}): Promise { + for (const m of opts.modes) if (!Object.hasOwn(MODES, m)) throw new Error(`unknown mode: ${m}`); + const demucs = new Uint8Array(await Bun.file(MODEL_PATHS.demucs).arrayBuffer()); + const basicPitch = new Uint8Array(await Bun.file(MODEL_PATHS.basicPitch).arrayBuffer()); + const pipeline = await createPipeline({ demucs, basicPitch }); // model load once per run + const run: BenchRun = { + runId: opts.runId, + date: new Date().toISOString(), + modes: opts.modes.map((name) => ({ name, flags: MODES[name]! })), + results: [], + meta: { + machine: `${os.cpus()[0]?.model ?? "unknown cpu"} x${os.cpus().length}, ${os.platform()} ${os.arch()}, bun ${Bun.version}`, + models: { "htdemucs_6s.onnx": await sha256(demucs), "nmp.onnx": await sha256(basicPitch) }, + soundFont: "alphaTab bundled sonivox.sf2 (Apache-2.0)", + }, + }; + for (const tier of opts.tiers) for (const seed of opts.seeds) { + const dir = await buildSong(tier, seed, opts.root); + const { truth, mix, stems } = await loadSongDir(dir); + const outDir = join(opts.out, opts.runId, tier, String(seed)); + for (const mode of run.modes) { + const t0 = performance.now(); + const res = await pipeline({ mix, oracle: { stems, truth } }, mode.flags); + const seconds = (performance.now() - t0) / 1000; + const score = scoreSong(truth, res.output, mode.flags); + const d = join(outDir, mode.name); + await mkdir(d, { recursive: true }); + await Bun.write(join(d, "notes.json"), JSON.stringify({ + grid: res.output.grid, tracks: res.output.tracks, warnings: res.warnings, flags: mode.flags, seconds, + })); + await Bun.write(join(d, "out.gp"), res.gp); + await Bun.write(join(d, "score.json"), JSON.stringify(score, null, 1)); + run.results.push({ tier, seed, mode: mode.name, score, seconds }); + console.log(`${tier}/${seed} ${mode.name}: guitar F1 ${score.guitarMerged.f1?.toFixed(3) ?? "n/a"} (${seconds.toFixed(1)}s)`); + } + } + const rdir = opts.resultsDir ?? "results"; + await mkdir(rdir, { recursive: true }); + await Bun.write(join(rdir, `${opts.runId}.json`), JSON.stringify(run, null, 1)); + await Bun.write(join(rdir, `${opts.runId}.md`), renderReport(run)); + return run; +} diff --git a/tests/bench/report.test.ts b/tests/bench/report.test.ts new file mode 100644 index 0000000..75ff698 --- /dev/null +++ b/tests/bench/report.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, test } from "bun:test"; +import { renderReport, type BenchRun } from "../../src/bench/report.ts"; +import type { SongScore } from "../../src/eval/score.ts"; + +const mr = (f1: number | null, oct = 0) => ({ tp: 0, fp: 0, fn: 0, precision: f1, recall: f1, f1, octaveErrors: oct, matches: [] as [number, number][] }); +function score(gf1: number, vocals: number | null, flags = { oracleStems: false, oracleTempo: false, oracleNotes: false }): SongScore { + return { + flags, + guitarMerged: { ...mr(gf1, 1), pass: gf1 >= 0.85 }, + partSplit: null, + perRole: { guitar: mr(gf1), vocals: mr(vocals), bass: mr(0.5) }, + drumsPerClass: null, + rhythm: { factor: 1, beatOffset: 0, onBeat: 0.9, durationOk: null }, + stringAgreement: null, + spuriousRate: 0.1, + }; +} +const modes = [ + { name: "full", flags: { oracleStems: false, oracleTempo: false, oracleNotes: false } }, + { name: "oracle-notes", flags: { oracleStems: true, oracleTempo: true, oracleNotes: true } }, +]; +const mk = (g: number): BenchRun => ({ + runId: "fx", date: "2026-10-30", modes, + results: [ + { tier: "t1a", seed: 1, mode: "full", score: score(g, null), seconds: 1 }, + { tier: "t1a", seed: 2, mode: "full", score: score(g, 0.4), seconds: 2 }, + { tier: "t1a", seed: 1, mode: "oracle-notes", score: score(1, 1, modes[1]!.flags), seconds: 1 }, + ], +}); + +describe("renderReport", () => { + test("column order, n/a, caveat", () => { + const md = renderReport(mk(0.9)); + expect(md.indexOf("Guitar")).toBeLessThan(md.indexOf("Vocals")); + expect(md.indexOf("Vocals")).toBeLessThan(md.indexOf("Bass")); + expect(md).toContain("n/a"); + expect(md).toContain("Voice Oohs"); + expect(md).toContain("fx"); + }); + test("tier pass line", () => { + expect(renderReport(mk(0.9))).toContain("PASS"); + const bad = renderReport(mk(0.5)); + expect(bad).toContain("FAIL"); + expect(bad).not.toContain("Tier t1a: PASS"); + }); +});