feat: benchmark runner and report

Co-Authored-By: Claude Sonnet 5.5 <[email protected]>
2026-10-30 05:14:41 +01:00
parent 4c89f0d664
commit 42d88a2401
4 changed files with 220 additions and 1 deletions
@@ -0,0 +1,69 @@
+// Benchmark orchestration: builds songs, runs the pipeline per oracle mode, scores, writes artefacts.
+import { join } from "node:path";
+import { mkdir } from "node:fs/promises";
+import os from "node:os";
+import type { TierId } from "../gen/tiers.ts";
+import { buildSong, loadSongDir } from "./dataset.ts";
+import { createPipeline } from "../pipeline/run.ts";
+import { scoreSong, type OracleFlags } from "../eval/score.ts";
+import { renderReport, type BenchRun } from "./report.ts";
+
+export const MODES: Record<string, OracleFlags> = {
+  "full": { oracleStems: false, oracleTempo: false, oracleNotes: false },
+  "oracle-stems": { oracleStems: true, oracleTempo: false, oracleNotes: false },
+  "oracle-tempo": { oracleStems: true, oracleTempo: true, oracleNotes: false },
+  "oracle-notes": { oracleStems: true, oracleTempo: true, oracleNotes: true },
+};
+
+const MODEL_PATHS = { demucs: "models/demucs/htdemucs_6s.onnx", basicPitch: "models/basic-pitch/nmp.onnx" };
+
+async function sha256(bytes: Uint8Array): Promise<string> {
+  const h = new Bun.CryptoHasher("sha256");
+  h.update(bytes);
+  return h.digest("hex");
+}
+
+export async function runBench(opts: {
+  tiers: TierId[]; seeds: number[]; modes: string[]; runId: string; root: string; out: string; resultsDir?: string;
+}): Promise<BenchRun> {
+  for (const m of opts.modes) if (!Object.hasOwn(MODES, m)) throw new Error(`unknown mode: ${m}`);
+  const demucs = new Uint8Array(await Bun.file(MODEL_PATHS.demucs).arrayBuffer());
+  const basicPitch = new Uint8Array(await Bun.file(MODEL_PATHS.basicPitch).arrayBuffer());
+  const pipeline = await createPipeline({ demucs, basicPitch }); // model load once per run
+  const run: BenchRun = {
+    runId: opts.runId,
+    date: new Date().toISOString(),
+    modes: opts.modes.map((name) => ({ name, flags: MODES[name]! })),
+    results: [],
+    meta: {
+      machine: `${os.cpus()[0]?.model ?? "unknown cpu"} x${os.cpus().length}, ${os.platform()} ${os.arch()}, bun ${Bun.version}`,
+      models: { "htdemucs_6s.onnx": await sha256(demucs), "nmp.onnx": await sha256(basicPitch) },
+      soundFont: "alphaTab bundled sonivox.sf2 (Apache-2.0)",
+    },
+  };
+  for (const tier of opts.tiers) for (const seed of opts.seeds) {
+    const dir = await buildSong(tier, seed, opts.root);
+    const { truth, mix, stems } = await loadSongDir(dir);
+    const outDir = join(opts.out, opts.runId, tier, String(seed));
+    for (const mode of run.modes) {
+      const t0 = performance.now();
+      const res = await pipeline({ mix, oracle: { stems, truth } }, mode.flags);
+      const seconds = (performance.now() - t0) / 1000;
+      const score = scoreSong(truth, res.output, mode.flags);
+      const d = join(outDir, mode.name);
+      await mkdir(d, { recursive: true });
+      await Bun.write(join(d, "notes.json"), JSON.stringify({
+        grid: res.output.grid, tracks: res.output.tracks, warnings: res.warnings, flags: mode.flags, seconds,
+      }));
+      await Bun.write(join(d, "out.gp"), res.gp);
+      await Bun.write(join(d, "score.json"), JSON.stringify(score, null, 1));
+      run.results.push({ tier, seed, mode: mode.name, score, seconds });
+      console.log(`${tier}/${seed} ${mode.name}: guitar F1 ${score.guitarMerged.f1?.toFixed(3) ?? "n/a"} (${seconds.toFixed(1)}s)`);
+    }
+  }
+  const rdir = opts.resultsDir ?? "results";
+  await mkdir(rdir, { recursive: true });
+  await Bun.write(join(rdir, `${opts.runId}.json`), JSON.stringify(run, null, 1));
+  await Bun.write(join(rdir, `${opts.runId}.md`), renderReport(run));
+  return run;
+}