feat: benchmark runner and report

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-30 05:14:41 +01:00
parent 4c89f0d664
commit 42d88a2401
4 changed files with 220 additions and 1 deletions
@@ -0,0 +1,46 @@
+import { describe, expect, test } from "bun:test";
+import { renderReport, type BenchRun } from "../../src/bench/report.ts";
+import type { SongScore } from "../../src/eval/score.ts";
+
+const mr = (f1: number | null, oct = 0) => ({ tp: 0, fp: 0, fn: 0, precision: f1, recall: f1, f1, octaveErrors: oct, matches: [] as [number, number][] });
+function score(gf1: number, vocals: number | null, flags = { oracleStems: false, oracleTempo: false, oracleNotes: false }): SongScore {
+  return {
+    flags,
+    guitarMerged: { ...mr(gf1, 1), pass: gf1 >= 0.85 },
+    partSplit: null,
+    perRole: { guitar: mr(gf1), vocals: mr(vocals), bass: mr(0.5) },
+    drumsPerClass: null,
+    rhythm: { factor: 1, beatOffset: 0, onBeat: 0.9, durationOk: null },
+    stringAgreement: null,
+    spuriousRate: 0.1,
+  };
+}
+const modes = [
+  { name: "full", flags: { oracleStems: false, oracleTempo: false, oracleNotes: false } },
+  { name: "oracle-notes", flags: { oracleStems: true, oracleTempo: true, oracleNotes: true } },
+];
+const mk = (g: number): BenchRun => ({
+  runId: "fx", date: "2026-10-30", modes,
+  results: [
+    { tier: "t1a", seed: 1, mode: "full", score: score(g, null), seconds: 1 },
+    { tier: "t1a", seed: 2, mode: "full", score: score(g, 0.4), seconds: 2 },
+    { tier: "t1a", seed: 1, mode: "oracle-notes", score: score(1, 1, modes[1]!.flags), seconds: 1 },
+  ],
+});
+
+describe("renderReport", () => {
+  test("column order, n/a, caveat", () => {
+    const md = renderReport(mk(0.9));
+    expect(md.indexOf("Guitar")).toBeLessThan(md.indexOf("Vocals"));
+    expect(md.indexOf("Vocals")).toBeLessThan(md.indexOf("Bass"));
+    expect(md).toContain("n/a");
+    expect(md).toContain("Voice Oohs");
+    expect(md).toContain("fx");
+  });
+  test("tier pass line", () => {
+    expect(renderReport(mk(0.9))).toContain("PASS");
+    const bad = renderReport(mk(0.5));
+    expect(bad).toContain("FAIL");
+    expect(bad).not.toContain("Tier t1a: PASS");
+  });
+});