diff --git a/src/bench/cli.ts b/src/bench/cli.ts index 3157f34..7a2c13b 100644 --- a/src/bench/cli.ts +++ b/src/bench/cli.ts @@ -1,4 +1,5 @@ import { runBench } from "./runner.ts"; +import { runReal } from "./real.ts"; import { buildSong } from "./dataset.ts"; import type { TierId } from "../gen/tiers.ts"; import { TIERS } from "../gen/tiers.ts"; @@ -47,8 +48,14 @@ async function main() { console.log(`results/${runId}.md`); break; } + case "real": { + const runId = arg(rest, "run-id", new Date().toISOString().replace(/[:.]/g, "-")); + await runReal({ runId, auto: rest.includes("--auto"), root: arg(rest, "root", "reference"), out: arg(rest, "out", "runs") }); + console.log(`results/real-${runId}.md`); + break; + } default: - console.error("usage: bun run bench gen [--tiers t1a,t1b] [--seeds 1-5] [--root data]\n bun run bench run [--tiers ..] [--seeds ..] [--modes full,oracle-stems,oracle-tempo,oracle-notes] [--run-id id] [--root data] [--out runs]"); + console.error("usage: bun run bench gen [--tiers t1a,t1b] [--seeds 1-5] [--root data]\n bun run bench run [--tiers ..] [--seeds ..] [--modes full,oracle-stems,oracle-tempo,oracle-notes] [--run-id id] [--root data] [--out runs]\n bun run bench real [--auto] [--run-id id] [--root reference] [--out runs]"); process.exit(1); } } diff --git a/src/bench/real.ts b/src/bench/real.ts new file mode 100644 index 0000000..9af3b29 --- /dev/null +++ b/src/bench/real.ts @@ -0,0 +1,133 @@ +// Real-song benchmark: full pipeline on a team recording, scored per role against the GP-exported MIDI warped through the saved alignment. +import { join } from "node:path"; +import { mkdir } from "node:fs/promises"; +import type { Role, TimedNote } from "../core/types.ts"; +import { createPipeline } from "../pipeline/run.ts"; +import { matchNotes, type MatchResult } from "../eval/notes.ts"; +import { DRUM_CLASSES, ROLE_ORDER, drumClass, toClassPitch, type DrumClass } from "../eval/score.ts"; +import { readManifest, readAlignment, loadSong, trackKeys } from "../ref/manifest.ts"; +import { makeWarp, type Anchor } from "../align/warp.ts"; +import { autoAlign, alignmentFit } from "../align/auto.ts"; +import { fitSummary, type FitReport } from "../align/fit.ts"; + +export interface RealSongResult { + id: string; title: string; + alignment: { source: "manual" | "auto"; manual: number; auto: number; savedAt: string | null }; + fit: FitReport; + perRole: Partial>; + /** F1 at the lenient +-100 ms tolerance, per role (null when the role has no notes on either side). */ + lenient: Partial>; + drumsPerClass: Record | null; + excludedTracks: string[]; + /** Capo per track from the manifest; informational only (not applied to scoring). */ + capo: Record; + seconds: number; +} + +const MODELS = { demucs: "models/demucs/htdemucs_6s.onnx", basicPitch: "models/basic-pitch/nmp.onnx" }; +const SF2 = join(import.meta.dir, "../../node_modules/@coderline/alphatab/dist/soundfont/sonivox.sf2"); +const TOL = 0.05, LENIENT_TOL = 0.1; + +export async function runReal(opts: { runId: string; auto: boolean; root?: string; out?: string; resultsDir?: string }): Promise { + const root = opts.root ?? "reference", out = opts.out ?? "runs", rdir = opts.resultsDir ?? "results"; + const songs = await readManifest(root); + const pipeline = await createPipeline({ + demucs: new Uint8Array(await Bun.file(MODELS.demucs).arrayBuffer()), + basicPitch: new Uint8Array(await Bun.file(MODELS.basicPitch).arrayBuffer()), + }); + const sf2 = opts.auto ? new Uint8Array(await Bun.file(SF2).arrayBuffer()) : null; + const results: RealSongResult[] = []; + const skipped: string[] = []; + for (const entry of songs) { + const saved = await readAlignment(entry.id, root); + if (!saved && !opts.auto) { skipped.push(entry.id); console.log(`${entry.id}: skipped (no alignment.json; pass --auto to compute one)`); continue; } + const t0 = performance.now(); + const { smf, audio } = await loadSong(entry.id, root); + const keys = trackKeys(smf.tracks); + const roleOf = (i: number): Role | null => entry.roles[keys[i]!] ?? null; + + let anchors: Anchor[], savedAt: string | null, source: "manual" | "auto"; + if (saved) { anchors = saved.anchors; savedAt = saved.savedAt ?? null; source = "manual"; } + else { anchors = autoAlign(audio, smf, entry.roles, [], sf2!).anchors; savedAt = null; source = "auto"; } + const fit = alignmentFit(audio, smf, entry.roles, anchors); + const warp = makeWarp(anchors); + + const res = await pipeline({ mix: audio }, { oracleStems: false, oracleTempo: false, oracleNotes: false }); + const refOf = (role: Role): TimedNote[] => smf.tracks.flatMap((t, i) => roleOf(i) === role + ? t.notes.map((n) => ({ onset: warp.toAudio(n.onset), offset: warp.toAudio(n.offset), pitch: n.pitch })) : []); + const estOf = (role: Role): TimedNote[] => res.output.tracks.filter((t) => t.role === role).flatMap((t) => t.notes); + + const perRole: RealSongResult["perRole"] = {}, lenient: RealSongResult["lenient"] = {}; + let drumsPerClass: RealSongResult["drumsPerClass"] = null; + for (const role of ROLE_ORDER) { + let r = refOf(role), e = estOf(role); + if (role === "drums") { + const rawR = r, rawE = e; + r = r.map(toClassPitch); e = e.map(toClassPitch); + if (rawR.length + rawE.length > 0) { + drumsPerClass = {} as Record; + for (const c of Object.keys(DRUM_CLASSES) as DrumClass[]) { + drumsPerClass[c] = matchNotes(rawR.filter((n) => drumClass(n.pitch) === c).map(toClassPitch), rawE.filter((n) => drumClass(n.pitch) === c).map(toClassPitch), { onsetTol: TOL }); + } + } + } + perRole[role] = matchNotes(r, e, { onsetTol: TOL }); + lenient[role] = matchNotes(r, e, { onsetTol: LENIENT_TOL }).f1; + } + const seconds = (performance.now() - t0) / 1000; + const result: RealSongResult = { + id: entry.id, title: entry.title, + alignment: { source, manual: anchors.filter((a) => a.kind === "manual").length, auto: anchors.filter((a) => a.kind === "auto").length, savedAt }, + fit, perRole, lenient, drumsPerClass, + excludedTracks: smf.tracks.map((t, i) => (roleOf(i) ? null : keys[i]!)).filter((k): k is string => k !== null), + capo: entry.capo ?? {}, seconds, + }; + const d = join(out, `real-${opts.runId}`, entry.id); + await mkdir(d, { recursive: true }); + await Bun.write(join(d, "notes.json"), JSON.stringify({ grid: res.output.grid, tracks: res.output.tracks, warnings: res.warnings, anchors, seconds })); + await Bun.write(join(d, "out.gp"), res.gp); + await Bun.write(join(d, "score.json"), JSON.stringify(result, null, 1)); + results.push(result); + console.log(`${entry.id}: guitar F1 ${fmt(perRole.guitar?.f1)} (${seconds.toFixed(1)}s)`); + } + await mkdir(rdir, { recursive: true }); + await Bun.write(join(rdir, `real-${opts.runId}.json`), JSON.stringify(results, (k, v) => (k === "matches" ? undefined : v), 1)); + await Bun.write(join(rdir, `real-${opts.runId}.md`), renderRealReport(opts.runId, results, skipped)); + return results; +} + +const NA = "n/a"; +const fmt = (x: number | null | undefined) => (x == null ? NA : x.toFixed(2)); +const row = (cells: string[]) => `| ${cells.join(" | ")} |`; + +export function renderRealReport(runId: string, results: RealSongResult[], skipped: string[] = []): string { + const o: string[] = [`# STSheets real-song benchmark: ${runId}`, ""]; + o.push("> Reference = the team's Guitar Pro-exported MIDI warped onto the recording by the saved alignment; scores are note-onset matches (pitch must equal; drums compared by class). " + + "The fit line reports the *consistency of the alignment* (warped MIDI onsets vs onset-flux peaks of the recording), not an independent validation of it: " + + "a poor alignment lowers every score below. Capo is informational and not applied.", ""); + for (const r of results) { + const a = r.alignment; + const src = a.source === "auto" ? `auto (${a.manual} manual / ${a.auto} auto anchors, computed this run)` : `saved (${a.manual} manual / ${a.auto} auto anchors, saved ${a.savedAt ?? "unknown"})`; + o.push(`## ${r.title} (${r.id})`, ""); + o.push(`- Alignment: ${src}`, `- Consistency of the alignment: ${fitSummary(r.fit)}`); + const capos = Object.entries(r.capo); + o.push(`- Capo: ${capos.length ? capos.map(([k, v]) => `${k}: ${v}`).join(", ") : "none listed"}`); + o.push(`- Excluded tracks (no role): ${r.excludedTracks.length ? r.excludedTracks.join(", ") : "none"}`, `- Pipeline + scoring: ${r.seconds.toFixed(1)} s`, ""); + const head = ["Role", "Ref notes", "Est notes", "P ±50 ms", "R ±50 ms", "F1 ±50 ms", "F1 ±100 ms"]; + o.push(row(head), row(head.map(() => "---"))); + for (const role of ROLE_ORDER) { + const m = r.perRole[role]; + if (!m) continue; + o.push(row([role, String(m.tp + m.fn), String(m.tp + m.fp), fmt(m.precision), fmt(m.recall), fmt(m.f1), fmt(r.lenient[role])])); + } + o.push(""); + if (r.drumsPerClass) { + const dh = ["Drum class", "Ref", "Est", "F1 ±50 ms"]; + o.push(row(dh), row(dh.map(() => "---"))); + for (const [c, m] of Object.entries(r.drumsPerClass)) o.push(row([c, String(m.tp + m.fn), String(m.tp + m.fp), fmt(m.f1)])); + o.push(""); + } + } + if (skipped.length) o.push(`Skipped (no alignment.json): ${skipped.join(", ")}`, ""); + return o.join("\n"); +} diff --git a/src/eval/score.ts b/src/eval/score.ts index 6d7563a..a0c0ad6 100644 --- a/src/eval/score.ts +++ b/src/eval/score.ts @@ -30,12 +30,12 @@ export const DRUM_CLASSES: Record = { }; const PASS_F1 = 0.85; -function drumClass(pitch: number): DrumClass | null { +export function drumClass(pitch: number): DrumClass | null { for (const [c, ps] of Object.entries(DRUM_CLASSES)) if (ps.includes(pitch)) return c as DrumClass; return null; } // Drum pitches are compared by class: map to the class's first GM number; unknown pitches stay distinct. -const toClassPitch = (n: TimedNote): TimedNote => { +export const toClassPitch = (n: TimedNote): TimedNote => { const c = drumClass(n.pitch); return { ...n, pitch: c ? DRUM_CLASSES[c][0]! : 1000 + n.pitch }; }; diff --git a/tests/bench/real.test.ts b/tests/bench/real.test.ts new file mode 100644 index 0000000..e7435ff --- /dev/null +++ b/tests/bench/real.test.ts @@ -0,0 +1,92 @@ +import { describe, expect, test } from "bun:test"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { renderRealReport, runReal, type RealSongResult } from "../../src/bench/real.ts"; +import { buildSong, loadSongDir } from "../../src/bench/dataset.ts"; +import { writeSmf } from "../../src/align/smf.ts"; +import { writeAlignment, type SongEntry } from "../../src/ref/manifest.ts"; +import { createPipeline } from "../../src/pipeline/run.ts"; +import { scoreSong } from "../../src/eval/score.ts"; +import type { Role } from "../../src/core/types.ts"; + +const hasReference = await Bun.file("reference/songs.json").exists(); +const mr = (f1: number | null, tp = 8, fp = 2, fn = 2) => ({ tp, fp, fn, precision: f1, recall: f1, f1, octaveErrors: 0, matches: [] as [number, number][] }); +const fit = (rate: number) => ({ matchRate: rate, medianResidualMs: 12.3, windows: [{ startSec: 0, matchRate: rate, medianResidualMs: 12.3, flagged: false }, { startSec: 30, matchRate: 0.2, medianResidualMs: 80, flagged: true }] }); +const fixture = (): RealSongResult[] => [ + { + id: "a", title: "Song A", alignment: { source: "auto", manual: 0, auto: 40, savedAt: null }, fit: fit(0.8), + perRole: { bass: mr(0.5), guitar: mr(0.9), keys: mr(null, 0, 0, 0) }, lenient: { bass: 0.6, guitar: 0.95, keys: null }, + drumsPerClass: null, excludedTracks: ["Click", "Pad"], capo: { Lead: 2 }, seconds: 3, + }, + { + id: "b", title: "Song B", alignment: { source: "manual", manual: 5, auto: 30, savedAt: "2026-10-29T10:00:00Z" }, fit: fit(0.6), + perRole: { guitar: mr(0.3) }, lenient: { guitar: 0.4 }, drumsPerClass: null, excludedTracks: [], capo: {}, seconds: 4, + }, +]; + +describe("renderRealReport", () => { + const md = renderRealReport("fx", fixture()); + test("guitar row precedes the other roles; auto source and fit shown", () => { + expect(md).toContain("Song A"); + expect(md).toContain("computed this run"); + expect(md).toContain("2026-10-29T10:00:00Z"); + expect(md).toContain("0 manual / 40 auto"); + expect(md).toContain("80.0% of MIDI onsets matched"); + expect(md).toContain("consistency of the alignment"); + const a = md.slice(md.indexOf("Song A"), md.indexOf("Song B")); + expect(a.indexOf("| guitar")).toBeGreaterThan(-1); + expect(a.indexOf("| guitar")).toBeLessThan(a.indexOf("| bass")); + expect(a.indexOf("| bass")).toBeLessThan(a.indexOf("| keys")); + expect(a).toContain("0.90"); + expect(a).toContain("0.95"); + expect(a).toContain("Click, Pad"); + expect(a).toContain("Lead: 2"); + }); +}); + +describe("runReal", () => { + test("t1a fixture with identity alignment matches the synthetic full-mode score", async () => { + const root = await mkdtemp(join(tmpdir(), "stsheets-real-")); + try { + const dir = await buildSong("t1a", 1, join(root, "data")); + const { truth } = await loadSongDir(dir); + const prog: Record = { guitar: 25, bass: 33, keys: 0, vocals: 53, drums: 0 }; + const tracks = truth.tracks.map((t) => ({ name: t.name, program: prog[t.role], isDrums: t.role === "drums", notes: t.notes })); + const ref = join(root, "ref"); + await Bun.write(join(ref, "t1a-1", "song.mid"), writeSmf(tracks, truth.bpm)); + const entry: SongEntry = { + id: "t1a-1", title: "Synthetic t1a/1", audio: join(dir, "mix.wav"), midi: join(ref, "t1a-1", "song.mid"), + roles: Object.fromEntries(truth.tracks.map((t) => [t.name, t.role])), capo: {}, + }; + await Bun.write(join(ref, "songs.json"), JSON.stringify([entry, { ...entry, id: "noalign", title: "No alignment" }])); + const dur = 600; + await writeAlignment("t1a-1", { anchors: [{ midiSec: 0, audioSec: 0, kind: "manual" }, { midiSec: dur, audioSec: dur, kind: "manual" }], savedAt: "2026-10-30T00:00:00Z", fit: null }, ref); + + const results = await runReal({ runId: "t", auto: false, root: ref, out: join(root, "runs"), resultsDir: join(root, "results") }); + expect(results.map((r) => r.id)).toEqual(["t1a-1"]); + const r = results[0]!; + expect(r.alignment).toEqual({ source: "manual", manual: 2, auto: 0, savedAt: "2026-10-30T00:00:00Z" }); + expect(r.fit.matchRate).not.toBeNull(); + + const pipe = await createPipeline({ + demucs: new Uint8Array(await Bun.file("models/demucs/htdemucs_6s.onnx").arrayBuffer()), + basicPitch: new Uint8Array(await Bun.file("models/basic-pitch/nmp.onnx").arrayBuffer()), + }); + const flags = { oracleStems: false, oracleTempo: false, oracleNotes: false }; + const loaded = await loadSongDir(dir); + const synth = scoreSong(truth, (await pipe({ mix: loaded.mix }, flags)).output, flags); + expect(Math.abs((r.perRole.guitar!.f1 ?? 0) - (synth.guitarMerged.f1 ?? 0))).toBeLessThan(0.02); + + for (const f of ["notes.json", "out.gp", "score.json"]) expect(await Bun.file(join(root, "runs", "real-t", "t1a-1", f)).exists()).toBe(true); + const json = await Bun.file(join(root, "results", "real-t.json")).text(); + expect(json).not.toContain('"matches"'); + expect(await Bun.file(join(root, "results", "real-t.md")).text()).toContain("Synthetic t1a/1"); + } finally { await rm(root, { recursive: true, force: true }); } + }, 600_000); + + test.skipIf(!hasReference)("real reference manifest loads", async () => { + const { readManifest } = await import("../../src/ref/manifest.ts"); + expect((await readManifest()).length).toBeGreaterThan(0); + }); +});