Lucas WintherandClaude Opus 5.5
48ae9780a6
- Return an unchanged copy for a non-finite/zero grid (NaN downbeat used to
delete every guitar note).
- Skip an added note when a same-pitch note (bar, neighbouring bars or already
added) lies within tol; avoids near-duplicates across bar lines.
- bench run/real: --no-repeat-vote passes { repeatVote: false } to the pipeline.
- Tests: notes before a late downbeat, bpm 0 / NaN / Infinity, trailing partial
bar, half-tempo grid, neighbour-bar duplicate, pipeline-level on/off.
Eval unchanged: synthetic t1a 0.885 t1b 0.840 t1c 0.752 t1d 0.737, lnoe 0.507.
Co-Authored-By: Claude Opus 5.5 <[email protected]>
72 lines
3.6 KiB
TypeScript
72 lines
3.6 KiB
TypeScript
// Benchmark orchestration: builds songs, runs the pipeline per oracle mode, scores, writes artefacts.
|
|
import { join } from "node:path";
|
|
import { mkdir } from "node:fs/promises";
|
|
import os from "node:os";
|
|
import type { TierId } from "../gen/tiers.ts";
|
|
import { buildSong, loadSongDir } from "./dataset.ts";
|
|
import { createPipeline } from "../pipeline/run.ts";
|
|
import { scoreSong, type OracleFlags } from "../eval/score.ts";
|
|
import { renderReport, resultsJson, type BenchRun } from "./report.ts";
|
|
|
|
export const MODES: Record<string, OracleFlags> = {
|
|
"full": { oracleStems: false, oracleTempo: false, oracleNotes: false },
|
|
"oracle-stems": { oracleStems: true, oracleTempo: false, oracleNotes: false },
|
|
"oracle-tempo": { oracleStems: true, oracleTempo: true, oracleNotes: false },
|
|
"oracle-notes": { oracleStems: true, oracleTempo: true, oracleNotes: true },
|
|
};
|
|
|
|
const MODEL_PATHS = { demucs: "models/demucs/htdemucs_6s.onnx", basicPitch: "models/basic-pitch/nmp.onnx" };
|
|
|
|
async function sha256(bytes: Uint8Array): Promise<string> {
|
|
const h = new Bun.CryptoHasher("sha256");
|
|
h.update(bytes);
|
|
return h.digest("hex");
|
|
}
|
|
|
|
export async function runBench(opts: {
|
|
tiers: TierId[]; seeds: number[]; modes: string[]; runId: string; root: string; out: string; resultsDir?: string;
|
|
/** Guitar repetition voting (default on); false = --no-repeat-vote. */
|
|
repeatVote?: boolean;
|
|
}): Promise<BenchRun> {
|
|
for (const m of opts.modes) if (!Object.hasOwn(MODES, m)) throw new Error(`unknown mode: ${m}`);
|
|
const demucs = new Uint8Array(await Bun.file(MODEL_PATHS.demucs).arrayBuffer());
|
|
const basicPitch = new Uint8Array(await Bun.file(MODEL_PATHS.basicPitch).arrayBuffer());
|
|
const pipeline = await createPipeline({ demucs, basicPitch }, { repeatVote: opts.repeatVote ?? true }); // model load once per run
|
|
const run: BenchRun = {
|
|
runId: opts.runId,
|
|
date: new Date().toISOString(),
|
|
modes: opts.modes.map((name) => ({ name, flags: MODES[name]! })),
|
|
results: [],
|
|
meta: {
|
|
machine: `${os.cpus()[0]?.model ?? "unknown cpu"} x${os.cpus().length}, ${os.platform()} ${os.arch()}, bun ${Bun.version}`,
|
|
models: { "htdemucs_6s.onnx": await sha256(demucs), "nmp.onnx": await sha256(basicPitch) },
|
|
soundFont: "alphaTab bundled sonivox.sf2 (Apache-2.0)",
|
|
},
|
|
};
|
|
for (const tier of opts.tiers) for (const seed of opts.seeds) {
|
|
const dir = await buildSong(tier, seed, opts.root);
|
|
const { truth, mix, stems } = await loadSongDir(dir);
|
|
const outDir = join(opts.out, opts.runId, tier, String(seed));
|
|
for (const mode of run.modes) {
|
|
const t0 = performance.now();
|
|
const res = await pipeline({ mix, oracle: { stems, truth } }, mode.flags);
|
|
const seconds = (performance.now() - t0) / 1000;
|
|
const score = scoreSong(truth, res.output, mode.flags, res.gp);
|
|
const d = join(outDir, mode.name);
|
|
await mkdir(d, { recursive: true });
|
|
await Bun.write(join(d, "notes.json"), JSON.stringify({
|
|
grid: res.output.grid, tracks: res.output.tracks, warnings: res.warnings, flags: mode.flags, seconds,
|
|
}));
|
|
await Bun.write(join(d, "out.gp"), res.gp);
|
|
await Bun.write(join(d, "score.json"), JSON.stringify(score, null, 1));
|
|
run.results.push({ tier, seed, mode: mode.name, score, seconds });
|
|
console.log(`${tier}/${seed} ${mode.name}: guitar F1 ${score.guitarMerged.f1?.toFixed(3) ?? "n/a"} (${seconds.toFixed(1)}s)`);
|
|
}
|
|
}
|
|
const rdir = opts.resultsDir ?? "results";
|
|
await mkdir(rdir, { recursive: true });
|
|
await Bun.write(join(rdir, `${opts.runId}.json`), resultsJson(run));
|
|
await Bun.write(join(rdir, `${opts.runId}.md`), renderReport(run));
|
|
return run;
|
|
}
|