diff --git a/.superpowers/sdd/2026-10-30-synthetic-benchmark/final-fix-report.md b/.superpowers/sdd/2026-10-30-synthetic-benchmark/final-fix-report.md new file mode 100644 index 0000000..09ed28b --- /dev/null +++ b/.superpowers/sdd/2026-10-30-synthetic-benchmark/final-fix-report.md @@ -0,0 +1,13 @@ +# Final-review fixes report + +Verification: `bun test --timeout 300000` -> 89 pass, 1 skip, 0 fail (90 tests, 19 files). `bun run typecheck` -> clean. Each fix was test-first (quantize and dataset tests seen red before the fix; report/score tests written before the implementation). + +1. Per-mode tick (src/bench/report.ts): uses median merged-guitar F1 >= 0.85 and appends "k/n pass". Test "per-mode tick uses the median rule with a k/n pass count" (4/5 pass, median passes -> "✓ 4/5"; 1/5 -> "✗ 1/5"). +2. R12 (src/pipeline/quantize.ts): chord duration = median of members' snapped durations (even count: rounded mean of the middle two), min 1 step. Tests: updated chord test (8,4,8 steps -> 8) plus a 70 ms ghost-note case that keeps a 16-step chord. +3. R11: SongScore gains `gpPitch` (MatchResult from truthFromGp(outGp), onsets/offsets + grid.firstDownbeat) and `barAgreement` {refBars,outBars,timeSigMatch}; scoreSong takes optional `outGp`; runner passes res.gp. truthFromGp now also returns `bars` (masterBars count). Report columns: "GP-file guitar F1", "Bar count match", "Time sig match", "Techniques" (= "n/a (tier 1 has no technique marks)"). Tests in tests/eval/score.test.ts (downbeat shift, null without GP, bar-count mismatch) and tests/bench/report.test.ts. +4. results/.json: new `resultsJson()` in report.ts drops `matches`; runner uses it (score.json untouched). Test "results JSON omits per-note matches". +5. `resample` moved to src/dsp/resample.ts, re-exported from src/audio/wav.ts; transcribe.ts imports src/dsp/mono.ts and src/dsp/resample.ts. Grep for "audio/wav" under src/pipeline, dsp, score, gen, eval finds only comments. +6. src/bench/dataset.ts: `RENDER_VERSION = 1` (bump comment), written to `/render-version` before song.json; buildSong rebuilds if missing/different. Test: old and missing marker both trigger a rebuild (mp3 mtime advances). +7. README caveat on oracle-stems; models/demucs/README.md with provenance. + +Note: the report column count grew (3 extra metric columns + Techniques); committed results/ baseline files were not regenerated (controller re-runs the benchmark). diff --git a/README.md b/README.md index afe3b11..6740a74 100644 --- a/README.md +++ b/README.md @@ -32,5 +32,6 @@ The baseline is in `results/baseline-2026-10-30.md` (tier summaries plus a per-s - Vocals are rendered with a GM "Voice Oohs" patch, not real singing, so vocal numbers are indicative only. - Drums use a heuristic baseline transcriber. +- `oracle-stems` mode feeds the true per-track stems, which reveals the track count and roles, so part-split F1 under oracle-stems (and the oracle modes built on it) is not meaningful. - The SONiVOX guitar is partly routed to Demucs' piano stem, a synthetic-timbre gap that lowers full-mode guitar scores. - Fingering (string agreement) is informational, not pass/fail. diff --git a/models/demucs/README.md b/models/demucs/README.md new file mode 100644 index 0000000..3957801 --- /dev/null +++ b/models/demucs/README.md @@ -0,0 +1,7 @@ +# Demucs ONNX model provenance + +`htdemucs_6s.onnx` (git-ignored; fetched by `bun run fetch-models`, sha256 recorded in `manifest.json`) is the +6-stem Hybrid Transformer Demucs exported to ONNX. + +- Source: Hugging Face `Kani95/htdemucs-6s-ort` (ONNX Runtime export). License per its model card: Apache-2.0. +- Original Demucs weights and code (facebookresearch/demucs): MIT. diff --git a/src/audio/wav.ts b/src/audio/wav.ts index 1431dfb..9a54f79 100644 --- a/src/audio/wav.ts +++ b/src/audio/wav.ts @@ -68,30 +68,5 @@ export async function writeWav(path: string, a: Audio): Promise { } export { toMono } from "../dsp/mono.ts"; +export { resample } from "../dsp/resample.ts"; -const sinc = (x: number) => (x === 0 ? 1 : Math.sin(Math.PI * x) / (Math.PI * x)); - -/** Windowed-sinc (Blackman) resampler; low-passes at the lower Nyquist when downsampling. */ -export function resample(x: Float32Array, from: number, to: number): Float32Array { - if (from === to) return x.slice(); - const ratio = to / from; - const outLen = Math.round(x.length * ratio); - const out = new Float32Array(outLen); - const cutoff = Math.min(1, ratio); // in units of input Nyquist - const half = 16 / cutoff; // kernel half-width in input samples (16 zero-crossings) - for (let j = 0; j < outLen; j++) { - const t = j / ratio; - const lo = Math.max(0, Math.ceil(t - half)), hi = Math.min(x.length - 1, Math.floor(t + half)); - let acc = 0, wsum = 0; - for (let i = lo; i <= hi; i++) { - const d = i - t; - const w = 0.42 + 0.5 * Math.cos((Math.PI * d) / half) + 0.08 * Math.cos((2 * Math.PI * d) / half); - const k = cutoff * sinc(cutoff * d) * w; - acc += x[i]! * k; - wsum += k; - } - // Normalise to unity DC gain so edges and non-integer ratios stay calibrated. - out[j] = wsum !== 0 ? acc / wsum : 0; - } - return out; -} diff --git a/src/bench/dataset.ts b/src/bench/dataset.ts index 9f1579d..2f3c089 100644 --- a/src/bench/dataset.ts +++ b/src/bench/dataset.ts @@ -14,7 +14,14 @@ const SF2_PATH = join(import.meta.dir, "../../node_modules/@coderline/alphatab/d const SR = 44100; /** - * Build one song into `///`. Idempotent: skipped when mix.mp3 exists and song.json matches. + * Bump whenever the rendered audio could change for the same song.json: i.e. any change to + * src/audio/render.ts, fx.ts, mix.ts, the SoundFont, or the alphaTab version. Stored in /render-version + * (written last, with song.json); buildSong rebuilds a dir whose stored version differs or is missing. + */ +export const RENDER_VERSION = 1; + +/** + * Build one song into `///`. Idempotent: skipped when mix.mp3 exists, song.json matches and render-version equals RENDER_VERSION. * stems/.wav are exactly each track's contribution to mix.wav (post-fx, post-pan, gain, normalisation), * so the stems sum to the mix up to PCM16 rounding. */ @@ -23,7 +30,9 @@ export async function buildSong(tier: TierId, seed: number, root = "data"): Prom const song = generateSong(tier, seed); const songJson = JSON.stringify(song, null, 1); const jsonFile = Bun.file(join(dir, "song.json")); - if ((await Bun.file(join(dir, "mix.mp3")).exists()) && (await jsonFile.exists()) && (await jsonFile.text()) === songJson) { + const verFile = Bun.file(join(dir, "render-version")); + if ((await Bun.file(join(dir, "mix.mp3")).exists()) && (await jsonFile.exists()) && (await jsonFile.text()) === songJson + && (await verFile.exists()) && (await verFile.text()).trim() === String(RENDER_VERSION)) { return dir; } await mkdir(join(dir, "stems"), { recursive: true }); @@ -39,6 +48,7 @@ export async function buildSong(tier: TierId, seed: number, root = "data"): Prom for (let i = 0; i < song.tracks.length; i++) await writeWav(join(dir, "stems", `${song.tracks[i]!.name}.wav`), scaled[i]!); await writeWav(join(dir, "mix.wav"), mix); await encodeMp3(join(dir, "mix.wav"), join(dir, "mix.mp3")); + await Bun.write(join(dir, "render-version"), String(RENDER_VERSION)); await Bun.write(join(dir, "song.json"), songJson); // written last: marks the build complete return dir; } diff --git a/src/bench/report.ts b/src/bench/report.ts index 0770d6c..a9d00cb 100644 --- a/src/bench/report.ts +++ b/src/bench/report.ts @@ -13,6 +13,7 @@ export interface BenchRun { const PASS_F1 = 0.85; const NA = "n/a"; +const TECHNIQUES_NA = "n/a (tier 1 has no technique marks)"; export function median(xs: (number | null | undefined)[]): number | null { const v = xs.filter((x): x is number => x != null && !Number.isNaN(x)).sort((a, b) => a - b); @@ -53,6 +54,9 @@ const COLS: Col[] = [ { head: "Tempo factor", get: (s) => s.rhythm?.factor ?? null }, { head: "Beat offset", get: (s) => s.rhythm?.beatOffset ?? null, d: 0 }, { head: "String agreement", get: (s) => s.stringAgreement }, + { head: "GP-file guitar F1", get: (s) => f1OrNA(s.gpPitch) }, + { head: "Bar count match", get: (s) => (s.barAgreement ? Number(s.barAgreement.refBars === s.barAgreement.outBars) : null), d: 0 }, + { head: "Time sig match", get: (s) => (s.barAgreement ? Number(s.barAgreement.timeSigMatch) : null), d: 0 }, ]; const withSpur = (cell: string, c: Col, scores: SongScore[]) => { @@ -82,11 +86,14 @@ export function renderReport(run: BenchRun): string { out.push(`## Tier ${tier}`, ""); const rows = run.modes.filter((m) => rs.some((r) => r.mode === m.name)).map((m) => { const ms = rs.filter((r) => r.mode === m.name); - const guitarCell = `${stat(ms.map((r) => r.score.guitarMerged.f1))} ${ms.every((r) => r.score.guitarMerged.pass) ? "✓" : "✗"}`; + const f1s = ms.map((r) => r.score.guitarMerged.f1); + const medF1 = median(f1s); + const nPass = ms.filter((r) => r.score.guitarMerged.pass).length; + const guitarCell = `${stat(f1s)} ${medF1 != null && medF1 >= PASS_F1 ? "✓" : "✗"} ${nPass}/${ms.length}`; return [m.name, String(ms.length), guitarCell, - ...COLS.slice(1).map((c) => withSpur(stat(ms.map((r) => c.get(r.score)), c.d ?? 2), c, ms.map((r) => r.score)))]; + ...COLS.slice(1).map((c) => withSpur(stat(ms.map((r) => c.get(r.score)), c.d ?? 2), c, ms.map((r) => r.score))), TECHNIQUES_NA]; }); - out.push(table(["Mode", "Songs", ...COLS.map((c) => c.head)], rows), ""); + out.push(table(["Mode", "Songs", ...COLS.map((c) => c.head), "Techniques"], rows), ""); const full = rs.filter((r) => r.mode === "full"); const med = median(full.map((r) => r.score.guitarMerged.f1)); out.push(med == null @@ -96,8 +103,13 @@ export function renderReport(run: BenchRun): string { out.push("## Per-song appendix", ""); const sorted = [...run.results].sort((a, b) => a.tier.localeCompare(b.tier) || a.seed - b.seed || a.mode.localeCompare(b.mode)); - out.push(table(["Tier", "Seed", "Mode", "Guitar pass", ...COLS.map((c) => c.head), "Seconds"], + out.push(table(["Tier", "Seed", "Mode", "Guitar pass", ...COLS.map((c) => c.head), "Techniques", "Seconds"], sorted.map((r) => [r.tier, String(r.seed), r.mode, r.score.guitarMerged.pass ? "✓" : "✗", - ...COLS.map((c) => withSpur(fmt(c.get(r.score), c.d ?? 2), c, [r.score])), r.seconds.toFixed(1)])), ""); + ...COLS.map((c) => withSpur(fmt(c.get(r.score), c.d ?? 2), c, [r.score])), TECHNIQUES_NA, r.seconds.toFixed(1)])), ""); return out.join("\n"); } + +/** Serialisable copy of a run for results/.json: per-note `matches` arrays are dropped (they stay in runs/.../score.json). */ +export function resultsJson(run: BenchRun): string { + return JSON.stringify(run, (k, v) => (k === "matches" ? undefined : v), 1); +} diff --git a/src/bench/runner.ts b/src/bench/runner.ts index 439c69b..f819a3f 100644 --- a/src/bench/runner.ts +++ b/src/bench/runner.ts @@ -6,7 +6,7 @@ import type { TierId } from "../gen/tiers.ts"; import { buildSong, loadSongDir } from "./dataset.ts"; import { createPipeline } from "../pipeline/run.ts"; import { scoreSong, type OracleFlags } from "../eval/score.ts"; -import { renderReport, type BenchRun } from "./report.ts"; +import { renderReport, resultsJson, type BenchRun } from "./report.ts"; export const MODES: Record = { "full": { oracleStems: false, oracleTempo: false, oracleNotes: false }, @@ -49,7 +49,7 @@ export async function runBench(opts: { const t0 = performance.now(); const res = await pipeline({ mix, oracle: { stems, truth } }, mode.flags); const seconds = (performance.now() - t0) / 1000; - const score = scoreSong(truth, res.output, mode.flags); + const score = scoreSong(truth, res.output, mode.flags, res.gp); const d = join(outDir, mode.name); await mkdir(d, { recursive: true }); await Bun.write(join(d, "notes.json"), JSON.stringify({ @@ -63,7 +63,7 @@ export async function runBench(opts: { } const rdir = opts.resultsDir ?? "results"; await mkdir(rdir, { recursive: true }); - await Bun.write(join(rdir, `${opts.runId}.json`), JSON.stringify(run, null, 1)); + await Bun.write(join(rdir, `${opts.runId}.json`), resultsJson(run)); await Bun.write(join(rdir, `${opts.runId}.md`), renderReport(run)); return run; } diff --git a/src/dsp/resample.ts b/src/dsp/resample.ts new file mode 100644 index 0000000..a7433d9 --- /dev/null +++ b/src/dsp/resample.ts @@ -0,0 +1,28 @@ +// Pure windowed-sinc resampler. Re-exported from src/audio/wav.ts for existing importers. + +const sinc = (x: number) => (x === 0 ? 1 : Math.sin(Math.PI * x) / (Math.PI * x)); + +/** Windowed-sinc (Blackman) resampler; low-passes at the lower Nyquist when downsampling. */ +export function resample(x: Float32Array, from: number, to: number): Float32Array { + if (from === to) return x.slice(); + const ratio = to / from; + const outLen = Math.round(x.length * ratio); + const out = new Float32Array(outLen); + const cutoff = Math.min(1, ratio); // in units of input Nyquist + const half = 16 / cutoff; // kernel half-width in input samples (16 zero-crossings) + for (let j = 0; j < outLen; j++) { + const t = j / ratio; + const lo = Math.max(0, Math.ceil(t - half)), hi = Math.min(x.length - 1, Math.floor(t + half)); + let acc = 0, wsum = 0; + for (let i = lo; i <= hi; i++) { + const d = i - t; + const w = 0.42 + 0.5 * Math.cos((Math.PI * d) / half) + 0.08 * Math.cos((2 * Math.PI * d) / half); + const k = cutoff * sinc(cutoff * d) * w; + acc += x[i]! * k; + wsum += k; + } + // Normalise to unity DC gain so edges and non-integer ratios stay calibrated. + out[j] = wsum !== 0 ? acc / wsum : 0; + } + return out; +} diff --git a/src/eval/score.ts b/src/eval/score.ts index bcbdc91..6d7563a 100644 --- a/src/eval/score.ts +++ b/src/eval/score.ts @@ -1,6 +1,7 @@ import { secondsToTicks } from "../core/time.ts"; import type { Grid, Role, ScoreNote, TimedNote, TruthTrack } from "../core/types.ts"; import { matchNotes, maxAssignment, prf, type MatchResult } from "./notes.ts"; +import { truthFromGp } from "../score/gp.ts"; import { alignGrid, rhythmScore } from "./rhythm.ts"; export interface OracleFlags { oracleStems: boolean; oracleTempo: boolean; oracleNotes: boolean } @@ -16,6 +17,10 @@ export interface SongScore { rhythm: { factor: number; beatOffset: number; onBeat: number | null; durationOk: number | null } | null; stringAgreement: number | null; spuriousRate: number | null; + /** Merged-guitar pitch match computed from the re-imported output GP file (null without outGp). */ + gpPitch: MatchResult | null; + /** Truth vs output-GP bar count and time signature (null without outGp). */ + barAgreement: { refBars: number; outBars: number; timeSigMatch: boolean } | null; } export const ROLE_ORDER: Role[] = ["guitar", "vocals", "bass", "drums", "keys"]; @@ -39,8 +44,8 @@ const notesOf = (tracks: { role: Role; notes: TimedNote[] }[], role: Role) => tracks.filter((t) => t.role === role).flatMap((t) => t.notes); export function scoreSong( - truth: { bpm: number; timeSig: [number, number]; tracks: TruthTrack[] }, - out: PipelineOutput, flags: OracleFlags, + truth: { bpm: number; timeSig: [number, number]; bars?: number; tracks: TruthTrack[] }, + out: PipelineOutput, flags: OracleFlags, outGp?: Uint8Array, ): SongScore { const refGuitar = notesOf(truth.tracks, "guitar"); const outGuitar = notesOf(out.tracks, "guitar"); @@ -101,9 +106,23 @@ export function scoreSong( if (a !== undefined && b !== undefined) { withString++; if (a === b) agree++; } } + let gpPitch: SongScore["gpPitch"] = null; + let barAgreement: SongScore["barAgreement"] = null; + if (outGp) { + const re = truthFromGp(outGp); + const shift = out.grid.firstDownbeat; // the GP's time zero is the grid's first downbeat + const gpGuitar = notesOf(re.tracks, "guitar").map((n) => ({ ...n, onset: n.onset + shift, offset: n.offset + shift })); + gpPitch = matchNotes(refGuitar, gpGuitar); + const barSec = (60 / truth.bpm) * truth.timeSig[0] * (4 / truth.timeSig[1]); + const lastEnd = Math.max(0, ...truth.tracks.flatMap((t) => t.notes.map((n) => n.offset))); + const refBars = truth.bars ?? Math.ceil(lastEnd / barSec - 1e-9); + barAgreement = { refBars, outBars: re.bars, timeSigMatch: truth.timeSig[0] === re.timeSig[0] && truth.timeSig[1] === re.timeSig[1] }; + } + return { flags, guitarMerged, partSplit, perRole, drumsPerClass, rhythm, stringAgreement: withString ? agree / withString : null, spuriousRate: outGuitar.length ? gm.fp / outGuitar.length : null, + gpPitch, barAgreement, }; } diff --git a/src/pipeline/quantize.ts b/src/pipeline/quantize.ts index 9d1d2cf..b5bb6c5 100644 --- a/src/pipeline/quantize.ts +++ b/src/pipeline/quantize.ts @@ -29,7 +29,7 @@ export function snapTicks(notes: TimedNote[], grid: Grid, stepsPerBeat = 4): num /** * Snap timed notes to a step grid (default 16ths) relative to grid.firstDownbeat. * Notes starting within 35 ms of a cluster's first onset share one tick; every note in a - * tick group gets the group's shortest duration (clipping to the next onset is left to the score writer); exact + * tick group gets the group's median duration (R12: one short ghost note cannot collapse a chord) (clipping to the next onset is left to the score writer); exact * duplicates (same tick + pitch) are dropped; dur is always >= 1 step. */ export function quantize(notes: TimedNote[], grid: Grid, stepsPerBeat = 4): ScoreNote[] { @@ -50,7 +50,7 @@ export function quantize(notes: TimedNote[], grid: Grid, stepsPerBeat = 4): Scor qs.push({ step: s0, durSteps: Math.max(1, s1 - s0), pitch: n.pitch }); }); - // Group by step; dedupe by pitch; chord duration = shortest. + // Group by step; dedupe by pitch; chord duration = median. const groups = new Map(); for (const q of qs) { const g = groups.get(q.step); @@ -61,8 +61,9 @@ export function quantize(notes: TimedNote[], grid: Grid, stepsPerBeat = 4): Scor const out: ScoreNote[] = []; steps.forEach((s) => { const g = groups.get(s)!; - let d = Math.min(...g.map((x) => x.durSteps)); - d = Math.max(1, d); + const ds = g.map((x) => x.durSteps).sort((a, b) => a - b); + const m = ds.length >> 1; + const d = Math.max(1, ds.length % 2 ? ds[m]! : Math.round((ds[m - 1]! + ds[m]!) / 2)); for (const x of g.sort((a, b) => a.pitch - b.pitch)) out.push({ tick: s * step, dur: d * step, pitch: x.pitch }); }); return out; diff --git a/src/pipeline/transcribe.ts b/src/pipeline/transcribe.ts index eeeb25b..80c550b 100644 --- a/src/pipeline/transcribe.ts +++ b/src/pipeline/transcribe.ts @@ -5,7 +5,8 @@ // Pure: no fs / Bun APIs; only ORT APIs that also exist in onnxruntime-web (InferenceSession.create, Tensor, run). import * as ort from "onnxruntime-node"; import type { Audio, TimedNote } from "../core/types.ts"; -import { toMono, resample } from "../audio/wav.ts"; +import { toMono } from "../dsp/mono.ts"; +import { resample } from "../dsp/resample.ts"; export interface TranscribeOpts { onsetThresh?: number; // 0.5 diff --git a/src/score/gp.ts b/src/score/gp.ts index 1fd73b1..d8c8f4f 100644 --- a/src/score/gp.ts +++ b/src/score/gp.ts @@ -43,7 +43,7 @@ export function songToMidi(song: Song): at.midi.MidiFile { const ROLES: readonly string[] = ["guitar", "bass", "keys", "vocals", "drums"]; -export function truthFromGp(bytes: Uint8Array): { bpm: number; timeSig: [number, number]; tracks: TruthTrack[] } { +export function truthFromGp(bytes: Uint8Array): { bpm: number; timeSig: [number, number]; bars: number; tracks: TruthTrack[] } { const score = at.importer.ScoreLoader.loadScoreFromBytes(bytes, new at.Settings()); const bpm = score.tempo; const mb0 = score.masterBars[0]!; @@ -80,5 +80,5 @@ export function truthFromGp(bytes: Uint8Array): { bpm: number; timeSig: [number, if (!track.isPercussion) tr.tuning = [...staff.tuning]; return tr; }); - return { bpm, timeSig: [mb0.timeSignatureNumerator, mb0.timeSignatureDenominator], tracks }; + return { bpm, timeSig: [mb0.timeSignatureNumerator, mb0.timeSignatureDenominator], bars: score.masterBars.length, tracks }; } diff --git a/tests/bench/dataset.test.ts b/tests/bench/dataset.test.ts index ff76f8c..0763a59 100644 --- a/tests/bench/dataset.test.ts +++ b/tests/bench/dataset.test.ts @@ -2,7 +2,7 @@ import { test, expect } from "bun:test"; import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { buildSong, loadSongDir } from "../../src/bench/dataset.ts"; +import { buildSong, loadSongDir, RENDER_VERSION } from "../../src/bench/dataset.ts"; test("buildSong writes all artefacts; loadSongDir reads them; stems sum to mix; idempotent", async () => { const root = await mkdtemp(join(tmpdir(), "stsheets-ds-")); @@ -55,3 +55,27 @@ test("multi-track t1b: stems on disk sum to mix.wav", async () => { await rm(root, { recursive: true, force: true }); } }, 120_000); + +test("a dir built under an older/missing RENDER_VERSION is rebuilt", async () => { + const root = await mkdtemp(join(tmpdir(), "stsheets-ds-")); + try { + const dir = await buildSong("t1a", 2, root); + const mp3 = join(dir, "mix.mp3"); + const v = Bun.file(join(dir, "render-version")); + expect((await v.text()).trim()).toBe(String(RENDER_VERSION)); + const m0 = (await Bun.file(mp3).stat()).mtimeMs; + await Bun.write(join(dir, "render-version"), String(RENDER_VERSION - 1)); // simulate an old build + await Bun.sleep(20); + await buildSong("t1a", 2, root); + const m1 = (await Bun.file(mp3).stat()).mtimeMs; + expect(m1).toBeGreaterThan(m0); + expect((await Bun.file(join(dir, "render-version")).text()).trim()).toBe(String(RENDER_VERSION)); + // missing marker file also rebuilds + await rm(join(dir, "render-version")); + await Bun.sleep(20); + await buildSong("t1a", 2, root); + expect((await Bun.file(mp3).stat()).mtimeMs).toBeGreaterThan(m1); + } finally { + await rm(root, { recursive: true, force: true }); + } +}, 180_000); diff --git a/tests/bench/report.test.ts b/tests/bench/report.test.ts index b09e21d..21287fd 100644 --- a/tests/bench/report.test.ts +++ b/tests/bench/report.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from "bun:test"; -import { renderReport, type BenchRun } from "../../src/bench/report.ts"; +import { renderReport, resultsJson, type BenchRun } from "../../src/bench/report.ts"; import type { SongScore } from "../../src/eval/score.ts"; const mr = (f1: number | null, oct = 0) => ({ tp: 0, fp: 0, fn: 0, precision: f1, recall: f1, f1, octaveErrors: oct, matches: [] as [number, number][] }); @@ -13,6 +13,8 @@ function score(gf1: number, vocals: number | null, flags = { oracleStems: false, rhythm: { factor: 1, beatOffset: 0, onBeat: 0.9, durationOk: null }, stringAgreement: null, spuriousRate: 0.1, + gpPitch: { ...mr(gf1), tp: 0, fp: 0, fn: 1 }, + barAgreement: { refBars: 8, outBars: 8, timeSigMatch: true }, }; } const modes = [ @@ -44,6 +46,23 @@ describe("renderReport", () => { expect(bad).toContain("FAIL"); expect(bad).not.toContain("Tier t1a: PASS"); }); + test("per-mode tick uses the median rule with a k/n pass count", () => { + const gs = [0.9, 0.9, 0.9, 0.9, 0.3]; + const run: BenchRun = { runId: "fx", date: "d", modes: [modes[0]!], results: gs.map((g, i) => ({ tier: "t1a" as const, seed: i, mode: "full", score: score(g, null), seconds: 1 })) }; + const row = renderReport(run).split("\n").find((l) => l.startsWith("| full"))!; + expect(row).toContain("✓ 4/5"); + expect(row).not.toContain("✗"); + const bad = renderReport({ ...run, results: gs.map((g, i) => ({ tier: "t1a" as const, seed: i, mode: "full", score: score(g === 0.9 ? 0.3 : 0.9, null), seconds: 1 })) }); + expect(bad.split("\n").find((l) => l.startsWith("| full"))).toContain("✗ 1/5"); + }); + test("secondary metric columns: GP-file F1, bar agreement, techniques n/a", () => { + const md = renderReport(mk(0.9)); + const header = md.split("\n").find((l) => l.startsWith("| Mode"))!; + expect(header).toContain("GP-file guitar F1"); + expect(header).toContain("Bar count match"); + expect(header).toContain("Techniques"); + expect(md).toContain("n/a (tier 1 has no technique marks)"); + }); test("role with empty truth renders n/a with spurious count", () => { const run = mk(0.9); const s = run.results[0]!.score; @@ -53,3 +72,12 @@ describe("renderReport", () => { expect(md).not.toContain("0.00 (0.00"); }); }); + +test("results JSON omits per-note matches", () => { + const run = mk(0.9); + run.results[0]!.score.guitarMerged.matches = [[0, 1]]; + const json = resultsJson(run); + expect(json).not.toContain('"matches"'); + expect(JSON.parse(json).results).toHaveLength(3); + expect(JSON.stringify(run)).toContain('"matches"'); // in-memory run (and score.json) keep them +}); diff --git a/tests/eval/score.test.ts b/tests/eval/score.test.ts index cea3456..100d18e 100644 --- a/tests/eval/score.test.ts +++ b/tests/eval/score.test.ts @@ -49,3 +49,27 @@ test("drums per class", () => { expect(s.drumsPerClass!.hihat.precision).toBe(0); expect(s.drumsPerClass!.tom.f1).toBeNull(); }); + +test("gpPitch and barAgreement come from the output GP file (offset by grid.firstDownbeat)", async () => { + const { songToGp, truthFromGp } = await import("../../src/score/gp.ts"); + const { generateSong } = await import("../../src/gen/generate.ts"); + const song = generateSong("t1a", 1); + const gp = songToGp(song); + const truthGp = truthFromGp(gp); + const outNotes = truthGp.tracks.filter((t) => t.role === "guitar").flatMap((t) => t.notes); + const mkOut = (firstDownbeat: number): PipelineOutput => + ({ grid: { bpm: truthGp.bpm, firstDownbeat, beatsPerBar: 4 }, tracks: [{ name: "o", role: "guitar", notes: outNotes, scoreNotes: [] }] }); + const s = scoreSong(truthGp, mkOut(0), flags, gp); + expect(s.gpPitch!.f1).toBe(1); + expect(s.barAgreement).toEqual({ refBars: truthGp.bars, outBars: truthGp.bars, timeSigMatch: true }); + // GP time zero is the grid's first downbeat: a 0.25 s downbeat shifts the GP notes beyond the 50 ms tolerance + expect(scoreSong(truthGp, mkOut(0.25), flags, gp).gpPitch!.f1!).toBeLessThan(0.5); + // the truth shifted by the same downbeat matches again + const shifted = { ...truthGp, tracks: truthGp.tracks.map((t) => ({ ...t, notes: t.notes.map((x) => ({ ...x, onset: x.onset + 0.25, offset: x.offset + 0.25 })) })) }; + expect(scoreSong(shifted, mkOut(0.25), flags, gp).gpPitch!.f1).toBe(1); + // without an output GP both are null; a different bar count is reported + const none = scoreSong(truthGp, mkOut(0), flags); + expect(none.gpPitch).toBeNull(); + expect(none.barAgreement).toBeNull(); + expect(scoreSong({ ...truthGp, bars: truthGp.bars + 2 }, mkOut(0), flags, gp).barAgreement).toEqual({ refBars: truthGp.bars + 2, outBars: truthGp.bars, timeSigMatch: true }); +}); diff --git a/tests/pipeline/quantize.test.ts b/tests/pipeline/quantize.test.ts index 9ea4ea1..3f02802 100644 --- a/tests/pipeline/quantize.test.ts +++ b/tests/pipeline/quantize.test.ts @@ -15,11 +15,19 @@ test("drops notes before the first downbeat", () => { expect(q.map((n) => n.pitch)).toEqual([62]); }); -test("merges chord notes within 35 ms, with one consistent duration", () => { +test("merges chord notes within 35 ms, with one consistent (median) duration", () => { + // snapped durations: 8, 4, 8 steps -> median 8 const q = quantize([t(0.6, 1.6, 60), t(0.625, 1.1, 64), t(0.63, 1.6, 67)], g); expect(new Set(q.map((n) => n.tick)).size).toBe(1); expect(new Set(q.map((n) => n.dur)).size).toBe(1); expect(q).toHaveLength(3); + expect(q[0]!.dur).toBe(8 * 240); +}); + +test("one 70 ms ghost note does not collapse a long chord (median duration)", () => { + const q = quantize([t(0.5, 2.5, 60), t(0.5, 2.5, 64), t(0.5, 0.57, 67)], g); + expect(q).toHaveLength(3); + for (const n of q) expect(n.dur).toBe(16 * 240); }); test("collapses exact duplicates, enforces min duration", () => {