feat: evaluation metrics

Co-Authored-By: Claude Sonnet 5.5 <[email protected]>

tests/eval/notes.test.ts

@@ -0,0 +1,37 @@
+import { expect, test } from "bun:test";
+import { matchNotes, maxAssignment } from "../../src/eval/notes.ts";
+import type { TimedNote } from "../../src/core/types.ts";
+
+const n = (onset: number, pitch: number, string?: number): TimedNote => ({ onset, offset: onset + 0.2, pitch, string });
+const ref = [n(1.0, 60), n(1.5, 62), n(2.0, 64)];
+
+test("tp / octave errors", () => {
+  const est = [n(1.03, 60), n(1.5, 74), n(2.2, 64)];
+  expect(matchNotes(ref, est).tp).toBe(1);
+  expect(matchNotes(ref, est).octaveErrors).toBe(1);
+});
+test("perfect and empty cases", () => {
+  expect(matchNotes(ref, ref).f1).toBe(1);
+  expect(matchNotes([], []).f1).toBeNull();
+  expect(matchNotes(ref, []).recall).toBe(0);
+  expect(matchNotes(ref, []).precision).toBeNull();
+  expect(matchNotes([], ref).recall).toBeNull();
+  expect(matchNotes([], ref).f1).toBe(0);
+});
+test("matching is maximum, not greedy", () => {
+  // greedy nearest would pair r1-e1 (0.00) leaving r2 unmatched; optimum pairs r1-e2, r2-e1
+  const r = [n(1.0, 60), n(1.04, 60)];
+  const e = [n(1.0, 60), n(0.96, 60)];
+  expect(matchNotes(r, e, { onsetTol: 0.045 }).tp).toBe(2);
+  const r2 = [n(0.0, 60), n(0.08, 60)];
+  const e2 = [n(0.04, 60), n(-0.01, 60)];
+  expect(matchNotes(r2, e2, { onsetTol: 0.05 }).tp).toBe(2);
+});
+test("pitchString key", () => {
+  expect(matchNotes([n(1, 60, 1)], [n(1, 60, 2)], { key: "pitchString" }).tp).toBe(0);
+  expect(matchNotes([n(1, 60, 1)], [n(1, 60, 2)]).tp).toBe(1);
+});
+test("maxAssignment", () => {
+  expect(maxAssignment([[5, 4], [4, 0]])).toEqual([1, 0]);
+  expect(maxAssignment([[3], [5]])).toEqual([-1, 0]);
+});

tests/eval/rhythm.test.ts

@@ -0,0 +1,21 @@
+import { expect, test } from "bun:test";
+import { alignGrid, rhythmScore } from "../../src/eval/rhythm.ts";
+import type { ScoreNote } from "../../src/core/types.ts";
+
+const sn = (a: number[][]): ScoreNote[] => a.map(([tick, dur, pitch]) => ({ tick: tick!, dur: dur!, pitch: pitch! }));
+
+test("grid alignment: double tempo and one beat shift", () => {
+  const r = sn([[0, 240, 60], [960, 960, 62]]);
+  const e = sn([[960, 480, 60], [2880, 1920, 62]]);
+  expect(alignGrid(r, e)).toEqual({ factor: 2, beatOffset: 1 });
+  expect(rhythmScore(r, e, { factor: 2, beatOffset: 1 })).toEqual({ onBeat: 1, durationOk: 1 });
+});
+test("identity preferred and partial scores", () => {
+  const r = sn([[0, 480, 60], [960, 480, 62]]);
+  expect(alignGrid(r, r)).toEqual({ factor: 1, beatOffset: 0 });
+  const e = sn([[0, 480, 60], [1000, 240, 62]]);
+  expect(rhythmScore(r, e, { factor: 1, beatOffset: 0 })).toEqual({ onBeat: 0.5, durationOk: 0.5 });
+});
+test("no pitch matches -> null", () => {
+  expect(rhythmScore(sn([[0, 1, 60]]), sn([[0, 1, 61]]), { factor: 1, beatOffset: 0 })).toEqual({ onBeat: null, durationOk: null });
+});

tests/eval/score.test.ts

@@ -0,0 +1,51 @@
+import { expect, test } from "bun:test";
+import { scoreSong, type PipelineOutput } from "../../src/eval/score.ts";
+import { secondsToTicks } from "../../src/core/time.ts";
+import type { TimedNote, ScoreNote, TruthTrack } from "../../src/core/types.ts";
+
+const flags = { oracleStems: false, oracleTempo: false, oracleNotes: false };
+const bpm = 120; // 0.5 s per beat
+const n = (onset: number, pitch: number, string?: number): TimedNote => ({ onset, offset: onset + 0.5, pitch, string });
+const sc = (t: TimedNote[]): ScoreNote[] => t.map((x) => ({
+  tick: Math.round(secondsToTicks(x.onset, bpm)), dur: Math.round(secondsToTicks(x.offset - x.onset, bpm)), pitch: x.pitch, string: x.string,
+}));
+const grid = { bpm, firstDownbeat: 0, beatsPerBar: 4 };
+
+test("two ref guitars, one merged out track", () => {
+  const g1 = [n(0, 64, 1), n(0.5, 62, 2)];
+  const g2 = [n(0, 40, 6), n(0.5, 45, 5)];
+  const truth = { bpm, timeSig: [4, 4] as [number, number], tracks: [
+    { name: "g1", role: "guitar", notes: g1 }, { name: "g2", role: "guitar", notes: g2 } ] as TruthTrack[] };
+  const merged = [...g1, ...g2];
+  const out: PipelineOutput = { grid, tracks: [{ name: "o", role: "guitar", notes: merged, scoreNotes: sc(merged) }] };
+  const s = scoreSong(truth, out, flags);
+  expect(s.guitarMerged.f1).toBe(1);
+  expect(s.guitarMerged.pass).toBe(true);
+  expect(s.partSplit).not.toBeNull();
+  expect(s.partSplit!.outTracks).toBe(1);
+  expect(s.partSplit!.refTracks).toBe(2);
+  expect(s.partSplit!.f1).toBeCloseTo(0.5, 6);
+  expect(s.rhythm).toEqual({ factor: 1, beatOffset: 0, onBeat: 1, durationOk: 1 });
+  expect(s.stringAgreement).toBe(1);
+  expect(s.spuriousRate).toBe(0);
+  expect(s.perRole.vocals!.f1).toBeNull();
+  expect(s.drumsPerClass).toBeNull();
+});
+test("partSplit null with one ref guitar; rhythm null without guitar", () => {
+  const truth = { bpm, timeSig: [4, 4] as [number, number], tracks: [{ name: "g", role: "guitar", notes: [n(0, 60)] }] as TruthTrack[] };
+  const s = scoreSong(truth, { grid, tracks: [] }, flags);
+  expect(s.partSplit).toBeNull();
+  expect(s.guitarMerged.recall).toBe(0);
+  expect(s.guitarMerged.pass).toBe(false);
+  expect(s.rhythm).toBeNull();
+  expect(s.spuriousRate).toBeNull();
+});
+test("drums per class", () => {
+  const truth = { bpm, timeSig: [4, 4] as [number, number], tracks: [{ name: "d", role: "drums", notes: [n(0, 36), n(0.5, 38)] }] as TruthTrack[] };
+  const est = [n(0, 35), n(0.5, 42)];
+  const s = scoreSong(truth, { grid, tracks: [{ name: "d", role: "drums", notes: est, scoreNotes: [] }] }, flags);
+  expect(s.drumsPerClass!.kick.f1).toBe(1);
+  expect(s.drumsPerClass!.snare.recall).toBe(0);
+  expect(s.drumsPerClass!.hihat.precision).toBe(0);
+  expect(s.drumsPerClass!.tom.f1).toBeNull();
+});