diff --git a/src/eval/notes.ts b/src/eval/notes.ts new file mode 100644 index 0000000..80b99c4 --- /dev/null +++ b/src/eval/notes.ts @@ -0,0 +1,138 @@ +import type { TimedNote } from "../core/types.ts"; + +export interface MatchResult { + tp: number; fp: number; fn: number; + precision: number | null; recall: number | null; f1: number | null; + octaveErrors: number; + matches: [number, number][]; // [refIndex, estIndex] +} + +export interface Keyed { key: string | number; x: number } + +const EPS = 1e-9; + +/** Adjacency list ref -> est: equal key and |x_ref - x_est| <= tol. */ +export function buildAdjacency(refs: Keyed[], ests: Keyed[], tol: number): number[][] { + const groups = new Map(); + ests.forEach((e, i) => { + let g = groups.get(e.key); + if (!g) groups.set(e.key, (g = [])); + g.push({ x: e.x, i }); + }); + for (const g of groups.values()) g.sort((a, b) => a.x - b.x); + return refs.map((r) => { + const g = groups.get(r.key); + if (!g) return []; + let lo = 0, hi = g.length; + while (lo < hi) { const mid = (lo + hi) >> 1; if (g[mid]!.x < r.x - tol - EPS) lo = mid + 1; else hi = mid; } + const out: number[] = []; + for (let k = lo; k < g.length && g[k]!.x <= r.x + tol + EPS; k++) out.push(g[k]!.i); + return out; + }); +} + +/** Exact maximum bipartite matching (Hopcroft-Karp). Returns, per left node, the matched right index or -1. */ +export function maxMatching(nL: number, nR: number, adj: number[][]): number[] { + const matchL = new Array(nL).fill(-1); + const matchR = new Array(nR).fill(-1); + const dist = new Array(nL).fill(0); + const INF = Number.MAX_SAFE_INTEGER; + const bfs = (): boolean => { + const q: number[] = []; + for (let u = 0; u < nL; u++) { if (matchL[u] === -1) { dist[u] = 0; q.push(u); } else dist[u] = INF; } + let found = false; + for (let h = 0; h < q.length; h++) { + const u = q[h]!; + for (const v of adj[u]!) { + const w = matchR[v]!; + if (w === -1) found = true; + else if (dist[w] === INF) { dist[w] = dist[u]! + 1; q.push(w); } + } + } + return found; + }; + const dfs = (u: number): boolean => { + for (const v of adj[u]!) { + const w = matchR[v]!; + if (w === -1 || (dist[w] === dist[u]! + 1 && dfs(w))) { matchL[u] = v; matchR[v] = u; return true; } + } + dist[u] = INF; + return false; + }; + while (bfs()) for (let u = 0; u < nL; u++) if (matchL[u] === -1) dfs(u); + return matchL; +} + +export function prf(tp: number, fp: number, fn: number): { precision: number | null; recall: number | null; f1: number | null } { + const precision = tp + fp === 0 ? null : tp / (tp + fp); + const recall = tp + fn === 0 ? null : tp / (tp + fn); + const f1 = tp + fp + fn === 0 ? null : (2 * tp) / (2 * tp + fp + fn); + return { precision, recall, f1 }; +} + +export function matchNotes( + ref: TimedNote[], est: TimedNote[], opts: { onsetTol?: number; key?: "pitch" | "pitchString" } = {}, +): MatchResult { + const tol = opts.onsetTol ?? 0.05; + const keyOf = opts.key === "pitchString" ? (n: TimedNote) => `${n.pitch}:${n.string ?? ""}` : (n: TimedNote) => n.pitch; + const adj = buildAdjacency(ref.map((n) => ({ key: keyOf(n), x: n.onset })), est.map((n) => ({ key: keyOf(n), x: n.onset })), tol); + const matchL = maxMatching(ref.length, est.length, adj); + const matches: [number, number][] = []; + const estUsed = new Array(est.length).fill(false); + matchL.forEach((j, i) => { if (j >= 0) { matches.push([i, j]); estUsed[j] = true; } }); + const tp = matches.length, fp = est.length - tp, fn = ref.length - tp; + + // octave errors: unmatched est within tol of an unmatched ref a whole octave away (each ref used once) + const refFree = matchL.map((j) => j === -1); + let octaveErrors = 0; + est.forEach((e, j) => { + if (estUsed[j]) return; + for (let i = 0; i < ref.length; i++) { + if (refFree[i] && Math.abs(ref[i]!.pitch - e.pitch) === 12 && Math.abs(ref[i]!.onset - e.onset) <= tol + EPS) { + refFree[i] = false; octaveErrors++; return; + } + } + }); + return { tp, fp, fn, ...prf(tp, fp, fn), octaveErrors, matches }; +} + +/** Hungarian algorithm, maximising total weight. gain is rows x cols (any shape). Returns col per row or -1. */ +export function maxAssignment(gain: number[][]): number[] { + const R = gain.length; + const C = R === 0 ? 0 : gain[0]!.length; + const N = Math.max(R, C); + if (N === 0) return []; + let maxV = 0; + for (const row of gain) for (const v of row) maxV = Math.max(maxV, v); + const cost = (i: number, j: number) => (i < R && j < C ? maxV - gain[i]![j]! : maxV); + const u = new Array(N + 1).fill(0), v = new Array(N + 1).fill(0); + const p = new Array(N + 1).fill(0), way = new Array(N + 1).fill(0); + for (let i = 1; i <= N; i++) { + p[0] = i; + let j0 = 0; + const minv = new Array(N + 1).fill(Infinity); + const used = new Array(N + 1).fill(false); + do { + used[j0] = true; + const i0 = p[j0]!; + let delta = Infinity, j1 = 0; + for (let j = 1; j <= N; j++) { + if (used[j]) continue; + const cur = cost(i0 - 1, j - 1) - u[i0]! - v[j]!; + if (cur < minv[j]!) { minv[j] = cur; way[j] = j0; } + if (minv[j]! < delta) { delta = minv[j]!; j1 = j; } + } + for (let j = 0; j <= N; j++) { + if (used[j]) { u[p[j]!] = u[p[j]!]! + delta; v[j] = v[j]! - delta; } else minv[j] = minv[j]! - delta; + } + j0 = j1; + } while (p[j0] !== 0); + do { const j1 = way[j0]!; p[j0] = p[j1]!; j0 = j1; } while (j0); + } + const res = new Array(R).fill(-1); + for (let j = 1; j <= N; j++) { + const i = p[j]! - 1; + if (i >= 0 && i < R && j - 1 < C) res[i] = j - 1; + } + return res; +} diff --git a/src/eval/rhythm.ts b/src/eval/rhythm.ts new file mode 100644 index 0000000..57b6f26 --- /dev/null +++ b/src/eval/rhythm.ts @@ -0,0 +1,62 @@ +import { TPQ, type ScoreNote } from "../core/types.ts"; +import { buildAdjacency, maxMatching } from "./notes.ts"; + +export interface GridAlignment { factor: 0.5 | 1 | 2; beatOffset: number } + +const FACTORS: (0.5 | 1 | 2)[] = [1, 0.5, 2]; +const OFFSETS = [0, -1, 1, -2, 2, -3, 3]; + +// estTick = refTick * factor + beatOffset * TPQ +const alignedTick = (n: ScoreNote, a: GridAlignment) => Math.round(n.tick * a.factor + a.beatOffset * TPQ); +const alignedDur = (n: ScoreNote, a: GridAlignment) => Math.round(n.dur * a.factor); + +export function alignGrid(refNotes: ScoreNote[], estNotes: ScoreNote[]): GridAlignment { + const counts = new Map(); + for (const e of estNotes) { const k = `${e.pitch}|${e.tick}`; counts.set(k, (counts.get(k) ?? 0) + 1); } + let best: GridAlignment = { factor: 1, beatOffset: 0 }; + let bestScore = -1; // strict > keeps the identity-first preference on ties + for (const factor of FACTORS) { + for (const beatOffset of OFFSETS) { + const a = { factor, beatOffset }; + const left = new Map(counts); + let s = 0; + for (const r of refNotes) { + const k = `${r.pitch}|${alignedTick(r, a)}`; + const c = left.get(k) ?? 0; + if (c > 0) { left.set(k, c - 1); s++; } + } + if (s > bestScore) { bestScore = s; best = a; } + } + } + return best; +} + +export function rhythmScore(ref: ScoreNote[], est: ScoreNote[], align: GridAlignment): { onBeat: number | null; durationOk: number | null } { + const pairs: [number, number][] = []; + const refDone = new Array(ref.length).fill(false); + const estDone = new Array(est.length).fill(false); + // Pitch-matched pairing, strictest to loosest; each stage is an exact maximum matching on the leftovers. + const stages: { keyR: (n: ScoreNote) => string | number; keyE: (n: ScoreNote) => string | number; tol: number }[] = [ + { keyR: (n) => `${n.pitch}|${alignedDur(n, align)}`, keyE: (n) => `${n.pitch}|${n.dur}`, tol: 0 }, + { keyR: (n) => n.pitch, keyE: (n) => n.pitch, tol: 0 }, + { keyR: (n) => n.pitch, keyE: (n) => n.pitch, tol: TPQ / 2 }, + ]; + for (const st of stages) { + const ri = ref.map((_, i) => i).filter((i) => !refDone[i]); + const ei = est.map((_, i) => i).filter((i) => !estDone[i]); + const adj = buildAdjacency( + ri.map((i) => ({ key: st.keyR(ref[i]!), x: alignedTick(ref[i]!, align) })), + ei.map((i) => ({ key: st.keyE(est[i]!), x: est[i]!.tick })), + st.tol, + ); + const m = maxMatching(ri.length, ei.length, adj); + m.forEach((j, a) => { if (j >= 0) { pairs.push([ri[a]!, ei[j]!]); refDone[ri[a]!] = true; estDone[ei[j]!] = true; } }); + } + if (pairs.length === 0) return { onBeat: null, durationOk: null }; + let on = 0, dur = 0; + for (const [i, j] of pairs) { + if (alignedTick(ref[i]!, align) === est[j]!.tick) on++; + if (alignedDur(ref[i]!, align) === est[j]!.dur) dur++; + } + return { onBeat: on / pairs.length, durationOk: dur / pairs.length }; +} diff --git a/src/eval/score.ts b/src/eval/score.ts new file mode 100644 index 0000000..bcbdc91 --- /dev/null +++ b/src/eval/score.ts @@ -0,0 +1,109 @@ +import { secondsToTicks } from "../core/time.ts"; +import type { Grid, Role, ScoreNote, TimedNote, TruthTrack } from "../core/types.ts"; +import { matchNotes, maxAssignment, prf, type MatchResult } from "./notes.ts"; +import { alignGrid, rhythmScore } from "./rhythm.ts"; + +export interface OracleFlags { oracleStems: boolean; oracleTempo: boolean; oracleNotes: boolean } +export interface OutTrack { name: string; role: Role; notes: TimedNote[]; scoreNotes: ScoreNote[] } +export interface PipelineOutput { grid: Grid; tracks: OutTrack[] } +export type DrumClass = "kick" | "snare" | "hihat" | "cymbal" | "tom"; +export interface SongScore { + flags: OracleFlags; + guitarMerged: MatchResult & { pass: boolean }; + partSplit: { f1: number | null; outTracks: number; refTracks: number } | null; + perRole: Partial>; + drumsPerClass: Record | null; + rhythm: { factor: number; beatOffset: number; onBeat: number | null; durationOk: number | null } | null; + stringAgreement: number | null; + spuriousRate: number | null; +} + +export const ROLE_ORDER: Role[] = ["guitar", "vocals", "bass", "drums", "keys"]; +export const DRUM_CLASSES: Record = { + kick: [35, 36], snare: [37, 38, 40], hihat: [42, 44, 46], + cymbal: [49, 51, 52, 53, 55, 57, 59], tom: [41, 43, 45, 47, 48, 50], +}; +const PASS_F1 = 0.85; + +function drumClass(pitch: number): DrumClass | null { + for (const [c, ps] of Object.entries(DRUM_CLASSES)) if (ps.includes(pitch)) return c as DrumClass; + return null; +} +// Drum pitches are compared by class: map to the class's first GM number; unknown pitches stay distinct. +const toClassPitch = (n: TimedNote): TimedNote => { + const c = drumClass(n.pitch); + return { ...n, pitch: c ? DRUM_CLASSES[c][0]! : 1000 + n.pitch }; +}; + +const notesOf = (tracks: { role: Role; notes: TimedNote[] }[], role: Role) => + tracks.filter((t) => t.role === role).flatMap((t) => t.notes); + +export function scoreSong( + truth: { bpm: number; timeSig: [number, number]; tracks: TruthTrack[] }, + out: PipelineOutput, flags: OracleFlags, +): SongScore { + const refGuitar = notesOf(truth.tracks, "guitar"); + const outGuitar = notesOf(out.tracks, "guitar"); + const gm = matchNotes(refGuitar, outGuitar); + const guitarMerged = { ...gm, pass: gm.f1 !== null && gm.f1 >= PASS_F1 }; + + // partSplit (R3): null only with <2 ref guitar tracks + const refG = truth.tracks.filter((t) => t.role === "guitar"); + const outG = out.tracks.filter((t) => t.role === "guitar"); + let partSplit: SongScore["partSplit"] = null; + if (refG.length >= 2) { + const gain = refG.map((r) => outG.map((o) => matchNotes(r.notes, o.notes).tp)); + const asg = outG.length ? maxAssignment(gain) : refG.map(() => -1); + let tp = 0; + asg.forEach((j, i) => { if (j >= 0) tp += gain[i]![j]!; }); + const fn = refG.reduce((s, t) => s + t.notes.length, 0) - tp; + const fp = outG.reduce((s, t) => s + t.notes.length, 0) - tp; + partSplit = { f1: prf(tp, fp, fn).f1, outTracks: outG.length, refTracks: refG.length }; + } + + // per role (drums compared by class) + const perRole: Partial> = {}; + for (const role of ROLE_ORDER) { + if (role === "guitar") { perRole.guitar = gm; continue; } + let r = notesOf(truth.tracks, role), e = notesOf(out.tracks, role); + if (role === "drums") { r = r.map(toClassPitch); e = e.map(toClassPitch); } + perRole[role] = matchNotes(r, e); + } + + let drumsPerClass: SongScore["drumsPerClass"] = null; + const refDrums = notesOf(truth.tracks, "drums"), outDrums = notesOf(out.tracks, "drums"); + if (refDrums.length + outDrums.length > 0) { + drumsPerClass = {} as Record; + for (const c of Object.keys(DRUM_CLASSES) as DrumClass[]) { + drumsPerClass[c] = matchNotes( + refDrums.filter((n) => drumClass(n.pitch) === c).map(toClassPitch), + outDrums.filter((n) => drumClass(n.pitch) === c).map(toClassPitch), + ); + } + } + + // rhythm on merged guitar (R2: truth seconds -> ticks) + let rhythm: SongScore["rhythm"] = null; + const outGuitarScore = outG.flatMap((t) => t.scoreNotes); + if (refGuitar.length > 0 && outGuitarScore.length > 0) { + const refScore: ScoreNote[] = refGuitar.map((n) => ({ + tick: Math.round(secondsToTicks(n.onset, truth.bpm)), + dur: Math.round(secondsToTicks(n.offset - n.onset, truth.bpm)), + pitch: n.pitch, string: n.string, fret: n.fret, + })); + const align = alignGrid(refScore, outGuitarScore); + rhythm = { factor: align.factor, beatOffset: align.beatOffset, ...rhythmScore(refScore, outGuitarScore, align) }; + } + + let withString = 0, agree = 0; + for (const [i, j] of gm.matches) { + const a = refGuitar[i]!.string, b = outGuitar[j]!.string; + if (a !== undefined && b !== undefined) { withString++; if (a === b) agree++; } + } + + return { + flags, guitarMerged, partSplit, perRole, drumsPerClass, rhythm, + stringAgreement: withString ? agree / withString : null, + spuriousRate: outGuitar.length ? gm.fp / outGuitar.length : null, + }; +} diff --git a/tests/eval/notes.test.ts b/tests/eval/notes.test.ts new file mode 100644 index 0000000..862fdc4 --- /dev/null +++ b/tests/eval/notes.test.ts @@ -0,0 +1,37 @@ +import { expect, test } from "bun:test"; +import { matchNotes, maxAssignment } from "../../src/eval/notes.ts"; +import type { TimedNote } from "../../src/core/types.ts"; + +const n = (onset: number, pitch: number, string?: number): TimedNote => ({ onset, offset: onset + 0.2, pitch, string }); +const ref = [n(1.0, 60), n(1.5, 62), n(2.0, 64)]; + +test("tp / octave errors", () => { + const est = [n(1.03, 60), n(1.5, 74), n(2.2, 64)]; + expect(matchNotes(ref, est).tp).toBe(1); + expect(matchNotes(ref, est).octaveErrors).toBe(1); +}); +test("perfect and empty cases", () => { + expect(matchNotes(ref, ref).f1).toBe(1); + expect(matchNotes([], []).f1).toBeNull(); + expect(matchNotes(ref, []).recall).toBe(0); + expect(matchNotes(ref, []).precision).toBeNull(); + expect(matchNotes([], ref).recall).toBeNull(); + expect(matchNotes([], ref).f1).toBe(0); +}); +test("matching is maximum, not greedy", () => { + // greedy nearest would pair r1-e1 (0.00) leaving r2 unmatched; optimum pairs r1-e2, r2-e1 + const r = [n(1.0, 60), n(1.04, 60)]; + const e = [n(1.0, 60), n(0.96, 60)]; + expect(matchNotes(r, e, { onsetTol: 0.045 }).tp).toBe(2); + const r2 = [n(0.0, 60), n(0.08, 60)]; + const e2 = [n(0.04, 60), n(-0.01, 60)]; + expect(matchNotes(r2, e2, { onsetTol: 0.05 }).tp).toBe(2); +}); +test("pitchString key", () => { + expect(matchNotes([n(1, 60, 1)], [n(1, 60, 2)], { key: "pitchString" }).tp).toBe(0); + expect(matchNotes([n(1, 60, 1)], [n(1, 60, 2)]).tp).toBe(1); +}); +test("maxAssignment", () => { + expect(maxAssignment([[5, 4], [4, 0]])).toEqual([1, 0]); + expect(maxAssignment([[3], [5]])).toEqual([-1, 0]); +}); diff --git a/tests/eval/rhythm.test.ts b/tests/eval/rhythm.test.ts new file mode 100644 index 0000000..e8f58ec --- /dev/null +++ b/tests/eval/rhythm.test.ts @@ -0,0 +1,21 @@ +import { expect, test } from "bun:test"; +import { alignGrid, rhythmScore } from "../../src/eval/rhythm.ts"; +import type { ScoreNote } from "../../src/core/types.ts"; + +const sn = (a: number[][]): ScoreNote[] => a.map(([tick, dur, pitch]) => ({ tick: tick!, dur: dur!, pitch: pitch! })); + +test("grid alignment: double tempo and one beat shift", () => { + const r = sn([[0, 240, 60], [960, 960, 62]]); + const e = sn([[960, 480, 60], [2880, 1920, 62]]); + expect(alignGrid(r, e)).toEqual({ factor: 2, beatOffset: 1 }); + expect(rhythmScore(r, e, { factor: 2, beatOffset: 1 })).toEqual({ onBeat: 1, durationOk: 1 }); +}); +test("identity preferred and partial scores", () => { + const r = sn([[0, 480, 60], [960, 480, 62]]); + expect(alignGrid(r, r)).toEqual({ factor: 1, beatOffset: 0 }); + const e = sn([[0, 480, 60], [1000, 240, 62]]); + expect(rhythmScore(r, e, { factor: 1, beatOffset: 0 })).toEqual({ onBeat: 0.5, durationOk: 0.5 }); +}); +test("no pitch matches -> null", () => { + expect(rhythmScore(sn([[0, 1, 60]]), sn([[0, 1, 61]]), { factor: 1, beatOffset: 0 })).toEqual({ onBeat: null, durationOk: null }); +}); diff --git a/tests/eval/score.test.ts b/tests/eval/score.test.ts new file mode 100644 index 0000000..cea3456 --- /dev/null +++ b/tests/eval/score.test.ts @@ -0,0 +1,51 @@ +import { expect, test } from "bun:test"; +import { scoreSong, type PipelineOutput } from "../../src/eval/score.ts"; +import { secondsToTicks } from "../../src/core/time.ts"; +import type { TimedNote, ScoreNote, TruthTrack } from "../../src/core/types.ts"; + +const flags = { oracleStems: false, oracleTempo: false, oracleNotes: false }; +const bpm = 120; // 0.5 s per beat +const n = (onset: number, pitch: number, string?: number): TimedNote => ({ onset, offset: onset + 0.5, pitch, string }); +const sc = (t: TimedNote[]): ScoreNote[] => t.map((x) => ({ + tick: Math.round(secondsToTicks(x.onset, bpm)), dur: Math.round(secondsToTicks(x.offset - x.onset, bpm)), pitch: x.pitch, string: x.string, +})); +const grid = { bpm, firstDownbeat: 0, beatsPerBar: 4 }; + +test("two ref guitars, one merged out track", () => { + const g1 = [n(0, 64, 1), n(0.5, 62, 2)]; + const g2 = [n(0, 40, 6), n(0.5, 45, 5)]; + const truth = { bpm, timeSig: [4, 4] as [number, number], tracks: [ + { name: "g1", role: "guitar", notes: g1 }, { name: "g2", role: "guitar", notes: g2 } ] as TruthTrack[] }; + const merged = [...g1, ...g2]; + const out: PipelineOutput = { grid, tracks: [{ name: "o", role: "guitar", notes: merged, scoreNotes: sc(merged) }] }; + const s = scoreSong(truth, out, flags); + expect(s.guitarMerged.f1).toBe(1); + expect(s.guitarMerged.pass).toBe(true); + expect(s.partSplit).not.toBeNull(); + expect(s.partSplit!.outTracks).toBe(1); + expect(s.partSplit!.refTracks).toBe(2); + expect(s.partSplit!.f1).toBeCloseTo(0.5, 6); + expect(s.rhythm).toEqual({ factor: 1, beatOffset: 0, onBeat: 1, durationOk: 1 }); + expect(s.stringAgreement).toBe(1); + expect(s.spuriousRate).toBe(0); + expect(s.perRole.vocals!.f1).toBeNull(); + expect(s.drumsPerClass).toBeNull(); +}); +test("partSplit null with one ref guitar; rhythm null without guitar", () => { + const truth = { bpm, timeSig: [4, 4] as [number, number], tracks: [{ name: "g", role: "guitar", notes: [n(0, 60)] }] as TruthTrack[] }; + const s = scoreSong(truth, { grid, tracks: [] }, flags); + expect(s.partSplit).toBeNull(); + expect(s.guitarMerged.recall).toBe(0); + expect(s.guitarMerged.pass).toBe(false); + expect(s.rhythm).toBeNull(); + expect(s.spuriousRate).toBeNull(); +}); +test("drums per class", () => { + const truth = { bpm, timeSig: [4, 4] as [number, number], tracks: [{ name: "d", role: "drums", notes: [n(0, 36), n(0.5, 38)] }] as TruthTrack[] }; + const est = [n(0, 35), n(0.5, 42)]; + const s = scoreSong(truth, { grid, tracks: [{ name: "d", role: "drums", notes: est, scoreNotes: [] }] }, flags); + expect(s.drumsPerClass!.kick.f1).toBe(1); + expect(s.drumsPerClass!.snare.recall).toBe(0); + expect(s.drumsPerClass!.hihat.precision).toBe(0); + expect(s.drumsPerClass!.tom.f1).toBeNull(); +});