feat: evaluation metrics
Co-Authored-By: Claude Sonnet 5.5 <[email protected]>
This commit is contained in:
Lucas Winther
co-authored by
Claude Sonnet 5.5
Claude Sonnet 5.5
parent
32d96cadcc
commit
6bd8ab9cee
@@ -0,0 +1,138 @@
|
||||
import type { TimedNote } from "../core/types.ts";
|
||||
|
||||
export interface MatchResult {
|
||||
tp: number; fp: number; fn: number;
|
||||
precision: number | null; recall: number | null; f1: number | null;
|
||||
octaveErrors: number;
|
||||
matches: [number, number][]; // [refIndex, estIndex]
|
||||
}
|
||||
|
||||
export interface Keyed { key: string | number; x: number }
|
||||
|
||||
const EPS = 1e-9;
|
||||
|
||||
/** Adjacency list ref -> est: equal key and |x_ref - x_est| <= tol. */
|
||||
export function buildAdjacency(refs: Keyed[], ests: Keyed[], tol: number): number[][] {
|
||||
const groups = new Map<string | number, { x: number; i: number }[]>();
|
||||
ests.forEach((e, i) => {
|
||||
let g = groups.get(e.key);
|
||||
if (!g) groups.set(e.key, (g = []));
|
||||
g.push({ x: e.x, i });
|
||||
});
|
||||
for (const g of groups.values()) g.sort((a, b) => a.x - b.x);
|
||||
return refs.map((r) => {
|
||||
const g = groups.get(r.key);
|
||||
if (!g) return [];
|
||||
let lo = 0, hi = g.length;
|
||||
while (lo < hi) { const mid = (lo + hi) >> 1; if (g[mid]!.x < r.x - tol - EPS) lo = mid + 1; else hi = mid; }
|
||||
const out: number[] = [];
|
||||
for (let k = lo; k < g.length && g[k]!.x <= r.x + tol + EPS; k++) out.push(g[k]!.i);
|
||||
return out;
|
||||
});
|
||||
}
|
||||
|
||||
/** Exact maximum bipartite matching (Hopcroft-Karp). Returns, per left node, the matched right index or -1. */
|
||||
export function maxMatching(nL: number, nR: number, adj: number[][]): number[] {
|
||||
const matchL = new Array<number>(nL).fill(-1);
|
||||
const matchR = new Array<number>(nR).fill(-1);
|
||||
const dist = new Array<number>(nL).fill(0);
|
||||
const INF = Number.MAX_SAFE_INTEGER;
|
||||
const bfs = (): boolean => {
|
||||
const q: number[] = [];
|
||||
for (let u = 0; u < nL; u++) { if (matchL[u] === -1) { dist[u] = 0; q.push(u); } else dist[u] = INF; }
|
||||
let found = false;
|
||||
for (let h = 0; h < q.length; h++) {
|
||||
const u = q[h]!;
|
||||
for (const v of adj[u]!) {
|
||||
const w = matchR[v]!;
|
||||
if (w === -1) found = true;
|
||||
else if (dist[w] === INF) { dist[w] = dist[u]! + 1; q.push(w); }
|
||||
}
|
||||
}
|
||||
return found;
|
||||
};
|
||||
const dfs = (u: number): boolean => {
|
||||
for (const v of adj[u]!) {
|
||||
const w = matchR[v]!;
|
||||
if (w === -1 || (dist[w] === dist[u]! + 1 && dfs(w))) { matchL[u] = v; matchR[v] = u; return true; }
|
||||
}
|
||||
dist[u] = INF;
|
||||
return false;
|
||||
};
|
||||
while (bfs()) for (let u = 0; u < nL; u++) if (matchL[u] === -1) dfs(u);
|
||||
return matchL;
|
||||
}
|
||||
|
||||
export function prf(tp: number, fp: number, fn: number): { precision: number | null; recall: number | null; f1: number | null } {
|
||||
const precision = tp + fp === 0 ? null : tp / (tp + fp);
|
||||
const recall = tp + fn === 0 ? null : tp / (tp + fn);
|
||||
const f1 = tp + fp + fn === 0 ? null : (2 * tp) / (2 * tp + fp + fn);
|
||||
return { precision, recall, f1 };
|
||||
}
|
||||
|
||||
export function matchNotes(
|
||||
ref: TimedNote[], est: TimedNote[], opts: { onsetTol?: number; key?: "pitch" | "pitchString" } = {},
|
||||
): MatchResult {
|
||||
const tol = opts.onsetTol ?? 0.05;
|
||||
const keyOf = opts.key === "pitchString" ? (n: TimedNote) => `${n.pitch}:${n.string ?? ""}` : (n: TimedNote) => n.pitch;
|
||||
const adj = buildAdjacency(ref.map((n) => ({ key: keyOf(n), x: n.onset })), est.map((n) => ({ key: keyOf(n), x: n.onset })), tol);
|
||||
const matchL = maxMatching(ref.length, est.length, adj);
|
||||
const matches: [number, number][] = [];
|
||||
const estUsed = new Array<boolean>(est.length).fill(false);
|
||||
matchL.forEach((j, i) => { if (j >= 0) { matches.push([i, j]); estUsed[j] = true; } });
|
||||
const tp = matches.length, fp = est.length - tp, fn = ref.length - tp;
|
||||
|
||||
// octave errors: unmatched est within tol of an unmatched ref a whole octave away (each ref used once)
|
||||
const refFree = matchL.map((j) => j === -1);
|
||||
let octaveErrors = 0;
|
||||
est.forEach((e, j) => {
|
||||
if (estUsed[j]) return;
|
||||
for (let i = 0; i < ref.length; i++) {
|
||||
if (refFree[i] && Math.abs(ref[i]!.pitch - e.pitch) === 12 && Math.abs(ref[i]!.onset - e.onset) <= tol + EPS) {
|
||||
refFree[i] = false; octaveErrors++; return;
|
||||
}
|
||||
}
|
||||
});
|
||||
return { tp, fp, fn, ...prf(tp, fp, fn), octaveErrors, matches };
|
||||
}
|
||||
|
||||
/** Hungarian algorithm, maximising total weight. gain is rows x cols (any shape). Returns col per row or -1. */
|
||||
export function maxAssignment(gain: number[][]): number[] {
|
||||
const R = gain.length;
|
||||
const C = R === 0 ? 0 : gain[0]!.length;
|
||||
const N = Math.max(R, C);
|
||||
if (N === 0) return [];
|
||||
let maxV = 0;
|
||||
for (const row of gain) for (const v of row) maxV = Math.max(maxV, v);
|
||||
const cost = (i: number, j: number) => (i < R && j < C ? maxV - gain[i]![j]! : maxV);
|
||||
const u = new Array<number>(N + 1).fill(0), v = new Array<number>(N + 1).fill(0);
|
||||
const p = new Array<number>(N + 1).fill(0), way = new Array<number>(N + 1).fill(0);
|
||||
for (let i = 1; i <= N; i++) {
|
||||
p[0] = i;
|
||||
let j0 = 0;
|
||||
const minv = new Array<number>(N + 1).fill(Infinity);
|
||||
const used = new Array<boolean>(N + 1).fill(false);
|
||||
do {
|
||||
used[j0] = true;
|
||||
const i0 = p[j0]!;
|
||||
let delta = Infinity, j1 = 0;
|
||||
for (let j = 1; j <= N; j++) {
|
||||
if (used[j]) continue;
|
||||
const cur = cost(i0 - 1, j - 1) - u[i0]! - v[j]!;
|
||||
if (cur < minv[j]!) { minv[j] = cur; way[j] = j0; }
|
||||
if (minv[j]! < delta) { delta = minv[j]!; j1 = j; }
|
||||
}
|
||||
for (let j = 0; j <= N; j++) {
|
||||
if (used[j]) { u[p[j]!] = u[p[j]!]! + delta; v[j] = v[j]! - delta; } else minv[j] = minv[j]! - delta;
|
||||
}
|
||||
j0 = j1;
|
||||
} while (p[j0] !== 0);
|
||||
do { const j1 = way[j0]!; p[j0] = p[j1]!; j0 = j1; } while (j0);
|
||||
}
|
||||
const res = new Array<number>(R).fill(-1);
|
||||
for (let j = 1; j <= N; j++) {
|
||||
const i = p[j]! - 1;
|
||||
if (i >= 0 && i < R && j - 1 < C) res[i] = j - 1;
|
||||
}
|
||||
return res;
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
import { TPQ, type ScoreNote } from "../core/types.ts";
|
||||
import { buildAdjacency, maxMatching } from "./notes.ts";
|
||||
|
||||
export interface GridAlignment { factor: 0.5 | 1 | 2; beatOffset: number }
|
||||
|
||||
const FACTORS: (0.5 | 1 | 2)[] = [1, 0.5, 2];
|
||||
const OFFSETS = [0, -1, 1, -2, 2, -3, 3];
|
||||
|
||||
// estTick = refTick * factor + beatOffset * TPQ
|
||||
const alignedTick = (n: ScoreNote, a: GridAlignment) => Math.round(n.tick * a.factor + a.beatOffset * TPQ);
|
||||
const alignedDur = (n: ScoreNote, a: GridAlignment) => Math.round(n.dur * a.factor);
|
||||
|
||||
export function alignGrid(refNotes: ScoreNote[], estNotes: ScoreNote[]): GridAlignment {
|
||||
const counts = new Map<string, number>();
|
||||
for (const e of estNotes) { const k = `${e.pitch}|${e.tick}`; counts.set(k, (counts.get(k) ?? 0) + 1); }
|
||||
let best: GridAlignment = { factor: 1, beatOffset: 0 };
|
||||
let bestScore = -1; // strict > keeps the identity-first preference on ties
|
||||
for (const factor of FACTORS) {
|
||||
for (const beatOffset of OFFSETS) {
|
||||
const a = { factor, beatOffset };
|
||||
const left = new Map(counts);
|
||||
let s = 0;
|
||||
for (const r of refNotes) {
|
||||
const k = `${r.pitch}|${alignedTick(r, a)}`;
|
||||
const c = left.get(k) ?? 0;
|
||||
if (c > 0) { left.set(k, c - 1); s++; }
|
||||
}
|
||||
if (s > bestScore) { bestScore = s; best = a; }
|
||||
}
|
||||
}
|
||||
return best;
|
||||
}
|
||||
|
||||
export function rhythmScore(ref: ScoreNote[], est: ScoreNote[], align: GridAlignment): { onBeat: number | null; durationOk: number | null } {
|
||||
const pairs: [number, number][] = [];
|
||||
const refDone = new Array<boolean>(ref.length).fill(false);
|
||||
const estDone = new Array<boolean>(est.length).fill(false);
|
||||
// Pitch-matched pairing, strictest to loosest; each stage is an exact maximum matching on the leftovers.
|
||||
const stages: { keyR: (n: ScoreNote) => string | number; keyE: (n: ScoreNote) => string | number; tol: number }[] = [
|
||||
{ keyR: (n) => `${n.pitch}|${alignedDur(n, align)}`, keyE: (n) => `${n.pitch}|${n.dur}`, tol: 0 },
|
||||
{ keyR: (n) => n.pitch, keyE: (n) => n.pitch, tol: 0 },
|
||||
{ keyR: (n) => n.pitch, keyE: (n) => n.pitch, tol: TPQ / 2 },
|
||||
];
|
||||
for (const st of stages) {
|
||||
const ri = ref.map((_, i) => i).filter((i) => !refDone[i]);
|
||||
const ei = est.map((_, i) => i).filter((i) => !estDone[i]);
|
||||
const adj = buildAdjacency(
|
||||
ri.map((i) => ({ key: st.keyR(ref[i]!), x: alignedTick(ref[i]!, align) })),
|
||||
ei.map((i) => ({ key: st.keyE(est[i]!), x: est[i]!.tick })),
|
||||
st.tol,
|
||||
);
|
||||
const m = maxMatching(ri.length, ei.length, adj);
|
||||
m.forEach((j, a) => { if (j >= 0) { pairs.push([ri[a]!, ei[j]!]); refDone[ri[a]!] = true; estDone[ei[j]!] = true; } });
|
||||
}
|
||||
if (pairs.length === 0) return { onBeat: null, durationOk: null };
|
||||
let on = 0, dur = 0;
|
||||
for (const [i, j] of pairs) {
|
||||
if (alignedTick(ref[i]!, align) === est[j]!.tick) on++;
|
||||
if (alignedDur(ref[i]!, align) === est[j]!.dur) dur++;
|
||||
}
|
||||
return { onBeat: on / pairs.length, durationOk: dur / pairs.length };
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
import { secondsToTicks } from "../core/time.ts";
|
||||
import type { Grid, Role, ScoreNote, TimedNote, TruthTrack } from "../core/types.ts";
|
||||
import { matchNotes, maxAssignment, prf, type MatchResult } from "./notes.ts";
|
||||
import { alignGrid, rhythmScore } from "./rhythm.ts";
|
||||
|
||||
export interface OracleFlags { oracleStems: boolean; oracleTempo: boolean; oracleNotes: boolean }
|
||||
export interface OutTrack { name: string; role: Role; notes: TimedNote[]; scoreNotes: ScoreNote[] }
|
||||
export interface PipelineOutput { grid: Grid; tracks: OutTrack[] }
|
||||
export type DrumClass = "kick" | "snare" | "hihat" | "cymbal" | "tom";
|
||||
export interface SongScore {
|
||||
flags: OracleFlags;
|
||||
guitarMerged: MatchResult & { pass: boolean };
|
||||
partSplit: { f1: number | null; outTracks: number; refTracks: number } | null;
|
||||
perRole: Partial<Record<Role, MatchResult>>;
|
||||
drumsPerClass: Record<DrumClass, MatchResult> | null;
|
||||
rhythm: { factor: number; beatOffset: number; onBeat: number | null; durationOk: number | null } | null;
|
||||
stringAgreement: number | null;
|
||||
spuriousRate: number | null;
|
||||
}
|
||||
|
||||
export const ROLE_ORDER: Role[] = ["guitar", "vocals", "bass", "drums", "keys"];
|
||||
export const DRUM_CLASSES: Record<DrumClass, number[]> = {
|
||||
kick: [35, 36], snare: [37, 38, 40], hihat: [42, 44, 46],
|
||||
cymbal: [49, 51, 52, 53, 55, 57, 59], tom: [41, 43, 45, 47, 48, 50],
|
||||
};
|
||||
const PASS_F1 = 0.85;
|
||||
|
||||
function drumClass(pitch: number): DrumClass | null {
|
||||
for (const [c, ps] of Object.entries(DRUM_CLASSES)) if (ps.includes(pitch)) return c as DrumClass;
|
||||
return null;
|
||||
}
|
||||
// Drum pitches are compared by class: map to the class's first GM number; unknown pitches stay distinct.
|
||||
const toClassPitch = (n: TimedNote): TimedNote => {
|
||||
const c = drumClass(n.pitch);
|
||||
return { ...n, pitch: c ? DRUM_CLASSES[c][0]! : 1000 + n.pitch };
|
||||
};
|
||||
|
||||
const notesOf = (tracks: { role: Role; notes: TimedNote[] }[], role: Role) =>
|
||||
tracks.filter((t) => t.role === role).flatMap((t) => t.notes);
|
||||
|
||||
export function scoreSong(
|
||||
truth: { bpm: number; timeSig: [number, number]; tracks: TruthTrack[] },
|
||||
out: PipelineOutput, flags: OracleFlags,
|
||||
): SongScore {
|
||||
const refGuitar = notesOf(truth.tracks, "guitar");
|
||||
const outGuitar = notesOf(out.tracks, "guitar");
|
||||
const gm = matchNotes(refGuitar, outGuitar);
|
||||
const guitarMerged = { ...gm, pass: gm.f1 !== null && gm.f1 >= PASS_F1 };
|
||||
|
||||
// partSplit (R3): null only with <2 ref guitar tracks
|
||||
const refG = truth.tracks.filter((t) => t.role === "guitar");
|
||||
const outG = out.tracks.filter((t) => t.role === "guitar");
|
||||
let partSplit: SongScore["partSplit"] = null;
|
||||
if (refG.length >= 2) {
|
||||
const gain = refG.map((r) => outG.map((o) => matchNotes(r.notes, o.notes).tp));
|
||||
const asg = outG.length ? maxAssignment(gain) : refG.map(() => -1);
|
||||
let tp = 0;
|
||||
asg.forEach((j, i) => { if (j >= 0) tp += gain[i]![j]!; });
|
||||
const fn = refG.reduce((s, t) => s + t.notes.length, 0) - tp;
|
||||
const fp = outG.reduce((s, t) => s + t.notes.length, 0) - tp;
|
||||
partSplit = { f1: prf(tp, fp, fn).f1, outTracks: outG.length, refTracks: refG.length };
|
||||
}
|
||||
|
||||
// per role (drums compared by class)
|
||||
const perRole: Partial<Record<Role, MatchResult>> = {};
|
||||
for (const role of ROLE_ORDER) {
|
||||
if (role === "guitar") { perRole.guitar = gm; continue; }
|
||||
let r = notesOf(truth.tracks, role), e = notesOf(out.tracks, role);
|
||||
if (role === "drums") { r = r.map(toClassPitch); e = e.map(toClassPitch); }
|
||||
perRole[role] = matchNotes(r, e);
|
||||
}
|
||||
|
||||
let drumsPerClass: SongScore["drumsPerClass"] = null;
|
||||
const refDrums = notesOf(truth.tracks, "drums"), outDrums = notesOf(out.tracks, "drums");
|
||||
if (refDrums.length + outDrums.length > 0) {
|
||||
drumsPerClass = {} as Record<DrumClass, MatchResult>;
|
||||
for (const c of Object.keys(DRUM_CLASSES) as DrumClass[]) {
|
||||
drumsPerClass[c] = matchNotes(
|
||||
refDrums.filter((n) => drumClass(n.pitch) === c).map(toClassPitch),
|
||||
outDrums.filter((n) => drumClass(n.pitch) === c).map(toClassPitch),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// rhythm on merged guitar (R2: truth seconds -> ticks)
|
||||
let rhythm: SongScore["rhythm"] = null;
|
||||
const outGuitarScore = outG.flatMap((t) => t.scoreNotes);
|
||||
if (refGuitar.length > 0 && outGuitarScore.length > 0) {
|
||||
const refScore: ScoreNote[] = refGuitar.map((n) => ({
|
||||
tick: Math.round(secondsToTicks(n.onset, truth.bpm)),
|
||||
dur: Math.round(secondsToTicks(n.offset - n.onset, truth.bpm)),
|
||||
pitch: n.pitch, string: n.string, fret: n.fret,
|
||||
}));
|
||||
const align = alignGrid(refScore, outGuitarScore);
|
||||
rhythm = { factor: align.factor, beatOffset: align.beatOffset, ...rhythmScore(refScore, outGuitarScore, align) };
|
||||
}
|
||||
|
||||
let withString = 0, agree = 0;
|
||||
for (const [i, j] of gm.matches) {
|
||||
const a = refGuitar[i]!.string, b = outGuitar[j]!.string;
|
||||
if (a !== undefined && b !== undefined) { withString++; if (a === b) agree++; }
|
||||
}
|
||||
|
||||
return {
|
||||
flags, guitarMerged, partSplit, perRole, drumsPerClass, rhythm,
|
||||
stringAgreement: withString ? agree / withString : null,
|
||||
spuriousRate: outGuitar.length ? gm.fp / outGuitar.length : null,
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user