diff --git a/src/align/auto.ts b/src/align/auto.ts new file mode 100644 index 0000000..96155a0 --- /dev/null +++ b/src/align/auto.ts @@ -0,0 +1,228 @@ +// Automatic MIDI -> recording alignment (coarse-to-fine DTW on chroma + onset features) and its fit quality. +// Pure: no I/O, no Bun/process/fs. The SoundFont bytes are passed in. +import type { Audio, Role } from "../core/types.ts"; +import type { Smf } from "./smf.ts"; +import { makeWarp, type Anchor, type Warp } from "./warp.ts"; +import { renderMidi } from "./render.ts"; +import { features, pickOnsets, type Features } from "./features.ts"; +import { dtw } from "./dtw.ts"; +import { trackKeys } from "./roles.ts"; +import { fitQuality, type FitReport } from "./fit.ts"; + +const FINE_RATE = 50; +const COARSE_FACTOR = 5; // coarse pass at 10 Hz +const LAMBDA = 1; // weight of the onset-strength term in the frame cost +const FINE_BANDS_SEC = [0.5, 1, 2]; // fine pass band half-widths tried in turn before falling back to the coarse path +const ANCHOR_HALF_WIN_SEC = 1; // path points within ±1 s (warped time) of a bar estimate its offset +const MIN_POINTS = 10; // diagonal path points required to place a bar anchor +const MANUAL_CLEARANCE_SEC = 1; // no auto anchor within 1 s (MIDI time) of a manual anchor +const ZERO = 1e-9; + +/** Bar start times (MIDI seconds) from the tempo map and time signatures (4/4 when none), up to durationSec. */ +export function barTimes(smf: Smf): number[] { + const tempos = smf.tempoMap.length ? [...smf.tempoMap].sort((a, b) => a.sec - b.sec) : [{ sec: 0, bpm: 120 }]; + const sigs = [...smf.timeSigs].sort((a, b) => a.sec - b.sec); + const bpmAt = (t: number) => { let b = tempos[0]!.bpm; for (const p of tempos) if (p.sec <= t + ZERO) b = p.bpm; return b; }; + // time after advancing `q` quarter notes from `t`, honouring tempo changes on the way + const advance = (t: number, q: number): number => { + while (q > ZERO) { + const spq = 60 / bpmAt(t); + const next = tempos.find((p) => p.sec > t + ZERO); + const span = next ? (next.sec - t) / spq : Infinity; + if (q <= span) return t + q * spq; + q -= span; t = next!.sec; + } + return t; + }; + const out: number[] = []; + let t = 0; + while (t < smf.durationSec - ZERO && out.length < 100000) { + out.push(t); + let num = 4, den = 4; + for (const s of sigs) if (s.sec <= t + ZERO) { num = s.num; den = s.den; } + let next = advance(t, (num * 4) / den); + const change = sigs.find((s) => s.sec > t + ZERO && s.sec < next - ZERO); // signature change mid-bar: restart there + if (change) next = change.sec; + if (!(next > t)) break; + t = next; + } + return out; +} + +/** Aggregate 50 Hz features by `k` frames (centred): chroma energy summed and re-normalised, onset max, re-z-scored. */ +function downsample(f: Features, k: number): Features { + const n = Math.max(1, Math.floor((f.chroma.length - 1) / k) + 1); + const chroma: Float32Array[] = [], onset = new Float32Array(n); + for (let o = 0; o < n; o++) { + const c = new Float32Array(12); + let fl = -Infinity; + for (let t = Math.max(0, o * k - (k >> 1)); t <= Math.min(f.chroma.length - 1, o * k + (k >> 1)); t++) { + const s = f.chroma[t]!; + for (let q = 0; q < 12; q++) c[q]! += s[q]! * s[q]!; + fl = Math.max(fl, f.onset[t]!); + } + let norm = 0; + for (let q = 0; q < 12; q++) { c[q] = Math.sqrt(c[q]!); norm += c[q]! * c[q]!; } + norm = Math.sqrt(norm); + if (norm > 1e-6) for (let q = 0; q < 12; q++) c[q]! /= norm; else c.fill(0); + chroma.push(c); onset[o] = fl; + } + let mean = 0; for (const v of onset) mean += v; mean /= n; + let v2 = 0; for (const v of onset) v2 += (v - mean) ** 2; + const sd = Math.sqrt(v2 / n) || 1; + for (let o = 0; o < n; o++) onset[o] = (onset[o]! - mean) / sd; + return { rate: f.rate / k, onset, chroma }; +} + +/** Frame cost (1 − cos) + λ|Δonset|. Two silent (all-zero chroma) frames match perfectly; silence vs sound costs 1. */ +function costFn(A: Features, B: Features): (i: number, j: number) => number { + const flat = (f: Features) => { const x = new Float32Array(f.chroma.length * 12); f.chroma.forEach((c, i) => x.set(c, i * 12)); return x; }; + const zero = (f: Features) => Uint8Array.from(f.chroma, (c) => (c.every((v) => v === 0) ? 1 : 0)); + const ca = flat(A), cb = flat(B), za = zero(A), zb = zero(B), oa = A.onset, ob = B.onset; + return (i, j) => { + let c: number; + if (za[i] || zb[j]) c = za[i] && zb[j] ? 0 : 1; + else { + let d = 0; + for (let q = 0, p = i * 12, r = j * 12; q < 12; q++) d += ca[p + q]! * cb[r + q]!; + c = 1 - d; + } + return c + LAMBDA * Math.abs(oa[i]! - ob[j]!); + }; +} + +type Path = [number, number][]; + +/** Align rendered frames [i0, i1] with recording frames [j0, j1] (inclusive, fine rate), coarse then banded fine. */ +function alignSegment(fine: [Features, Features], coarse: [Features, Features], i0: number, i1: number, j0: number, j1: number): Path { + const K = COARSE_FACTOR; + const ci0 = Math.round(i0 / K), ci1 = Math.max(ci0, Math.min(coarse[0].chroma.length - 1, Math.round(i1 / K))); + const cj0 = Math.round(j0 / K), cj1 = Math.max(cj0, Math.min(coarse[1].chroma.length - 1, Math.round(j1 / K))); + const cc = costFn(coarse[0], coarse[1]); + const cp = dtw((i, j) => cc(ci0 + i, cj0 + j), ci1 - ci0 + 1, cj1 - cj0 + 1); + // coarse map: mean recording frame per rendered frame, in segment-relative coarse units + const sum = new Float64Array(ci1 - ci0 + 1), cnt = new Float64Array(ci1 - ci0 + 1); + for (const [i, j] of cp) { sum[i]! += j; cnt[i]! += 1; } + const cmap = Array.from(sum, (s, i) => s / cnt[i]!); + const centre = (i: number): number => { // fine row (segment-relative) -> fine column (segment-relative) + const x = Math.min(cmap.length - 1, Math.max(0, (i0 + i) / K - ci0)); + const a = Math.floor(x), b = Math.min(cmap.length - 1, a + 1); + const y = cmap[a]! + (x - a) * (cmap[b]! - cmap[a]!); + return (y + cj0) * K - j0; + }; + const fc = costFn(fine[0], fine[1]); + const p = bandedWithRetry((i, j) => fc(i0 + i, j0 + j), i1 - i0 + 1, j1 - j0 + 1, centre); + if (p) return p.map(([i, j]) => [i0 + i, j0 + j]); + // fallback: the coarse path at fine resolution + return cp.map(([i, j]) => [Math.min(i1, (ci0 + i) * K), Math.min(j1, (cj0 + j) * K)]); +} + +/** Banded fine DTW, widening the band (±0.5 s, ±1 s, ±2 s) when the end cell is unreachable; null if all fail. */ +export function bandedWithRetry(costAt: (i: number, j: number) => number, n: number, m: number, centre: (i: number) => number): Path | null { + for (const sec of FINE_BANDS_SEC) { + try { + return dtw(costAt, n, m, { center: centre, width: Math.round(sec * FINE_RATE) }); + } catch (e) { + if (!(e instanceof Error) || !/unreachable/.test(e.message)) throw e; + } + } + return null; +} + +/** Recording time for warped time `w`: w + median offset of diagonal path steps within ±1 s, or null if too few. */ +function offsetAt(diag: { w: number; off: number }[], w: number): number | null { + let lo = 0, hi = diag.length; + while (lo < hi) { const mid = (lo + hi) >> 1; if (diag[mid]!.w < w - ANCHOR_HALF_WIN_SEC) lo = mid + 1; else hi = mid; } + const offs: number[] = []; + for (let k = lo; k < diag.length && diag[k]!.w <= w + ANCHOR_HALF_WIN_SEC; k++) offs.push(diag[k]!.off); + if (offs.length < MIN_POINTS) return null; + offs.sort((a, b) => a - b); + const h = offs.length >> 1; + return offs.length % 2 ? offs[h]! : (offs[h - 1]! + offs[h]!) / 2; +} + +/** Keep manual anchors; keep an auto anchor only if it is strictly monotone against its kept neighbours. */ +function compose(manual: Anchor[], auto: Anchor[]): Anchor[] { + const all = [...manual, ...auto].sort((a, b) => a.midiSec - b.midiSec || (a.kind === "manual" ? -1 : 1)); + const out: Anchor[] = []; + for (let k = 0; k < all.length; k++) { + const a = all[k]!; + if (a.kind === "manual") { + while (out.length && out[out.length - 1]!.kind === "auto" && !(out[out.length - 1]!.audioSec < a.audioSec && out[out.length - 1]!.midiSec < a.midiSec)) out.pop(); + out.push(a); continue; + } + const prev = out[out.length - 1]; + if (prev && !(a.midiSec > prev.midiSec && a.audioSec > prev.audioSec)) continue; + const nextManual = all.slice(k + 1).find((x) => x.kind === "manual"); + if (nextManual && !(a.audioSec < nextManual.audioSec)) continue; + out.push(a); + } + return out; +} + +const durationOf = (a: Audio) => (a.channels[0]?.length ?? 0) / a.sampleRate; + +/** Fit of `warp` against recording onsets: warped onsets of scored-role notes (roles keyed by `trackKeys`). */ +function fitFor(recOnsets: number[], smf: Smf, roles: Record, warp: Warp, durSec: number): FitReport { + const keys = trackKeys(smf.tracks); + const midi = smf.tracks.flatMap((t, i) => (roles[keys[i]!] ? t.notes.map((n) => warp.toAudio(n.onset)) : [])); + return fitQuality(recOnsets, midi, durSec); +} + +/** Fit quality of any alignment (e.g. a hand-made one): onset-flux peaks of the raw recording vs the warped MIDI. */ +export function alignmentFit(audio: Audio, smf: Smf, roles: Record, anchors: Anchor[]): FitReport { + const onsets = pickOnsets(features(audio, FINE_RATE).onset, FINE_RATE); + return fitFor(onsets, smf, roles, makeWarp(anchors), durationOf(audio)); +} + +/** + * Align `smf` to `audio`, honouring `manual` anchors exactly. The MIDI is rendered under the manual warp, then each + * stretch between consecutive manual anchors (plus start/end) is aligned by a 10 Hz DTW refined at 50 Hz in a band + * around it. One auto anchor is placed per bar (skipping bars within 1 s of a manual anchor). `roles` is keyed by + * `trackKeys`; tracks without a role are rendered but excluded from the fit. + */ +export function autoAlign( + audio: Audio, smf: Smf, roles: Record, manual: Anchor[], sf2: Uint8Array, +): { anchors: Anchor[]; fit: FitReport } { + const man = makeWarp(manual).anchors.map((a) => ({ ...a, kind: "manual" as const })); + const W0 = makeWarp(man); + const audioDur = durationOf(audio); + const renderDur = Math.max(audioDur, W0.toAudio(smf.durationSec)); + const rendered = renderMidi(smf.tracks, W0, sf2, { mono: true, durationSec: renderDur }); + const fR = features(rendered, FINE_RATE), fA = features(audio, FINE_RATE); + const cR = downsample(fR, COARSE_FACTOR), cA = downsample(fA, COARSE_FACTOR); + const lastR = fR.chroma.length - 1, lastA = fA.chroma.length - 1; + + // segment boundaries: manual anchors map warped time == recording time, so both axes split at the same instants + const cuts = man.map((a) => Math.round(a.audioSec * FINE_RATE)).filter((f) => f > 0 && f < Math.min(lastR, lastA)); + const bounds = [...new Set(cuts)].sort((a, b) => a - b); + const path: Path = []; + let prev = 0; + for (const b of [...bounds, -1]) { + const i1 = b < 0 ? lastR : b, j1 = b < 0 ? lastA : b; + path.push(...alignSegment([fR, fA], [cR, cA], prev, i1, prev, j1)); + prev = b; + } + + // diagonal steps carry timing; horizontal/vertical runs (stalls, silence, unmatched material) do not + const diag: { w: number; off: number }[] = []; + for (let k = 1; k < path.length; k++) { + const [i, j] = path[k]!, [pi, pj] = path[k - 1]!; + if (i === pi + 1 && j === pj + 1) diag.push({ w: i / FINE_RATE, off: (j - i) / FINE_RATE }); + } + diag.sort((a, b) => a.w - b.w); + + const auto: Anchor[] = []; + for (const m of barTimes(smf)) { + if (man.some((a) => Math.abs(a.midiSec - m) < MANUAL_CLEARANCE_SEC)) continue; + const w = W0.toAudio(m); + const off = offsetAt(diag, w); + if (off === null) continue; + const a = w + off; + if (a < 0 || a > audioDur) continue; + auto.push({ midiSec: m, audioSec: a, kind: "auto" }); + } + const anchors = compose(man, auto); + const fit = fitFor(pickOnsets(fA.onset, FINE_RATE), smf, roles, makeWarp(anchors), audioDur); + return { anchors, fit }; +} diff --git a/src/align/fit.ts b/src/align/fit.ts new file mode 100644 index 0000000..567c631 --- /dev/null +++ b/src/align/fit.ts @@ -0,0 +1,57 @@ +// Fit quality of an alignment: how well warped MIDI onsets land on onsets picked from the raw recording. +// Pure: no I/O. Recording onsets must come from the audio itself (onset flux), never from transcription. + +export interface FitWindow { startSec: number; matchRate: number | null; medianResidualMs: number | null; flagged: boolean } +export interface FitReport { matchRate: number | null; medianResidualMs: number | null; windows: FitWindow[] } + +export const MATCH_TOL_SEC = 0.07; +export const WINDOW_SEC = 30; +const FLAG_MIN_RATE = 0.5; +const FLAG_MAX_RESIDUAL_MS = 60; +const UNIQUE_EPS = 1e-6; + +function median(xs: number[]): number | null { + if (!xs.length) return null; + const s = [...xs].sort((a, b) => a - b), m = s.length >> 1; + return s.length % 2 ? s[m]! : (s[m - 1]! + s[m]!) / 2; +} + +/** |residual| in seconds to the nearest recording onset, or null when none lies within ±70 ms. */ +function nearest(sorted: number[], t: number): number | null { + let lo = 0, hi = sorted.length; + while (lo < hi) { const m = (lo + hi) >> 1; if (sorted[m]! < t) lo = m + 1; else hi = m; } + let best = Infinity; + if (lo < sorted.length) best = Math.min(best, Math.abs(sorted[lo]! - t)); + if (lo > 0) best = Math.min(best, Math.abs(sorted[lo - 1]! - t)); + return best <= MATCH_TOL_SEC + 1e-9 ? best : null; +} + +function summarise(residuals: (number | null)[]): { matchRate: number | null; medianResidualMs: number | null } { + if (!residuals.length) return { matchRate: null, medianResidualMs: null }; + const hits = residuals.filter((r): r is number => r !== null); + const med = median(hits); + return { matchRate: hits.length / residuals.length, medianResidualMs: med === null ? null : med * 1000 }; +} + +/** + * Match each unique MIDI onset (already warped to audio time) to the nearest recording onset within ±70 ms + * (non-exclusive). Reports match rate and median |residual| overall and per 30 s window (by MIDI onset time). + * A window is flagged when its match rate < 0.5 or its median residual > 60 ms; empty windows are null, unflagged. + */ +export function fitQuality(audioOnsets: number[], midiOnsets: number[], durSec: number): FitReport { + const audio = [...audioOnsets].sort((a, b) => a - b); + const midi: number[] = []; + for (const t of [...midiOnsets].sort((a, b) => a - b)) if (!midi.length || t - midi[midi.length - 1]! > UNIQUE_EPS) midi.push(t); + const res = midi.map((t) => nearest(audio, t)); + + const lastOnset = midi.length ? midi[midi.length - 1]! : 0; + const nWin = Math.max(1, Math.ceil(Math.max(durSec, lastOnset + 1e-9) / WINDOW_SEC)); + const buckets: (number | null)[][] = Array.from({ length: nWin }, () => []); + midi.forEach((t, k) => buckets[Math.min(nWin - 1, Math.max(0, Math.floor(t / WINDOW_SEC)))]!.push(res[k]!)); + const windows = buckets.map((b, w): FitWindow => { + const s = summarise(b); + const flagged = s.matchRate !== null && (s.matchRate < FLAG_MIN_RATE || (s.medianResidualMs ?? 0) > FLAG_MAX_RESIDUAL_MS); + return { startSec: w * WINDOW_SEC, ...s, flagged }; + }); + return { ...summarise(res), windows }; +} diff --git a/src/align/roles.ts b/src/align/roles.ts index afe8ab1..e988b7d 100644 --- a/src/align/roles.ts +++ b/src/align/roles.ts @@ -15,3 +15,14 @@ export function guessRole(t: RefTrack): Role | null { if ((p >= 0 && p <= 7) || (p >= 16 && p <= 23) || (p >= 88 && p <= 95)) return "keys"; return null; } + +// Role-map key for a track: its name, or "track N" when unnamed; disambiguated on collision. +export function trackKeys(tracks: { index: number; name: string }[]): string[] { + const used = new Set(); + return tracks.map((t) => { + let k = t.name || `track ${t.index}`; + if (used.has(k)) k = `${k} #${t.index}`; + used.add(k); + return k; + }); +} diff --git a/src/ref/manifest.ts b/src/ref/manifest.ts index 73fbb34..e5a3223 100644 --- a/src/ref/manifest.ts +++ b/src/ref/manifest.ts @@ -2,7 +2,7 @@ import { join, resolve } from "node:path"; import { mkdir } from "node:fs/promises"; import type { Role } from "../core/types.ts"; import { parseSmf } from "../align/smf.ts"; -import { guessRole } from "../align/roles.ts"; +import { guessRole, trackKeys } from "../align/roles.ts"; export interface SongEntry { id: string; title: string; audio: string; midi: string; @@ -22,16 +22,7 @@ export async function readManifest(root = "reference"): Promise { return (await f.json()) as SongEntry[]; } -// Role-map key for a track: its name, or "track N" when unnamed; disambiguated on collision. -export function trackKeys(tracks: { index: number; name: string }[]): string[] { - const used = new Set(); - return tracks.map((t) => { - let k = t.name || `track ${t.index}`; - if (used.has(k)) k = `${k} #${t.index}`; - used.add(k); - return k; - }); -} +export { trackKeys }; export async function addSong(e: Omit & { roles?: SongEntry["roles"] }, root = "reference"): Promise { const dir = songDir(e.id, root); diff --git a/tests/align/auto.test.ts b/tests/align/auto.test.ts new file mode 100644 index 0000000..0d12966 --- /dev/null +++ b/tests/align/auto.test.ts @@ -0,0 +1,135 @@ +import { test, expect, beforeAll } from "bun:test"; +import { alignmentFit, autoAlign, bandedWithRetry, barTimes } from "../../src/align/auto.ts"; +import { makeWarp, type Anchor, type Warp } from "../../src/align/warp.ts"; +import type { RefTrack, Smf } from "../../src/align/smf.ts"; +import type { Audio, Role, TruthTrack } from "../../src/core/types.ts"; +import { buildSong, loadSongDir } from "../../src/bench/dataset.ts"; + +const SF2 = new Uint8Array(await Bun.file("node_modules/@coderline/alphatab/dist/soundfont/sonivox.sf2").arrayBuffer()); +const BIG = 60_000; + +let song: Awaited>; +beforeAll(async () => { song = await loadSongDir(await buildSong("t1c", 1)); }, BIG); + +/** In-memory Smf from truth tracks: `map` turns truth (= recording) time into MIDI time; `reps` tiles the song. */ +function smfFrom(truth: TruthTrack[], programs: number[], bpm: number, map: (t: number) => number, reps = 1, period = 0): Smf { + const tracks: RefTrack[] = truth.map((t, i) => ({ + index: i + 1, name: t.name, program: programs[i]!, isDrums: t.role === "drums", + notes: Array.from({ length: reps }, (_, r) => t.notes.map((n) => ({ + onset: map(n.onset + r * period), offset: map(n.offset + r * period), pitch: n.pitch, velocity: n.velocity ?? 96, + channel: t.role === "drums" ? 9 : i, + }))).flat(), + })); + const durationSec = Math.max(...tracks.flatMap((t) => t.notes.map((n) => n.offset))); + return { durationSec, tracks, tempoMap: [{ sec: 0, bpm }], timeSigs: [{ sec: 0, num: 4, den: 4 }] }; +} +const rolesOf = (truth: TruthTrack[]): Record => Object.fromEntries(truth.map((t) => [t.name, t.role])); +const durOf = (a: Audio) => a.channels[0]!.length / a.sampleRate; +const median = (xs: number[]) => { const s = [...xs].sort((a, b) => a - b); return s[s.length >> 1]!; }; + +/** Known distortion recording time -> MIDI time: +0.4 s, then bars 4–10 at ×1.06 tempo, then ≈ −0.8 s (continuous). */ +function distortion(bpm: number): Warp { + const bar = (4 * 60) / bpm, b1 = 4 * bar, b2 = 10 * bar; + const m2 = b1 + 0.4 + (b2 - b1) / 1.06; + return makeWarp([ + { midiSec: 0, audioSec: 0.4, kind: "manual" }, + { midiSec: b1, audioSec: b1 + 0.4, kind: "manual" }, + { midiSec: b2, audioSec: m2, kind: "manual" }, + { midiSec: b2 + 100, audioSec: m2 + 100, kind: "manual" }, + ]); // toAudio: recording -> MIDI, toMidi: MIDI -> recording (the truth the aligner must recover) +} + +function recoveryErrors(smf: Smf, got: Anchor[], trueMidiToAudio: (m: number) => number, audioDur: number): number[] { + const w = makeWarp(got); + const onsets = [...new Set(smf.tracks.flatMap((t) => t.notes.map((n) => n.onset)))]; + return onsets.filter((m) => { const a = trueMidiToAudio(m); return a >= 0 && a <= audioDur; }) + .map((m) => Math.abs(w.toAudio(m) - trueMidiToAudio(m))); +} + +test("recovers a known piecewise warp: median error < 40 ms, fit match rate > 0.7", () => { + const D = distortion(song.song.bpm); + const smf = smfFrom(song.truth.tracks, song.song.tracks.map((t) => t.program), song.song.bpm, D.toAudio); + const t0 = performance.now(); + const { anchors, fit } = autoAlign(song.mix, smf, rolesOf(song.truth.tracks), [], SF2); + const ms = performance.now() - t0; + const err = recoveryErrors(smf, anchors, D.toMidi, durOf(song.mix)); + console.log(`synthetic warp: ${anchors.length} anchors, median err ${(median(err) * 1000).toFixed(1)} ms, p90 ${(err.sort((a, b) => a - b)[Math.floor(err.length * 0.9)]! * 1000).toFixed(1)} ms, fit ${fit.matchRate?.toFixed(3)} / ${fit.medianResidualMs?.toFixed(1)} ms, ${ms.toFixed(0)} ms`); + expect(anchors.every((a) => a.kind === "auto")).toBe(true); + expect(median(err)).toBeLessThan(0.04); + expect(fit.matchRate!).toBeGreaterThan(0.7); + expect(fit.windows.length).toBe(2); +}, BIG); + +test("a correct manual anchor mid-song is honoured exactly", () => { + const D = distortion(song.song.bpm); + const smf = smfFrom(song.truth.tracks, song.song.tracks.map((t) => t.program), song.song.bpm, D.toAudio); + const manual: Anchor = { midiSec: D.toAudio(30), audioSec: 30, kind: "manual" }; + const { anchors } = autoAlign(song.mix, smf, rolesOf(song.truth.tracks), [manual], SF2); + expect(anchors.filter((a) => a.kind === "manual")).toEqual([manual]); + expect(makeWarp(anchors).toAudio(manual.midiSec)).toBe(30); + // no auto anchor within 1 s (MIDI time) of the manual one + expect(anchors.some((a) => a.kind === "auto" && Math.abs(a.midiSec - manual.midiSec) < 1)).toBe(false); + expect(anchors.length).toBeGreaterThan(5); + const err = recoveryErrors(smf, anchors, D.toMidi, durOf(song.mix)); + expect(median(err)).toBeLessThan(0.04); +}, BIG); + +test("a silent intro in the recording does not drag the path", () => { + const pad = 4 * song.mix.sampleRate; + const mix: Audio = { sampleRate: song.mix.sampleRate, channels: song.mix.channels.map((c) => { const o = new Float32Array(c.length + pad); o.set(c, pad); return o; }) }; + const smf = smfFrom(song.truth.tracks, song.song.tracks.map((t) => t.program), song.song.bpm, (t) => t); + const { anchors, fit } = autoAlign(mix, smf, rolesOf(song.truth.tracks), [], SF2); + const err = recoveryErrors(smf, anchors, (m) => m + 4, durOf(mix)); + expect(median(err)).toBeLessThan(0.04); + expect(fit.matchRate!).toBeGreaterThan(0.7); +}, BIG); + +test("5-minute input aligns in < 15 s", () => { + const reps = 5, period = durOf(song.mix); + const mix: Audio = { sampleRate: song.mix.sampleRate, channels: song.mix.channels.map((c) => { const o = new Float32Array(c.length * reps); for (let r = 0; r < reps; r++) o.set(c, r * c.length); return o; }) }; + const smf = smfFrom(song.truth.tracks, song.song.tracks.map((t) => t.program), song.song.bpm, (t) => t + 0.3, reps, period); + const t0 = performance.now(); + const { anchors, fit } = autoAlign(mix, smf, rolesOf(song.truth.tracks), [], SF2); + const sec = (performance.now() - t0) / 1000; + console.log(`5-min (${durOf(mix).toFixed(0)} s) input: ${sec.toFixed(2)} s, ${anchors.length} anchors, fit ${fit.matchRate?.toFixed(3)}`); + expect(durOf(mix)).toBeGreaterThan(290); + expect(sec).toBeLessThan(15); + expect(fit.windows.length).toBe(10); +}, BIG); + +test("barTimes follows tempo and time-signature changes", () => { + const smf: Smf = { + durationSec: 10, tracks: [], + tempoMap: [{ sec: 0, bpm: 120 }, { sec: 4, bpm: 60 }], + timeSigs: [{ sec: 0, num: 4, den: 4 }, { sec: 4, num: 3, den: 4 }], + }; + // 4/4 @120: bars of 2 s at 0, 2; then 3/4 @60: bars of 3 s at 4, 7 + expect(barTimes(smf)).toEqual([0, 2, 4, 7]); + expect(barTimes({ durationSec: 3, tracks: [], tempoMap: [{ sec: 0, bpm: 120 }], timeSigs: [] })).toEqual([0, 2]); // default 4/4 + expect(barTimes({ durationSec: 4, tracks: [], tempoMap: [{ sec: 0, bpm: 120 }], timeSigs: [{ sec: 0, num: 6, den: 8 }] })).toEqual([0, 1.5, 3]); +}); + +test("fine pass widens the band when the end cell is unreachable, then gives up with null", () => { + const cost = (i: number, j: number) => Math.abs(i - j); + // centre jumps by 120 frames mid-way: ±25 and ±50 leave a gap between rows, ±100 bridges it + const jump = (i: number) => (i < 200 ? i : i + 120); + const p = bandedWithRetry(cost, 400, 600, jump); + expect(p).not.toBeNull(); + expect(p![p!.length - 1]).toEqual([399, 599]); + // a 400-frame jump defeats even ±2 s (±100 frames) + expect(bandedWithRetry(cost, 400, 1000, (i) => (i < 200 ? i : i + 400))).toBeNull(); +}); + +test("fit exposes a bad alignment instead of hiding it", () => { + const D = distortion(song.song.bpm); + const smf = smfFrom(song.truth.tracks, song.song.tracks.map((t) => t.program), song.song.bpm, D.toAudio); + const roles = rolesOf(song.truth.tracks); + const bad = alignmentFit(song.mix, smf, roles, []); // identity: off by 0.4–0.8 s throughout + expect(bad.matchRate!).toBeLessThan(0.5); + // the −0.8 s section is exposed; the +0.4 s section is not, as 0.4 s is ≈ 40 ms from two 16ths at 68 bpm (grid alias) + expect(bad.windows[1]!.flagged).toBe(true); + const good = alignmentFit(song.mix, smf, roles, autoAlign(song.mix, smf, roles, [], SF2).anchors); + expect(good.windows.some((w) => w.flagged)).toBe(false); + // unmapped tracks are excluded: with no scored roles there is nothing to measure + expect(alignmentFit(song.mix, smf, Object.fromEntries(Object.keys(roles).map((k) => [k, null])), []).matchRate).toBeNull(); +}, BIG); diff --git a/tests/align/fit.test.ts b/tests/align/fit.test.ts new file mode 100644 index 0000000..209dd02 --- /dev/null +++ b/tests/align/fit.test.ts @@ -0,0 +1,64 @@ +import { test, expect } from "bun:test"; +import { fitQuality } from "../../src/align/fit.ts"; + +test("perfect fit: match rate 1, residual 0, one window per 30 s", () => { + const on = [1, 2, 3, 31, 32, 61]; + const r = fitQuality(on, on, 75); + expect(r.matchRate).toBe(1); + expect(r.medianResidualMs).toBe(0); + expect(r.windows.map((w) => w.startSec)).toEqual([0, 30, 60]); + expect(r.windows.every((w) => !w.flagged && w.matchRate === 1)).toBe(true); +}); + +test("match within ±70 ms only; residual is median |audio − midi| of matched onsets", () => { + const midi = [1, 2, 3, 4]; + const audio = [1.01, 2.03, 3.069, 4.2]; // last is 200 ms away: unmatched + const r = fitQuality(audio, midi, 10); + expect(r.matchRate).toBe(0.75); + expect(r.medianResidualMs!).toBeCloseTo(30, 6); +}); + +test("nearest recording onset is used and may be shared by several MIDI onsets", () => { + const r = fitQuality([1.0], [0.98, 1.0, 1.02], 5); + expect(r.matchRate).toBe(1); + expect(r.medianResidualMs!).toBeCloseTo(20, 6); +}); + +test("empty window gives nulls and is not flagged", () => { + const r = fitQuality([1, 2, 65], [1, 2, 65], 90); + expect(r.windows).toHaveLength(3); + const w = r.windows[1]!; + expect(w).toEqual({ startSec: 30, matchRate: null, medianResidualMs: null, flagged: false }); +}); + +test("no MIDI onsets at all gives overall nulls", () => { + const r = fitQuality([1, 2], [], 10); + expect(r.matchRate).toBeNull(); + expect(r.medianResidualMs).toBeNull(); + expect(r.windows).toEqual([{ startSec: 0, matchRate: null, medianResidualMs: null, flagged: false }]); +}); + +test("window flagged when match rate < 0.5", () => { + const midi = [1, 2, 3, 4, 31, 32]; + const audio = [1, 31, 32]; // window 0: 1/4 matched + const r = fitQuality(audio, midi, 60); + expect(r.windows[0]!.matchRate).toBe(0.25); + expect(r.windows[0]!.flagged).toBe(true); + expect(r.windows[1]!.flagged).toBe(false); + expect(r.matchRate).toBe(0.5); +}); + +test("window flagged when median residual > 60 ms even with full match", () => { + const midi = [1, 2, 3]; + const r = fitQuality(midi.map((t) => t + 0.065), midi, 10); + expect(r.matchRate).toBe(1); + expect(r.windows[0]!.medianResidualMs!).toBeCloseTo(65, 6); + expect(r.windows[0]!.flagged).toBe(true); +}); + +test("unsorted and duplicate MIDI onsets: duplicates count once", () => { + const r = fitQuality([2, 1], [2, 1, 1, 1], 5); + expect(r.matchRate).toBe(1); + const s = fitQuality([1], [1, 1, 1, 3], 5); + expect(s.matchRate).toBe(0.5); // unique onsets {1, 3} +});