diff --git a/src/align/auto.ts b/src/align/auto.ts index 96155a0..7f717eb 100644 --- a/src/align/auto.ts +++ b/src/align/auto.ts @@ -94,7 +94,7 @@ function costFn(A: Features, B: Features): (i: number, j: number) => number { type Path = [number, number][]; /** Align rendered frames [i0, i1] with recording frames [j0, j1] (inclusive, fine rate), coarse then banded fine. */ -function alignSegment(fine: [Features, Features], coarse: [Features, Features], i0: number, i1: number, j0: number, j1: number): Path { +function alignSegment(fine: [Features, Features], coarse: [Features, Features], i0: number, i1: number, j0: number, j1: number, bandsSec: number[]): Path { const K = COARSE_FACTOR; const ci0 = Math.round(i0 / K), ci1 = Math.max(ci0, Math.min(coarse[0].chroma.length - 1, Math.round(i1 / K))); const cj0 = Math.round(j0 / K), cj1 = Math.max(cj0, Math.min(coarse[1].chroma.length - 1, Math.round(j1 / K))); @@ -111,15 +111,40 @@ function alignSegment(fine: [Features, Features], coarse: [Features, Features], return (y + cj0) * K - j0; }; const fc = costFn(fine[0], fine[1]); - const p = bandedWithRetry((i, j) => fc(i0 + i, j0 + j), i1 - i0 + 1, j1 - j0 + 1, centre); + const p = bandedWithRetry((i, j) => fc(i0 + i, j0 + j), i1 - i0 + 1, j1 - j0 + 1, centre, bandsSec); if (p) return p.map(([i, j]) => [i0 + i, j0 + j]); - // fallback: the coarse path at fine resolution - return cp.map(([i, j]) => [Math.min(i1, (ci0 + i) * K), Math.min(j1, (cj0 + j) * K)]); + return densify(cp, ci0, cj0, i0, i1, j0, j1); +} + +/** + * Fallback: the coarse path expanded to a contiguous fine path from (i0, j0) to (i1, j1). Each coarse step becomes K + * fine steps in the same direction (so coarse diagonals stay diagonal and still yield anchors), clamped to the + * segment; any remainder at the end is closed with horizontal/vertical steps. + */ +function densify(cp: Path, ci0: number, cj0: number, i0: number, i1: number, j0: number, j1: number): Path { + const K = COARSE_FACTOR; + const out: Path = [[i0, j0]]; + let i = i0, j = j0; + // the coarse grid origin may sit up to K/2 fine frames off (i0, j0); align to it first + const step = (di: number, dj: number) => { + const ni = Math.min(i1, i + di), nj = Math.min(j1, j + dj); + if (ni !== i || nj !== j) { i = ni; j = nj; out.push([i, j]); } + }; + while (i < Math.min(i1, ci0 * K)) step(1, 0); + while (j < Math.min(j1, cj0 * K)) step(0, 1); + for (let k = 1; k < cp.length; k++) { + const di = cp[k]![0] - cp[k - 1]![0], dj = cp[k]![1] - cp[k - 1]![1]; + for (let q = 0; q < K; q++) step(di, dj); + } + while (i < i1 || j < j1) step(i < i1 ? 1 : 0, j < j1 ? 1 : 0); + return out; } /** Banded fine DTW, widening the band (±0.5 s, ±1 s, ±2 s) when the end cell is unreachable; null if all fail. */ -export function bandedWithRetry(costAt: (i: number, j: number) => number, n: number, m: number, centre: (i: number) => number): Path | null { - for (const sec of FINE_BANDS_SEC) { +export function bandedWithRetry( + costAt: (i: number, j: number) => number, n: number, m: number, centre: (i: number) => number, bandsSec = FINE_BANDS_SEC, +): Path | null { + for (const sec of bandsSec) { try { return dtw(costAt, n, m, { center: centre, width: Math.round(sec * FINE_RATE) }); } catch (e) { @@ -129,12 +154,14 @@ export function bandedWithRetry(costAt: (i: number, j: number) => number, n: num return null; } -/** Recording time for warped time `w`: w + median offset of diagonal path steps within ±1 s, or null if too few. */ -function offsetAt(diag: { w: number; off: number }[], w: number): number | null { +/** Offset (recording − warped) at warped time `w`: median over diagonal path steps within ±1 s, clipped to the + * segment [segLo, segHi] (warped seconds); null if too few points. */ +function offsetAt(diag: { w: number; off: number }[], w: number, segLo: number, segHi: number): number | null { + const from = Math.max(segLo, w - ANCHOR_HALF_WIN_SEC), to = Math.min(segHi, w + ANCHOR_HALF_WIN_SEC); let lo = 0, hi = diag.length; - while (lo < hi) { const mid = (lo + hi) >> 1; if (diag[mid]!.w < w - ANCHOR_HALF_WIN_SEC) lo = mid + 1; else hi = mid; } + while (lo < hi) { const mid = (lo + hi) >> 1; if (diag[mid]!.w < from) lo = mid + 1; else hi = mid; } const offs: number[] = []; - for (let k = lo; k < diag.length && diag[k]!.w <= w + ANCHOR_HALF_WIN_SEC; k++) offs.push(diag[k]!.off); + for (let k = lo; k < diag.length && diag[k]!.w <= to; k++) offs.push(diag[k]!.off); if (offs.length < MIN_POINTS) return null; offs.sort((a, b) => a - b); const h = offs.length >> 1; @@ -183,6 +210,7 @@ export function alignmentFit(audio: Audio, smf: Smf, roles: Record, manual: Anchor[], sf2: Uint8Array, + opts: { fineBandsSec?: number[] } = {}, // band half-widths for the fine pass; [] forces the coarse-path fallback ): { anchors: Anchor[]; fit: FitReport } { const man = makeWarp(manual).anchors.map((a) => ({ ...a, kind: "manual" as const })); const W0 = makeWarp(man); @@ -196,30 +224,31 @@ export function autoAlign( // segment boundaries: manual anchors map warped time == recording time, so both axes split at the same instants const cuts = man.map((a) => Math.round(a.audioSec * FINE_RATE)).filter((f) => f > 0 && f < Math.min(lastR, lastA)); const bounds = [...new Set(cuts)].sort((a, b) => a - b); - const path: Path = []; + // per segment: warped-time span and diagonal path steps (they carry timing; horizontal/vertical runs are stalls, + // silence or unmatched material and do not) + const segs: { lo: number; hi: number; diag: { w: number; off: number }[] }[] = []; let prev = 0; for (const b of [...bounds, -1]) { const i1 = b < 0 ? lastR : b, j1 = b < 0 ? lastA : b; - path.push(...alignSegment([fR, fA], [cR, cA], prev, i1, prev, j1)); + const path = alignSegment([fR, fA], [cR, cA], prev, i1, prev, j1, opts.fineBandsSec ?? FINE_BANDS_SEC); + const diag: { w: number; off: number }[] = []; + for (let k = 1; k < path.length; k++) { + const [i, j] = path[k]!, [pi, pj] = path[k - 1]!; + if (i === pi + 1 && j === pj + 1) diag.push({ w: i / FINE_RATE, off: (j - i) / FINE_RATE }); + } + segs.push({ lo: prev / FINE_RATE, hi: b < 0 ? Infinity : i1 / FINE_RATE, diag }); prev = b; } - // diagonal steps carry timing; horizontal/vertical runs (stalls, silence, unmatched material) do not - const diag: { w: number; off: number }[] = []; - for (let k = 1; k < path.length; k++) { - const [i, j] = path[k]!, [pi, pj] = path[k - 1]!; - if (i === pi + 1 && j === pj + 1) diag.push({ w: i / FINE_RATE, off: (j - i) / FINE_RATE }); - } - diag.sort((a, b) => a.w - b.w); - const auto: Anchor[] = []; for (const m of barTimes(smf)) { if (man.some((a) => Math.abs(a.midiSec - m) < MANUAL_CLEARANCE_SEC)) continue; const w = W0.toAudio(m); - const off = offsetAt(diag, w); + const seg = segs.find((s) => w < s.hi) ?? segs[segs.length - 1]!; + const off = offsetAt(seg.diag, w, seg.lo, seg.hi); if (off === null) continue; const a = w + off; - if (a < 0 || a > audioDur) continue; + if ((m > 0 ? a <= 0 : a < 0) || a > audioDur) continue; // no degenerate first stretch auto.push({ midiSec: m, audioSec: a, kind: "auto" }); } const anchors = compose(man, auto); diff --git a/tests/align/auto.test.ts b/tests/align/auto.test.ts index 0d12966..d2253a3 100644 --- a/tests/align/auto.test.ts +++ b/tests/align/auto.test.ts @@ -26,6 +26,9 @@ function smfFrom(truth: TruthTrack[], programs: number[], bpm: number, map: (t: const rolesOf = (truth: TruthTrack[]): Record => Object.fromEntries(truth.map((t) => [t.name, t.role])); const durOf = (a: Audio) => a.channels[0]!.length / a.sampleRate; const median = (xs: number[]) => { const s = [...xs].sort((a, b) => a - b); return s[s.length >> 1]!; }; +const p90 = (xs: number[]) => { const s = [...xs].sort((a, b) => a - b); return s[Math.floor(s.length * 0.9)]!; }; +const debug = (msg: string) => { if (process.env.ALIGN_DEBUG) console.log(msg); }; +const strictlyMonotone = (A: Anchor[]) => A.every((a, k) => k === 0 || (a.midiSec > A[k - 1]!.midiSec && a.audioSec > A[k - 1]!.audioSec)); /** Known distortion recording time -> MIDI time: +0.4 s, then bars 4–10 at ×1.06 tempo, then ≈ −0.8 s (continuous). */ function distortion(bpm: number): Warp { @@ -53,11 +56,13 @@ test("recovers a known piecewise warp: median error < 40 ms, fit match rate > 0. const { anchors, fit } = autoAlign(song.mix, smf, rolesOf(song.truth.tracks), [], SF2); const ms = performance.now() - t0; const err = recoveryErrors(smf, anchors, D.toMidi, durOf(song.mix)); - console.log(`synthetic warp: ${anchors.length} anchors, median err ${(median(err) * 1000).toFixed(1)} ms, p90 ${(err.sort((a, b) => a - b)[Math.floor(err.length * 0.9)]! * 1000).toFixed(1)} ms, fit ${fit.matchRate?.toFixed(3)} / ${fit.medianResidualMs?.toFixed(1)} ms, ${ms.toFixed(0)} ms`); + debug(`synthetic warp: ${anchors.length} anchors, median err ${(median(err) * 1000).toFixed(1)} ms, p90 ${(p90(err) * 1000).toFixed(1)} ms, fit ${fit.matchRate?.toFixed(3)} / ${fit.medianResidualMs?.toFixed(1)} ms, ${ms.toFixed(0)} ms`); expect(anchors.every((a) => a.kind === "auto")).toBe(true); expect(median(err)).toBeLessThan(0.04); + expect(p90(err)).toBeLessThan(0.08); expect(fit.matchRate!).toBeGreaterThan(0.7); expect(fit.windows.length).toBe(2); + expect(anchors.every((a) => a.audioSec > 0 || a.midiSec === 0)).toBe(true); }, BIG); test("a correct manual anchor mid-song is honoured exactly", () => { @@ -91,7 +96,7 @@ test("5-minute input aligns in < 15 s", () => { const t0 = performance.now(); const { anchors, fit } = autoAlign(mix, smf, rolesOf(song.truth.tracks), [], SF2); const sec = (performance.now() - t0) / 1000; - console.log(`5-min (${durOf(mix).toFixed(0)} s) input: ${sec.toFixed(2)} s, ${anchors.length} anchors, fit ${fit.matchRate?.toFixed(3)}`); + debug(`5-min (${durOf(mix).toFixed(0)} s) input: ${sec.toFixed(2)} s, ${anchors.length} anchors, fit ${fit.matchRate?.toFixed(3)}`); expect(durOf(mix)).toBeGreaterThan(290); expect(sec).toBeLessThan(15); expect(fit.windows.length).toBe(10); @@ -133,3 +138,34 @@ test("fit exposes a bad alignment instead of hiding it", () => { // unmapped tracks are excluded: with no scored roles there is nothing to measure expect(alignmentFit(song.mix, smf, Object.fromEntries(Object.keys(roles).map((k) => [k, null])), []).matchRate).toBeNull(); }, BIG); + +test("a deliberately wrong manual anchor (+0.3 s) is kept exactly and the result stays monotonic", () => { + const D = distortion(song.song.bpm); + const smf = smfFrom(song.truth.tracks, song.song.tracks.map((t) => t.program), song.song.bpm, D.toAudio); + const manual: Anchor = { midiSec: D.toAudio(30), audioSec: 30.3, kind: "manual" }; + const { anchors } = autoAlign(song.mix, smf, rolesOf(song.truth.tracks), [manual], SF2); + expect(anchors.filter((a) => a.kind === "manual")).toEqual([manual]); + expect(makeWarp(anchors).toAudio(manual.midiSec)).toBe(30.3); + expect(strictlyMonotone(anchors)).toBe(true); + expect(anchors.every((a) => a.audioSec > 0 || a.midiSec === 0)).toBe(true); + // away from the bad anchor (> 4 s), the autos still follow the truth + const far = anchors.filter((a) => a.kind === "auto" && Math.abs(a.audioSec - 30) > 4); + expect(far.length).toBeGreaterThan(5); + expect(median(far.map((a) => Math.abs(a.audioSec - D.toMidi(a.midiSec))))).toBeLessThan(0.04); +}, BIG); + +test("when every fine band fails, the coarse-path fallback still yields auto anchors", () => { + const D = distortion(song.song.bpm); + const smf = smfFrom(song.truth.tracks, song.song.tracks.map((t) => t.program), song.song.bpm, D.toAudio); + const manual: Anchor = { midiSec: D.toAudio(30), audioSec: 30, kind: "manual" }; + const { anchors, fit } = autoAlign(song.mix, smf, rolesOf(song.truth.tracks), [manual], SF2, { fineBandsSec: [] }); + const autos = anchors.filter((a) => a.kind === "auto"); + // both segments (before and after the manual anchor) get anchors + expect(autos.filter((a) => a.audioSec < 30).length).toBeGreaterThan(3); + expect(autos.filter((a) => a.audioSec > 30).length).toBeGreaterThan(3); + expect(strictlyMonotone(anchors)).toBe(true); + const err = recoveryErrors(smf, anchors, D.toMidi, durOf(song.mix)); + debug(`coarse fallback: ${autos.length} autos, median err ${(median(err) * 1000).toFixed(1)} ms, p90 ${(p90(err) * 1000).toFixed(1)} ms, fit ${fit.matchRate?.toFixed(3)}`); + expect(median(err)).toBeLessThan(0.06); // 10 Hz resolution: coarser than the fine pass, still hand-correctable + expect(p90(err)).toBeLessThan(0.12); +}, BIG);