fix(align): densify coarse fallback path; per-segment offset pooling; no degenerate first stretch
The coarse-path fallback now expands each 10 Hz step to five 50 Hz steps so its diagonals still yield bar anchors; fine band widths are injectable to test it. Offset pooling is clipped to the current segment, and auto anchors need audioSec > 0 unless midiSec is 0. Tests: forced fallback, wrong manual anchor, p90 bound; debug output gated behind ALIGN_DEBUG. Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
Lucas Winther
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
8cad85e103
commit
99a1076547
+51
-22
@@ -94,7 +94,7 @@ function costFn(A: Features, B: Features): (i: number, j: number) => number {
|
||||
type Path = [number, number][];
|
||||
|
||||
/** Align rendered frames [i0, i1] with recording frames [j0, j1] (inclusive, fine rate), coarse then banded fine. */
|
||||
function alignSegment(fine: [Features, Features], coarse: [Features, Features], i0: number, i1: number, j0: number, j1: number): Path {
|
||||
function alignSegment(fine: [Features, Features], coarse: [Features, Features], i0: number, i1: number, j0: number, j1: number, bandsSec: number[]): Path {
|
||||
const K = COARSE_FACTOR;
|
||||
const ci0 = Math.round(i0 / K), ci1 = Math.max(ci0, Math.min(coarse[0].chroma.length - 1, Math.round(i1 / K)));
|
||||
const cj0 = Math.round(j0 / K), cj1 = Math.max(cj0, Math.min(coarse[1].chroma.length - 1, Math.round(j1 / K)));
|
||||
@@ -111,15 +111,40 @@ function alignSegment(fine: [Features, Features], coarse: [Features, Features],
|
||||
return (y + cj0) * K - j0;
|
||||
};
|
||||
const fc = costFn(fine[0], fine[1]);
|
||||
const p = bandedWithRetry((i, j) => fc(i0 + i, j0 + j), i1 - i0 + 1, j1 - j0 + 1, centre);
|
||||
const p = bandedWithRetry((i, j) => fc(i0 + i, j0 + j), i1 - i0 + 1, j1 - j0 + 1, centre, bandsSec);
|
||||
if (p) return p.map(([i, j]) => [i0 + i, j0 + j]);
|
||||
// fallback: the coarse path at fine resolution
|
||||
return cp.map(([i, j]) => [Math.min(i1, (ci0 + i) * K), Math.min(j1, (cj0 + j) * K)]);
|
||||
return densify(cp, ci0, cj0, i0, i1, j0, j1);
|
||||
}
|
||||
|
||||
/**
|
||||
* Fallback: the coarse path expanded to a contiguous fine path from (i0, j0) to (i1, j1). Each coarse step becomes K
|
||||
* fine steps in the same direction (so coarse diagonals stay diagonal and still yield anchors), clamped to the
|
||||
* segment; any remainder at the end is closed with horizontal/vertical steps.
|
||||
*/
|
||||
function densify(cp: Path, ci0: number, cj0: number, i0: number, i1: number, j0: number, j1: number): Path {
|
||||
const K = COARSE_FACTOR;
|
||||
const out: Path = [[i0, j0]];
|
||||
let i = i0, j = j0;
|
||||
// the coarse grid origin may sit up to K/2 fine frames off (i0, j0); align to it first
|
||||
const step = (di: number, dj: number) => {
|
||||
const ni = Math.min(i1, i + di), nj = Math.min(j1, j + dj);
|
||||
if (ni !== i || nj !== j) { i = ni; j = nj; out.push([i, j]); }
|
||||
};
|
||||
while (i < Math.min(i1, ci0 * K)) step(1, 0);
|
||||
while (j < Math.min(j1, cj0 * K)) step(0, 1);
|
||||
for (let k = 1; k < cp.length; k++) {
|
||||
const di = cp[k]![0] - cp[k - 1]![0], dj = cp[k]![1] - cp[k - 1]![1];
|
||||
for (let q = 0; q < K; q++) step(di, dj);
|
||||
}
|
||||
while (i < i1 || j < j1) step(i < i1 ? 1 : 0, j < j1 ? 1 : 0);
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Banded fine DTW, widening the band (±0.5 s, ±1 s, ±2 s) when the end cell is unreachable; null if all fail. */
|
||||
export function bandedWithRetry(costAt: (i: number, j: number) => number, n: number, m: number, centre: (i: number) => number): Path | null {
|
||||
for (const sec of FINE_BANDS_SEC) {
|
||||
export function bandedWithRetry(
|
||||
costAt: (i: number, j: number) => number, n: number, m: number, centre: (i: number) => number, bandsSec = FINE_BANDS_SEC,
|
||||
): Path | null {
|
||||
for (const sec of bandsSec) {
|
||||
try {
|
||||
return dtw(costAt, n, m, { center: centre, width: Math.round(sec * FINE_RATE) });
|
||||
} catch (e) {
|
||||
@@ -129,12 +154,14 @@ export function bandedWithRetry(costAt: (i: number, j: number) => number, n: num
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Recording time for warped time `w`: w + median offset of diagonal path steps within ±1 s, or null if too few. */
|
||||
function offsetAt(diag: { w: number; off: number }[], w: number): number | null {
|
||||
/** Offset (recording − warped) at warped time `w`: median over diagonal path steps within ±1 s, clipped to the
|
||||
* segment [segLo, segHi] (warped seconds); null if too few points. */
|
||||
function offsetAt(diag: { w: number; off: number }[], w: number, segLo: number, segHi: number): number | null {
|
||||
const from = Math.max(segLo, w - ANCHOR_HALF_WIN_SEC), to = Math.min(segHi, w + ANCHOR_HALF_WIN_SEC);
|
||||
let lo = 0, hi = diag.length;
|
||||
while (lo < hi) { const mid = (lo + hi) >> 1; if (diag[mid]!.w < w - ANCHOR_HALF_WIN_SEC) lo = mid + 1; else hi = mid; }
|
||||
while (lo < hi) { const mid = (lo + hi) >> 1; if (diag[mid]!.w < from) lo = mid + 1; else hi = mid; }
|
||||
const offs: number[] = [];
|
||||
for (let k = lo; k < diag.length && diag[k]!.w <= w + ANCHOR_HALF_WIN_SEC; k++) offs.push(diag[k]!.off);
|
||||
for (let k = lo; k < diag.length && diag[k]!.w <= to; k++) offs.push(diag[k]!.off);
|
||||
if (offs.length < MIN_POINTS) return null;
|
||||
offs.sort((a, b) => a - b);
|
||||
const h = offs.length >> 1;
|
||||
@@ -183,6 +210,7 @@ export function alignmentFit(audio: Audio, smf: Smf, roles: Record<string, Role
|
||||
*/
|
||||
export function autoAlign(
|
||||
audio: Audio, smf: Smf, roles: Record<string, Role | null>, manual: Anchor[], sf2: Uint8Array,
|
||||
opts: { fineBandsSec?: number[] } = {}, // band half-widths for the fine pass; [] forces the coarse-path fallback
|
||||
): { anchors: Anchor[]; fit: FitReport } {
|
||||
const man = makeWarp(manual).anchors.map((a) => ({ ...a, kind: "manual" as const }));
|
||||
const W0 = makeWarp(man);
|
||||
@@ -196,30 +224,31 @@ export function autoAlign(
|
||||
// segment boundaries: manual anchors map warped time == recording time, so both axes split at the same instants
|
||||
const cuts = man.map((a) => Math.round(a.audioSec * FINE_RATE)).filter((f) => f > 0 && f < Math.min(lastR, lastA));
|
||||
const bounds = [...new Set(cuts)].sort((a, b) => a - b);
|
||||
const path: Path = [];
|
||||
// per segment: warped-time span and diagonal path steps (they carry timing; horizontal/vertical runs are stalls,
|
||||
// silence or unmatched material and do not)
|
||||
const segs: { lo: number; hi: number; diag: { w: number; off: number }[] }[] = [];
|
||||
let prev = 0;
|
||||
for (const b of [...bounds, -1]) {
|
||||
const i1 = b < 0 ? lastR : b, j1 = b < 0 ? lastA : b;
|
||||
path.push(...alignSegment([fR, fA], [cR, cA], prev, i1, prev, j1));
|
||||
const path = alignSegment([fR, fA], [cR, cA], prev, i1, prev, j1, opts.fineBandsSec ?? FINE_BANDS_SEC);
|
||||
const diag: { w: number; off: number }[] = [];
|
||||
for (let k = 1; k < path.length; k++) {
|
||||
const [i, j] = path[k]!, [pi, pj] = path[k - 1]!;
|
||||
if (i === pi + 1 && j === pj + 1) diag.push({ w: i / FINE_RATE, off: (j - i) / FINE_RATE });
|
||||
}
|
||||
segs.push({ lo: prev / FINE_RATE, hi: b < 0 ? Infinity : i1 / FINE_RATE, diag });
|
||||
prev = b;
|
||||
}
|
||||
|
||||
// diagonal steps carry timing; horizontal/vertical runs (stalls, silence, unmatched material) do not
|
||||
const diag: { w: number; off: number }[] = [];
|
||||
for (let k = 1; k < path.length; k++) {
|
||||
const [i, j] = path[k]!, [pi, pj] = path[k - 1]!;
|
||||
if (i === pi + 1 && j === pj + 1) diag.push({ w: i / FINE_RATE, off: (j - i) / FINE_RATE });
|
||||
}
|
||||
diag.sort((a, b) => a.w - b.w);
|
||||
|
||||
const auto: Anchor[] = [];
|
||||
for (const m of barTimes(smf)) {
|
||||
if (man.some((a) => Math.abs(a.midiSec - m) < MANUAL_CLEARANCE_SEC)) continue;
|
||||
const w = W0.toAudio(m);
|
||||
const off = offsetAt(diag, w);
|
||||
const seg = segs.find((s) => w < s.hi) ?? segs[segs.length - 1]!;
|
||||
const off = offsetAt(seg.diag, w, seg.lo, seg.hi);
|
||||
if (off === null) continue;
|
||||
const a = w + off;
|
||||
if (a < 0 || a > audioDur) continue;
|
||||
if ((m > 0 ? a <= 0 : a < 0) || a > audioDur) continue; // no degenerate first stretch
|
||||
auto.push({ midiSec: m, audioSec: a, kind: "auto" });
|
||||
}
|
||||
const anchors = compose(man, auto);
|
||||
|
||||
@@ -26,6 +26,9 @@ function smfFrom(truth: TruthTrack[], programs: number[], bpm: number, map: (t:
|
||||
const rolesOf = (truth: TruthTrack[]): Record<string, Role | null> => Object.fromEntries(truth.map((t) => [t.name, t.role]));
|
||||
const durOf = (a: Audio) => a.channels[0]!.length / a.sampleRate;
|
||||
const median = (xs: number[]) => { const s = [...xs].sort((a, b) => a - b); return s[s.length >> 1]!; };
|
||||
const p90 = (xs: number[]) => { const s = [...xs].sort((a, b) => a - b); return s[Math.floor(s.length * 0.9)]!; };
|
||||
const debug = (msg: string) => { if (process.env.ALIGN_DEBUG) console.log(msg); };
|
||||
const strictlyMonotone = (A: Anchor[]) => A.every((a, k) => k === 0 || (a.midiSec > A[k - 1]!.midiSec && a.audioSec > A[k - 1]!.audioSec));
|
||||
|
||||
/** Known distortion recording time -> MIDI time: +0.4 s, then bars 4–10 at ×1.06 tempo, then ≈ −0.8 s (continuous). */
|
||||
function distortion(bpm: number): Warp {
|
||||
@@ -53,11 +56,13 @@ test("recovers a known piecewise warp: median error < 40 ms, fit match rate > 0.
|
||||
const { anchors, fit } = autoAlign(song.mix, smf, rolesOf(song.truth.tracks), [], SF2);
|
||||
const ms = performance.now() - t0;
|
||||
const err = recoveryErrors(smf, anchors, D.toMidi, durOf(song.mix));
|
||||
console.log(`synthetic warp: ${anchors.length} anchors, median err ${(median(err) * 1000).toFixed(1)} ms, p90 ${(err.sort((a, b) => a - b)[Math.floor(err.length * 0.9)]! * 1000).toFixed(1)} ms, fit ${fit.matchRate?.toFixed(3)} / ${fit.medianResidualMs?.toFixed(1)} ms, ${ms.toFixed(0)} ms`);
|
||||
debug(`synthetic warp: ${anchors.length} anchors, median err ${(median(err) * 1000).toFixed(1)} ms, p90 ${(p90(err) * 1000).toFixed(1)} ms, fit ${fit.matchRate?.toFixed(3)} / ${fit.medianResidualMs?.toFixed(1)} ms, ${ms.toFixed(0)} ms`);
|
||||
expect(anchors.every((a) => a.kind === "auto")).toBe(true);
|
||||
expect(median(err)).toBeLessThan(0.04);
|
||||
expect(p90(err)).toBeLessThan(0.08);
|
||||
expect(fit.matchRate!).toBeGreaterThan(0.7);
|
||||
expect(fit.windows.length).toBe(2);
|
||||
expect(anchors.every((a) => a.audioSec > 0 || a.midiSec === 0)).toBe(true);
|
||||
}, BIG);
|
||||
|
||||
test("a correct manual anchor mid-song is honoured exactly", () => {
|
||||
@@ -91,7 +96,7 @@ test("5-minute input aligns in < 15 s", () => {
|
||||
const t0 = performance.now();
|
||||
const { anchors, fit } = autoAlign(mix, smf, rolesOf(song.truth.tracks), [], SF2);
|
||||
const sec = (performance.now() - t0) / 1000;
|
||||
console.log(`5-min (${durOf(mix).toFixed(0)} s) input: ${sec.toFixed(2)} s, ${anchors.length} anchors, fit ${fit.matchRate?.toFixed(3)}`);
|
||||
debug(`5-min (${durOf(mix).toFixed(0)} s) input: ${sec.toFixed(2)} s, ${anchors.length} anchors, fit ${fit.matchRate?.toFixed(3)}`);
|
||||
expect(durOf(mix)).toBeGreaterThan(290);
|
||||
expect(sec).toBeLessThan(15);
|
||||
expect(fit.windows.length).toBe(10);
|
||||
@@ -133,3 +138,34 @@ test("fit exposes a bad alignment instead of hiding it", () => {
|
||||
// unmapped tracks are excluded: with no scored roles there is nothing to measure
|
||||
expect(alignmentFit(song.mix, smf, Object.fromEntries(Object.keys(roles).map((k) => [k, null])), []).matchRate).toBeNull();
|
||||
}, BIG);
|
||||
|
||||
test("a deliberately wrong manual anchor (+0.3 s) is kept exactly and the result stays monotonic", () => {
|
||||
const D = distortion(song.song.bpm);
|
||||
const smf = smfFrom(song.truth.tracks, song.song.tracks.map((t) => t.program), song.song.bpm, D.toAudio);
|
||||
const manual: Anchor = { midiSec: D.toAudio(30), audioSec: 30.3, kind: "manual" };
|
||||
const { anchors } = autoAlign(song.mix, smf, rolesOf(song.truth.tracks), [manual], SF2);
|
||||
expect(anchors.filter((a) => a.kind === "manual")).toEqual([manual]);
|
||||
expect(makeWarp(anchors).toAudio(manual.midiSec)).toBe(30.3);
|
||||
expect(strictlyMonotone(anchors)).toBe(true);
|
||||
expect(anchors.every((a) => a.audioSec > 0 || a.midiSec === 0)).toBe(true);
|
||||
// away from the bad anchor (> 4 s), the autos still follow the truth
|
||||
const far = anchors.filter((a) => a.kind === "auto" && Math.abs(a.audioSec - 30) > 4);
|
||||
expect(far.length).toBeGreaterThan(5);
|
||||
expect(median(far.map((a) => Math.abs(a.audioSec - D.toMidi(a.midiSec))))).toBeLessThan(0.04);
|
||||
}, BIG);
|
||||
|
||||
test("when every fine band fails, the coarse-path fallback still yields auto anchors", () => {
|
||||
const D = distortion(song.song.bpm);
|
||||
const smf = smfFrom(song.truth.tracks, song.song.tracks.map((t) => t.program), song.song.bpm, D.toAudio);
|
||||
const manual: Anchor = { midiSec: D.toAudio(30), audioSec: 30, kind: "manual" };
|
||||
const { anchors, fit } = autoAlign(song.mix, smf, rolesOf(song.truth.tracks), [manual], SF2, { fineBandsSec: [] });
|
||||
const autos = anchors.filter((a) => a.kind === "auto");
|
||||
// both segments (before and after the manual anchor) get anchors
|
||||
expect(autos.filter((a) => a.audioSec < 30).length).toBeGreaterThan(3);
|
||||
expect(autos.filter((a) => a.audioSec > 30).length).toBeGreaterThan(3);
|
||||
expect(strictlyMonotone(anchors)).toBe(true);
|
||||
const err = recoveryErrors(smf, anchors, D.toMidi, durOf(song.mix));
|
||||
debug(`coarse fallback: ${autos.length} autos, median err ${(median(err) * 1000).toFixed(1)} ms, p90 ${(p90(err) * 1000).toFixed(1)} ms, fit ${fit.matchRate?.toFixed(3)}`);
|
||||
expect(median(err)).toBeLessThan(0.06); // 10 Hz resolution: coarser than the fine pass, still hand-correctable
|
||||
expect(p90(err)).toBeLessThan(0.12);
|
||||
}, BIG);
|
||||
|
||||
Reference in New Issue
Block a user