lucasw89/STSheets
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(align): densify coarse fallback path; per-segment offset pooling; no degenerate first stretch

Browse Source 
The coarse-path fallback now expands each 10 Hz step to five 50 Hz steps so its
diagonals still yield bar anchors; fine band widths are injectable to test it.
Offset pooling is clipped to the current segment, and auto anchors need
audioSec > 0 unless midiSec is 0. Tests: forced fallback, wrong manual anchor,
p90 bound; debug output gated behind ALIGN_DEBUG.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Lucas Winther
2026-10-30 06:57:47 +01:00
co-authored by Claude Opus 5.5
parent 8cad85e103
commit 99a1076547
2 changed files with 89 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/align/auto.ts
+51 -22
View File
@@ -94,7 +94,7 @@ function costFn(A: Features, B: Features): (i: number, j: number) => number {
type Path = [number, number][];
/** Align rendered frames [i0, i1] with recording frames [j0, j1] (inclusive, fine rate), coarse then banded fine. */
function alignSegment(fine: [Features, Features], coarse: [Features, Features], i0: number, i1: number, j0: number, j1: number): Path {
function alignSegment(fine: [Features, Features], coarse: [Features, Features], i0: number, i1: number, j0: number, j1: number, bandsSec: number[]): Path {
const K = COARSE_FACTOR;
const ci0 = Math.round(i0 / K), ci1 = Math.max(ci0, Math.min(coarse[0].chroma.length - 1, Math.round(i1 / K)));
const cj0 = Math.round(j0 / K), cj1 = Math.max(cj0, Math.min(coarse[1].chroma.length - 1, Math.round(j1 / K)));
@@ -111,15 +111,40 @@ function alignSegment(fine: [Features, Features], coarse: [Features, Features],
return (y + cj0) * K - j0;
};
const fc = costFn(fine[0], fine[1]);
const p = bandedWithRetry((i, j) => fc(i0 + i, j0 + j), i1 - i0 + 1, j1 - j0 + 1, centre);
const p = bandedWithRetry((i, j) => fc(i0 + i, j0 + j), i1 - i0 + 1, j1 - j0 + 1, centre, bandsSec);
if (p) return p.map(([i, j]) => [i0 + i, j0 + j]);
// fallback: the coarse path at fine resolution
return cp.map(([i, j]) => [Math.min(i1, (ci0 + i) * K), Math.min(j1, (cj0 + j) * K)]);
return densify(cp, ci0, cj0, i0, i1, j0, j1);
}
/**
* Fallback: the coarse path expanded to a contiguous fine path from (i0, j0) to (i1, j1). Each coarse step becomes K
* fine steps in the same direction (so coarse diagonals stay diagonal and still yield anchors), clamped to the
* segment; any remainder at the end is closed with horizontal/vertical steps.
*/
function densify(cp: Path, ci0: number, cj0: number, i0: number, i1: number, j0: number, j1: number): Path {
const K = COARSE_FACTOR;
const out: Path = [[i0, j0]];
let i = i0, j = j0;
// the coarse grid origin may sit up to K/2 fine frames off (i0, j0); align to it first
const step = (di: number, dj: number) => {
const ni = Math.min(i1, i + di), nj = Math.min(j1, j + dj);
if (ni !== i || nj !== j) { i = ni; j = nj; out.push([i, j]); }
};
while (i < Math.min(i1, ci0 * K)) step(1, 0);
while (j < Math.min(j1, cj0 * K)) step(0, 1);
for (let k = 1; k < cp.length; k++) {
const di = cp[k]![0] - cp[k - 1]![0], dj = cp[k]![1] - cp[k - 1]![1];
for (let q = 0; q < K; q++) step(di, dj);
}
while (i < i1 || j < j1) step(i < i1 ? 1 : 0, j < j1 ? 1 : 0);
return out;
}
/** Banded fine DTW, widening the band (±0.5 s, ±1 s, ±2 s) when the end cell is unreachable; null if all fail. */
export function bandedWithRetry(costAt: (i: number, j: number) => number, n: number, m: number, centre: (i: number) => number): Path | null {
for (const sec of FINE_BANDS_SEC) {
export function bandedWithRetry(
costAt: (i: number, j: number) => number, n: number, m: number, centre: (i: number) => number, bandsSec = FINE_BANDS_SEC,
): Path | null {
for (const sec of bandsSec) {
try {
return dtw(costAt, n, m, { center: centre, width: Math.round(sec * FINE_RATE) });
} catch (e) {
@@ -129,12 +154,14 @@ export function bandedWithRetry(costAt: (i: number, j: number) => number, n: num
return null;
}
/** Recording time for warped time `w`: w + median offset of diagonal path steps within ±1 s, or null if too few. */
function offsetAt(diag: { w: number; off: number }[], w: number): number | null {
/** Offset (recording − warped) at warped time `w`: median over diagonal path steps within ±1 s, clipped to the
* segment [segLo, segHi] (warped seconds); null if too few points. */
function offsetAt(diag: { w: number; off: number }[], w: number, segLo: number, segHi: number): number | null {
const from = Math.max(segLo, w - ANCHOR_HALF_WIN_SEC), to = Math.min(segHi, w + ANCHOR_HALF_WIN_SEC);
let lo = 0, hi = diag.length;
while (lo < hi) { const mid = (lo + hi) >> 1; if (diag[mid]!.w < w - ANCHOR_HALF_WIN_SEC) lo = mid + 1; else hi = mid; }
while (lo < hi) { const mid = (lo + hi) >> 1; if (diag[mid]!.w < from) lo = mid + 1; else hi = mid; }
const offs: number[] = [];
for (let k = lo; k < diag.length && diag[k]!.w <= w + ANCHOR_HALF_WIN_SEC; k++) offs.push(diag[k]!.off);
for (let k = lo; k < diag.length && diag[k]!.w <= to; k++) offs.push(diag[k]!.off);
if (offs.length < MIN_POINTS) return null;
offs.sort((a, b) => a - b);
const h = offs.length >> 1;
@@ -183,6 +210,7 @@ export function alignmentFit(audio: Audio, smf: Smf, roles: Record<string, Role
*/
export function autoAlign(
audio: Audio, smf: Smf, roles: Record<string, Role | null>, manual: Anchor[], sf2: Uint8Array,
opts: { fineBandsSec?: number[] } = {}, // band half-widths for the fine pass; [] forces the coarse-path fallback
): { anchors: Anchor[]; fit: FitReport } {
const man = makeWarp(manual).anchors.map((a) => ({ ...a, kind: "manual" as const }));
const W0 = makeWarp(man);
@@ -196,30 +224,31 @@ export function autoAlign(
// segment boundaries: manual anchors map warped time == recording time, so both axes split at the same instants
const cuts = man.map((a) => Math.round(a.audioSec * FINE_RATE)).filter((f) => f > 0 && f < Math.min(lastR, lastA));
const bounds = [...new Set(cuts)].sort((a, b) => a - b);
const path: Path = [];
// per segment: warped-time span and diagonal path steps (they carry timing; horizontal/vertical runs are stalls,
// silence or unmatched material and do not)
const segs: { lo: number; hi: number; diag: { w: number; off: number }[] }[] = [];
let prev = 0;
for (const b of [...bounds, -1]) {
const i1 = b < 0 ? lastR : b, j1 = b < 0 ? lastA : b;
path.push(...alignSegment([fR, fA], [cR, cA], prev, i1, prev, j1));
const path = alignSegment([fR, fA], [cR, cA], prev, i1, prev, j1, opts.fineBandsSec ?? FINE_BANDS_SEC);
const diag: { w: number; off: number }[] = [];
for (let k = 1; k < path.length; k++) {
const [i, j] = path[k]!, [pi, pj] = path[k - 1]!;
if (i === pi + 1 && j === pj + 1) diag.push({ w: i / FINE_RATE, off: (j - i) / FINE_RATE });
}
segs.push({ lo: prev / FINE_RATE, hi: b < 0 ? Infinity : i1 / FINE_RATE, diag });
prev = b;
}
// diagonal steps carry timing; horizontal/vertical runs (stalls, silence, unmatched material) do not
const diag: { w: number; off: number }[] = [];
for (let k = 1; k < path.length; k++) {
const [i, j] = path[k]!, [pi, pj] = path[k - 1]!;
if (i === pi + 1 && j === pj + 1) diag.push({ w: i / FINE_RATE, off: (j - i) / FINE_RATE });
}
diag.sort((a, b) => a.w - b.w);
const auto: Anchor[] = [];
for (const m of barTimes(smf)) {
if (man.some((a) => Math.abs(a.midiSec - m) < MANUAL_CLEARANCE_SEC)) continue;
const w = W0.toAudio(m);
const off = offsetAt(diag, w);
const seg = segs.find((s) => w < s.hi) ?? segs[segs.length - 1]!;
const off = offsetAt(seg.diag, w, seg.lo, seg.hi);
if (off === null) continue;
const a = w + off;
if (a < 0 || a > audioDur) continue;
if ((m > 0 ? a <= 0 : a < 0) || a > audioDur) continue; // no degenerate first stretch
auto.push({ midiSec: m, audioSec: a, kind: "auto" });
}
const anchors = compose(man, auto);