feat(align): automatic alignment and fit quality

Coarse (10 Hz) DTW per stretch between manual anchors, refined at 50 Hz in a
band that widens 0.5 -> 1 -> 2 s on unreachable end cells (coarse path as last
resort); one auto anchor per bar from median offsets of diagonal path steps.
Fit quality from raw-audio onset peaks vs warped scored-role MIDI onsets
(+-70 ms, 30 s windows, flagged on rate < 0.5 or median > 60 ms).
trackKeys moves to src/align/roles.ts (pure) and is re-exported by manifest.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

src/align/auto.ts

@@ -0,0 +1,228 @@
+// Automatic MIDI -> recording alignment (coarse-to-fine DTW on chroma + onset features) and its fit quality.
+// Pure: no I/O, no Bun/process/fs. The SoundFont bytes are passed in.
+import type { Audio, Role } from "../core/types.ts";
+import type { Smf } from "./smf.ts";
+import { makeWarp, type Anchor, type Warp } from "./warp.ts";
+import { renderMidi } from "./render.ts";
+import { features, pickOnsets, type Features } from "./features.ts";
+import { dtw } from "./dtw.ts";
+import { trackKeys } from "./roles.ts";
+import { fitQuality, type FitReport } from "./fit.ts";
+
+const FINE_RATE = 50;
+const COARSE_FACTOR = 5; // coarse pass at 10 Hz
+const LAMBDA = 1; // weight of the onset-strength term in the frame cost
+const FINE_BANDS_SEC = [0.5, 1, 2]; // fine pass band half-widths tried in turn before falling back to the coarse path
+const ANCHOR_HALF_WIN_SEC = 1; // path points within ±1 s (warped time) of a bar estimate its offset
+const MIN_POINTS = 10; // diagonal path points required to place a bar anchor
+const MANUAL_CLEARANCE_SEC = 1; // no auto anchor within 1 s (MIDI time) of a manual anchor
+const ZERO = 1e-9;
+
+/** Bar start times (MIDI seconds) from the tempo map and time signatures (4/4 when none), up to durationSec. */
+export function barTimes(smf: Smf): number[] {
+  const tempos = smf.tempoMap.length ? [...smf.tempoMap].sort((a, b) => a.sec - b.sec) : [{ sec: 0, bpm: 120 }];
+  const sigs = [...smf.timeSigs].sort((a, b) => a.sec - b.sec);
+  const bpmAt = (t: number) => { let b = tempos[0]!.bpm; for (const p of tempos) if (p.sec <= t + ZERO) b = p.bpm; return b; };
+  // time after advancing `q` quarter notes from `t`, honouring tempo changes on the way
+  const advance = (t: number, q: number): number => {
+    while (q > ZERO) {
+      const spq = 60 / bpmAt(t);
+      const next = tempos.find((p) => p.sec > t + ZERO);
+      const span = next ? (next.sec - t) / spq : Infinity;
+      if (q <= span) return t + q * spq;
+      q -= span; t = next!.sec;
+    }
+    return t;
+  };
+  const out: number[] = [];
+  let t = 0;
+  while (t < smf.durationSec - ZERO && out.length < 100000) {
+    out.push(t);
+    let num = 4, den = 4;
+    for (const s of sigs) if (s.sec <= t + ZERO) { num = s.num; den = s.den; }
+    let next = advance(t, (num * 4) / den);
+    const change = sigs.find((s) => s.sec > t + ZERO && s.sec < next - ZERO); // signature change mid-bar: restart there
+    if (change) next = change.sec;
+    if (!(next > t)) break;
+    t = next;
+  }
+  return out;
+}
+
+/** Aggregate 50 Hz features by `k` frames (centred): chroma energy summed and re-normalised, onset max, re-z-scored. */
+function downsample(f: Features, k: number): Features {
+  const n = Math.max(1, Math.floor((f.chroma.length - 1) / k) + 1);
+  const chroma: Float32Array[] = [], onset = new Float32Array(n);
+  for (let o = 0; o < n; o++) {
+    const c = new Float32Array(12);
+    let fl = -Infinity;
+    for (let t = Math.max(0, o * k - (k >> 1)); t <= Math.min(f.chroma.length - 1, o * k + (k >> 1)); t++) {
+      const s = f.chroma[t]!;
+      for (let q = 0; q < 12; q++) c[q]! += s[q]! * s[q]!;
+      fl = Math.max(fl, f.onset[t]!);
+    }
+    let norm = 0;
+    for (let q = 0; q < 12; q++) { c[q] = Math.sqrt(c[q]!); norm += c[q]! * c[q]!; }
+    norm = Math.sqrt(norm);
+    if (norm > 1e-6) for (let q = 0; q < 12; q++) c[q]! /= norm; else c.fill(0);
+    chroma.push(c); onset[o] = fl;
+  }
+  let mean = 0; for (const v of onset) mean += v; mean /= n;
+  let v2 = 0; for (const v of onset) v2 += (v - mean) ** 2;
+  const sd = Math.sqrt(v2 / n) || 1;
+  for (let o = 0; o < n; o++) onset[o] = (onset[o]! - mean) / sd;
+  return { rate: f.rate / k, onset, chroma };
+}
+
+/** Frame cost (1 − cos) + λ|Δonset|. Two silent (all-zero chroma) frames match perfectly; silence vs sound costs 1. */
+function costFn(A: Features, B: Features): (i: number, j: number) => number {
+  const flat = (f: Features) => { const x = new Float32Array(f.chroma.length * 12); f.chroma.forEach((c, i) => x.set(c, i * 12)); return x; };
+  const zero = (f: Features) => Uint8Array.from(f.chroma, (c) => (c.every((v) => v === 0) ? 1 : 0));
+  const ca = flat(A), cb = flat(B), za = zero(A), zb = zero(B), oa = A.onset, ob = B.onset;
+  return (i, j) => {
+    let c: number;
+    if (za[i] || zb[j]) c = za[i] && zb[j] ? 0 : 1;
+    else {
+      let d = 0;
+      for (let q = 0, p = i * 12, r = j * 12; q < 12; q++) d += ca[p + q]! * cb[r + q]!;
+      c = 1 - d;
+    }
+    return c + LAMBDA * Math.abs(oa[i]! - ob[j]!);
+  };
+}
+
+type Path = [number, number][];
+
+/** Align rendered frames [i0, i1] with recording frames [j0, j1] (inclusive, fine rate), coarse then banded fine. */
+function alignSegment(fine: [Features, Features], coarse: [Features, Features], i0: number, i1: number, j0: number, j1: number): Path {
+  const K = COARSE_FACTOR;
+  const ci0 = Math.round(i0 / K), ci1 = Math.max(ci0, Math.min(coarse[0].chroma.length - 1, Math.round(i1 / K)));
+  const cj0 = Math.round(j0 / K), cj1 = Math.max(cj0, Math.min(coarse[1].chroma.length - 1, Math.round(j1 / K)));
+  const cc = costFn(coarse[0], coarse[1]);
+  const cp = dtw((i, j) => cc(ci0 + i, cj0 + j), ci1 - ci0 + 1, cj1 - cj0 + 1);
+  // coarse map: mean recording frame per rendered frame, in segment-relative coarse units
+  const sum = new Float64Array(ci1 - ci0 + 1), cnt = new Float64Array(ci1 - ci0 + 1);
+  for (const [i, j] of cp) { sum[i]! += j; cnt[i]! += 1; }
+  const cmap = Array.from(sum, (s, i) => s / cnt[i]!);
+  const centre = (i: number): number => { // fine row (segment-relative) -> fine column (segment-relative)
+    const x = Math.min(cmap.length - 1, Math.max(0, (i0 + i) / K - ci0));
+    const a = Math.floor(x), b = Math.min(cmap.length - 1, a + 1);
+    const y = cmap[a]! + (x - a) * (cmap[b]! - cmap[a]!);
+    return (y + cj0) * K - j0;
+  };
+  const fc = costFn(fine[0], fine[1]);
+  const p = bandedWithRetry((i, j) => fc(i0 + i, j0 + j), i1 - i0 + 1, j1 - j0 + 1, centre);
+  if (p) return p.map(([i, j]) => [i0 + i, j0 + j]);
+  // fallback: the coarse path at fine resolution
+  return cp.map(([i, j]) => [Math.min(i1, (ci0 + i) * K), Math.min(j1, (cj0 + j) * K)]);
+}
+
+/** Banded fine DTW, widening the band (±0.5 s, ±1 s, ±2 s) when the end cell is unreachable; null if all fail. */
+export function bandedWithRetry(costAt: (i: number, j: number) => number, n: number, m: number, centre: (i: number) => number): Path | null {
+  for (const sec of FINE_BANDS_SEC) {
+    try {
+      return dtw(costAt, n, m, { center: centre, width: Math.round(sec * FINE_RATE) });
+    } catch (e) {
+      if (!(e instanceof Error) || !/unreachable/.test(e.message)) throw e;
+    }
+  }
+  return null;
+}
+
+/** Recording time for warped time `w`: w + median offset of diagonal path steps within ±1 s, or null if too few. */
+function offsetAt(diag: { w: number; off: number }[], w: number): number | null {
+  let lo = 0, hi = diag.length;
+  while (lo < hi) { const mid = (lo + hi) >> 1; if (diag[mid]!.w < w - ANCHOR_HALF_WIN_SEC) lo = mid + 1; else hi = mid; }
+  const offs: number[] = [];
+  for (let k = lo; k < diag.length && diag[k]!.w <= w + ANCHOR_HALF_WIN_SEC; k++) offs.push(diag[k]!.off);
+  if (offs.length < MIN_POINTS) return null;
+  offs.sort((a, b) => a - b);
+  const h = offs.length >> 1;
+  return offs.length % 2 ? offs[h]! : (offs[h - 1]! + offs[h]!) / 2;
+}
+
+/** Keep manual anchors; keep an auto anchor only if it is strictly monotone against its kept neighbours. */
+function compose(manual: Anchor[], auto: Anchor[]): Anchor[] {
+  const all = [...manual, ...auto].sort((a, b) => a.midiSec - b.midiSec || (a.kind === "manual" ? -1 : 1));
+  const out: Anchor[] = [];
+  for (let k = 0; k < all.length; k++) {
+    const a = all[k]!;
+    if (a.kind === "manual") {
+      while (out.length && out[out.length - 1]!.kind === "auto" && !(out[out.length - 1]!.audioSec < a.audioSec && out[out.length - 1]!.midiSec < a.midiSec)) out.pop();
+      out.push(a); continue;
+    }
+    const prev = out[out.length - 1];
+    if (prev && !(a.midiSec > prev.midiSec && a.audioSec > prev.audioSec)) continue;
+    const nextManual = all.slice(k + 1).find((x) => x.kind === "manual");
+    if (nextManual && !(a.audioSec < nextManual.audioSec)) continue;
+    out.push(a);
+  }
+  return out;
+}
+
+const durationOf = (a: Audio) => (a.channels[0]?.length ?? 0) / a.sampleRate;
+
+/** Fit of `warp` against recording onsets: warped onsets of scored-role notes (roles keyed by `trackKeys`). */
+function fitFor(recOnsets: number[], smf: Smf, roles: Record<string, Role | null>, warp: Warp, durSec: number): FitReport {
+  const keys = trackKeys(smf.tracks);
+  const midi = smf.tracks.flatMap((t, i) => (roles[keys[i]!] ? t.notes.map((n) => warp.toAudio(n.onset)) : []));
+  return fitQuality(recOnsets, midi, durSec);
+}
+
+/** Fit quality of any alignment (e.g. a hand-made one): onset-flux peaks of the raw recording vs the warped MIDI. */
+export function alignmentFit(audio: Audio, smf: Smf, roles: Record<string, Role | null>, anchors: Anchor[]): FitReport {
+  const onsets = pickOnsets(features(audio, FINE_RATE).onset, FINE_RATE);
+  return fitFor(onsets, smf, roles, makeWarp(anchors), durationOf(audio));
+}
+
+/**
+ * Align `smf` to `audio`, honouring `manual` anchors exactly. The MIDI is rendered under the manual warp, then each
+ * stretch between consecutive manual anchors (plus start/end) is aligned by a 10 Hz DTW refined at 50 Hz in a band
+ * around it. One auto anchor is placed per bar (skipping bars within 1 s of a manual anchor). `roles` is keyed by
+ * `trackKeys`; tracks without a role are rendered but excluded from the fit.
+ */
+export function autoAlign(
+  audio: Audio, smf: Smf, roles: Record<string, Role | null>, manual: Anchor[], sf2: Uint8Array,
+): { anchors: Anchor[]; fit: FitReport } {
+  const man = makeWarp(manual).anchors.map((a) => ({ ...a, kind: "manual" as const }));
+  const W0 = makeWarp(man);
+  const audioDur = durationOf(audio);
+  const renderDur = Math.max(audioDur, W0.toAudio(smf.durationSec));
+  const rendered = renderMidi(smf.tracks, W0, sf2, { mono: true, durationSec: renderDur });
+  const fR = features(rendered, FINE_RATE), fA = features(audio, FINE_RATE);
+  const cR = downsample(fR, COARSE_FACTOR), cA = downsample(fA, COARSE_FACTOR);
+  const lastR = fR.chroma.length - 1, lastA = fA.chroma.length - 1;
+
+  // segment boundaries: manual anchors map warped time == recording time, so both axes split at the same instants
+  const cuts = man.map((a) => Math.round(a.audioSec * FINE_RATE)).filter((f) => f > 0 && f < Math.min(lastR, lastA));
+  const bounds = [...new Set(cuts)].sort((a, b) => a - b);
+  const path: Path = [];
+  let prev = 0;
+  for (const b of [...bounds, -1]) {
+    const i1 = b < 0 ? lastR : b, j1 = b < 0 ? lastA : b;
+    path.push(...alignSegment([fR, fA], [cR, cA], prev, i1, prev, j1));
+    prev = b;
+  }
+
+  // diagonal steps carry timing; horizontal/vertical runs (stalls, silence, unmatched material) do not
+  const diag: { w: number; off: number }[] = [];
+  for (let k = 1; k < path.length; k++) {
+    const [i, j] = path[k]!, [pi, pj] = path[k - 1]!;
+    if (i === pi + 1 && j === pj + 1) diag.push({ w: i / FINE_RATE, off: (j - i) / FINE_RATE });
+  }
+  diag.sort((a, b) => a.w - b.w);
+
+  const auto: Anchor[] = [];
+  for (const m of barTimes(smf)) {
+    if (man.some((a) => Math.abs(a.midiSec - m) < MANUAL_CLEARANCE_SEC)) continue;
+    const w = W0.toAudio(m);
+    const off = offsetAt(diag, w);
+    if (off === null) continue;
+    const a = w + off;
+    if (a < 0 || a > audioDur) continue;
+    auto.push({ midiSec: m, audioSec: a, kind: "auto" });
+  }
+  const anchors = compose(man, auto);
+  const fit = fitFor(pickOnsets(fA.onset, FINE_RATE), smf, roles, makeWarp(anchors), audioDur);
+  return { anchors, fit };
+}

src/align/fit.ts

@@ -0,0 +1,57 @@
+// Fit quality of an alignment: how well warped MIDI onsets land on onsets picked from the raw recording.
+// Pure: no I/O. Recording onsets must come from the audio itself (onset flux), never from transcription.
+
+export interface FitWindow { startSec: number; matchRate: number | null; medianResidualMs: number | null; flagged: boolean }
+export interface FitReport { matchRate: number | null; medianResidualMs: number | null; windows: FitWindow[] }
+
+export const MATCH_TOL_SEC = 0.07;
+export const WINDOW_SEC = 30;
+const FLAG_MIN_RATE = 0.5;
+const FLAG_MAX_RESIDUAL_MS = 60;
+const UNIQUE_EPS = 1e-6;
+
+function median(xs: number[]): number | null {
+  if (!xs.length) return null;
+  const s = [...xs].sort((a, b) => a - b), m = s.length >> 1;
+  return s.length % 2 ? s[m]! : (s[m - 1]! + s[m]!) / 2;
+}
+
+/** |residual| in seconds to the nearest recording onset, or null when none lies within ±70 ms. */
+function nearest(sorted: number[], t: number): number | null {
+  let lo = 0, hi = sorted.length;
+  while (lo < hi) { const m = (lo + hi) >> 1; if (sorted[m]! < t) lo = m + 1; else hi = m; }
+  let best = Infinity;
+  if (lo < sorted.length) best = Math.min(best, Math.abs(sorted[lo]! - t));
+  if (lo > 0) best = Math.min(best, Math.abs(sorted[lo - 1]! - t));
+  return best <= MATCH_TOL_SEC + 1e-9 ? best : null;
+}
+
+function summarise(residuals: (number | null)[]): { matchRate: number | null; medianResidualMs: number | null } {
+  if (!residuals.length) return { matchRate: null, medianResidualMs: null };
+  const hits = residuals.filter((r): r is number => r !== null);
+  const med = median(hits);
+  return { matchRate: hits.length / residuals.length, medianResidualMs: med === null ? null : med * 1000 };
+}
+
+/**
+ * Match each unique MIDI onset (already warped to audio time) to the nearest recording onset within ±70 ms
+ * (non-exclusive). Reports match rate and median |residual| overall and per 30 s window (by MIDI onset time).
+ * A window is flagged when its match rate < 0.5 or its median residual > 60 ms; empty windows are null, unflagged.
+ */
+export function fitQuality(audioOnsets: number[], midiOnsets: number[], durSec: number): FitReport {
+  const audio = [...audioOnsets].sort((a, b) => a - b);
+  const midi: number[] = [];
+  for (const t of [...midiOnsets].sort((a, b) => a - b)) if (!midi.length || t - midi[midi.length - 1]! > UNIQUE_EPS) midi.push(t);
+  const res = midi.map((t) => nearest(audio, t));
+
+  const lastOnset = midi.length ? midi[midi.length - 1]! : 0;
+  const nWin = Math.max(1, Math.ceil(Math.max(durSec, lastOnset + 1e-9) / WINDOW_SEC));
+  const buckets: (number | null)[][] = Array.from({ length: nWin }, () => []);
+  midi.forEach((t, k) => buckets[Math.min(nWin - 1, Math.max(0, Math.floor(t / WINDOW_SEC)))]!.push(res[k]!));
+  const windows = buckets.map((b, w): FitWindow => {
+    const s = summarise(b);
+    const flagged = s.matchRate !== null && (s.matchRate < FLAG_MIN_RATE || (s.medianResidualMs ?? 0) > FLAG_MAX_RESIDUAL_MS);
+    return { startSec: w * WINDOW_SEC, ...s, flagged };
+  });
+  return { ...summarise(res), windows };
+}

src/align/roles.ts

@@ -15,3 +15,14 @@ export function guessRole(t: RefTrack): Role | null {
   if ((p >= 0 && p <= 7) || (p >= 16 && p <= 23) || (p >= 88 && p <= 95)) return "keys";
   return null;
 }
+
+// Role-map key for a track: its name, or "track N" when unnamed; disambiguated on collision.
+export function trackKeys(tracks: { index: number; name: string }[]): string[] {
+  const used = new Set<string>();
+  return tracks.map((t) => {
+    let k = t.name || `track ${t.index}`;
+    if (used.has(k)) k = `${k} #${t.index}`;
+    used.add(k);
+    return k;
+  });
+}

src/ref/manifest.ts

@@ -2,7 +2,7 @@ import { join, resolve } from "node:path";
 import { mkdir } from "node:fs/promises";
 import type { Role } from "../core/types.ts";
 import { parseSmf } from "../align/smf.ts";
-import { guessRole } from "../align/roles.ts";
+import { guessRole, trackKeys } from "../align/roles.ts";
 
 export interface SongEntry {
   id: string; title: string; audio: string; midi: string;
@@ -22,16 +22,7 @@ export async function readManifest(root = "reference"): Promise<SongEntry[]> {
   return (await f.json()) as SongEntry[];
 }
 
-// Role-map key for a track: its name, or "track N" when unnamed; disambiguated on collision.
-export function trackKeys(tracks: { index: number; name: string }[]): string[] {
-  const used = new Set<string>();
-  return tracks.map((t) => {
-    let k = t.name || `track ${t.index}`;
-    if (used.has(k)) k = `${k} #${t.index}`;
-    used.add(k);
-    return k;
-  });
-}
+export { trackKeys };
 
 export async function addSong(e: Omit<SongEntry, "roles"> & { roles?: SongEntry["roles"] }, root = "reference"): Promise<SongEntry> {
   const dir = songDir(e.id, root);