diff --git a/src/pipeline/repeat.ts b/src/pipeline/repeat.ts new file mode 100644 index 0000000..3c8b1eb --- /dev/null +++ b/src/pipeline/repeat.ts @@ -0,0 +1,133 @@ +// Repetition voting. Songs repeat bars (riffs, verses, choruses); a transcriber misses or invents different notes +// in each repeat. Bars whose transcriptions are similar are treated as repeats of one another, and notes are voted +// per (pitch, position in bar): notes that almost never recur across a bar's repeats are dropped (precision) and +// notes present in enough repeats but missing from the bar are added (recall). Repetition is estimated from the +// transcription itself under the estimated constant-tempo grid. Pure: no I/O. +import type { Grid, TimedNote } from "../core/types.ts"; + +export interface RepeatOpts { + /** Minimum coarse similarity (16th-slot occupancy cosine, smoothed along the diagonal) for two bars to be repeats. */ + minSim?: number; + /** Minimum exact similarity (equal pitch, onset within tol) for two bars to be repeats. Guards against tempo + * drift: a slightly wrong grid makes distant bars coarse-similar but misaligned, and voting must not run then. */ + minExact?: number; + /** Bars of context on each side used to smooth coarse similarity along the repetition diagonal. */ + context?: number; + /** Minimum number of repeats (other bars) a bar needs before voting is applied. */ + minRepeats?: number; + /** Drop a note when it recurs in fewer than this fraction of the bar's repeats. */ + keepFrac?: number; + /** Add a note when it occurs in at least this fraction of the bar's repeats ... */ + addFrac?: number; + /** ... and in at least this many repeats. */ + minAddSupport?: number; + /** Onset tolerance (s) when matching notes across repeats. */ + tol?: number; + drop?: boolean; + add?: boolean; +} + +/** Defaults chosen in experiment E5 (docs/experiments/2026-10-31-accuracy.md). */ +export const REPEAT_DEFAULTS: Required = { + minSim: 0.6, minExact: 0.3, context: 1, minRepeats: 2, keepFrac: 0.15, addFrac: 0.25, minAddSupport: 3, tol: 0.05, + drop: true, add: true, +}; + +const SLOTS = 16; + +interface Placed { n: TimedNote; rel: number } + +/** Group notes into bars of the grid; rel = onset relative to the bar's downbeat (s). bar0 = time of bar 0. */ +function barsOf(notes: TimedNote[], grid: Grid): { byBar: Placed[][]; barSec: number; bar0: number } { + const barSec = (60 / grid.bpm) * grid.beatsPerBar; + const half = barSec / SLOTS / 2; // a note just before a downbeat belongs to that bar + const startBar = notes.reduce((m, n) => Math.min(m, Math.floor((n.onset - grid.firstDownbeat + half) / barSec)), 0); + const bar0 = grid.firstDownbeat + startBar * barSec; + const byBar: Placed[][] = []; + for (const n of notes) { + const b = Math.floor((n.onset - bar0 + half) / barSec); + (byBar[b] ??= []).push({ n, rel: n.onset - (bar0 + b * barSec) }); + } + for (let b = 0; b < byBar.length; b++) byBar[b] ??= []; + return { byBar, barSec, bar0 }; +} + +/** + * Bar x bar similarities. coarse: cosine of (16th slot, pitch) occupancy, a key counting as shared at the same or an + * adjacent slot, averaged along the diagonal over +-context bars. exact: fraction of notes with an equal-pitch note + * within tol at the same position, normalised by sqrt(|i| |j|). + */ +export function barSimilarity( + byBar: { pitch: number; rel: number }[][], barSec: number, tol: number, context: number, +): { coarse: Float64Array[]; exact: Float64Array[] } { + const nB = byBar.length; + const keys = byBar.map((ps) => new Set(ps.map((p) => Math.min(SLOTS - 1, Math.max(0, Math.round((p.rel / barSec) * SLOTS))) * 128 + p.pitch))); + const raw = keys.map((ki) => Float64Array.from(keys, (kj) => { + if (!ki.size || !kj.size) return 0; + let dot = 0; + for (const k of ki) if (kj.has(k) || kj.has(k - 128) || kj.has(k + 128)) dot++; + return dot / Math.sqrt(ki.size * kj.size); + })); + const coarse = raw.map((_, i) => Float64Array.from(raw, (__, j) => { + let s = 0, c = 0; + for (let k = -context; k <= context; k++) { + const a = i + k, b = j + k; + if (a >= 0 && b >= 0 && a < nB && b < nB) { s += raw[a]![b]!; c++; } + } + return c ? s / c : 0; + })); + const exact = byBar.map((A) => Float64Array.from(byBar, (B) => { + if (!A.length || !B.length) return 0; + let m = 0; + for (const p of A) if (B.some((q) => q.pitch === p.pitch && Math.abs(q.rel - p.rel) <= tol)) m++; + return m / Math.sqrt(A.length * B.length); + })); + return { coarse, exact }; +} + +/** Vote notes across repeated bars. Returns a new, onset-sorted note list; input notes are not modified. */ +export function repeatVote(notes: TimedNote[], grid: Grid, opts: RepeatOpts = {}): TimedNote[] { + const o = { ...REPEAT_DEFAULTS, ...opts }; + if (notes.length === 0 || !(grid.bpm > 0) || !(grid.beatsPerBar > 0)) return notes.slice(); + const { byBar, barSec, bar0 } = barsOf(notes, grid); + const nB = byBar.length; + const { coarse, exact } = barSimilarity(byBar.map((ps) => ps.map((p) => ({ pitch: p.n.pitch, rel: p.rel }))), barSec, o.tol, o.context); + const has = (bar: number, pitch: number, rel: number) => byBar[bar]!.some((q) => q.n.pitch === pitch && Math.abs(q.rel - rel) <= o.tol); + + const out: TimedNote[] = []; + for (let i = 0; i < nB; i++) { + const bar = byBar[i]!; + const reps: number[] = []; + for (let j = 0; j < nB; j++) if (j !== i && coarse[i]![j]! >= o.minSim && exact[i]![j]! >= o.minExact) reps.push(j); + if (reps.length < o.minRepeats) { for (const p of bar) out.push(p.n); continue; } + + for (const p of bar) { + if (!o.drop) { out.push(p.n); continue; } + const support = reps.filter((j) => has(j, p.n.pitch, p.rel)).length; + if (support / reps.length >= o.keepFrac) out.push(p.n); + } + if (!o.add) continue; + + // Candidates from the repeats, clustered by pitch and position; one vote per repeat. + const cands: { pitch: number; rel: number; rels: number[]; durs: number[]; bars: Set }[] = []; + for (const j of reps) for (const q of byBar[j]!) { + if (has(i, q.n.pitch, q.rel)) continue; + let c = cands.find((x) => x.pitch === q.n.pitch && Math.abs(x.rel - q.rel) <= o.tol); + if (!c) cands.push((c = { pitch: q.n.pitch, rel: q.rel, rels: [], durs: [], bars: new Set() })); + if (c.bars.has(j)) continue; + c.bars.add(j); c.rels.push(q.rel); c.durs.push(q.n.offset - q.n.onset); + } + for (const c of cands) { + if (c.bars.size < o.minAddSupport || c.bars.size / reps.length < o.addFrac) continue; + const onset = bar0 + i * barSec + median(c.rels); + if (onset >= 0) out.push({ onset, offset: onset + median(c.durs), pitch: c.pitch }); + } + } + return out.sort((a, b) => a.onset - b.onset || a.pitch - b.pitch); +} + +function median(a: number[]): number { + const s = [...a].sort((x, y) => x - y); + const m = s.length >> 1; + return s.length % 2 ? s[m]! : (s[m - 1]! + s[m]!) / 2; +} diff --git a/src/pipeline/run.ts b/src/pipeline/run.ts index bee9768..63d7126 100644 --- a/src/pipeline/run.ts +++ b/src/pipeline/run.ts @@ -10,12 +10,18 @@ import { transcribeDrums } from "./drums.ts"; import { estimateGrid, gridFromTruth } from "./tempo.ts"; import { quantize, snapTicks } from "./quantize.ts"; import { assignTab } from "./tab.ts"; +import { repeatVote } from "./repeat.ts"; export interface PipelineInput { mix: Audio; oracle?: { stems: Map; truth: { bpm: number; timeSig: [number, number]; tracks: TruthTrack[] } }; } export interface PipelineResult { output: PipelineOutput; gp: Uint8Array; warnings: string[] } +export interface PipelineOpts { + /** Vote guitar notes across repeated bars (see repeat.ts). Default on: experiment E5 raised guitar F1 on + * synthetic t1a–t1d (+0.02 median) and on "Last Night on Earth" (0.403 → 0.507). */ + repeatVote?: boolean; +} const PROGRAM: Record = { guitar: 27, bass: 33, keys: 0, vocals: 53, drums: 0 }; const RANGE: Record, TranscribeOpts> = { @@ -55,6 +61,7 @@ export function sumAudio(...as: Audio[]): Audio { export async function createPipeline( models: { demucs: Uint8Array; basicPitch: Uint8Array }, + opts: PipelineOpts = {}, ): Promise<(input: PipelineInput, flags: OracleFlags) => Promise> { const separate = await createSeparator(models.demucs); const transcribe = await createTranscriber(models.basicPitch); @@ -103,7 +110,10 @@ export async function createPipeline( else if (flags.oracleNotes) { const src = job.truthNotes ?? oracle!.truth.tracks.filter((t) => t.role === job.role).flatMap((t) => t.notes); notes = src.map((n) => ({ onset: n.onset, offset: n.offset, pitch: n.pitch, velocity: n.velocity })); - } else notes = await transcribe(job.audio!, RANGE[job.role]); + } else { + notes = await transcribe(job.audio!, RANGE[job.role]); + if (job.role === "guitar" && (opts.repeatVote ?? true)) notes = repeatVote(notes, grid); + } let scoreNotes = quantize(notes, grid); const tuning = TUNING[job.role]; diff --git a/tests/pipeline/repeat.test.ts b/tests/pipeline/repeat.test.ts new file mode 100644 index 0000000..4b8bce6 --- /dev/null +++ b/tests/pipeline/repeat.test.ts @@ -0,0 +1,49 @@ +import { expect, test } from "bun:test"; +import type { Grid, TimedNote } from "../../src/core/types.ts"; +import { repeatVote } from "../../src/pipeline/repeat.ts"; + +const grid: Grid = { bpm: 120, firstDownbeat: 0, beatsPerBar: 4 }; // 2 s bars +const riff: [number, number][] = [[0, 52], [0.5, 55], [1, 57], [1.5, 59], [1.75, 60]]; // [pos s, pitch] + +function bars(n: number, f: (bar: number, pos: number, pitch: number) => boolean = () => true): TimedNote[] { + const out: TimedNote[] = []; + for (let b = 0; b < n; b++) for (const [pos, pitch] of riff) { + if (f(b, pos, pitch)) out.push({ onset: b * 2 + pos, offset: b * 2 + pos + 0.2, pitch }); + } + return out; +} +const key = (n: TimedNote) => `${n.onset.toFixed(3)}:${n.pitch}`; + +test("adds a note missing from one repeat", () => { + const notes = bars(6, (b, pos) => !(b === 3 && pos === 1)); + const out = repeatVote(notes, grid); + expect(out.length).toBe(30); + const added = out.find((n) => Math.abs(n.onset - 7) < 1e-6 && n.pitch === 57); + expect(added).toBeDefined(); + expect(added!.offset - added!.onset).toBeCloseTo(0.2, 6); +}); + +test("drops a note that never recurs in a bar's repeats", () => { + const notes = [...bars(6), { onset: 4.25, offset: 4.4, pitch: 70 }]; + const out = repeatVote(notes, grid); + expect(out.map(key)).toEqual(bars(6).map(key)); +}); + +test("leaves non-repeating material unchanged", () => { + const notes: TimedNote[] = []; + for (let b = 0; b < 6; b++) notes.push({ onset: b * 2 + 0.3 * (b % 4), offset: b * 2 + 1, pitch: 45 + 3 * b }); + expect(repeatVote(notes, grid).map(key)).toEqual(notes.map(key)); +}); + +test("add/drop can be disabled and empty input is handled", () => { + const notes = [...bars(6, (b, pos) => !(b === 3 && pos === 1)), { onset: 4.25, offset: 4.4, pitch: 70 }]; + expect(repeatVote(notes, grid, { add: false, drop: false }).length).toBe(notes.length); + expect(repeatVote([], grid)).toEqual([]); +}); + +test("bars before a late first downbeat are handled", () => { + const g: Grid = { ...grid, firstDownbeat: 0.5 }; + const notes = bars(6, (b, pos) => !(b === 2 && pos === 0.5)).map((n) => ({ ...n, onset: n.onset + 0.5, offset: n.offset + 0.5 })); + const out = repeatVote(notes, g); + expect(out.some((n) => Math.abs(n.onset - 5) < 1e-6 && n.pitch === 55)).toBe(true); +});