feat(pipeline): vote guitar notes across repeated bars (E5)
Bars whose transcriptions are similar (16th-slot occupancy cosine >= 0.6, diagonal-smoothed, and exact pitch/onset match >= 0.3) are treated as repeats. Notes recurring in < 15% of a bar's repeats are dropped; notes in >= 25% and >= 3 repeats but missing from the bar are added. Default on for guitar. Synthetic median guitar F1 t1a 0.866->0.885, t1b 0.814->0.840, t1c 0.734->0.752, t1d 0.721->0.737; lnoe 0.403->0.507. Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
Lucas Winther
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
19ac52c93a
commit
132659a99c
@@ -0,0 +1,133 @@
|
||||
// Repetition voting. Songs repeat bars (riffs, verses, choruses); a transcriber misses or invents different notes
|
||||
// in each repeat. Bars whose transcriptions are similar are treated as repeats of one another, and notes are voted
|
||||
// per (pitch, position in bar): notes that almost never recur across a bar's repeats are dropped (precision) and
|
||||
// notes present in enough repeats but missing from the bar are added (recall). Repetition is estimated from the
|
||||
// transcription itself under the estimated constant-tempo grid. Pure: no I/O.
|
||||
import type { Grid, TimedNote } from "../core/types.ts";
|
||||
|
||||
export interface RepeatOpts {
|
||||
/** Minimum coarse similarity (16th-slot occupancy cosine, smoothed along the diagonal) for two bars to be repeats. */
|
||||
minSim?: number;
|
||||
/** Minimum exact similarity (equal pitch, onset within tol) for two bars to be repeats. Guards against tempo
|
||||
* drift: a slightly wrong grid makes distant bars coarse-similar but misaligned, and voting must not run then. */
|
||||
minExact?: number;
|
||||
/** Bars of context on each side used to smooth coarse similarity along the repetition diagonal. */
|
||||
context?: number;
|
||||
/** Minimum number of repeats (other bars) a bar needs before voting is applied. */
|
||||
minRepeats?: number;
|
||||
/** Drop a note when it recurs in fewer than this fraction of the bar's repeats. */
|
||||
keepFrac?: number;
|
||||
/** Add a note when it occurs in at least this fraction of the bar's repeats ... */
|
||||
addFrac?: number;
|
||||
/** ... and in at least this many repeats. */
|
||||
minAddSupport?: number;
|
||||
/** Onset tolerance (s) when matching notes across repeats. */
|
||||
tol?: number;
|
||||
drop?: boolean;
|
||||
add?: boolean;
|
||||
}
|
||||
|
||||
/** Defaults chosen in experiment E5 (docs/experiments/2026-10-31-accuracy.md). */
|
||||
export const REPEAT_DEFAULTS: Required<RepeatOpts> = {
|
||||
minSim: 0.6, minExact: 0.3, context: 1, minRepeats: 2, keepFrac: 0.15, addFrac: 0.25, minAddSupport: 3, tol: 0.05,
|
||||
drop: true, add: true,
|
||||
};
|
||||
|
||||
const SLOTS = 16;
|
||||
|
||||
interface Placed { n: TimedNote; rel: number }
|
||||
|
||||
/** Group notes into bars of the grid; rel = onset relative to the bar's downbeat (s). bar0 = time of bar 0. */
|
||||
function barsOf(notes: TimedNote[], grid: Grid): { byBar: Placed[][]; barSec: number; bar0: number } {
|
||||
const barSec = (60 / grid.bpm) * grid.beatsPerBar;
|
||||
const half = barSec / SLOTS / 2; // a note just before a downbeat belongs to that bar
|
||||
const startBar = notes.reduce((m, n) => Math.min(m, Math.floor((n.onset - grid.firstDownbeat + half) / barSec)), 0);
|
||||
const bar0 = grid.firstDownbeat + startBar * barSec;
|
||||
const byBar: Placed[][] = [];
|
||||
for (const n of notes) {
|
||||
const b = Math.floor((n.onset - bar0 + half) / barSec);
|
||||
(byBar[b] ??= []).push({ n, rel: n.onset - (bar0 + b * barSec) });
|
||||
}
|
||||
for (let b = 0; b < byBar.length; b++) byBar[b] ??= [];
|
||||
return { byBar, barSec, bar0 };
|
||||
}
|
||||
|
||||
/**
|
||||
* Bar x bar similarities. coarse: cosine of (16th slot, pitch) occupancy, a key counting as shared at the same or an
|
||||
* adjacent slot, averaged along the diagonal over +-context bars. exact: fraction of notes with an equal-pitch note
|
||||
* within tol at the same position, normalised by sqrt(|i| |j|).
|
||||
*/
|
||||
export function barSimilarity(
|
||||
byBar: { pitch: number; rel: number }[][], barSec: number, tol: number, context: number,
|
||||
): { coarse: Float64Array[]; exact: Float64Array[] } {
|
||||
const nB = byBar.length;
|
||||
const keys = byBar.map((ps) => new Set(ps.map((p) => Math.min(SLOTS - 1, Math.max(0, Math.round((p.rel / barSec) * SLOTS))) * 128 + p.pitch)));
|
||||
const raw = keys.map((ki) => Float64Array.from(keys, (kj) => {
|
||||
if (!ki.size || !kj.size) return 0;
|
||||
let dot = 0;
|
||||
for (const k of ki) if (kj.has(k) || kj.has(k - 128) || kj.has(k + 128)) dot++;
|
||||
return dot / Math.sqrt(ki.size * kj.size);
|
||||
}));
|
||||
const coarse = raw.map((_, i) => Float64Array.from(raw, (__, j) => {
|
||||
let s = 0, c = 0;
|
||||
for (let k = -context; k <= context; k++) {
|
||||
const a = i + k, b = j + k;
|
||||
if (a >= 0 && b >= 0 && a < nB && b < nB) { s += raw[a]![b]!; c++; }
|
||||
}
|
||||
return c ? s / c : 0;
|
||||
}));
|
||||
const exact = byBar.map((A) => Float64Array.from(byBar, (B) => {
|
||||
if (!A.length || !B.length) return 0;
|
||||
let m = 0;
|
||||
for (const p of A) if (B.some((q) => q.pitch === p.pitch && Math.abs(q.rel - p.rel) <= tol)) m++;
|
||||
return m / Math.sqrt(A.length * B.length);
|
||||
}));
|
||||
return { coarse, exact };
|
||||
}
|
||||
|
||||
/** Vote notes across repeated bars. Returns a new, onset-sorted note list; input notes are not modified. */
|
||||
export function repeatVote(notes: TimedNote[], grid: Grid, opts: RepeatOpts = {}): TimedNote[] {
|
||||
const o = { ...REPEAT_DEFAULTS, ...opts };
|
||||
if (notes.length === 0 || !(grid.bpm > 0) || !(grid.beatsPerBar > 0)) return notes.slice();
|
||||
const { byBar, barSec, bar0 } = barsOf(notes, grid);
|
||||
const nB = byBar.length;
|
||||
const { coarse, exact } = barSimilarity(byBar.map((ps) => ps.map((p) => ({ pitch: p.n.pitch, rel: p.rel }))), barSec, o.tol, o.context);
|
||||
const has = (bar: number, pitch: number, rel: number) => byBar[bar]!.some((q) => q.n.pitch === pitch && Math.abs(q.rel - rel) <= o.tol);
|
||||
|
||||
const out: TimedNote[] = [];
|
||||
for (let i = 0; i < nB; i++) {
|
||||
const bar = byBar[i]!;
|
||||
const reps: number[] = [];
|
||||
for (let j = 0; j < nB; j++) if (j !== i && coarse[i]![j]! >= o.minSim && exact[i]![j]! >= o.minExact) reps.push(j);
|
||||
if (reps.length < o.minRepeats) { for (const p of bar) out.push(p.n); continue; }
|
||||
|
||||
for (const p of bar) {
|
||||
if (!o.drop) { out.push(p.n); continue; }
|
||||
const support = reps.filter((j) => has(j, p.n.pitch, p.rel)).length;
|
||||
if (support / reps.length >= o.keepFrac) out.push(p.n);
|
||||
}
|
||||
if (!o.add) continue;
|
||||
|
||||
// Candidates from the repeats, clustered by pitch and position; one vote per repeat.
|
||||
const cands: { pitch: number; rel: number; rels: number[]; durs: number[]; bars: Set<number> }[] = [];
|
||||
for (const j of reps) for (const q of byBar[j]!) {
|
||||
if (has(i, q.n.pitch, q.rel)) continue;
|
||||
let c = cands.find((x) => x.pitch === q.n.pitch && Math.abs(x.rel - q.rel) <= o.tol);
|
||||
if (!c) cands.push((c = { pitch: q.n.pitch, rel: q.rel, rels: [], durs: [], bars: new Set() }));
|
||||
if (c.bars.has(j)) continue;
|
||||
c.bars.add(j); c.rels.push(q.rel); c.durs.push(q.n.offset - q.n.onset);
|
||||
}
|
||||
for (const c of cands) {
|
||||
if (c.bars.size < o.minAddSupport || c.bars.size / reps.length < o.addFrac) continue;
|
||||
const onset = bar0 + i * barSec + median(c.rels);
|
||||
if (onset >= 0) out.push({ onset, offset: onset + median(c.durs), pitch: c.pitch });
|
||||
}
|
||||
}
|
||||
return out.sort((a, b) => a.onset - b.onset || a.pitch - b.pitch);
|
||||
}
|
||||
|
||||
function median(a: number[]): number {
|
||||
const s = [...a].sort((x, y) => x - y);
|
||||
const m = s.length >> 1;
|
||||
return s.length % 2 ? s[m]! : (s[m - 1]! + s[m]!) / 2;
|
||||
}
|
||||
+11
-1
@@ -10,12 +10,18 @@ import { transcribeDrums } from "./drums.ts";
|
||||
import { estimateGrid, gridFromTruth } from "./tempo.ts";
|
||||
import { quantize, snapTicks } from "./quantize.ts";
|
||||
import { assignTab } from "./tab.ts";
|
||||
import { repeatVote } from "./repeat.ts";
|
||||
|
||||
export interface PipelineInput {
|
||||
mix: Audio;
|
||||
oracle?: { stems: Map<string, Audio>; truth: { bpm: number; timeSig: [number, number]; tracks: TruthTrack[] } };
|
||||
}
|
||||
export interface PipelineResult { output: PipelineOutput; gp: Uint8Array; warnings: string[] }
|
||||
export interface PipelineOpts {
|
||||
/** Vote guitar notes across repeated bars (see repeat.ts). Default on: experiment E5 raised guitar F1 on
|
||||
* synthetic t1a–t1d (+0.02 median) and on "Last Night on Earth" (0.403 → 0.507). */
|
||||
repeatVote?: boolean;
|
||||
}
|
||||
|
||||
const PROGRAM: Record<Role, number> = { guitar: 27, bass: 33, keys: 0, vocals: 53, drums: 0 };
|
||||
const RANGE: Record<Exclude<Role, "drums">, TranscribeOpts> = {
|
||||
@@ -55,6 +61,7 @@ export function sumAudio(...as: Audio[]): Audio {
|
||||
|
||||
export async function createPipeline(
|
||||
models: { demucs: Uint8Array; basicPitch: Uint8Array },
|
||||
opts: PipelineOpts = {},
|
||||
): Promise<(input: PipelineInput, flags: OracleFlags) => Promise<PipelineResult>> {
|
||||
const separate = await createSeparator(models.demucs);
|
||||
const transcribe = await createTranscriber(models.basicPitch);
|
||||
@@ -103,7 +110,10 @@ export async function createPipeline(
|
||||
else if (flags.oracleNotes) {
|
||||
const src = job.truthNotes ?? oracle!.truth.tracks.filter((t) => t.role === job.role).flatMap((t) => t.notes);
|
||||
notes = src.map((n) => ({ onset: n.onset, offset: n.offset, pitch: n.pitch, velocity: n.velocity }));
|
||||
} else notes = await transcribe(job.audio!, RANGE[job.role]);
|
||||
} else {
|
||||
notes = await transcribe(job.audio!, RANGE[job.role]);
|
||||
if (job.role === "guitar" && (opts.repeatVote ?? true)) notes = repeatVote(notes, grid);
|
||||
}
|
||||
|
||||
let scoreNotes = quantize(notes, grid);
|
||||
const tuning = TUNING[job.role];
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
import { expect, test } from "bun:test";
|
||||
import type { Grid, TimedNote } from "../../src/core/types.ts";
|
||||
import { repeatVote } from "../../src/pipeline/repeat.ts";
|
||||
|
||||
const grid: Grid = { bpm: 120, firstDownbeat: 0, beatsPerBar: 4 }; // 2 s bars
|
||||
const riff: [number, number][] = [[0, 52], [0.5, 55], [1, 57], [1.5, 59], [1.75, 60]]; // [pos s, pitch]
|
||||
|
||||
function bars(n: number, f: (bar: number, pos: number, pitch: number) => boolean = () => true): TimedNote[] {
|
||||
const out: TimedNote[] = [];
|
||||
for (let b = 0; b < n; b++) for (const [pos, pitch] of riff) {
|
||||
if (f(b, pos, pitch)) out.push({ onset: b * 2 + pos, offset: b * 2 + pos + 0.2, pitch });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
const key = (n: TimedNote) => `${n.onset.toFixed(3)}:${n.pitch}`;
|
||||
|
||||
test("adds a note missing from one repeat", () => {
|
||||
const notes = bars(6, (b, pos) => !(b === 3 && pos === 1));
|
||||
const out = repeatVote(notes, grid);
|
||||
expect(out.length).toBe(30);
|
||||
const added = out.find((n) => Math.abs(n.onset - 7) < 1e-6 && n.pitch === 57);
|
||||
expect(added).toBeDefined();
|
||||
expect(added!.offset - added!.onset).toBeCloseTo(0.2, 6);
|
||||
});
|
||||
|
||||
test("drops a note that never recurs in a bar's repeats", () => {
|
||||
const notes = [...bars(6), { onset: 4.25, offset: 4.4, pitch: 70 }];
|
||||
const out = repeatVote(notes, grid);
|
||||
expect(out.map(key)).toEqual(bars(6).map(key));
|
||||
});
|
||||
|
||||
test("leaves non-repeating material unchanged", () => {
|
||||
const notes: TimedNote[] = [];
|
||||
for (let b = 0; b < 6; b++) notes.push({ onset: b * 2 + 0.3 * (b % 4), offset: b * 2 + 1, pitch: 45 + 3 * b });
|
||||
expect(repeatVote(notes, grid).map(key)).toEqual(notes.map(key));
|
||||
});
|
||||
|
||||
test("add/drop can be disabled and empty input is handled", () => {
|
||||
const notes = [...bars(6, (b, pos) => !(b === 3 && pos === 1)), { onset: 4.25, offset: 4.4, pitch: 70 }];
|
||||
expect(repeatVote(notes, grid, { add: false, drop: false }).length).toBe(notes.length);
|
||||
expect(repeatVote([], grid)).toEqual([]);
|
||||
});
|
||||
|
||||
test("bars before a late first downbeat are handled", () => {
|
||||
const g: Grid = { ...grid, firstDownbeat: 0.5 };
|
||||
const notes = bars(6, (b, pos) => !(b === 2 && pos === 0.5)).map((n) => ({ ...n, onset: n.onset + 0.5, offset: n.offset + 0.5 }));
|
||||
const out = repeatVote(notes, g);
|
||||
expect(out.some((n) => Math.abs(n.onset - 5) < 1e-6 && n.pitch === 55)).toBe(true);
|
||||
});
|
||||
Reference in New Issue
Block a user