From 1d490f6897359c1177957a1e4e8c68d8718bc44d Mon Sep 17 00:00:00 2001 From: Lucas Winther Date: Fri, 30 Oct 2026 04:46:09 +0100 Subject: [PATCH] fix: tempo octave choice, start-of-music anchoring, mean-normalised combs, pure toMono Co-Authored-By: Claude Sonnet 5.5 --- src/audio/wav.ts | 11 +- src/dsp/mono.ts | 13 +++ src/pipeline/tempo.ts | 192 +++++++++++++++++++++-------------- tests/pipeline/tempo.test.ts | 77 +++++++++++--- 4 files changed, 194 insertions(+), 99 deletions(-) create mode 100644 src/dsp/mono.ts diff --git a/src/audio/wav.ts b/src/audio/wav.ts index 9c68599..1431dfb 100644 --- a/src/audio/wav.ts +++ b/src/audio/wav.ts @@ -67,16 +67,7 @@ export async function writeWav(path: string, a: Audio): Promise { await Bun.write(path, out); } -/** Average all channels into one mono signal. */ -export function toMono(a: Audio): Float32Array { - const n = a.channels[0]?.length ?? 0; - if (a.channels.length === 1) return a.channels[0]!.slice(); - const out = new Float32Array(n); - for (const ch of a.channels) for (let i = 0; i < n; i++) out[i]! += ch[i]!; - const k = 1 / a.channels.length; - for (let i = 0; i < n; i++) out[i]! *= k; - return out; -} +export { toMono } from "../dsp/mono.ts"; const sinc = (x: number) => (x === 0 ? 1 : Math.sin(Math.PI * x) / (Math.PI * x)); diff --git a/src/dsp/mono.ts b/src/dsp/mono.ts new file mode 100644 index 0000000..1328cbe --- /dev/null +++ b/src/dsp/mono.ts @@ -0,0 +1,13 @@ +// Pure channel downmix. Re-exported from src/audio/wav.ts for existing importers. +import type { Audio } from "../core/types.ts"; + +/** Average all channels into one mono signal. */ +export function toMono(a: Audio): Float32Array { + const n = a.channels[0]?.length ?? 0; + if (a.channels.length === 1) return a.channels[0]!.slice(); + const out = new Float32Array(n); + for (const ch of a.channels) for (let i = 0; i < n; i++) out[i]! += ch[i]!; + const k = 1 / a.channels.length; + for (let i = 0; i < n; i++) out[i]! *= k; + return out; +} diff --git a/src/pipeline/tempo.ts b/src/pipeline/tempo.ts index dc027c8..8a3ffd9 100644 --- a/src/pipeline/tempo.ts +++ b/src/pipeline/tempo.ts @@ -1,7 +1,7 @@ // Tempo and downbeat estimation for constant-tempo material. Pure: no I/O, browser-portable. import type { Audio, Grid } from "../core/types.ts"; import { stft } from "../dsp/fft.ts"; -import { toMono } from "../audio/wav.ts"; +import { toMono } from "../dsp/mono.ts"; export interface GridOpts { minBpm?: number; maxBpm?: number; beatsPerBar?: number } @@ -14,6 +14,9 @@ const N_FFT = 2048; const PRIOR_CENTRE_BPM = 100; const PRIOR_SIGMA_OCT = 0.7; const LOW_HZ = 150; +const TIE_RATIO = 0.95; +const WEAK_PULSE_RATIO = 0.93; +const START_TIEBREAK = 0.1; /** Half-wave rectified log-magnitude spectral flux: full band and low band (< 150 Hz). */ function onsetEnvelopes(x: Float32Array, sr: number, hop: number): { all: Float64Array; low: Float64Array; tonal: Float64Array[] } { @@ -60,80 +63,18 @@ function detrend(e: Float64Array): Float64Array { return out; } + function sampleAt(e: Float64Array, pos: number): number { if (pos < 0 || pos > e.length - 1) return 0; const i = Math.floor(pos), fr = pos - i; return e[i]! * (1 - fr) + (i + 1 < e.length ? e[i + 1]! : e[i]!) * fr; } -/** Sum of envelope on the comb t = phase + n * period (all in frames, fractional). */ -function combSum(e: Float64Array, period: number, phase: number, stride = 1): number { - let s = 0; - for (let t = phase; t <= e.length - 1; t += period * stride) s += sampleAt(e, t); - return s; -} - -export function estimateGrid(audio: Audio, opts: GridOpts = {}): Grid { - const minBpm = opts.minBpm ?? 60; - const maxBpm = opts.maxBpm ?? 180; - const beatsPerBar = opts.beatsPerBar ?? 4; - const sr = audio.sampleRate; - const hop = Math.max(1, Math.round((512 * sr) / 44100)); - const fps = sr / hop; - const x = toMono(audio); - const { all, low, tonal } = onsetEnvelopes(x, sr, hop); - const env = detrend(all); - const lowEnv = detrend(low); - - // Tempo: autocorrelation of the envelope weighted by a log-Gaussian prior. - const n = env.length; - const acf = (lag: number): number => { - let s = 0; - const l = Math.round(lag); - for (let i = 0; i + l < n; i++) s += env[i]! * env[i + l]!; - return s / Math.max(1, n - l); - }; - const prior = (bpm: number) => Math.exp(-0.5 * (Math.log2(bpm / PRIOR_CENTRE_BPM) / PRIOR_SIGMA_OCT) ** 2); - let bestBpm = PRIOR_CENTRE_BPM, bestScore = -Infinity; - for (let bpm = minBpm; bpm <= maxBpm; bpm += 0.25) { - const lag = (60 * fps) / bpm; - // Reward harmonic support at 2 and 4 beats (bar structure). - const s = (acf(lag) + 0.5 * acf(2 * lag) + 0.25 * acf(4 * lag)) * prior(bpm); - if (s > bestScore) { bestScore = s; bestBpm = bpm; } - } - const scoreAt = (bpm: number): number => { const lag = (60 * fps) / bpm; return acf(lag) + 0.5 * acf(2 * lag) + 0.25 * acf(4 * lag); }; - // Octave rule for slow pop: a fast peak whose half-tempo peak is nearly as strong is taken as double-time. - if (bestBpm > 120 && bestBpm / 2 >= minBpm && scoreAt(bestBpm / 2) >= 0.9 * scoreAt(bestBpm)) bestBpm /= 2; - - // Joint refinement of tempo and phase on the comb (fine grid, +-4% around the coarse peak). - let rb = bestBpm, rp = 0, rs = -Infinity; - for (let bpm = bestBpm * 0.96; bpm <= bestBpm * 1.04; bpm += bestBpm * 0.001) { - const period = (60 * fps) / bpm; - const steps = Math.ceil(period); - for (let k = 0; k < steps; k++) { - const phase = (k * period) / steps; - const s = combSum(env, period, phase); - if (s > rs) { rs = s; rb = bpm; rp = phase; } - } - } - - // Downbeat: the beat phase whose bar-spaced comb has the most low-frequency flux. - const period = (60 * fps) / rb; - // Score each candidate by low-band flux plus harmonic novelty (chords change on bar lines), each normalised, - // with a small prior for the first candidate (music tends to start on a downbeat). - const lowS: number[] = [], novS: number[] = []; - for (let k = 0; k < beatsPerBar; k++) { - lowS.push(combSum(lowEnv, period, rp + k * period, beatsPerBar)); - novS.push(noveltyComb(tonal, period, rp + k * period, beatsPerBar)); - } - const lowMax = Math.max(...lowS) || 1, novMax = Math.max(...novS) || 1; - let bestK = 0, bestLow = -Infinity; - for (let k = 0; k < beatsPerBar; k++) { - const s = lowS[k]! / lowMax + novS[k]! / novMax + (k === 0 ? 0.2 : 0); - if (s > bestLow) { bestLow = s; bestK = k; } - } - const down = (rp + bestK * period) / fps; - return { bpm: rb, firstDownbeat: down, beatsPerBar }; +/** Mean envelope value on the comb t = phase + n * period * stride (frames, fractional). */ +function combMean(e: Float64Array, period: number, phase: number, stride = 1): number { + let s = 0, count = 0; + for (let t = phase; t <= e.length - 1; t += period * stride) { s += sampleAt(e, t); count++; } + return count ? s / count : 0; } /** Mean tonal spectrum over frames [a, b), L2-normalised. */ @@ -150,17 +91,120 @@ function meanSpec(tonal: Float64Array[], a: number, b: number): Float64Array | n } /** Harmonic novelty at time t (frames): 1 - cosine between the spectrum before and after, skipping the attack. */ -function harmonicNovelty(tonal: Float64Array[], t: number, period: number): number { +function harmonicNovelty(tonal: Float64Array[], t: number, period: number): number | null { const a = meanSpec(tonal, t - 0.9 * period, t - 0.1 * period); const b = meanSpec(tonal, t + 0.25 * period, t + 0.9 * period); - if (!a || !b) return 0; + if (!a || !b) return null; let dot = 0; for (let k = 0; k < a.length; k++) dot += a[k]! * b[k]!; return 1 - dot; } +/** Mean harmonic novelty over the valid comb positions. */ function noveltyComb(tonal: Float64Array[], period: number, phase: number, stride: number): number { - let s = 0; - for (let t = phase; t < tonal.length; t += period * stride) s += harmonicNovelty(tonal, t, period); - return s; + let s = 0, count = 0; + for (let t = phase; t < tonal.length; t += period * stride) { + const v = harmonicNovelty(tonal, t, period); + if (v !== null) { s += v; count++; } + } + return count ? s / count : 0; +} + +export function estimateGrid(audio: Audio, opts: GridOpts = {}): Grid { + const minBpm = opts.minBpm ?? 60; + const maxBpm = opts.maxBpm ?? 180; + const beatsPerBar = opts.beatsPerBar ?? 4; + const sr = audio.sampleRate; + const hop = Math.max(1, Math.round((512 * sr) / 44100)); + const fps = sr / hop; + const x = toMono(audio); + if (x.length <= N_FFT) throw new Error(`estimateGrid: audio too short (${x.length} samples, need > ${N_FFT})`); + let peak = 0; + for (let i = 0; i < x.length; i++) peak = Math.max(peak, Math.abs(x[i]!)); + if (peak < 1e-6) throw new Error("estimateGrid: silent input"); + const { all, low, tonal } = onsetEnvelopes(x, sr, hop); + const env = detrend(all); + const lowEnv = detrend(low); + + // Tempo: autocorrelation of the envelope (beat, 2-beat and bar-ish lags) weighted by a log-Gaussian prior. + const n = env.length; + const acf = (lag: number): number => { + let s = 0; + const l = Math.round(lag); + for (let i = 0; i + l < n; i++) s += env[i]! * env[i + l]!; + return s / Math.max(1, n - l); + }; + const scoreAt = (bpm: number): number => { + const lag = (60 * fps) / bpm; + return acf(lag) + 0.5 * acf(2 * lag) + 0.25 * acf(4 * lag); + }; + const prior = (bpm: number) => Math.exp(-0.5 * (Math.log2(bpm / PRIOR_CENTRE_BPM) / PRIOR_SIGMA_OCT) ** 2); + let coarse = PRIOR_CENTRE_BPM, bestScore = -Infinity; + for (let bpm = minBpm; bpm <= maxBpm; bpm += 0.25) { + const s = scoreAt(bpm) * prior(bpm); + if (s > bestScore) { bestScore = s; coarse = bpm; } + } + + // Joint refinement of tempo and phase on the beat comb (fine grid, +-4% around the coarse peak). + const refine = (centre: number) => { + let rb = centre, rp = 0, rs = -Infinity; + for (let bpm = centre * 0.96; bpm <= centre * 1.04; bpm += centre * 0.001) { + const period = (60 * fps) / bpm; + const steps = Math.ceil(period); + for (let k = 0; k < steps; k++) { + const phase = (k * period) / steps; + const s = combMean(env, period, phase); + if (s > rs) { rs = s; rb = bpm; rp = phase; } + } + } + return { bpm: rb, phase: rp }; + }; + let fit = refine(coarse); + + // Octave choice (prior-driven): a fast candidate is taken as double-time only when the data barely prefers it + // (autocorrelation within 5% of the half tempo) AND its pulses are clearly weaker than the half-tempo pulses + // (offbeat subdivisions rather than beats). Clear data preference, or equally strong pulses, keeps the fast tempo. + if (coarse / 2 >= minBpm && scoreAt(coarse / 2) >= TIE_RATIO * scoreAt(coarse)) { + const slow = refine(coarse / 2); + const pf = pulseMean(env, (60 * fps) / fit.bpm, fit.phase); + const ps = pulseMean(env, (60 * fps) / slow.bpm, slow.phase); + if (pf < WEAK_PULSE_RATIO * ps) fit = slow; + } + const { bpm, phase } = fit; + const period = (60 * fps) / bpm; + + // Downbeat: score each of the beatsPerBar candidate phases by mean low-band flux plus mean harmonic novelty + // (chords change on bar lines), each normalised to the best candidate. A small tie-break favours the + // candidate nearest the first onset above the noise floor (the start of the music). + const envMax = Math.max(...env); + let firstOnset = 0; + while (firstOnset < n && env[firstOnset]! < 0.05 * envMax) firstOnset++; + const barFrames = period * beatsPerBar; + const lowS: number[] = [], novS: number[] = [], near: number[] = []; + for (let k = 0; k < beatsPerBar; k++) { + const t = phase + k * period; + lowS.push(combMean(lowEnv, period, t, beatsPerBar)); + novS.push(noveltyComb(tonal, period, t, beatsPerBar)); + const d = (((firstOnset - t) % barFrames) + barFrames) % barFrames; + near.push(Math.min(d, barFrames - d)); + } + const lowMax = Math.max(...lowS) || 1, novMax = Math.max(...novS) || 1; + const nearest = near.indexOf(Math.min(...near)); + let bestK = 0, bestS = -Infinity; + for (let k = 0; k < beatsPerBar; k++) { + const s = lowS[k]! / lowMax + novS[k]! / novMax + (k === nearest ? START_TIEBREAK : 0); + if (s > bestS) { bestS = s; bestK = k; } + } + return { bpm, firstDownbeat: (phase + bestK * period) / fps, beatsPerBar }; +} + +/** Like combMean but each pulse takes the local maximum within +-1 frame (robust to sub-frame timing jitter). */ +function pulseMean(e: Float64Array, period: number, phase: number, stride = 1): number { + let s = 0, count = 0; + for (let t = phase; t <= e.length - 1; t += period * stride) { + const c = Math.round(t); + s += Math.max(e[Math.max(0, c - 1)]!, e[Math.min(e.length - 1, c)]!, e[Math.min(e.length - 1, c + 1)]!); + count++; + } + return count ? s / count : 0; } diff --git a/tests/pipeline/tempo.test.ts b/tests/pipeline/tempo.test.ts index 7aa837d..154c8a0 100644 --- a/tests/pipeline/tempo.test.ts +++ b/tests/pipeline/tempo.test.ts @@ -1,10 +1,11 @@ -import { test, expect, beforeAll } from "bun:test"; +import { test, expect, beforeAll, afterAll } from "bun:test"; import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { buildSong, loadSongDir } from "../../src/bench/dataset.ts"; import { estimateGrid, gridFromTruth } from "../../src/pipeline/tempo.ts"; import type { TierId } from "../../src/gen/tiers.ts"; +import type { Audio } from "../../src/core/types.ts"; type Row = { id: string; trueBpm: number; estBpm: number; factor: number; beatErrMs: number; barErrMs: number }; const rows: Row[] = []; @@ -19,28 +20,79 @@ beforeAll(async () => { root = await mkdtemp(join(tmpdir(), "stsheets-tempo-")); for (const tier of ["t1b", "t1c"] as TierId[]) { for (let seed = 1; seed <= 5; seed++) { - const dir = await buildSong(tier, seed, root); - const { song, mix } = await loadSongDir(dir); + const { song, mix } = await loadSongDir(await buildSong(tier, seed, root)); const g = estimateGrid(mix); - const beat = 60 / song.bpm; const f = [0.5, 1, 2].find((c) => Math.abs(g.bpm / song.bpm - c) / c < 0.02) ?? 0; - // Compare against the estimated beat period so a 2x/0.5x grid is judged on its own beat. - const estBeat = 60 / g.bpm; + const estBeat = 60 / g.bpm; // judged on the estimated beat so a 2x/0.5x grid is compared on its own beat rows.push({ id: `${tier}-${seed}`, trueBpm: song.bpm, estBpm: g.bpm, factor: f, beatErrMs: circ(g.firstDownbeat, estBeat) * 1000, barErrMs: circ(g.firstDownbeat, estBeat * g.beatsPerBar) * 1000, }); - void beat; } } console.log(rows.map((r) => `${r.id} true=${r.trueBpm} est=${r.estBpm.toFixed(2)} f=${r.factor} beatErr=${r.beatErrMs.toFixed(1)}ms barErr=${r.barErrMs.toFixed(1)}ms`).join("\n")); }, 600_000); +afterAll(async () => { if (root) await rm(root, { recursive: true, force: true }); }); + +// ---- synthetic drum/chord patterns ------------------------------------------------------------------------------- +const SR = 44100; +function synth(opts: { bpm: number; bars: number; lead: number; pickupBeats?: number; kicks: number[]; snares: number[]; eighthHats: boolean }): Audio { + const beat = 60 / opts.bpm; + const total = opts.lead + ((opts.pickupBeats ?? 0) + opts.bars * 4) * beat + 0.5; + const x = new Float32Array(Math.floor(total * SR)); + let seed = 12345; + const rnd = () => ((seed = (seed * 1664525 + 1013904223) >>> 0) / 2 ** 32) * 2 - 1; + const add = (t: number, len: number, f: (i: number, tt: number) => number) => { + const s = Math.floor(t * SR); + for (let i = 0; i < len * SR && s + i < x.length; i++) x[s + i]! += f(i, i / SR); + }; + const kick = (t: number) => add(t, 0.25, (_, tt) => 0.9 * Math.sin(2 * Math.PI * (50 + 80 * Math.exp(-tt * 40)) * tt) * Math.exp(-tt * 14) + 1.2 * rnd() * Math.exp(-tt * 150)); + const snare = (t: number) => add(t, 0.15, (_, tt) => 1.5 * rnd() * Math.exp(-tt * 18)); + const hat = (t: number, a: number) => add(t, 0.04, (_, tt) => a * rnd() * Math.exp(-tt * 120)); + const t0 = opts.lead + (opts.pickupBeats ?? 0) * beat; // true first downbeat + for (let p = 0; p < (opts.pickupBeats ?? 0); p++) hat(opts.lead + p * beat, 0.2); + for (let b = 0; b < opts.bars; b++) { + const barT = t0 + b * 4 * beat; + // alternating two-chord progression: a sustained chord per bar (harmonic change on the bar line) + const chord = b % 2 === 0 ? [220, 277.2, 329.6] : [196, 246.9, 293.7]; + add(barT, 4 * beat, (_, tt) => chord.reduce((s, f) => s + 0.04 * Math.sin(2 * Math.PI * f * tt), 0) * Math.min(1, tt * 4)); + for (const k of opts.kicks) kick(barT + k * beat); + for (const k of opts.snares) snare(barT + k * beat); + for (let h = 0; h < (opts.eighthHats ? 8 : 4); h++) hat(barT + (h * beat) / (opts.eighthHats ? 2 : 1), 0.12); + } + return { sampleRate: SR, channels: [x] }; +} + test("gridFromTruth is the oracle grid", () => { expect(gridFromTruth(80, 4)).toEqual({ bpm: 80, firstDownbeat: 0, beatsPerBar: 4 }); }); +test("leading silence: downbeat found at the true downbeat, not at the first beat after t=0", () => { + const g = estimateGrid(synth({ bpm: 90, bars: 8, lead: 0.7, kicks: [0], snares: [], eighthHats: false })); + expect(Math.abs(g.bpm / 90 - 1)).toBeLessThan(0.02); + expect(circ(g.firstDownbeat - 0.7, (60 / g.bpm) * 4) * 1000).toBeLessThan(50); +}); + +test("one-beat pickup: bar phase is correct", () => { + const beat = 60 / 90; + const g = estimateGrid(synth({ bpm: 90, bars: 8, lead: 0, pickupBeats: 1, kicks: [0], snares: [], eighthHats: false })); + expect(Math.abs(g.bpm / 90 - 1)).toBeLessThan(0.02); + expect(circ(g.firstDownbeat - beat, (60 / g.bpm) * 4) * 1000).toBeLessThan(50); +}); + +test("genuine fast tempo (140 bpm kick/snare/8th-hat rock) is not halved", () => { + const g = estimateGrid(synth({ bpm: 140, bars: 12, lead: 0, kicks: [0, 2], snares: [1, 3], eighthHats: true })); + expect(Math.abs(g.bpm / 140 - 1)).toBeLessThan(0.02); +}); + +test("rejects too-short and silent input", () => { + expect(() => estimateGrid({ sampleRate: SR, channels: [new Float32Array(100)] })).toThrow(/too short/); + expect(() => estimateGrid({ sampleRate: SR, channels: [new Float32Array(SR * 5)] })).toThrow(/silent/); +}); + +// ---- tier-1 mixes ------------------------------------------------------------------------------------------------ test("tempo within 2% (allowing 0.5x/1x/2x) in >= 8 of 10 songs", () => { expect(rows.filter((r) => r.factor !== 0).length).toBeGreaterThanOrEqual(8); }); @@ -49,12 +101,7 @@ test("beat phase within 50 ms of 0 in >= 8 of 10 songs", () => { expect(rows.filter((r) => r.beatErrMs <= 50).length).toBeGreaterThanOrEqual(8); }); -test("report exact 1x and downbeat-phase hit counts", () => { - const exact = rows.filter((r) => r.factor === 1).length; - const bar = rows.filter((r) => r.factor === 1 && r.barErrMs <= 50).length; - console.log(`exact 1x: ${exact}/10, bar-phase right: ${bar}/10`); - expect(exact).toBeGreaterThanOrEqual(0); +test("exact 1x tempo and correct bar phase in >= 8 of 10 songs", () => { + expect(rows.filter((r) => r.factor === 1).length).toBeGreaterThanOrEqual(8); + expect(rows.filter((r) => r.factor === 1 && r.barErrMs <= 50).length).toBeGreaterThanOrEqual(8); }); - -import { afterAll } from "bun:test"; -afterAll(async () => { if (root) await rm(root, { recursive: true, force: true }); });