93 lines
5.4 KiB
TypeScript
93 lines
5.4 KiB
TypeScript
import { describe, expect, test } from "bun:test";
|
|
import { mkdtemp, rm } from "node:fs/promises";
|
|
import { tmpdir } from "node:os";
|
|
import { join } from "node:path";
|
|
import { renderRealReport, runReal, type RealSongResult } from "../../src/bench/real.ts";
|
|
import { buildSong, loadSongDir } from "../../src/bench/dataset.ts";
|
|
import { writeSmf } from "../../src/align/smf.ts";
|
|
import { writeAlignment, type SongEntry } from "../../src/ref/manifest.ts";
|
|
import { createPipeline } from "../../src/pipeline/run.ts";
|
|
import { scoreSong } from "../../src/eval/score.ts";
|
|
import type { Role } from "../../src/core/types.ts";
|
|
|
|
const hasReference = await Bun.file("reference/songs.json").exists();
|
|
const mr = (f1: number | null, tp = 8, fp = 2, fn = 2) => ({ tp, fp, fn, precision: f1, recall: f1, f1, octaveErrors: 0, matches: [] as [number, number][] });
|
|
const fit = (rate: number) => ({ matchRate: rate, medianResidualMs: 12.3, windows: [{ startSec: 0, matchRate: rate, medianResidualMs: 12.3, flagged: false }, { startSec: 30, matchRate: 0.2, medianResidualMs: 80, flagged: true }] });
|
|
const fixture = (): RealSongResult[] => [
|
|
{
|
|
id: "a", title: "Song A", alignment: { source: "auto", manual: 0, auto: 40, savedAt: null }, fit: fit(0.8),
|
|
perRole: { bass: mr(0.5), guitar: mr(0.9), keys: mr(null, 0, 0, 0) }, lenient: { bass: 0.6, guitar: 0.95, keys: null },
|
|
drumsPerClass: null, excludedTracks: ["Click", "Pad"], capo: { Lead: 2 }, seconds: 3,
|
|
},
|
|
{
|
|
id: "b", title: "Song B", alignment: { source: "manual", manual: 5, auto: 30, savedAt: "2026-10-29T10:00:00Z" }, fit: fit(0.6),
|
|
perRole: { guitar: mr(0.3) }, lenient: { guitar: 0.4 }, drumsPerClass: null, excludedTracks: [], capo: {}, seconds: 4,
|
|
},
|
|
];
|
|
|
|
describe("renderRealReport", () => {
|
|
const md = renderRealReport("fx", fixture());
|
|
test("guitar row precedes the other roles; auto source and fit shown", () => {
|
|
expect(md).toContain("Song A");
|
|
expect(md).toContain("computed this run");
|
|
expect(md).toContain("2026-10-29T10:00:00Z");
|
|
expect(md).toContain("0 manual / 40 auto");
|
|
expect(md).toContain("80.0% of MIDI onsets matched");
|
|
expect(md).toContain("consistency of the alignment");
|
|
const a = md.slice(md.indexOf("Song A"), md.indexOf("Song B"));
|
|
expect(a.indexOf("| guitar")).toBeGreaterThan(-1);
|
|
expect(a.indexOf("| guitar")).toBeLessThan(a.indexOf("| bass"));
|
|
expect(a.indexOf("| bass")).toBeLessThan(a.indexOf("| keys"));
|
|
expect(a).toContain("0.90");
|
|
expect(a).toContain("0.95");
|
|
expect(a).toContain("Click, Pad");
|
|
expect(a).toContain("Lead: 2");
|
|
});
|
|
});
|
|
|
|
describe("runReal", () => {
|
|
test("t1a fixture with identity alignment matches the synthetic full-mode score", async () => {
|
|
const root = await mkdtemp(join(tmpdir(), "stsheets-real-"));
|
|
try {
|
|
const dir = await buildSong("t1a", 1, join(root, "data"));
|
|
const { truth } = await loadSongDir(dir);
|
|
const prog: Record<Role, number> = { guitar: 25, bass: 33, keys: 0, vocals: 53, drums: 0 };
|
|
const tracks = truth.tracks.map((t) => ({ name: t.name, program: prog[t.role], isDrums: t.role === "drums", notes: t.notes }));
|
|
const ref = join(root, "ref");
|
|
await Bun.write(join(ref, "t1a-1", "song.mid"), writeSmf(tracks, truth.bpm));
|
|
const entry: SongEntry = {
|
|
id: "t1a-1", title: "Synthetic t1a/1", audio: join(dir, "mix.wav"), midi: join(ref, "t1a-1", "song.mid"),
|
|
roles: Object.fromEntries(truth.tracks.map((t) => [t.name, t.role])), capo: {},
|
|
};
|
|
await Bun.write(join(ref, "songs.json"), JSON.stringify([entry, { ...entry, id: "noalign", title: "No alignment" }]));
|
|
const dur = 600;
|
|
await writeAlignment("t1a-1", { anchors: [{ midiSec: 0, audioSec: 0, kind: "manual" }, { midiSec: dur, audioSec: dur, kind: "manual" }], savedAt: "2026-10-30T00:00:00Z", fit: null }, ref);
|
|
|
|
const results = await runReal({ runId: "t", auto: false, root: ref, out: join(root, "runs"), resultsDir: join(root, "results") });
|
|
expect(results.map((r) => r.id)).toEqual(["t1a-1"]);
|
|
const r = results[0]!;
|
|
expect(r.alignment).toEqual({ source: "manual", manual: 2, auto: 0, savedAt: "2026-10-30T00:00:00Z" });
|
|
expect(r.fit.matchRate).not.toBeNull();
|
|
|
|
const pipe = await createPipeline({
|
|
demucs: new Uint8Array(await Bun.file("models/demucs/htdemucs_6s.onnx").arrayBuffer()),
|
|
basicPitch: new Uint8Array(await Bun.file("models/basic-pitch/nmp.onnx").arrayBuffer()),
|
|
});
|
|
const flags = { oracleStems: false, oracleTempo: false, oracleNotes: false };
|
|
const loaded = await loadSongDir(dir);
|
|
const synth = scoreSong(truth, (await pipe({ mix: loaded.mix }, flags)).output, flags);
|
|
expect(Math.abs((r.perRole.guitar!.f1 ?? 0) - (synth.guitarMerged.f1 ?? 0))).toBeLessThan(0.02);
|
|
|
|
for (const f of ["notes.json", "out.gp", "score.json"]) expect(await Bun.file(join(root, "runs", "real-t", "t1a-1", f)).exists()).toBe(true);
|
|
const json = await Bun.file(join(root, "results", "real-t.json")).text();
|
|
expect(json).not.toContain('"matches"');
|
|
expect(await Bun.file(join(root, "results", "real-t.md")).text()).toContain("Synthetic t1a/1");
|
|
} finally { await rm(root, { recursive: true, force: true }); }
|
|
}, 600_000);
|
|
|
|
test.skipIf(!hasReference)("real reference manifest loads", async () => {
|
|
const { readManifest } = await import("../../src/ref/manifest.ts");
|
|
expect((await readManifest()).length).toBeGreaterThan(0);
|
|
});
|
|
});
|