perf(align): render previews on worker threads in parallel chunks, send Opus
The MIDI preview render (~2 s for a 5 min song) and auto-align ran synchronously on the server's event loop, and each preview was a 12.4 MB WAV. Renders now run as 4 parallel time chunks (2 s pre-roll, 10 ms cross-fade) on worker threads, auto-align on its own worker, and previews are mono Ogg Opus 48 kb/s via ffmpeg (1.5 MB, same timing as the WAV in Chrome). ?format=wav keeps the PCM output. Swimming: render 2.0 s -> 0.65 s + 0.6 s encode; event loop blocked 2.0 s -> <0.1 s per render. Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
7b7a1cc6c9
commit
3ce58e1e05
@@ -0,0 +1,64 @@
|
||||
// One worker thread (./jobs.ts) running jobs one at a time in FIFO order. Jobs whose AbortSignal fired while queued
|
||||
// are dropped without running (e.g. a preview render superseded by a newer edit). A crashed worker is replaced.
|
||||
import type { Job } from "./jobs.ts";
|
||||
|
||||
interface Pending { job: Job; signal?: AbortSignal; resolve(v: unknown): void; reject(e: Error): void }
|
||||
|
||||
export class JobWorker {
|
||||
private w: Worker | null = null;
|
||||
private queue: Pending[] = [];
|
||||
private current: { id: number; p: Pending } | null = null;
|
||||
private nextId = 1;
|
||||
private closed = false;
|
||||
|
||||
run<T>(job: Job, signal?: AbortSignal): Promise<T> {
|
||||
if (this.closed) return Promise.reject(new Error("job worker stopped"));
|
||||
return new Promise<T>((resolve, reject) => {
|
||||
this.queue.push({ job, signal, resolve: resolve as (v: unknown) => void, reject });
|
||||
this.pump();
|
||||
});
|
||||
}
|
||||
|
||||
/** Jobs queued or running. */
|
||||
get load(): number { return this.queue.length + (this.current ? 1 : 0); }
|
||||
|
||||
terminate(): void {
|
||||
this.closed = true;
|
||||
this.w?.terminate(); this.w = null;
|
||||
const err = new Error("job worker stopped");
|
||||
this.current?.p.reject(err); this.current = null;
|
||||
for (const p of this.queue.splice(0)) p.reject(err);
|
||||
}
|
||||
|
||||
private worker(): Worker {
|
||||
if (this.w) return this.w;
|
||||
const w = new Worker(new URL("./jobs.ts", import.meta.url).href);
|
||||
(w as Worker & { unref?(): void }).unref?.(); // never keeps the process alive
|
||||
w.onmessage = (e: MessageEvent<{ id: number; result?: unknown; error?: string }>) => {
|
||||
const c = this.current;
|
||||
if (!c || c.id !== e.data.id) return;
|
||||
this.current = null;
|
||||
if (e.data.error !== undefined) c.p.reject(new Error(e.data.error)); else c.p.resolve(e.data.result);
|
||||
this.pump();
|
||||
};
|
||||
w.onerror = (e) => {
|
||||
e.preventDefault?.();
|
||||
const c = this.current;
|
||||
this.current = null;
|
||||
if (this.w === w) { w.terminate(); this.w = null; }
|
||||
c?.p.reject(new Error(`job worker crashed: ${e.message}`));
|
||||
this.pump();
|
||||
};
|
||||
return (this.w = w);
|
||||
}
|
||||
|
||||
private pump(): void {
|
||||
while (!this.current && this.queue.length && !this.closed) {
|
||||
const p = this.queue.shift()!;
|
||||
if (p.signal?.aborted) { p.reject(new DOMException("aborted", "AbortError")); continue; }
|
||||
const id = this.nextId++;
|
||||
this.current = { id, p };
|
||||
this.worker().postMessage({ id, job: p.job });
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
// Worker thread for the alignment server's heavy, synchronous jobs (MIDI preview render, automatic alignment), so the
|
||||
// server's event loop keeps answering other requests (page, fit, audio) while they run. Started by ./jobs-pool.ts.
|
||||
/// <reference lib="webworker" />
|
||||
import { renderMidi } from "./render.ts";
|
||||
import { autoAlign } from "./auto.ts";
|
||||
import { makeWarp, type Anchor } from "./warp.ts";
|
||||
import type { Audio, Role } from "../core/types.ts";
|
||||
import type { RefTrack, Smf } from "./smf.ts";
|
||||
|
||||
export type Job =
|
||||
/** Mono, not normalised, audio time [startSec, startSec + durationSec) -> Float32Array. */
|
||||
| { type: "render"; sf2Path: string; tracks: RefTrack[]; anchors: Anchor[]; sampleRate: number; startSec: number; durationSec: number }
|
||||
| { type: "auto"; sf2Path: string; audio: Audio; smf: Smf; roles: Record<string, Role | null>; manual: Anchor[] };
|
||||
|
||||
const sf2s = new Map<string, Promise<Uint8Array>>();
|
||||
const sf2 = (p: string) => { let x = sf2s.get(p); if (!x) sf2s.set(p, (x = Bun.file(p).arrayBuffer().then((b) => new Uint8Array(b)))); return x; };
|
||||
|
||||
async function run(job: Job): Promise<unknown> {
|
||||
if (job.type === "render") {
|
||||
const a = renderMidi(job.tracks, makeWarp(job.anchors), await sf2(job.sf2Path), {
|
||||
sampleRate: job.sampleRate, mono: true, startSec: job.startSec, durationSec: job.durationSec, normalize: false,
|
||||
});
|
||||
return a.channels[0]!; // transferred back
|
||||
}
|
||||
return autoAlign(job.audio, job.smf, job.roles, job.manual, await sf2(job.sf2Path));
|
||||
}
|
||||
|
||||
self.onmessage = async (e: MessageEvent<{ id: number; job: Job }>) => {
|
||||
const { id, job } = e.data;
|
||||
try {
|
||||
const result = await run(job);
|
||||
self.postMessage({ id, result }, ArrayBuffer.isView(result) ? [result.buffer as ArrayBuffer] : []);
|
||||
} catch (err) {
|
||||
self.postMessage({ id, error: (err as Error).message ?? String(err) });
|
||||
}
|
||||
};
|
||||
@@ -0,0 +1,52 @@
|
||||
// MIDI preview rendering split into time chunks that render in parallel (one per worker thread), then stitched.
|
||||
// Each chunk is rendered with a pre-roll (notes already sounding are re-attacked inside it and it is discarded) and a
|
||||
// short overlap that is cross-faded into the next chunk, so seams are inaudible for a preview. Pure: no I/O.
|
||||
|
||||
export interface Chunk {
|
||||
/** Audio time of the chunk's first output sample (includes the pre-roll). */
|
||||
renderStart: number;
|
||||
/** Seconds to render from `renderStart`. */
|
||||
renderDur: number;
|
||||
/** The part of the timeline this chunk owns: [from, to). */
|
||||
from: number; to: number;
|
||||
}
|
||||
|
||||
export const PREROLL_SEC = 2;
|
||||
export const XFADE_SEC = 0.01;
|
||||
/** Chunks shorter than this are not worth a worker (synth start-up, pre-roll overhead). */
|
||||
export const MIN_CHUNK_SEC = 20;
|
||||
|
||||
/** Splits [0, durSec) into at most `maxChunks` equal chunks of at least MIN_CHUNK_SEC (always at least one). */
|
||||
export function planChunks(durSec: number, maxChunks: number): Chunk[] {
|
||||
const n = Math.max(1, Math.min(maxChunks, Math.floor(durSec / MIN_CHUNK_SEC)));
|
||||
const out: Chunk[] = [];
|
||||
for (let k = 0; k < n; k++) {
|
||||
const from = (durSec * k) / n, to = k === n - 1 ? durSec : (durSec * (k + 1)) / n;
|
||||
const renderStart = k === 0 ? 0 : from - PREROLL_SEC;
|
||||
const end = k === n - 1 ? durSec : to + XFADE_SEC;
|
||||
out.push({ renderStart, renderDur: end - renderStart, from, to });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Joins rendered chunks (mono, `sampleRate`, as planned by `planChunks`) into one signal of round(durSec·sr) samples:
|
||||
* each chunk contributes [from, to), and its XFADE_SEC overlap past `to` fades out while the next chunk fades in.
|
||||
*/
|
||||
export function stitchChunks(chunks: Chunk[], pcm: Float32Array[], sampleRate: number, durSec: number): Float32Array {
|
||||
const n = Math.round(durSec * sampleRate), out = new Float32Array(n);
|
||||
const xf = Math.max(1, Math.round(XFADE_SEC * sampleRate));
|
||||
chunks.forEach((c, k) => {
|
||||
const x = pcm[k]!, s0 = Math.round(c.from * sampleRate), s1 = k === chunks.length - 1 ? n : Math.round(c.to * sampleRate);
|
||||
const off = Math.round(c.renderStart * sampleRate); // output index of x[0]
|
||||
for (let i = s0; i < s1; i++) {
|
||||
const v = x[i - off] ?? 0;
|
||||
// first xf samples of a later chunk fade in; the previous chunk's overlap was already added (fading out)
|
||||
out[i] = k > 0 && i - s0 < xf ? out[i]! + v * ((i - s0) / xf) : v;
|
||||
}
|
||||
if (k < chunks.length - 1) {
|
||||
for (let j = 0; j < xf && s1 + j < n; j++) out[s1 + j] = (x[s1 + j - off] ?? 0) * (1 - j / xf);
|
||||
}
|
||||
});
|
||||
return out;
|
||||
}
|
||||
+14
-5
@@ -14,12 +14,16 @@ const MELODIC_CHANNELS = [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13, 14, 15];
|
||||
* Render reference tracks to audio with `warp` applied (MIDI time -> audio time). Notes are placed on a fixed-tempo
|
||||
* tick grid so alphaSynth output time equals audio time. Peak-normalised to -1 dBFS; `durationSec` pads/trims.
|
||||
* Pure: no I/O. Notes landing before audio t=0 are clamped to 0 (dropped if they end before it).
|
||||
* `startSec` renders a window: output sample 0 is audio time `startSec`; notes sounding at `startSec` are re-attacked
|
||||
* there (callers render a pre-roll and discard it) and notes starting after the window are skipped.
|
||||
* `normalize: false` skips the peak normalisation (e.g. for chunks normalised together later).
|
||||
*/
|
||||
export function renderMidi(
|
||||
tracks: RefTrack[], warp: Warp, sf2: Uint8Array,
|
||||
opts: { sampleRate?: number; mono?: boolean; durationSec?: number } = {},
|
||||
opts: { sampleRate?: number; mono?: boolean; durationSec?: number; startSec?: number; normalize?: boolean } = {},
|
||||
): Audio {
|
||||
const sampleRate = opts.sampleRate ?? RENDER_SR;
|
||||
const shift = opts.startSec ?? 0, endSec = opts.durationSec ?? Infinity;
|
||||
const midi = new at.midi.MidiFile();
|
||||
midi.division = 960;
|
||||
midi.addEvent(new at.midi.TempoChangeEvent(0, 500000));
|
||||
@@ -30,9 +34,9 @@ export function renderMidi(
|
||||
const ch = t.isDrums ? 9 : MELODIC_CHANNELS[next++ % MELODIC_CHANNELS.length]!;
|
||||
if (!seen.has(ch)) { seen.add(ch); midi.addEvent(new at.midi.ProgramChangeEvent(0, 0, ch, t.isDrums ? 0 : t.program)); }
|
||||
for (const n of t.notes) {
|
||||
const offSec = warp.toAudio(n.offset);
|
||||
if (offSec <= 0) continue;
|
||||
const on = Math.round(Math.max(0, warp.toAudio(n.onset)) * TPS);
|
||||
const offSec = warp.toAudio(n.offset) - shift, onSec = warp.toAudio(n.onset) - shift;
|
||||
if (offSec <= 0 || onSec >= endSec) continue;
|
||||
const on = Math.round(Math.max(0, onSec) * TPS);
|
||||
const off = Math.max(on + 1, Math.round(offSec * TPS));
|
||||
midi.addEvent(new at.midi.NoteOnEvent(0, on, ch, n.pitch, n.velocity));
|
||||
midi.addEvent(new at.midi.NoteOffEvent(0, off, ch, n.pitch, 0));
|
||||
@@ -49,8 +53,13 @@ export function renderMidi(
|
||||
chans = [L.map((v, i) => (v + R[i]!) / 2)];
|
||||
}
|
||||
for (const c of chans) for (let i = 0; i < c.length; i++) if (!Number.isFinite(c[i]!)) throw new Error("renderMidi: non-finite samples from synth");
|
||||
if (opts.normalize !== false) normalizePeak(chans);
|
||||
return { sampleRate, channels: chans };
|
||||
}
|
||||
|
||||
/** Scales in place so the peak is -1 dBFS (silence stays silent). */
|
||||
export function normalizePeak(chans: Float32Array[]): void {
|
||||
let peak = 0;
|
||||
for (const c of chans) for (let i = 0; i < c.length; i++) { const a = Math.abs(c[i]!); if (a > peak) peak = a; }
|
||||
if (peak > 0) { const g = PEAK / peak; for (const c of chans) for (let i = 0; i < c.length; i++) c[i]! *= g; }
|
||||
return { sampleRate, channels: chans };
|
||||
}
|
||||
|
||||
+84
-23
@@ -8,8 +8,10 @@ import type { Role } from "../core/types.ts";
|
||||
import { audioOgg, loadSong, readAlignment, readManifest, writeAlignment, type AlignmentFile, type LoadedSong } from "../ref/manifest.ts";
|
||||
import { trackKeys } from "./roles.ts";
|
||||
import { makeWarp, stretches, type Anchor } from "./warp.ts";
|
||||
import { autoAlign, fitFromCues, recordingCues, type AutoResult, type RecordingCues } from "./auto.ts";
|
||||
import { renderMidi } from "./render.ts";
|
||||
import { fitFromCues, recordingCues, type AutoResult, type RecordingCues } from "./auto.ts";
|
||||
import { normalizePeak } from "./render.ts";
|
||||
import { JobWorker } from "./jobs-pool.ts";
|
||||
import { planChunks, stitchChunks } from "./preview.ts";
|
||||
import { peaks } from "./peaks.ts";
|
||||
import { stretchInfo } from "./tempo.ts";
|
||||
import type { FitReport } from "./fit.ts";
|
||||
@@ -24,7 +26,11 @@ const MAX_BODY_BYTES = 1 << 20;
|
||||
const PEAKS_PER_SEC = 2000 / 60;
|
||||
const PEAKS_VERSION = 1;
|
||||
const RENDER_SR = 22050;
|
||||
const RENDER_CACHE = 6;
|
||||
const RENDER_CACHE = 24; // Opus previews are ~1.5 MB per 5 min
|
||||
/** Parallel render workers (each renders one time chunk of a preview). */
|
||||
const RENDER_WORKERS = Math.max(1, Math.min(4, (navigator.hardwareConcurrency ?? 2) - 1));
|
||||
// Preview transport: mono Opus. 48 kb/s at compression level 0 encodes 5 min in ~0.6 s to ~1.5 MB (vs 12 MB WAV).
|
||||
const OPUS_ARGS = ["-c:a", "libopus", "-b:a", "48k", "-compression_level", "0", "-f", "ogg", "pipe:1"];
|
||||
const SONG_CACHE = 2;
|
||||
|
||||
export interface ServeOptions { root?: string; host?: string; port?: number; urlHost?: string; quiet?: boolean; sf2Path?: string }
|
||||
@@ -34,11 +40,26 @@ interface SongState extends LoadedSong {
|
||||
roles: Record<string, Role | null>;
|
||||
cues: RecordingCues;
|
||||
peaks: { bucketSec: number; min: number[]; max: number[] };
|
||||
auto?: AutoResult; // first automatic pass, reused while no alignment.json exists
|
||||
auto?: Promise<AutoResult>; // first automatic pass, reused while no alignment.json exists
|
||||
}
|
||||
|
||||
class HttpError extends Error { constructor(readonly status: number, msg: string) { super(msg); } }
|
||||
|
||||
/** Mono float PCM -> Ogg Opus via ffmpeg (a separate process: the event loop stays free). */
|
||||
async function encodeOpus(pcm: Float32Array, sampleRate: number): Promise<Uint8Array<ArrayBuffer>> {
|
||||
const p = Bun.spawn(["ffmpeg", "-nostdin", "-loglevel", "error", "-f", "f32le", "-ar", String(sampleRate), "-ac", "1", "-i", "pipe:0", ...OPUS_ARGS], {
|
||||
stdin: new Uint8Array(pcm.buffer, pcm.byteOffset, pcm.byteLength), stdout: "pipe", stderr: "pipe",
|
||||
});
|
||||
const [out, err, code] = await Promise.all([new Response(p.stdout).arrayBuffer(), new Response(p.stderr).text(), p.exited]);
|
||||
if (code !== 0) throw new Error(`ffmpeg (opus) failed (${code}): ${err.trim()}`);
|
||||
return new Uint8Array(out);
|
||||
}
|
||||
|
||||
/** Copies audio into SharedArrayBuffers so worker jobs can read it without a copy per job. */
|
||||
function shareAudio(a: LoadedSong["audio"]): LoadedSong["audio"] {
|
||||
return { sampleRate: a.sampleRate, channels: a.channels.map((c) => { const x = new Float32Array(new SharedArrayBuffer(c.byteLength)); x.set(c); return x; }) };
|
||||
}
|
||||
|
||||
/** Small insertion-ordered LRU. */
|
||||
class Lru<V> {
|
||||
private m = new Map<string, V>();
|
||||
@@ -84,10 +105,15 @@ export async function startServer(opts: ServeOptions = {}): Promise<AlignServer>
|
||||
const host = opts.host ?? "127.0.0.1";
|
||||
const log = (msg: string) => { if (!opts.quiet) console.log(msg); };
|
||||
const token = Buffer.from(crypto.getRandomValues(new Uint8Array(18))).toString("base64url");
|
||||
const sf2 = new Uint8Array(await Bun.file(opts.sf2Path ?? SF2_PATH).arrayBuffer());
|
||||
const sf2Path = opts.sf2Path ?? SF2_PATH;
|
||||
if (!(await Bun.file(sf2Path).exists())) throw new Error(`soundfont not found: ${sf2Path}`);
|
||||
// Heavy work runs on worker threads so this event loop keeps serving while a render or auto-align is in progress.
|
||||
const renderWorkers = Array.from({ length: RENDER_WORKERS }, () => new JobWorker());
|
||||
const autoWorker = new JobWorker();
|
||||
|
||||
const songs = new Lru<Promise<SongState>>(SONG_CACHE);
|
||||
const renders = new Lru<Uint8Array<ArrayBuffer>>(RENDER_CACHE);
|
||||
const inflight = new Map<string, Promise<Uint8Array<ArrayBuffer>>>();
|
||||
|
||||
async function entryExists(id: string): Promise<boolean> {
|
||||
return (await readManifest(root)).some((s) => s.id === id);
|
||||
@@ -95,7 +121,8 @@ export async function startServer(opts: ServeOptions = {}): Promise<AlignServer>
|
||||
|
||||
async function buildState(id: string): Promise<SongState> {
|
||||
const t0 = performance.now();
|
||||
const s = await loadSong(id, root);
|
||||
const loaded = await loadSong(id, root);
|
||||
const s = { ...loaded, audio: shareAudio(loaded.audio) };
|
||||
const keys = trackKeys(s.smf.tracks);
|
||||
const roles = Object.fromEntries(keys.map((k) => [k, s.entry.roles[k] ?? null]));
|
||||
const cues = recordingCues(s.audio);
|
||||
@@ -131,9 +158,9 @@ export async function startServer(opts: ServeOptions = {}): Promise<AlignServer>
|
||||
const stretchesOf = (s: SongState, anchors: Anchor[]) => stretchInfo(stretches(anchors, s.smf.durationSec), s.smf.tempoMap);
|
||||
const state = (s: SongState, anchors: Anchor[], fit = fitOf(s, anchors)) => ({ anchors, stretches: stretchesOf(s, anchors), fit });
|
||||
|
||||
function runAuto(s: SongState, manual: Anchor[]) {
|
||||
async function runAuto(s: SongState, manual: Anchor[]): Promise<AutoResult> {
|
||||
const t0 = performance.now();
|
||||
const r = autoAlign(s.audio, s.smf, s.roles, manual, sf2);
|
||||
const r = await autoWorker.run<AutoResult>({ type: "auto", sf2Path, audio: s.audio, smf: s.smf, roles: s.roles, manual });
|
||||
log(`auto-aligned ${s.entry.id}: ${r.anchors.length} anchors (${manual.length} manual, ${r.chosen}) in ${((performance.now() - t0) / 1000).toFixed(1)} s`);
|
||||
return r;
|
||||
}
|
||||
@@ -151,6 +178,25 @@ export async function startServer(opts: ServeOptions = {}): Promise<AlignServer>
|
||||
}
|
||||
}
|
||||
|
||||
/** Warped MIDI rendered in parallel time chunks on the render workers, stitched, normalised and encoded. */
|
||||
async function renderPreview(s: SongState, anchors: Anchor[], format: "wav" | "opus", signal: AbortSignal): Promise<Uint8Array<ArrayBuffer>> {
|
||||
const t0 = performance.now();
|
||||
const warp = makeWarp(anchors);
|
||||
const dur = Math.max(s.audioDur, Math.min(warp.toAudio(s.smf.durationSec) + 1, s.audioDur + 60));
|
||||
const chunks = planChunks(dur, renderWorkers.length);
|
||||
const pcm = await Promise.all(chunks.map((c) => {
|
||||
const w = renderWorkers.reduce((a, b) => (b.load < a.load ? b : a));
|
||||
return w.run<Float32Array>({ type: "render", sf2Path, tracks: s.smf.tracks, anchors, sampleRate: RENDER_SR, startSec: c.renderStart, durationSec: c.renderDur }, signal);
|
||||
}));
|
||||
const mono = stitchChunks(chunks, pcm, RENDER_SR, dur);
|
||||
for (let i = 0; i < mono.length; i++) if (!Number.isFinite(mono[i]!)) throw new Error("render: non-finite samples from synth");
|
||||
normalizePeak([mono]);
|
||||
const t1 = performance.now();
|
||||
const out = format === "wav" ? encodeWav({ sampleRate: RENDER_SR, channels: [mono] }) : await encodeOpus(mono, RENDER_SR);
|
||||
log(`rendered ${s.entry.id} (${anchors.length} anchors, ${chunks.length} chunks) in ${((t1 - t0) / 1000).toFixed(2)} s + ${format} ${((performance.now() - t1) / 1000).toFixed(2)} s, ${(out.length / 1e6).toFixed(2)} MB`);
|
||||
return out;
|
||||
}
|
||||
|
||||
async function api(req: Request, path: string): Promise<Response> {
|
||||
const method = req.method;
|
||||
if (path === "/api/songs" && method === "GET") {
|
||||
@@ -172,12 +218,17 @@ export async function startServer(opts: ServeOptions = {}): Promise<AlignServer>
|
||||
const { file: saved, error } = await savedAlignment(id);
|
||||
let anchors: Anchor[], fit: FitReport | undefined;
|
||||
if (saved) anchors = saved.anchors;
|
||||
else { s.auto ??= runAuto(s, []); anchors = s.auto.anchors; fit = s.auto.fit; }
|
||||
else {
|
||||
s.auto ??= runAuto(s, []);
|
||||
s.auto.catch(() => { s.auto = undefined; });
|
||||
const r = await s.auto;
|
||||
anchors = r.anchors; fit = r.fit;
|
||||
}
|
||||
return Response.json({
|
||||
id, title: s.entry.title, audioDur: s.audioDur, midiDur: s.smf.durationSec,
|
||||
tracks: trackKeys(s.smf.tracks).map((name) => ({ name, role: s.roles[name] ?? null })),
|
||||
recordingPeaks: s.peaks, ...state(s, anchors, fit), midiTempo: s.smf.tempoMap,
|
||||
source: saved ? "saved" : "auto", savedAt: saved?.savedAt ?? null, ...(saved ? {} : { auto: autoInfo(s.auto!) }),
|
||||
source: saved ? "saved" : "auto", savedAt: saved?.savedAt ?? null, ...(saved ? {} : { auto: autoInfo(await s.auto!) }),
|
||||
...(error ? { warning: `${error}. Showing the automatic alignment instead; saving will overwrite the file.` } : {}),
|
||||
});
|
||||
}
|
||||
@@ -186,21 +237,28 @@ export async function startServer(opts: ServeOptions = {}): Promise<AlignServer>
|
||||
return new Response(Bun.file(await audioOgg(e, root)), { headers: { "content-type": "audio/ogg", "cache-control": "private, max-age=3600" } });
|
||||
}
|
||||
if (route === "POST render") {
|
||||
// ?format=wav: 16-bit PCM (tests, debugging). Default: Ogg Opus, ~8x smaller, for the page over the LAN.
|
||||
const format = new URL(req.url).searchParams.get("format") === "wav" ? "wav" : "opus";
|
||||
const anchors = parseAnchors(await readJson(req));
|
||||
const s = await song(id);
|
||||
const hash = Bun.hash(JSON.stringify(anchors)).toString(16);
|
||||
const key = `${id}:${hash}`;
|
||||
let wav = renders.get(key);
|
||||
const hit = wav !== undefined;
|
||||
if (!wav) {
|
||||
const t0 = performance.now();
|
||||
const warp = makeWarp(anchors);
|
||||
const dur = Math.max(s.audioDur, Math.min(warp.toAudio(s.smf.durationSec) + 1, s.audioDur + 60));
|
||||
wav = encodeWav(renderMidi(s.smf.tracks, warp, sf2, { sampleRate: RENDER_SR, mono: true, durationSec: dur }));
|
||||
renders.set(key, wav);
|
||||
log(`rendered ${id} (${anchors.length} anchors) in ${((performance.now() - t0) / 1000).toFixed(1)} s`);
|
||||
const key = `${id}:${format}:${hash}`;
|
||||
let body = renders.get(key);
|
||||
const hit = body !== undefined;
|
||||
if (!body) {
|
||||
const start = () => {
|
||||
const p = renderPreview(s, anchors, format, req.signal).finally(() => { if (inflight.get(key) === p) inflight.delete(key); });
|
||||
inflight.set(key, p);
|
||||
return p;
|
||||
};
|
||||
try { body = await (inflight.get(key) ?? start()); } catch (e) {
|
||||
if ((e as Error).name !== "AbortError") throw e;
|
||||
if (req.signal.aborted) return new Response(null, { status: 499 }); // client gave up (superseded edit)
|
||||
body = await start(); // a request we shared the render with was cancelled: render for this one
|
||||
}
|
||||
renders.set(key, body);
|
||||
}
|
||||
return new Response(wav, { headers: { "content-type": "audio/wav", "x-anchors-hash": hash, "x-cache": hit ? "hit" : "miss" } });
|
||||
return new Response(body, { headers: { "content-type": format === "wav" ? "audio/wav" : "audio/ogg", "x-anchors-hash": hash, "x-cache": hit ? "hit" : "miss" } });
|
||||
}
|
||||
if (route === "POST fit") {
|
||||
const anchors = parseAnchors(await readJson(req));
|
||||
@@ -209,7 +267,7 @@ export async function startServer(opts: ServeOptions = {}): Promise<AlignServer>
|
||||
if (route === "POST auto") {
|
||||
const manual = parseAnchors(await readJson(req)).filter((a) => a.kind === "manual");
|
||||
const s = await song(id);
|
||||
const r = runAuto(s, manual);
|
||||
const r = await runAuto(s, manual);
|
||||
return Response.json({ ...state(s, r.anchors, r.fit), auto: autoInfo(r) });
|
||||
}
|
||||
if (route === "POST reset") return Response.json(state(await song(id), []));
|
||||
@@ -276,6 +334,9 @@ export async function startServer(opts: ServeOptions = {}): Promise<AlignServer>
|
||||
const origin = `http://${opts.urlHost ?? (host === "0.0.0.0" || host === "::" ? "localhost" : host)}:${outer.port}`;
|
||||
return {
|
||||
url: `${origin}/?key=${token}`, origin, token, port: outer.port!,
|
||||
async stop() { await outer.stop(true); await inner.stop(true); },
|
||||
async stop() {
|
||||
await outer.stop(true); await inner.stop(true);
|
||||
for (const w of [...renderWorkers, autoWorker]) w.terminate();
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
import { test, expect } from "bun:test";
|
||||
import { planChunks, stitchChunks, MIN_CHUNK_SEC, PREROLL_SEC } from "../../src/align/preview.ts";
|
||||
import { renderMidi, normalizePeak } from "../../src/align/render.ts";
|
||||
import { makeWarp } from "../../src/align/warp.ts";
|
||||
import type { RefNote, RefTrack } from "../../src/align/smf.ts";
|
||||
|
||||
test("planChunks covers [0, dur) contiguously, with pre-roll after the first chunk", () => {
|
||||
const c = planChunks(280, 4);
|
||||
expect(c).toHaveLength(4);
|
||||
expect(c[0]!.from).toBe(0);
|
||||
expect(c[3]!.to).toBe(280);
|
||||
for (let k = 1; k < 4; k++) {
|
||||
expect(c[k]!.from).toBe(c[k - 1]!.to);
|
||||
expect(c[k]!.renderStart).toBeCloseTo(c[k]!.from - PREROLL_SEC, 9);
|
||||
}
|
||||
expect(planChunks(MIN_CHUNK_SEC * 1.5, 4)).toHaveLength(1);
|
||||
expect(planChunks(5, 4)).toEqual([{ renderStart: 0, renderDur: 5, from: 0, to: 5 }]);
|
||||
});
|
||||
|
||||
test("stitchChunks reproduces a signal from overlapping slices of it", () => {
|
||||
const sr = 1000, dur = 90, n = dur * sr;
|
||||
const x = Float32Array.from({ length: n }, (_, i) => Math.sin(i / 37) * 0.5);
|
||||
const chunks = planChunks(dur, 3);
|
||||
const pcm = chunks.map((c) => x.slice(Math.round(c.renderStart * sr), Math.round((c.renderStart + c.renderDur) * sr)));
|
||||
const y = stitchChunks(chunks, pcm, sr, dur);
|
||||
expect(y.length).toBe(n);
|
||||
let maxErr = 0;
|
||||
for (let i = 0; i < n; i++) maxErr = Math.max(maxErr, Math.abs(y[i]! - x[i]!));
|
||||
expect(maxErr).toBeLessThan(1e-6);
|
||||
});
|
||||
|
||||
test("a chunked render matches the whole render (onsets in place, small seam error)", async () => {
|
||||
const SF2 = new Uint8Array(await Bun.file("node_modules/@coderline/alphatab/dist/soundfont/sonivox.sf2").arrayBuffer());
|
||||
const notes: RefNote[] = [];
|
||||
for (let t = 0.25; t < 58; t += 0.5) notes.push({ onset: t, offset: t + 0.4, pitch: 52 + ((t * 2) % 12), velocity: 100, channel: 0 });
|
||||
notes.push({ onset: 18, offset: 23, pitch: 40, velocity: 100, channel: 0 }); // sustains across the first seam (20 s)
|
||||
const tracks: RefTrack[] = [{ index: 0, name: "g", program: 25, isDrums: false, notes }];
|
||||
const warp = makeWarp([{ midiSec: 0, audioSec: 0.3, kind: "manual" }, { midiSec: 50, audioSec: 51, kind: "manual" }]);
|
||||
const sr = 22050, dur = 60;
|
||||
const whole = renderMidi(tracks, warp, SF2, { sampleRate: sr, mono: true, durationSec: dur }).channels[0]!;
|
||||
const chunks = planChunks(dur, 3);
|
||||
const pcm = chunks.map((c) => renderMidi(tracks, warp, SF2, { sampleRate: sr, mono: true, startSec: c.renderStart, durationSec: c.renderDur, normalize: false }).channels[0]!);
|
||||
const y = stitchChunks(chunks, pcm, sr, dur);
|
||||
normalizePeak([y]);
|
||||
// energy per 50 ms frame agrees closely everywhere (identical away from seams)
|
||||
const frame = Math.floor(sr / 20);
|
||||
let worst = 0;
|
||||
for (let f = 0; f + frame <= y.length; f += frame) {
|
||||
let a = 0, b = 0;
|
||||
for (let i = f; i < f + frame; i++) { a += whole[i]! ** 2; b += y[i]! ** 2; }
|
||||
worst = Math.max(worst, Math.abs(Math.sqrt(a / frame) - Math.sqrt(b / frame)));
|
||||
}
|
||||
expect(worst).toBeLessThan(0.05);
|
||||
}, 60_000);
|
||||
@@ -151,10 +151,23 @@ describe("API", () => {
|
||||
expect(String.fromCharCode(...b.subarray(0, 4))).toBe("OggS");
|
||||
expect(await Bun.file(join(root, "fix", "audio.ogg")).exists()).toBe(true);
|
||||
}, BIG);
|
||||
test("render: RIFF WAV, mono 22.05 kHz, cached by anchors", async () => {
|
||||
test("render: Ogg Opus by default (small), cached by anchors", async () => {
|
||||
const body = json({ anchors: first.anchors });
|
||||
const r = await api("/api/song/fix/render", body);
|
||||
expect(r.status).toBe(200);
|
||||
expect(r.headers.get("content-type")).toBe("audio/ogg");
|
||||
const b = new Uint8Array(await r.arrayBuffer());
|
||||
expect(String.fromCharCode(...b.subarray(0, 4))).toBe("OggS");
|
||||
expect(new TextDecoder().decode(b.subarray(28, 36))).toBe("OpusHead");
|
||||
expect(b.length).toBeLessThan(first.audioDur * 22050 * 2 / 4); // far below 16-bit PCM
|
||||
const again = await api("/api/song/fix/render", body);
|
||||
expect(again.headers.get("x-cache")).toBe("hit");
|
||||
expect(again.headers.get("x-anchors-hash")).toBe(r.headers.get("x-anchors-hash"));
|
||||
}, BIG);
|
||||
test("render ?format=wav: RIFF WAV, mono 22.05 kHz, cached by anchors", async () => {
|
||||
const body = json({ anchors: first.anchors });
|
||||
const r = await api("/api/song/fix/render?format=wav", body);
|
||||
expect(r.status).toBe(200);
|
||||
expect(r.headers.get("content-type")).toBe("audio/wav");
|
||||
const b = new Uint8Array(await r.arrayBuffer());
|
||||
const dv = new DataView(b.buffer);
|
||||
@@ -162,7 +175,7 @@ describe("API", () => {
|
||||
expect(dv.getUint16(22, true)).toBe(1);
|
||||
expect(dv.getUint32(24, true)).toBe(22050);
|
||||
expect(b.length).toBeGreaterThan(first.audioDur * 22050 * 2);
|
||||
const again = await api("/api/song/fix/render", body);
|
||||
const again = await api("/api/song/fix/render?format=wav", body);
|
||||
expect(again.headers.get("x-cache")).toBe("hit");
|
||||
expect(again.headers.get("x-anchors-hash")).toBe(r.headers.get("x-anchors-hash"));
|
||||
}, BIG);
|
||||
|
||||
Reference in New Issue
Block a user