Author SHA1 Message Date
Lucas WintherandClaude Opus 5.5 1a922e2508 feat(align-page): pair marks + Link, per-track volume, no main-thread stalls
Clicking the Recording / MIDI lane places a dashed mark (nothing else changes); the Pair box
shows both as editable times (m:ss.s or seconds), R / M drop a mark at the playhead, the ▶
buttons audition each track from its mark, Link (Enter) turns them into one manual anchor
(undoable; order conflicts with your anchors are explained and the marks kept), Esc clears.
The ruler sets the playback cursor; anchors are dragged by their handles. Recording and MIDI
volume sliders (0-150 %, Web Audio gain, remembered). Waveform pyramids are built in slices,
previews cached by anchors client-side, fit not re-requested when unchanged, debounce 200 ms.
Help panel updated.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-30 22:04:46 +01:00
Lucas WintherandClaude Opus 5.5 3ce58e1e05 perf(align): render previews on worker threads in parallel chunks, send Opus
The MIDI preview render (~2 s for a 5 min song) and auto-align ran synchronously on the
server's event loop, and each preview was a 12.4 MB WAV. Renders now run as 4 parallel time
chunks (2 s pre-roll, 10 ms cross-fade) on worker threads, auto-align on its own worker, and
previews are mono Ogg Opus 48 kb/s via ffmpeg (1.5 MB, same timing as the WAV in Chrome).
?format=wav keeps the PCM output. Swimming: render 2.0 s -> 0.65 s + 0.6 s encode; event loop
blocked 2.0 s -> <0.1 s per render.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-30 22:04:46 +01:00
13 changed files with 826 additions and 99 deletions
+64
View File
@@ -0,0 +1,64 @@
// One worker thread (./jobs.ts) running jobs one at a time in FIFO order. Jobs whose AbortSignal fired while queued
// are dropped without running (e.g. a preview render superseded by a newer edit). A crashed worker is replaced.
import type { Job } from "./jobs.ts";
interface Pending { job: Job; signal?: AbortSignal; resolve(v: unknown): void; reject(e: Error): void }
export class JobWorker {
private w: Worker | null = null;
private queue: Pending[] = [];
private current: { id: number; p: Pending } | null = null;
private nextId = 1;
private closed = false;
run<T>(job: Job, signal?: AbortSignal): Promise<T> {
if (this.closed) return Promise.reject(new Error("job worker stopped"));
return new Promise<T>((resolve, reject) => {
this.queue.push({ job, signal, resolve: resolve as (v: unknown) => void, reject });
this.pump();
});
}
/** Jobs queued or running. */
get load(): number { return this.queue.length + (this.current ? 1 : 0); }
terminate(): void {
this.closed = true;
this.w?.terminate(); this.w = null;
const err = new Error("job worker stopped");
this.current?.p.reject(err); this.current = null;
for (const p of this.queue.splice(0)) p.reject(err);
}
private worker(): Worker {
if (this.w) return this.w;
const w = new Worker(new URL("./jobs.ts", import.meta.url).href);
(w as Worker & { unref?(): void }).unref?.(); // never keeps the process alive
w.onmessage = (e: MessageEvent<{ id: number; result?: unknown; error?: string }>) => {
const c = this.current;
if (!c || c.id !== e.data.id) return;
this.current = null;
if (e.data.error !== undefined) c.p.reject(new Error(e.data.error)); else c.p.resolve(e.data.result);
this.pump();
};
w.onerror = (e) => {
e.preventDefault?.();
const c = this.current;
this.current = null;
if (this.w === w) { w.terminate(); this.w = null; }
c?.p.reject(new Error(`job worker crashed: ${e.message}`));
this.pump();
};
return (this.w = w);
}
private pump(): void {
while (!this.current && this.queue.length && !this.closed) {
const p = this.queue.shift()!;
if (p.signal?.aborted) { p.reject(new DOMException("aborted", "AbortError")); continue; }
const id = this.nextId++;
this.current = { id, p };
this.worker().postMessage({ id, job: p.job });
}
}
}
+36
View File
@@ -0,0 +1,36 @@
// Worker thread for the alignment server's heavy, synchronous jobs (MIDI preview render, automatic alignment), so the
// server's event loop keeps answering other requests (page, fit, audio) while they run. Started by ./jobs-pool.ts.
/// <reference lib="webworker" />
import { renderMidi } from "./render.ts";
import { autoAlign } from "./auto.ts";
import { makeWarp, type Anchor } from "./warp.ts";
import type { Audio, Role } from "../core/types.ts";
import type { RefTrack, Smf } from "./smf.ts";
export type Job =
/** Mono, not normalised, audio time [startSec, startSec + durationSec) -> Float32Array. */
| { type: "render"; sf2Path: string; tracks: RefTrack[]; anchors: Anchor[]; sampleRate: number; startSec: number; durationSec: number }
| { type: "auto"; sf2Path: string; audio: Audio; smf: Smf; roles: Record<string, Role | null>; manual: Anchor[] };
const sf2s = new Map<string, Promise<Uint8Array>>();
const sf2 = (p: string) => { let x = sf2s.get(p); if (!x) sf2s.set(p, (x = Bun.file(p).arrayBuffer().then((b) => new Uint8Array(b)))); return x; };
async function run(job: Job): Promise<unknown> {
if (job.type === "render") {
const a = renderMidi(job.tracks, makeWarp(job.anchors), await sf2(job.sf2Path), {
sampleRate: job.sampleRate, mono: true, startSec: job.startSec, durationSec: job.durationSec, normalize: false,
});
return a.channels[0]!; // transferred back
}
return autoAlign(job.audio, job.smf, job.roles, job.manual, await sf2(job.sf2Path));
}
self.onmessage = async (e: MessageEvent<{ id: number; job: Job }>) => {
const { id, job } = e.data;
try {
const result = await run(job);
self.postMessage({ id, result }, ArrayBuffer.isView(result) ? [result.buffer as ArrayBuffer] : []);
} catch (err) {
self.postMessage({ id, error: (err as Error).message ?? String(err) });
}
};
+256 -63
View File
@@ -1,23 +1,34 @@
/// <reference lib="dom" />
/// <reference lib="dom.iterable" />
// Alignment page (layout A): two lanes (recording, warped MIDI audio) on one zoomable timeline, anchors spanning both,
// stretch tempo labels, fit-flagged windows, Web Audio playback with a mix choice. Geometry/edit logic is in the pure
// modules ./timeline.ts and ./edit.ts; warp maths come from ../warp.ts so every edit redraws instantly.
// stretch tempo labels, fit-flagged windows, Web Audio playback with a mix choice and per-track volume. The main tool is
// pairing: click the recording lane and the MIDI lane to place two marks (nothing else changes), then Link them into one
// manual anchor. The ruler sets the playback cursor; anchors are dragged by their handles. Geometry/edit logic is in the
// pure modules ./timeline.ts and ./edit.ts; warp maths come from ../warp.ts so every edit redraws instantly.
import { AUTO_CLEAR_SEC, deleteAnchor, makeWarp, stretches, type Anchor, type Warp } from "../warp.ts";
import { stretchInfo, stretchLabel, type StretchInfo } from "../tempo.ts";
import { fitSummary, WINDOW_SEC, type FitReport } from "../fit.ts";
import type { TempoPoint } from "../smf.ts";
import {
buildPyramid, clampView, columnPeaks, fitView, follow, formatTime, pyramidFromPeaks, rulerStep, scrollBy, timeToX, xToTime, zoomAt,
buildPyramidSliced, clampView, columnPeaks, fitView, follow, formatTime, parseTime, pyramidFromPeaks, rulerStep, scrollBy, timeToX, xToTime, zoomAt,
type PeakLevel, type View,
} from "./timeline.ts";
import { anchorAtCursor, dragAudio, dragMidi, hitAnchor, type Edit } from "./edit.ts";
import {
anchorAtCursor, dragAudio, dragMidi, hitAnchor, linkMarks, NO_MARKS, parseVolumes, placeMark,
type Edit, type PairMarks, type PairSide, type Volumes,
} from "./edit.ts";
// ---------- layout (CSS px) ----------
const GUTTER = 92, RULER_H = 22, LABEL_H = 18, LANE_H = 118, LANE_GAP = 8, STRETCH_H = 26;
const REC_Y = RULER_H + LABEL_H, MIDI_Y = REC_Y + LANE_H + LANE_GAP, STRETCH_Y = MIDI_Y + LANE_H, HEIGHT = STRETCH_Y + STRETCH_H;
const C = { rec: "#5b8def", midi: "#e2873b", anchor: "#2bb673", anchorAuto: "#86cfa9", warn: "#d9534f", ink: "#222", muted: "#888", lane: "#f4f5f7", flag: "rgba(217,83,79,.13)", play: "#7a3cff" };
const HIT_PX = 7, DRAG_PX = 3, RENDER_DEBOUNCE_MS = 400;
const C = {
rec: "#5b8def", midi: "#e2873b", anchor: "#2bb673", anchorAuto: "#86cfa9", warn: "#d9534f", ink: "#222", muted: "#888", lane: "#f4f5f7",
flag: "rgba(217,83,79,.13)", play: "#7a3cff", ruler: "#eef0f3", recMark: "#1f5fd6", midiMark: "#c2410c", pair: "#7a3cff",
};
const HIT_PX = 7, DRAG_PX = 3, RENDER_DEBOUNCE_MS = 200;
/** Anchor handles: grab zones (CSS px) at the top of the recording lane and the bottom of the MIDI lane. */
const HANDLE_W = 10, HANDLE_H = 9, HANDLE_ZONE = 14;
const VOLUME_KEY = "stsheets.align.volume", RENDER_CLIENT_CACHE = 8;
// ---------- DOM ----------
const $ = <T extends HTMLElement>(id: string) => document.getElementById(id) as T;
@@ -27,7 +38,11 @@ const songSel = $<HTMLSelectElement>("song"), songInfo = $("songinfo"), saveStat
const btn = {
play: $<HTMLButtonElement>("play"), add: $<HTMLButtonElement>("add"), del: $<HTMLButtonElement>("del"), undo: $<HTMLButtonElement>("undo"),
auto: $<HTMLButtonElement>("auto"), save: $<HTMLButtonElement>("save"), reset: $<HTMLButtonElement>("reset"),
link: $<HTMLButtonElement>("link"), clearMarks: $<HTMLButtonElement>("clearmarks"), playRec: $<HTMLButtonElement>("prec"), playMidi: $<HTMLButtonElement>("pmidi"),
};
const markIn = { rec: $<HTMLInputElement>("mrec"), midi: $<HTMLInputElement>("mmidi") };
const volIn = { rec: $<HTMLInputElement>("volrec"), midi: $<HTMLInputElement>("volmidi") };
const volOut = { rec: $<HTMLOutputElement>("volrecv"), midi: $<HTMLOutputElement>("volmidiv") };
const mixButtons = [...document.querySelectorAll<HTMLButtonElement>(".mix button")];
// ---------- state ----------
@@ -46,13 +61,17 @@ let info: StretchInfo[] = [];
let fit: FitReport | null = null, fitKey = "";
let savedKey: string | null = null, savedAt: string | null = null;
let view: View = { start: 0, pxPerSec: 10, width: 800 };
let cursor = 0, selected = -1, hover = -1, hoverLane: "rec" | "midi" | null = null;
let cursor = 0, selected = -1, hover = -1, hoverLane: PairSide | null = null;
let recWave: Wave | null = null, midiWave: Wave | null = null, midiKey = "";
let recBuf: AudioBuffer | null = null, midiBuf: AudioBuffer | null = null;
let busy = false, rendering = false;
const undoStack: Anchor[][] = [], redoStack: Anchor[][] = [];
let mix: Mix = "both";
let loadGen = 0;
let marks: PairMarks = NO_MARKS;
let volumes: Volumes = parseVolumes(localStorage.getItem(VOLUME_KEY));
/** Encoded preview renders by anchor key, so undo/redo replays without a server round trip. */
const renderCache = new Map<string, ArrayBuffer>();
const keyOf = (A: Anchor[]) => JSON.stringify(A.map((a) => [a.midiSec, a.audioSec, a.kind]));
const dur = () => song?.audioDur ?? 1;
@@ -77,19 +96,32 @@ function showOverlay(msg: string | null) { overlay.hidden = msg === null; overla
// ---------- audio ----------
const actx = new AudioContext();
// source -> mix gain -> volume gain -> pan -> out
const recGain = actx.createGain(), midiGain = actx.createGain();
const recVol = actx.createGain(), midiVol = actx.createGain();
const recPan = actx.createStereoPanner(), midiPan = actx.createStereoPanner();
recGain.connect(recPan).connect(actx.destination);
midiGain.connect(midiPan).connect(actx.destination);
recGain.connect(recVol).connect(recPan).connect(actx.destination);
midiGain.connect(midiVol).connect(midiPan).connect(actx.destination);
let recSrc: AudioBufferSourceNode | null = null, midiSrc: AudioBufferSourceNode | null = null;
let playing = false, playCtx0 = 0, playPos0 = 0;
/** While auditioning a mark only that track plays (the mix choice is ignored). */
let solo: PairSide | null = null;
function applyVolumes() {
for (const side of ["rec", "midi"] as const) {
(side === "rec" ? recVol : midiVol).gain.setTargetAtTime(volumes[side] / 100, actx.currentTime, 0.015);
volIn[side].value = String(volumes[side]);
volOut[side].value = `${volumes[side]}%`;
}
}
function applyMix() {
const set = (p: AudioParam, v: number) => p.setTargetAtTime(v, actx.currentTime, 0.015);
set(recGain.gain, mix === "midi" ? 0 : 1);
set(midiGain.gain, mix === "rec" ? 0 : mix === "both" ? 0.8 : 1);
set(recPan.pan, mix === "split" ? -1 : 0);
set(midiPan.pan, mix === "split" ? 1 : 0);
const m: Mix = solo ?? mix;
set(recGain.gain, m === "midi" ? 0 : 1);
set(midiGain.gain, m === "rec" ? 0 : m === "both" ? 0.8 : 1);
set(recPan.pan, m === "split" ? -1 : 0);
set(midiPan.pan, m === "split" ? 1 : 0);
mixButtons.forEach((b) => { b.classList.toggle("on", b.dataset.mix === mix); b.setAttribute("aria-checked", String(b.dataset.mix === mix)); });
}
@@ -104,31 +136,42 @@ function startSource(buf: AudioBuffer, dest: AudioNode, when: number, offset: nu
const playPos = () => playPos0 + Math.max(0, actx.currentTime - playCtx0);
async function play() {
if (!recBuf && !midiBuf) { setStatus("Audio is still loading…"); return; }
/** Plays from `from` (default: the cursor); `only` plays just that track (auditioning a pair mark). */
async function play(from = cursor, only: PairSide | null = null) {
const need = only === "rec" ? recBuf : only === "midi" ? midiBuf : recBuf ?? midiBuf;
if (!need) { setStatus(only === "midi" ? "The MIDI is still rendering…" : "Audio is still loading…"); return; }
await actx.resume();
stop();
solo = only; applyMix();
const t0 = actx.currentTime + 0.06; // both sources start on the same context tick
playCtx0 = t0; playPos0 = cursor;
recSrc = recBuf ? startSource(recBuf, recGain, t0, cursor) : null;
midiSrc = midiBuf ? startSource(midiBuf, midiGain, t0, cursor) : null;
playCtx0 = t0; playPos0 = from;
recSrc = recBuf && only !== "midi" ? startSource(recBuf, recGain, t0, from) : null;
midiSrc = midiBuf && only !== "rec" ? startSource(midiBuf, midiGain, t0, from) : null;
playing = true;
if (!recBuf) setStatus("Recording still loading: playing MIDI only.");
else if (!midiBuf) setStatus("MIDI still rendering: playing the recording only.");
btn.play.textContent = "■ Stop";
if (!only && !recBuf) setStatus("Recording still loading: playing MIDI only.");
else if (!only && !midiBuf) setStatus("MIDI still rendering: playing the recording only.");
updatePlayButtons();
requestAnimationFrame(tick);
}
function stop() {
for (const s of [recSrc, midiSrc]) { try { s?.stop(); } catch { /* already stopped */ } s?.disconnect(); }
recSrc = midiSrc = null;
if (playing) { playing = false; btn.play.textContent = "▶ Play from cursor"; draw(); }
if (playing) { playing = false; draw(); }
if (solo) { solo = null; applyMix(); }
updatePlayButtons();
}
function updatePlayButtons() {
btn.play.textContent = playing && !solo ? "■ Stop" : "▶ Play from cursor";
btn.playRec.textContent = playing && solo === "rec" ? "■" : "▶";
btn.playMidi.textContent = playing && solo === "midi" ? "■" : "▶";
}
/** Replace the MIDI buffer; if playing, the new one takes over seamlessly at the current position. */
function swapMidi(buf: AudioBuffer) {
midiBuf = buf;
if (!playing) return;
if (!playing || solo === "rec") return;
const t = actx.currentTime + 0.03;
const next = startSource(buf, midiGain, t, playPos0 + (t - playCtx0));
try { midiSrc?.stop(t); } catch { /* ended */ }
@@ -138,16 +181,18 @@ function swapMidi(buf: AudioBuffer) {
function tick() {
if (!playing) return;
const p = playPos();
if (p >= Math.max(recBuf?.duration ?? 0, midiBuf?.duration ?? 0)) { stop(); return; }
const end = solo === "rec" ? recBuf?.duration ?? 0 : solo === "midi" ? midiBuf?.duration ?? 0 : Math.max(recBuf?.duration ?? 0, midiBuf?.duration ?? 0);
if (p >= end) { stop(); return; }
view = follow(view, p, dur());
draw();
requestAnimationFrame(tick);
}
function waveFrom(buf: AudioBuffer): Wave {
const n = buf.length, mono = new Float32Array(n);
for (let c = 0; c < buf.numberOfChannels; c++) { const d = buf.getChannelData(c); for (let i = 0; i < n; i++) mono[i]! += d[i]! / buf.numberOfChannels; }
const levels = buildPyramid(mono, 64);
const pause = () => new Promise<void>((r) => setTimeout(r, 0));
/** Waveform pyramid of a decoded buffer, built in slices so a long recording never freezes the page. */
async function waveFrom(buf: AudioBuffer): Promise<Wave> {
const chans = Array.from({ length: buf.numberOfChannels }, (_, c) => buf.getChannelData(c));
const levels = await buildPyramidSliced(chans, 64, pause);
return { levels, rate: buf.sampleRate, norm: peakNorm(levels) };
}
function peakNorm(levels: PeakLevel[]): number {
@@ -198,19 +243,31 @@ function scheduleRefresh(now = false) {
async function refresh() {
if (!song) return;
const A = anchors, key = keyOf(A), id = song.id, gen = ++fitGen;
http<{ fit: FitReport }>(songUrl("/fit"), send("POST", { anchors: A }))
.then((j) => { if (gen === fitGen && song?.id === id) { fit = j.fit; fitKey = key; updateUi(); draw(); } })
.catch((e) => setStatus(`Fit update failed: ${(e as Error).message}`, true));
if (key === midiKey) return;
if (key !== fitKey) {
http<{ fit: FitReport }>(songUrl("/fit"), send("POST", { anchors: A }))
.then((j) => { if (gen === fitGen && song?.id === id) { fit = j.fit; fitKey = key; updateUi(); draw(); } })
.catch((e) => setStatus(`Fit update failed: ${(e as Error).message}`, true));
}
if (key === midiKey) { renderAbort?.abort(); return; }
renderAbort?.abort();
const ac = (renderAbort = new AbortController());
rendering = true; draw();
try {
const wav = await http<ArrayBuffer>(songUrl("/render"), send("POST", { anchors: A }, ac.signal));
const buf = await actx.decodeAudioData(wav);
const ckey = `${id}\n${key}`;
let enc = renderCache.get(ckey);
if (!enc) {
enc = await http<ArrayBuffer>(songUrl("/render"), send("POST", { anchors: A }, ac.signal));
renderCache.set(ckey, enc);
while (renderCache.size > RENDER_CLIENT_CACHE) renderCache.delete(renderCache.keys().next().value!);
}
if (ac.signal.aborted) return;
const buf = await actx.decodeAudioData(enc.slice(0)); // decoding detaches its input: keep the cached copy intact
if (ac.signal.aborted || song?.id !== id) return;
midiWave = waveFrom(buf); midiKey = key;
swapMidi(buf);
swapMidi(buf); // audible first; the waveform follows
midiKey = key;
const w = await waveFrom(buf);
if (ac.signal.aborted || song?.id !== id) return;
midiWave = w;
} catch (e) {
if (!ac.signal.aborted) setStatus(`MIDI render failed: ${(e as Error).message}`, true);
} finally {
@@ -226,6 +283,7 @@ function undo(redo = false) {
selected = -1;
setAnchors(prev);
scheduleRefresh();
setStatus(redo ? "Redone." : "Undone.");
}
function addAtCursor() {
@@ -237,6 +295,83 @@ function addAtCursor() {
+ (removed > 0 ? ` ${removed} automatic anchors within ${AUTO_CLEAR_SEC} s of it were removed; "Auto-align around my anchors" fills them in again.` : ""));
}
// ---------- pairing ----------
/** Where a mark sits on the timeline (recording seconds); the MIDI mark is drawn where the current warp puts its MIDI moment. */
const markX = (side: PairSide): number | null => (marks[side] === null ? null : side === "rec" ? marks.rec : warp.toAudio(marks.midi!));
/** The playhead while playing, else the cursor. */
const playhead = () => (playing ? playPos() : cursor);
/** Puts a mark at timeline position `t` (recording seconds). Changes nothing else: no warp change, no re-render. */
function setMark(side: PairSide, t: number) {
if (!song) return;
const lane = Math.min(dur() + 60, Math.max(0, t));
marks = placeMark(marks, side, side === "rec" ? lane : warp.toMidi(lane));
markIn[side].classList.remove("invalid");
const other: PairSide = side === "rec" ? "midi" : "rec";
setStatus(marks[other] === null
? `${side === "rec" ? "Recording" : "MIDI"} mark at ${formatTime(lane, 2)}. Now click the same note in the ${other === "rec" ? "Recording" : "MIDI"} lane (or press ${other === "rec" ? "R" : "M"} at the playhead).`
: `Marks: recording ${formatTime(markX("rec")!, 2)} ⇄ MIDI ${formatTime(markX("midi")!, 2)}. Press Link (Enter) to pair them, or move either mark.`);
updatePair(); draw();
}
function clearMarks() {
if (marks.rec === null && marks.midi === null) return;
marks = NO_MARKS;
markIn.rec.classList.remove("invalid"); markIn.midi.classList.remove("invalid");
if (solo) stop();
updatePair(); draw();
}
function link() {
if (!song || busy) return;
if (marks.rec === null || marks.midi === null) { setStatus("Place both a recording mark and a MIDI mark first (click the two lanes).", true); return; }
const rec = marks.rec, midiLane = markX("midi")!;
const autosBefore = anchors.filter((a) => a.kind === "auto").length;
let r: Edit;
try { r = linkMarks(anchors, marks); } catch (e) { setStatus((e as Error).message, true); return; } // marks are kept
const before = anchors;
marks = NO_MARKS;
selected = r.index;
commit(r.anchors, before, true);
updatePair();
const removed = autosBefore - anchors.filter((a) => a.kind === "auto").length;
setStatus(`Linked: the MIDI at ${formatTime(midiLane, 2)} now plays at recording ${formatTime(rec, 2)}. Re-rendering the MIDI…`
+ (removed > 0 ? ` ${removed} automatic anchors within ${AUTO_CLEAR_SEC} s were removed; "Auto-align around my anchors" fills them in again.` : "")
+ " (Ctrl+Z undoes.)");
}
/** Typed mark time (m:ss.s or seconds), applied on Enter / leaving the field. */
function markTyped(side: PairSide) {
const el = markIn[side], text = el.value.trim();
if (!text) { if (marks[side] !== null) { marks = { ...marks, [side]: null, active: marks.active === side ? null : marks.active }; updatePair(); draw(); } return; }
const t = parseTime(text);
if (t === null) { el.classList.add("invalid"); setStatus(`"${text}" is not a time: type e.g. 0:23.2 or 23.2`, true); return; }
setMark(side, t);
el.value = formatTime(markX(side)!, 2);
view = follow(view, t, dur()); draw();
}
function auditionMark(side: PairSide) {
const t = markX(side);
if (t === null) return;
if (playing && solo === side) { stop(); return; }
void play(t, side);
}
function updatePair() {
for (const side of ["rec", "midi"] as const) {
const t = markX(side), el = markIn[side];
if (document.activeElement !== el) el.value = t === null ? "" : formatTime(t, 2);
el.classList.toggle("set", t !== null);
if (side === "midi") el.title = t === null ? "MIDI mark (position in the MIDI lane): click the MIDI lane, press M at the playhead, or type a time"
: `MIDI mark at ${formatTime(t, 2)} in the MIDI lane = ${formatTime(marks.midi!, 2)} in the MIDI file`;
}
btn.playRec.disabled = marks.rec === null;
btn.playMidi.disabled = marks.midi === null;
btn.link.disabled = !song || busy || marks.rec === null || marks.midi === null;
btn.clearMarks.disabled = marks.rec === null && marks.midi === null;
}
function deleteSelected() {
if (selected < 0) return;
const k = selected;
@@ -314,6 +449,7 @@ async function openSong(id: string) {
history.replaceState(null, "", `#${encodeURIComponent(id)}`);
song = null; recBuf = midiBuf = null; recWave = midiWave = null; midiKey = ""; fit = null; undoStack.length = redoStack.length = 0;
selected = hover = -1;
marks = NO_MARKS;
setAnchors([]);
showOverlay("Loading song… The first time a song is opened the server converts the recording and runs the automatic alignment (up to ~30 s).");
setStatus("");
@@ -340,7 +476,7 @@ async function openSong(id: string) {
scheduleRefresh(true); // render the MIDI for the current anchors
http<ArrayBuffer>(songUrl("/audio"))
.then((b) => actx.decodeAudioData(b))
.then((buf) => { if (gen !== loadGen) return; recBuf = buf; recWave = waveFrom(buf); draw(); })
.then(async (buf) => { if (gen !== loadGen) return; recBuf = buf; const w = await waveFrom(buf); if (gen === loadGen) { recWave = w; draw(); } })
.catch((e) => setStatus(`Could not load the recording audio: ${(e as Error).message}`, true));
} catch (e) {
if (gen !== loadGen) return;
@@ -356,6 +492,7 @@ function updateUi() {
btn.undo.disabled = busy || !undoStack.length;
btn.auto.disabled = btn.save.disabled = btn.reset.disabled = !has || busy;
btn.play.disabled = !has;
updatePair();
if (!has) { saveState.textContent = ""; fitEl.textContent = "Fit: –"; countsEl.textContent = ""; return; }
const dirty = isDirty();
saveState.className = `savestate ${dirty ? "dirty" : "clean"}`;
@@ -411,6 +548,7 @@ function paintWave(w: Wave | null, y: number, color: string, alpha: number) {
function paint() {
const W = cssW;
g.clearRect(0, 0, W, HEIGHT);
g.fillStyle = C.ruler; g.fillRect(GUTTER, 0, W - GUTTER, RULER_H); // the ruler sets the playback cursor
g.font = "11px system-ui, sans-serif"; g.textBaseline = "middle";
// ruler
const step = rulerStep(view.pxPerSec), dec = step < 0.1 ? 2 : step < 1 ? 1 : 0;
@@ -460,8 +598,12 @@ function paint() {
g.beginPath(); g.moveTo(x, REC_Y - 2); g.lineTo(x, MIDI_Y + LANE_H); g.stroke();
if (manual || sel || hov) { // grab handles at the recording (top) and MIDI (bottom) ends
g.fillStyle = sel ? "#16804c" : C.anchor;
g.fillRect(x - 4, REC_Y - 2, 8, 6);
g.fillRect(x - 4, MIDI_Y + LANE_H - 6, 8, 6);
g.fillRect(x - HANDLE_W / 2, REC_Y - 2, HANDLE_W, HANDLE_H);
g.fillRect(x - HANDLE_W / 2, MIDI_Y + LANE_H - HANDLE_H, HANDLE_W, HANDLE_H);
} else { // small stubs so automatic anchors show where to grab them
g.fillStyle = C.anchorAuto;
g.fillRect(x - 2.5, REC_Y - 2, 5, 5);
g.fillRect(x - 2.5, MIDI_Y + LANE_H - 5, 5, 5);
}
const label = formatTime(a.audioSec, view.pxPerSec > 200 ? 2 : 1), tw = g.measureText(label).width;
if ((manual || sel || hov || spacingOk(k)) && x - tw / 2 > lastLabelEnd + 6) {
@@ -475,7 +617,7 @@ function paint() {
// hover tooltip
if (hover >= 0 && hover < anchors.length) {
const a = anchors[hover]!;
const txt = `${a.kind === "manual" ? "Your anchor" : "Automatic anchor"} · recording ${formatTime(a.audioSec, 2)} = MIDI ${formatTime(a.midiSec, 2)} · drag in the ${hoverLane === "midi" ? "MIDI lane to slide the MIDI" : "recording lane to move it"}`;
const txt = `${a.kind === "manual" ? "Your anchor" : "Automatic anchor"} · recording ${formatTime(a.audioSec, 2)} = MIDI ${formatTime(a.midiSec, 2)} · drag this handle to ${hoverLane === "midi" ? "slide the MIDI under it" : "move it along the recording"}`;
const tw = g.measureText(txt).width + 12;
const x = Math.min(W - tw - 4, Math.max(GUTTER, X(a.audioSec) + 8));
const y = hoverLane === "midi" ? MIDI_Y + 6 : REC_Y + 6;
@@ -483,6 +625,26 @@ function paint() {
g.fillStyle = "#fff"; g.textAlign = "left"; g.fillText(txt, x + 6, y + 9);
}
// pair marks: dashed, each in its own lane, joined by a dashed connector when both are placed
const xr = markX("rec"), xm = markX("midi");
const mark = (t: number, y: number, color: string, label: string) => {
const x = Math.round(X(t)) + 0.5;
if (x < GUTTER || x > W) return;
g.strokeStyle = color; g.lineWidth = 2; g.setLineDash([6, 4]);
g.beginPath(); g.moveTo(x, y); g.lineTo(x, y + LANE_H); g.stroke();
g.setLineDash([]); g.lineWidth = 1;
const tw = g.measureText(label).width + 10, lx = Math.min(W - tw - 2, x + 3);
g.fillStyle = color; g.fillRect(lx, y + LANE_H - 19, tw, 16);
g.fillStyle = "#fff"; g.textAlign = "left"; g.fillText(label, lx + 5, y + LANE_H - 11);
};
if (xr !== null) mark(xr, REC_Y, C.recMark, `R ${formatTime(xr, 2)}`);
if (xm !== null) mark(xm, MIDI_Y, C.midiMark, `M ${formatTime(xm, 2)}`);
if (xr !== null && xm !== null) {
g.strokeStyle = C.pair; g.lineWidth = 1.5; g.setLineDash([3, 3]);
g.beginPath(); g.moveTo(X(xr), REC_Y + LANE_H); g.lineTo(X(xm), MIDI_Y); g.stroke();
g.setLineDash([]); g.lineWidth = 1;
}
// cursor and playhead
const line = (t: number, color: string, w: number) => {
const x = Math.round(X(t)) + 0.5;
@@ -501,8 +663,10 @@ function paint() {
g.fillText("Recording", 10, REC_Y + 16);
g.fillText("MIDI audio", 10, MIDI_Y + 16);
g.font = "11px system-ui, sans-serif"; g.fillStyle = C.muted;
g.fillText("drag = move", 10, REC_Y + 32);
g.fillText("drag = slide", 10, MIDI_Y + 32);
g.fillText("click = mark", 10, REC_Y + 32);
g.fillText("▮ = drag", 10, REC_Y + 46);
g.fillText("click = mark", 10, MIDI_Y + 32);
g.fillText("▮ = drag", 10, MIDI_Y + 46);
g.fillText("tempo", 10, STRETCH_Y + 14);
g.fillText(formatTime(cursor, 2), 10, RULER_H / 2);
}
@@ -513,47 +677,52 @@ function spacingOk(k: number): boolean {
}
// ---------- pointer ----------
// Ruler (top strip): click/drag = playback cursor. Lanes: click = pair mark, drag = scroll. Anchor handles (top of the
// recording lane, bottom of the MIDI lane): drag = move the anchor, click = select it.
type Drag =
| { kind: "pan"; x0: number; lastX: number; moved: boolean }
| { kind: "pan"; x0: number; lastX: number; moved: boolean; lane: PairSide | null }
| { kind: "cursor" }
| { kind: "rec" | "midi"; index: number; start: Anchor[]; grabTime: number; grabMidi: number; offset: number; x0: number; moved: boolean };
let drag: Drag | null = null;
const local = (e: PointerEvent | WheelEvent) => { const r = canvas.getBoundingClientRect(); return { x: e.clientX - r.left, y: e.clientY - r.top }; };
const laneAt = (y: number): "rec" | "midi" | null => (y >= REC_Y - 4 && y < MIDI_Y - LANE_GAP / 2 ? "rec" : y >= MIDI_Y - LANE_GAP / 2 && y <= MIDI_Y + LANE_H + 2 ? "midi" : null);
const laneAt = (y: number): PairSide | null => (y >= REC_Y && y < MIDI_Y - LANE_GAP / 2 ? "rec" : y >= MIDI_Y - LANE_GAP / 2 && y <= MIDI_Y + LANE_H ? "midi" : null);
/** Which end's handle zone `y` is in (null: neither). */
const handleZone = (y: number): PairSide | null =>
y >= REC_Y - 6 && y <= REC_Y + HANDLE_ZONE ? "rec" : y >= MIDI_Y + LANE_H - HANDLE_ZONE && y <= MIDI_Y + LANE_H + 4 ? "midi" : null;
const inRuler = (y: number) => y < REC_Y - 6;
canvas.addEventListener("pointerdown", (e) => {
if (!song || busy || e.button !== 0) return;
const { x, y } = local(e);
if (x < GUTTER) return;
canvas.setPointerCapture(e.pointerId);
const lx = x - GUTTER, lane = laneAt(y);
if (y < REC_Y - 4) { drag = { kind: "cursor" }; setCursor(xToTime(view, lx), false); return; }
const hit = lane ? hitAnchor(anchors, view, lx, HIT_PX) : -1;
if (hit >= 0 && lane) {
const lx = x - GUTTER, zone = handleZone(y);
const hit = zone ? hitAnchor(anchors, view, lx, HIT_PX) : -1;
if (hit >= 0 && zone) {
selected = hit;
const t = xToTime(view, lx);
drag = { kind: lane, index: hit, start: anchors, grabTime: t, grabMidi: warp.toMidi(t), offset: anchors[hit]!.audioSec - t, x0: x, moved: false };
drag = { kind: zone, index: hit, start: anchors, grabTime: t, grabMidi: warp.toMidi(t), offset: anchors[hit]!.audioSec - t, x0: x, moved: false };
updateUi(); draw();
return;
}
drag = { kind: "pan", x0: x, lastX: x, moved: false };
canvas.style.cursor = "grabbing";
if (inRuler(y)) { drag = { kind: "cursor" }; setCursor(xToTime(view, lx), false); return; }
drag = { kind: "pan", x0: x, lastX: x, moved: false, lane: laneAt(y) };
});
canvas.addEventListener("pointermove", (e) => {
if (!song) return;
const { x, y } = local(e), lx = x - GUTTER;
if (!drag) {
const lane = x >= GUTTER ? laneAt(y) : null;
const h = lane ? hitAnchor(anchors, view, lx, HIT_PX) : -1;
if (h !== hover || lane !== hoverLane) { hover = h; hoverLane = lane; draw(); }
canvas.style.cursor = h >= 0 ? "ew-resize" : y < REC_Y - 4 && x >= GUTTER ? "text" : "default";
const zone = x >= GUTTER ? handleZone(y) : null;
const h = zone ? hitAnchor(anchors, view, lx, HIT_PX) : -1;
if (h !== hover || zone !== hoverLane) { hover = h; hoverLane = zone; draw(); }
canvas.style.cursor = x < GUTTER ? "default" : h >= 0 ? "ew-resize" : inRuler(y) ? "col-resize" : laneAt(y) ? "crosshair" : "default";
return;
}
if (drag.kind === "cursor") { setCursor(xToTime(view, lx), false); return; }
if (drag.kind === "pan") {
if (Math.abs(x - drag.x0) > DRAG_PX) drag.moved = true;
if (Math.abs(x - drag.x0) > DRAG_PX) { drag.moved = true; canvas.style.cursor = "grabbing"; }
if (drag.moved) { view = scrollBy(view, x - drag.lastX, dur()); drag.lastX = x; draw(); }
return;
}
@@ -575,7 +744,7 @@ function endDrag(e: PointerEvent) {
canvas.style.cursor = "default";
const { x } = local(e);
if (d.kind === "cursor" && playing) void play(); // scrubbing seeks playback once, on release
else if (d.kind === "pan" && !d.moved && x >= GUTTER) { selected = -1; setCursor(xToTime(view, x - GUTTER)); updateUi(); }
else if (d.kind === "pan" && !d.moved && x >= GUTTER && d.lane && e.type === "pointerup") setMark(d.lane, xToTime(view, x - GUTTER));
else if ((d.kind === "rec" || d.kind === "midi") && d.moved) {
const now = anchors;
anchors = d.start; // commit from the drag-start state so undo returns there
@@ -608,8 +777,10 @@ canvas.addEventListener("wheel", (e) => {
// ---------- keyboard & buttons ----------
window.addEventListener("keydown", (e) => {
const tag = (e.target as HTMLElement).tagName;
if (tag === "SELECT" || tag === "INPUT" || tag === "TEXTAREA" || !song) return;
const el = e.target as HTMLElement, tag = el.tagName;
const slider = tag === "INPUT" && (el as HTMLInputElement).type === "range";
if (tag === "SELECT" || tag === "TEXTAREA" || (tag === "INPUT" && !slider) || !song) return;
if (slider && /^(Arrow|Page|Home|End)/.test(e.key)) return; // the slider's own keys
const mod = e.ctrlKey || e.metaKey;
if (mod && e.key.toLowerCase() === "z") { e.preventDefault(); undo(e.shiftKey); return; }
if (mod && e.key.toLowerCase() === "y") { e.preventDefault(); undo(true); return; }
@@ -617,17 +788,38 @@ window.addEventListener("keydown", (e) => {
if (mod || e.altKey || busy) return;
switch (e.key) {
case " ": e.preventDefault(); playing ? stop() : void play(); break;
case "r": case "R": setMark("rec", playhead()); break;
case "m": case "M": setMark("midi", playhead()); break;
case "Enter": e.preventDefault(); link(); break;
case "Delete": case "Backspace": e.preventDefault(); deleteSelected(); break;
case "a": case "A": addAtCursor(); break;
case "ArrowLeft": e.preventDefault(); nudge(e.shiftKey ? -0.1 : -0.01); break;
case "ArrowRight": e.preventDefault(); nudge(e.shiftKey ? 0.1 : 0.01); break;
case "Escape": selected = -1; updateUi(); draw(); break;
case "Escape": if (marks.rec !== null || marks.midi !== null) { clearMarks(); setStatus("Marks cleared."); } else { selected = -1; updateUi(); draw(); } break;
case "0": view = fitView(dur(), view.width); draw(); break;
case "1": case "2": case "3": case "4": mix = (["both", "rec", "midi", "split"] as const)[Number(e.key) - 1]!; applyMix(); break;
}
});
btn.play.addEventListener("click", () => (playing ? stop() : void play()));
btn.play.addEventListener("click", () => (playing && !solo ? stop() : void play()));
btn.link.addEventListener("click", link);
btn.clearMarks.addEventListener("click", clearMarks);
btn.playRec.addEventListener("click", () => auditionMark("rec"));
btn.playMidi.addEventListener("click", () => auditionMark("midi"));
for (const side of ["rec", "midi"] as const) {
const el = markIn[side];
el.addEventListener("change", () => markTyped(side));
el.addEventListener("keydown", (e) => {
if (e.key === "Enter") { e.preventDefault(); markTyped(side); el.blur(); }
else if (e.key === "Escape") { el.blur(); updatePair(); }
});
el.addEventListener("input", () => el.classList.remove("invalid"));
volIn[side].addEventListener("input", () => {
volumes = { ...volumes, [side]: Number(volIn[side].value) };
applyVolumes();
try { localStorage.setItem(VOLUME_KEY, JSON.stringify(volumes)); } catch { /* storage unavailable */ }
});
}
btn.add.addEventListener("click", addAtCursor);
btn.del.addEventListener("click", deleteSelected);
btn.undo.addEventListener("click", () => undo());
@@ -635,13 +827,14 @@ btn.auto.addEventListener("click", () => void autoAlign());
btn.save.addEventListener("click", () => void save());
btn.reset.addEventListener("click", reset);
for (const b of mixButtons) b.addEventListener("click", () => { mix = b.dataset.mix as Mix; applyMix(); });
// keep Space/Enter from re-triggering the last clicked button
for (const b of document.querySelectorAll("button")) b.addEventListener("keydown", (e) => { if (e.key === " ") e.preventDefault(); });
// keep Space/Enter from re-triggering the last clicked button (they are play/stop and Link)
for (const b of document.querySelectorAll("button")) b.addEventListener("keydown", (e) => { if (e.key === " " || e.key === "Enter") e.preventDefault(); });
songSel.addEventListener("change", () => void openSong(songSel.value));
window.addEventListener("beforeunload", (e) => { if (isDirty() && (undoStack.length || savedKey !== null)) e.preventDefault(); });
new ResizeObserver(resize).observe(wrap);
applyMix();
applyVolumes();
resize();
updateUi();
loadSongs().catch((e) => showOverlay(`Could not load the song list: ${(e as Error).message}`));
+72
View File
@@ -63,3 +63,75 @@ export function anchorAtCursor(anchors: Anchor[], cursorSec: number): Edit {
const out = addManual(anchors, { midiSec, audioSec: cursorSec });
return { anchors: out, index: out.findIndex((a) => a.audioSec === cursorSec && a.kind === "manual") };
}
// ---------- pair anchors: a recording mark and a MIDI mark, linked into one manual anchor ----------
export type PairSide = "rec" | "midi";
/**
* Marks placed in Pair mode. `rec` = recording time (s). `midi` = MIDI time (s): the MIDI lane shows the warped MIDI,
* so a click there at lane time t means MIDI moment warp.toMidi(t) (see `midiMarkAt`); storing the MIDI moment keeps
* the mark on the same note if the warp changes before linking. `active` = the mark placed last (Space plays from it).
*/
export interface PairMarks { rec: number | null; midi: number | null; active: PairSide | null }
export const NO_MARKS: PairMarks = { rec: null, midi: null, active: null };
/** MIDI moment under lane time `laneSec` on the (warped) MIDI lane. */
export const midiMarkAt = (anchors: Anchor[], laneSec: number) => makeWarp(anchors).toMidi(laneSec);
export function placeMark(m: PairMarks, side: PairSide, sec: number): PairMarks {
return { ...m, [side]: sec, active: side };
}
/** Lane time (recording seconds) at which to draw / play a mark under the current anchors. */
export function markLaneSec(anchors: Anchor[], m: PairMarks, side: PairSide): number | null {
const v = m[side];
return v === null ? null : side === "rec" ? v : makeWarp(anchors).toAudio(v);
}
const fmt = (s: number) => {
const t = Math.round(Math.abs(s) * 100) / 100, mm = Math.floor(t / 60);
return `${s < 0 && t > 0 ? "-" : ""}${mm}:${(t - mm * 60).toFixed(2).padStart(5, "0")}`;
};
/**
* Why a manual anchor (midiSec, audioSec) cannot be added, or null if it can: it must sit strictly between your
* (manual) anchors in both MIDI and recording time. Automatic anchors never block it (addManual drops the ones in the way).
* Times in the message are timeline positions as drawn (the MIDI mark where the warped MIDI lane shows it; an anchor at
* its recording time, where both lanes draw it), so they match what the user sees and typed.
*/
export function linkConflict(anchors: Anchor[], midiSec: number, audioSec: number): string | null {
const w = makeWarp(anchors), man = w.anchors.filter((a) => a.kind === "manual");
const same = man.find((a) => a.midiSec === midiSec || a.audioSec === audioSec);
if (same) return `Can't link: your anchor at ${fmt(same.audioSec)} already uses that ${same.midiSec === midiSec ? "MIDI" : "recording"} moment. Move a mark or delete that anchor.`;
const left = man.filter((a) => a.midiSec < midiSec).pop(), right = man.find((a) => a.midiSec > midiSec);
const bad = left && !(audioSec > left.audioSec) ? { a: left, side: "after" } : right && !(audioSec < right.audioSec) ? { a: right, side: "before" } : null;
if (!bad) return null;
const other = bad.side === "after" ? "before" : "after";
return `Can't link: the MIDI mark (${fmt(w.toAudio(midiSec))}) is ${bad.side} your anchor at ${fmt(bad.a.audioSec)}, but the recording mark (${fmt(audioSec)}) is ${other} it. `
+ "Anchors must keep the same order in both lanes: move a mark, or delete that anchor first.";
}
/** The manual anchor joining both marks; throws (with a user-facing message) if a mark is missing or the order would break. */
export function linkMarks(anchors: Anchor[], m: PairMarks): Edit {
if (m.rec === null || m.midi === null) throw new Error("Place both a recording mark and a MIDI mark first.");
const audioSec = m.rec, midiSec = m.midi;
const why = linkConflict(anchors, midiSec, audioSec);
if (why) throw new Error(why);
const out = addManual(anchors, { midiSec, audioSec });
return { anchors: out, index: out.findIndex((a) => a.kind === "manual" && a.audioSec === audioSec && a.midiSec === midiSec) };
}
// ---------- volume ----------
export interface Volumes { rec: number; midi: number } // percent, 0–150
export const DEFAULT_VOLUMES: Volumes = { rec: 100, midi: 100 };
const clampPct = (x: unknown, d: number) => (typeof x === "number" && Number.isFinite(x) ? Math.min(150, Math.max(0, Math.round(x))) : d);
/** Volumes from a stored JSON string; anything missing or invalid falls back to 100 %. */
export function parseVolumes(raw: string | null): Volumes {
try {
const j = JSON.parse(raw ?? "null") as Partial<Volumes> | null;
return { rec: clampPct(j?.rec, DEFAULT_VOLUMES.rec), midi: clampPct(j?.midi, DEFAULT_VOLUMES.midi) };
} catch { return { ...DEFAULT_VOLUMES }; }
}
+35 -6
View File
@@ -4,6 +4,7 @@
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<title>STSheets · MIDI alignment</title>
<link rel="icon" href="data:," />
<link rel="stylesheet" href="./style.css" />
</head>
<body>
@@ -25,6 +26,24 @@
<div id="overlay" class="overlay" hidden></div>
</div>
<section class="pair" aria-label="Pair a recording moment with a MIDI moment">
<span class="pairtitle">Pair</span>
<span class="mark rec">
<label for="mrec">Recording</label>
<input id="mrec" class="time" placeholder="not set" autocomplete="off" spellcheck="false" title="Recording mark: click the Recording lane, press R at the playhead, or type a time (m:ss.s or seconds)" />
<button id="prec" class="aud" title="Listen: the recording alone from this mark" disabled>▶</button>
</span>
<span class="pairarrow" aria-hidden="true">⇄</span>
<span class="mark midi">
<label for="mmidi">MIDI</label>
<input id="mmidi" class="time" placeholder="not set" autocomplete="off" spellcheck="false" title="MIDI mark (position in the MIDI lane): click the MIDI lane, press M at the playhead, or type a time" />
<button id="pmidi" class="aud" title="Listen: the MIDI alone from this mark" disabled>▶</button>
</span>
<button id="link" class="link" title="Enter" disabled>Link</button>
<button id="clearmarks" class="quiet2" title="Esc" disabled>Clear</button>
<span id="pairhint" class="pairhint">Click a note in the <b>Recording</b> lane, then the same note in the <b>MIDI</b> lane, then <b>Link</b> (Enter).</span>
</section>
<div class="row controls">
<button id="play" title="Space">▶ Play from cursor</button>
<div class="mix" role="radiogroup" aria-label="Mix">
@@ -34,6 +53,10 @@
<button data-mix="midi" title="3">MIDI only</button>
<button data-mix="split" title="4">rec L / MIDI R</button>
</div>
<label class="vol rec" title="Recording volume">Recording <input id="volrec" type="range" min="0" max="150" step="1" value="100" /> <output id="volrecv">100%</output></label>
<label class="vol midi" title="MIDI volume">MIDI <input id="volmidi" type="range" min="0" max="150" step="1" value="100" /> <output id="volmidiv">100%</output></label>
</div>
<div class="row controls">
<button id="add" title="A">+ Anchor at cursor</button>
<button id="del" title="Delete" disabled>Delete anchor</button>
<button id="undo" title="Ctrl+Z" disabled>Undo</button>
@@ -51,12 +74,18 @@
the label under the lanes shows MIDI bpm → resulting bpm. <b>⚠</b> = a stretch changed by more than 5%; that usually means an anchor sits on the wrong bar.
Thick lines are your (manual) anchors, thin ones were placed automatically. Red-shaded 30 s windows fit poorly.</p>
<ul>
<li><b>Click</b> the lanes to set the cursor (black line); click an anchor to select it. <b>Wheel</b> zooms around the mouse (Shift+wheel or a sideways swipe scrolls); <b>drag the background</b> to scroll.</li>
<li>Drag an anchor in the <b>Recording</b> lane to move it along the recording (the MIDI moment stays attached to it).
Drag it in the <b>MIDI audio</b> lane to slide the MIDI under it: the bit of MIDI you grabbed follows the mouse while the anchor stays at the same place in the recording.</li>
<li>Moving or adding an anchor removes the automatic anchors within 10 s of it (never past your other anchors), plus any that would now be out of order; <b>Auto-align around my anchors</b> fills them in again.</li>
<li>Keys: <kbd>Space</kbd> play/stop · <kbd>A</kbd> anchor at cursor · <kbd>Delete</kbd> delete selected · <kbd>←</kbd>/<kbd>→</kbd> nudge selected anchor 10 ms (<kbd>Shift</kbd> 100 ms; nothing selected: move cursor 1 s) ·
<kbd>1</kbd>–<kbd>4</kbd> mix · <kbd>Ctrl</kbd>+<kbd>Z</kbd> undo / <kbd>Ctrl</kbd>+<kbd>Shift</kbd>+<kbd>Z</kbd> redo · <kbd>Ctrl</kbd>+<kbd>S</kbd> save · <kbd>0</kbd> show whole song.</li>
<li><b>Pairing (the main tool).</b> Click a note in the <b>Recording</b> lane: a dashed blue <b>recording mark</b> appears. Click the same note in the <b>MIDI</b> lane: a dashed orange <b>MIDI mark</b>.
Placing marks changes nothing else. Fine-tune them by clicking again, typing a time in the Pair box (<code>0:23.2</code> or <code>23.2</code>), or pressing <kbd>R</kbd> / <kbd>M</kbd> to drop a mark at the playhead while listening.
The <b>▶</b> next to each mark plays just that track from the mark. <b>Link</b> (<kbd>Enter</kbd>) makes them one anchor: that MIDI moment now plays at the recording mark. <kbd>Esc</kbd> clears the marks.</li>
<li><b>Playback cursor:</b> click or drag in the <b>time ruler</b> along the top (black triangle). <kbd>Space</kbd> plays from it; <kbd>←</kbd>/<kbd>→</kbd> moves it 1 s when no anchor is selected.</li>
<li><b>Moving anchors:</b> grab an anchor by its <b>handle</b> (the small green block at the top of the Recording lane or the bottom of the MIDI lane). Top handle: move the anchor along the recording.
Bottom handle: slide the MIDI under it (the bit of MIDI you grabbed follows the mouse). Clicking a handle selects the anchor (<kbd>Delete</kbd> removes it).</li>
<li><b>Wheel</b> zooms around the mouse (Shift+wheel or a sideways swipe scrolls); <b>drag the lane background</b> to scroll.</li>
<li>Adding or moving an anchor removes the automatic anchors within 10 s of it (never past your other anchors), plus any that would now be out of order; <b>Auto-align around my anchors</b> fills them in again.
Your anchors must stay in the same order in both lanes; Link tells you if a pair would cross one of them.</li>
<li><b>Volume:</b> the Recording and MIDI sliders (0–150 %) work on top of the mix choice, also while playing, and are remembered in this browser.</li>
<li>Keys: <kbd>Space</kbd> play/stop · <kbd>R</kbd> / <kbd>M</kbd> recording / MIDI mark at the playhead · <kbd>Enter</kbd> link · <kbd>Esc</kbd> clear marks / selection · <kbd>A</kbd> anchor at cursor · <kbd>Delete</kbd> delete selected ·
<kbd>←</kbd>/<kbd>→</kbd> nudge selected anchor 10 ms (<kbd>Shift</kbd> 100 ms) · <kbd>1</kbd>–<kbd>4</kbd> mix · <kbd>Ctrl</kbd>+<kbd>Z</kbd> undo / <kbd>Ctrl</kbd>+<kbd>Shift</kbd>+<kbd>Z</kbd> redo · <kbd>Ctrl</kbd>+<kbd>S</kbd> save · <kbd>0</kbd> show whole song.</li>
</ul>
</details>
+20
View File
@@ -40,3 +40,23 @@ button:focus-visible { outline: 2px solid var(--rec); outline-offset: 1px; }
.help summary { cursor: pointer; color: var(--muted); }
.help li { margin: 4px 0; }
kbd { font: 11px ui-monospace, monospace; border: 1px solid #ccc; border-bottom-width: 2px; border-radius: 3px; padding: 0 4px; background: #fff; }
.pair { display: flex; align-items: center; gap: 8px; flex-wrap: wrap; margin-top: 10px; padding: 8px 10px; background: var(--panel); border: 1px solid var(--line); border-left: 4px solid #7a3cff; border-radius: 6px; }
.pairtitle { font-weight: 600; font-size: 13px; margin-right: 2px; }
.pair .mark { display: inline-flex; align-items: center; gap: 4px; font-size: 13px; }
.pair .mark.rec label { color: #1f5fd6; font-weight: 600; }
.pair .mark.midi label { color: #c2410c; font-weight: 600; }
.pair input.time { font: 13px ui-monospace, monospace; width: 10ch; padding: 3px 5px; border: 1px dashed #aaa; border-radius: 4px; }
.pair .mark.rec input.time.set { border-color: #1f5fd6; border-style: solid; }
.pair .mark.midi input.time.set { border-color: #c2410c; border-style: solid; }
.pair input.time.invalid { border-color: var(--warn); background: #fff1f0; }
.pair button.aud { padding: 3px 7px; }
.pairarrow { color: var(--muted); }
button.link { background: #7a3cff; border-color: #6a2ee6; color: #fff; font-weight: 600; padding: 5px 16px; }
button.link:hover:not(:disabled) { background: #6a2ee6; }
button.quiet2 { color: #555; }
.pairhint { font-size: 12px; color: var(--muted); flex: 1 1 260px; }
.vol { display: inline-flex; align-items: center; gap: 4px; font-size: 13px; color: #444; margin-left: 6px; }
.vol input[type="range"] { width: 110px; }
.vol output { font: 12px ui-monospace, monospace; width: 4.5ch; color: var(--muted); }
.vol.rec { color: #1f5fd6; } .vol.midi { color: #c2410c; }
code { font: 12px ui-monospace, monospace; background: #f1f1f1; padding: 0 3px; border-radius: 3px; }
+36
View File
@@ -119,3 +119,39 @@ export function columnPeaks(levels: PeakLevel[], sampleRate: number, v: View): {
}
return { min, max };
}
/**
* Parses a typed time: "m:ss.s" ("1:23.4", "0:05"), "h:mm:ss.s", or plain seconds ("83.4", "83,4"). A comma is
* accepted as the decimal separator. Returns null for anything else or a negative time.
*/
export function parseTime(text: string): number | null {
const s = text.trim().replace(",", ".");
if (!/^\d+(:\d{1,2}){0,2}(\.\d*)?$|^\.\d+$/.test(s)) return null;
const parts = s.split(":").map(Number);
if (parts.slice(1).some((p) => p >= 60)) return null;
const v = parts.reduce((acc, p) => acc * 60 + p, 0);
return Number.isFinite(v) ? v : null;
}
/**
* Like `buildPyramid` but over several channels at once (min/max across all of them) and cooperative: after every
* `sliceSamples` samples it awaits `pause()` (e.g. a zero timeout) so a long recording never blocks the page.
*/
export async function buildPyramidSliced(
channels: Float32Array[], base: number, pause: () => Promise<void>, sliceSamples = 1 << 20,
): Promise<PeakLevel[]> {
const len = channels[0]?.length ?? 0;
const n = Math.max(1, Math.ceil(len / base));
const min = new Float32Array(n), max = new Float32Array(n);
const perSlice = Math.max(1, Math.floor(sliceSamples / base / Math.max(1, channels.length)));
for (let b0 = 0; b0 < n; b0 += perSlice) {
for (let b = b0, bEnd = Math.min(n, b0 + perSlice); b < bEnd; b++) {
let lo = Infinity, hi = -Infinity;
const i0 = b * base, i1 = Math.min(len, i0 + base);
for (const x of channels) for (let i = i0; i < i1; i++) { const s = x[i]!; if (s < lo) lo = s; if (s > hi) hi = s; }
min[b] = lo === Infinity ? 0 : lo; max[b] = hi === -Infinity ? 0 : hi;
}
if (b0 + perSlice < n) await pause();
}
return levelsFrom({ bucket: base, min, max });
}
+52
View File
@@ -0,0 +1,52 @@
// MIDI preview rendering split into time chunks that render in parallel (one per worker thread), then stitched.
// Each chunk is rendered with a pre-roll (notes already sounding are re-attacked inside it and it is discarded) and a
// short overlap that is cross-faded into the next chunk, so seams are inaudible for a preview. Pure: no I/O.
export interface Chunk {
/** Audio time of the chunk's first output sample (includes the pre-roll). */
renderStart: number;
/** Seconds to render from `renderStart`. */
renderDur: number;
/** The part of the timeline this chunk owns: [from, to). */
from: number; to: number;
}
export const PREROLL_SEC = 2;
export const XFADE_SEC = 0.01;
/** Chunks shorter than this are not worth a worker (synth start-up, pre-roll overhead). */
export const MIN_CHUNK_SEC = 20;
/** Splits [0, durSec) into at most `maxChunks` equal chunks of at least MIN_CHUNK_SEC (always at least one). */
export function planChunks(durSec: number, maxChunks: number): Chunk[] {
const n = Math.max(1, Math.min(maxChunks, Math.floor(durSec / MIN_CHUNK_SEC)));
const out: Chunk[] = [];
for (let k = 0; k < n; k++) {
const from = (durSec * k) / n, to = k === n - 1 ? durSec : (durSec * (k + 1)) / n;
const renderStart = k === 0 ? 0 : from - PREROLL_SEC;
const end = k === n - 1 ? durSec : to + XFADE_SEC;
out.push({ renderStart, renderDur: end - renderStart, from, to });
}
return out;
}
/**
* Joins rendered chunks (mono, `sampleRate`, as planned by `planChunks`) into one signal of round(durSec·sr) samples:
* each chunk contributes [from, to), and its XFADE_SEC overlap past `to` fades out while the next chunk fades in.
*/
export function stitchChunks(chunks: Chunk[], pcm: Float32Array[], sampleRate: number, durSec: number): Float32Array {
const n = Math.round(durSec * sampleRate), out = new Float32Array(n);
const xf = Math.max(1, Math.round(XFADE_SEC * sampleRate));
chunks.forEach((c, k) => {
const x = pcm[k]!, s0 = Math.round(c.from * sampleRate), s1 = k === chunks.length - 1 ? n : Math.round(c.to * sampleRate);
const off = Math.round(c.renderStart * sampleRate); // output index of x[0]
for (let i = s0; i < s1; i++) {
const v = x[i - off] ?? 0;
// first xf samples of a later chunk fade in; the previous chunk's overlap was already added (fading out)
out[i] = k > 0 && i - s0 < xf ? out[i]! + v * ((i - s0) / xf) : v;
}
if (k < chunks.length - 1) {
for (let j = 0; j < xf && s1 + j < n; j++) out[s1 + j] = (x[s1 + j - off] ?? 0) * (1 - j / xf);
}
});
return out;
}
+14 -5
View File
@@ -14,12 +14,16 @@ const MELODIC_CHANNELS = [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13, 14, 15];
* Render reference tracks to audio with `warp` applied (MIDI time -> audio time). Notes are placed on a fixed-tempo
* tick grid so alphaSynth output time equals audio time. Peak-normalised to -1 dBFS; `durationSec` pads/trims.
* Pure: no I/O. Notes landing before audio t=0 are clamped to 0 (dropped if they end before it).
* `startSec` renders a window: output sample 0 is audio time `startSec`; notes sounding at `startSec` are re-attacked
* there (callers render a pre-roll and discard it) and notes starting after the window are skipped.
* `normalize: false` skips the peak normalisation (e.g. for chunks normalised together later).
*/
export function renderMidi(
tracks: RefTrack[], warp: Warp, sf2: Uint8Array,
opts: { sampleRate?: number; mono?: boolean; durationSec?: number } = {},
opts: { sampleRate?: number; mono?: boolean; durationSec?: number; startSec?: number; normalize?: boolean } = {},
): Audio {
const sampleRate = opts.sampleRate ?? RENDER_SR;
const shift = opts.startSec ?? 0, endSec = opts.durationSec ?? Infinity;
const midi = new at.midi.MidiFile();
midi.division = 960;
midi.addEvent(new at.midi.TempoChangeEvent(0, 500000));
@@ -30,9 +34,9 @@ export function renderMidi(
const ch = t.isDrums ? 9 : MELODIC_CHANNELS[next++ % MELODIC_CHANNELS.length]!;
if (!seen.has(ch)) { seen.add(ch); midi.addEvent(new at.midi.ProgramChangeEvent(0, 0, ch, t.isDrums ? 0 : t.program)); }
for (const n of t.notes) {
const offSec = warp.toAudio(n.offset);
if (offSec <= 0) continue;
const on = Math.round(Math.max(0, warp.toAudio(n.onset)) * TPS);
const offSec = warp.toAudio(n.offset) - shift, onSec = warp.toAudio(n.onset) - shift;
if (offSec <= 0 || onSec >= endSec) continue;
const on = Math.round(Math.max(0, onSec) * TPS);
const off = Math.max(on + 1, Math.round(offSec * TPS));
midi.addEvent(new at.midi.NoteOnEvent(0, on, ch, n.pitch, n.velocity));
midi.addEvent(new at.midi.NoteOffEvent(0, off, ch, n.pitch, 0));
@@ -49,8 +53,13 @@ export function renderMidi(
chans = [L.map((v, i) => (v + R[i]!) / 2)];
}
for (const c of chans) for (let i = 0; i < c.length; i++) if (!Number.isFinite(c[i]!)) throw new Error("renderMidi: non-finite samples from synth");
if (opts.normalize !== false) normalizePeak(chans);
return { sampleRate, channels: chans };
}
/** Scales in place so the peak is -1 dBFS (silence stays silent). */
export function normalizePeak(chans: Float32Array[]): void {
let peak = 0;
for (const c of chans) for (let i = 0; i < c.length; i++) { const a = Math.abs(c[i]!); if (a > peak) peak = a; }
if (peak > 0) { const g = PEAK / peak; for (const c of chans) for (let i = 0; i < c.length; i++) c[i]! *= g; }
return { sampleRate, channels: chans };
}
+84 -23
View File
@@ -8,8 +8,10 @@ import type { Role } from "../core/types.ts";
import { audioOgg, loadSong, readAlignment, readManifest, writeAlignment, type AlignmentFile, type LoadedSong } from "../ref/manifest.ts";
import { trackKeys } from "./roles.ts";
import { makeWarp, stretches, type Anchor } from "./warp.ts";
import { autoAlign, fitFromCues, recordingCues, type AutoResult, type RecordingCues } from "./auto.ts";
import { renderMidi } from "./render.ts";
import { fitFromCues, recordingCues, type AutoResult, type RecordingCues } from "./auto.ts";
import { normalizePeak } from "./render.ts";
import { JobWorker } from "./jobs-pool.ts";
import { planChunks, stitchChunks } from "./preview.ts";
import { peaks } from "./peaks.ts";
import { stretchInfo } from "./tempo.ts";
import type { FitReport } from "./fit.ts";
@@ -24,7 +26,11 @@ const MAX_BODY_BYTES = 1 << 20;
const PEAKS_PER_SEC = 2000 / 60;
const PEAKS_VERSION = 1;
const RENDER_SR = 22050;
const RENDER_CACHE = 6;
const RENDER_CACHE = 24; // Opus previews are ~1.5 MB per 5 min
/** Parallel render workers (each renders one time chunk of a preview). */
const RENDER_WORKERS = Math.max(1, Math.min(4, (navigator.hardwareConcurrency ?? 2) - 1));
// Preview transport: mono Opus. 48 kb/s at compression level 0 encodes 5 min in ~0.6 s to ~1.5 MB (vs 12 MB WAV).
const OPUS_ARGS = ["-c:a", "libopus", "-b:a", "48k", "-compression_level", "0", "-f", "ogg", "pipe:1"];
const SONG_CACHE = 2;
export interface ServeOptions { root?: string; host?: string; port?: number; urlHost?: string; quiet?: boolean; sf2Path?: string }
@@ -34,11 +40,26 @@ interface SongState extends LoadedSong {
roles: Record<string, Role | null>;
cues: RecordingCues;
peaks: { bucketSec: number; min: number[]; max: number[] };
auto?: AutoResult; // first automatic pass, reused while no alignment.json exists
auto?: Promise<AutoResult>; // first automatic pass, reused while no alignment.json exists
}
class HttpError extends Error { constructor(readonly status: number, msg: string) { super(msg); } }
/** Mono float PCM -> Ogg Opus via ffmpeg (a separate process: the event loop stays free). */
async function encodeOpus(pcm: Float32Array, sampleRate: number): Promise<Uint8Array<ArrayBuffer>> {
const p = Bun.spawn(["ffmpeg", "-nostdin", "-loglevel", "error", "-f", "f32le", "-ar", String(sampleRate), "-ac", "1", "-i", "pipe:0", ...OPUS_ARGS], {
stdin: new Uint8Array(pcm.buffer, pcm.byteOffset, pcm.byteLength), stdout: "pipe", stderr: "pipe",
});
const [out, err, code] = await Promise.all([new Response(p.stdout).arrayBuffer(), new Response(p.stderr).text(), p.exited]);
if (code !== 0) throw new Error(`ffmpeg (opus) failed (${code}): ${err.trim()}`);
return new Uint8Array(out);
}
/** Copies audio into SharedArrayBuffers so worker jobs can read it without a copy per job. */
function shareAudio(a: LoadedSong["audio"]): LoadedSong["audio"] {
return { sampleRate: a.sampleRate, channels: a.channels.map((c) => { const x = new Float32Array(new SharedArrayBuffer(c.byteLength)); x.set(c); return x; }) };
}
/** Small insertion-ordered LRU. */
class Lru<V> {
private m = new Map<string, V>();
@@ -84,10 +105,15 @@ export async function startServer(opts: ServeOptions = {}): Promise<AlignServer>
const host = opts.host ?? "127.0.0.1";
const log = (msg: string) => { if (!opts.quiet) console.log(msg); };
const token = Buffer.from(crypto.getRandomValues(new Uint8Array(18))).toString("base64url");
const sf2 = new Uint8Array(await Bun.file(opts.sf2Path ?? SF2_PATH).arrayBuffer());
const sf2Path = opts.sf2Path ?? SF2_PATH;
if (!(await Bun.file(sf2Path).exists())) throw new Error(`soundfont not found: ${sf2Path}`);
// Heavy work runs on worker threads so this event loop keeps serving while a render or auto-align is in progress.
const renderWorkers = Array.from({ length: RENDER_WORKERS }, () => new JobWorker());
const autoWorker = new JobWorker();
const songs = new Lru<Promise<SongState>>(SONG_CACHE);
const renders = new Lru<Uint8Array<ArrayBuffer>>(RENDER_CACHE);
const inflight = new Map<string, Promise<Uint8Array<ArrayBuffer>>>();
async function entryExists(id: string): Promise<boolean> {
return (await readManifest(root)).some((s) => s.id === id);
@@ -95,7 +121,8 @@ export async function startServer(opts: ServeOptions = {}): Promise<AlignServer>
async function buildState(id: string): Promise<SongState> {
const t0 = performance.now();
const s = await loadSong(id, root);
const loaded = await loadSong(id, root);
const s = { ...loaded, audio: shareAudio(loaded.audio) };
const keys = trackKeys(s.smf.tracks);
const roles = Object.fromEntries(keys.map((k) => [k, s.entry.roles[k] ?? null]));
const cues = recordingCues(s.audio);
@@ -131,9 +158,9 @@ export async function startServer(opts: ServeOptions = {}): Promise<AlignServer>
const stretchesOf = (s: SongState, anchors: Anchor[]) => stretchInfo(stretches(anchors, s.smf.durationSec), s.smf.tempoMap);
const state = (s: SongState, anchors: Anchor[], fit = fitOf(s, anchors)) => ({ anchors, stretches: stretchesOf(s, anchors), fit });
function runAuto(s: SongState, manual: Anchor[]) {
async function runAuto(s: SongState, manual: Anchor[]): Promise<AutoResult> {
const t0 = performance.now();
const r = autoAlign(s.audio, s.smf, s.roles, manual, sf2);
const r = await autoWorker.run<AutoResult>({ type: "auto", sf2Path, audio: s.audio, smf: s.smf, roles: s.roles, manual });
log(`auto-aligned ${s.entry.id}: ${r.anchors.length} anchors (${manual.length} manual, ${r.chosen}) in ${((performance.now() - t0) / 1000).toFixed(1)} s`);
return r;
}
@@ -151,6 +178,25 @@ export async function startServer(opts: ServeOptions = {}): Promise<AlignServer>
}
}
/** Warped MIDI rendered in parallel time chunks on the render workers, stitched, normalised and encoded. */
async function renderPreview(s: SongState, anchors: Anchor[], format: "wav" | "opus", signal: AbortSignal): Promise<Uint8Array<ArrayBuffer>> {
const t0 = performance.now();
const warp = makeWarp(anchors);
const dur = Math.max(s.audioDur, Math.min(warp.toAudio(s.smf.durationSec) + 1, s.audioDur + 60));
const chunks = planChunks(dur, renderWorkers.length);
const pcm = await Promise.all(chunks.map((c) => {
const w = renderWorkers.reduce((a, b) => (b.load < a.load ? b : a));
return w.run<Float32Array>({ type: "render", sf2Path, tracks: s.smf.tracks, anchors, sampleRate: RENDER_SR, startSec: c.renderStart, durationSec: c.renderDur }, signal);
}));
const mono = stitchChunks(chunks, pcm, RENDER_SR, dur);
for (let i = 0; i < mono.length; i++) if (!Number.isFinite(mono[i]!)) throw new Error("render: non-finite samples from synth");
normalizePeak([mono]);
const t1 = performance.now();
const out = format === "wav" ? encodeWav({ sampleRate: RENDER_SR, channels: [mono] }) : await encodeOpus(mono, RENDER_SR);
log(`rendered ${s.entry.id} (${anchors.length} anchors, ${chunks.length} chunks) in ${((t1 - t0) / 1000).toFixed(2)} s + ${format} ${((performance.now() - t1) / 1000).toFixed(2)} s, ${(out.length / 1e6).toFixed(2)} MB`);
return out;
}
async function api(req: Request, path: string): Promise<Response> {
const method = req.method;
if (path === "/api/songs" && method === "GET") {
@@ -172,12 +218,17 @@ export async function startServer(opts: ServeOptions = {}): Promise<AlignServer>
const { file: saved, error } = await savedAlignment(id);
let anchors: Anchor[], fit: FitReport | undefined;
if (saved) anchors = saved.anchors;
else { s.auto ??= runAuto(s, []); anchors = s.auto.anchors; fit = s.auto.fit; }
else {
s.auto ??= runAuto(s, []);
s.auto.catch(() => { s.auto = undefined; });
const r = await s.auto;
anchors = r.anchors; fit = r.fit;
}
return Response.json({
id, title: s.entry.title, audioDur: s.audioDur, midiDur: s.smf.durationSec,
tracks: trackKeys(s.smf.tracks).map((name) => ({ name, role: s.roles[name] ?? null })),
recordingPeaks: s.peaks, ...state(s, anchors, fit), midiTempo: s.smf.tempoMap,
source: saved ? "saved" : "auto", savedAt: saved?.savedAt ?? null, ...(saved ? {} : { auto: autoInfo(s.auto!) }),
source: saved ? "saved" : "auto", savedAt: saved?.savedAt ?? null, ...(saved ? {} : { auto: autoInfo(await s.auto!) }),
...(error ? { warning: `${error}. Showing the automatic alignment instead; saving will overwrite the file.` } : {}),
});
}
@@ -186,21 +237,28 @@ export async function startServer(opts: ServeOptions = {}): Promise<AlignServer>
return new Response(Bun.file(await audioOgg(e, root)), { headers: { "content-type": "audio/ogg", "cache-control": "private, max-age=3600" } });
}
if (route === "POST render") {
// ?format=wav: 16-bit PCM (tests, debugging). Default: Ogg Opus, ~8x smaller, for the page over the LAN.
const format = new URL(req.url).searchParams.get("format") === "wav" ? "wav" : "opus";
const anchors = parseAnchors(await readJson(req));
const s = await song(id);
const hash = Bun.hash(JSON.stringify(anchors)).toString(16);
const key = `${id}:${hash}`;
let wav = renders.get(key);
const hit = wav !== undefined;
if (!wav) {
const t0 = performance.now();
const warp = makeWarp(anchors);
const dur = Math.max(s.audioDur, Math.min(warp.toAudio(s.smf.durationSec) + 1, s.audioDur + 60));
wav = encodeWav(renderMidi(s.smf.tracks, warp, sf2, { sampleRate: RENDER_SR, mono: true, durationSec: dur }));
renders.set(key, wav);
log(`rendered ${id} (${anchors.length} anchors) in ${((performance.now() - t0) / 1000).toFixed(1)} s`);
const key = `${id}:${format}:${hash}`;
let body = renders.get(key);
const hit = body !== undefined;
if (!body) {
const start = () => {
const p = renderPreview(s, anchors, format, req.signal).finally(() => { if (inflight.get(key) === p) inflight.delete(key); });
inflight.set(key, p);
return p;
};
try { body = await (inflight.get(key) ?? start()); } catch (e) {
if ((e as Error).name !== "AbortError") throw e;
if (req.signal.aborted) return new Response(null, { status: 499 }); // client gave up (superseded edit)
body = await start(); // a request we shared the render with was cancelled: render for this one
}
renders.set(key, body);
}
return new Response(wav, { headers: { "content-type": "audio/wav", "x-anchors-hash": hash, "x-cache": hit ? "hit" : "miss" } });
return new Response(body, { headers: { "content-type": format === "wav" ? "audio/wav" : "audio/ogg", "x-anchors-hash": hash, "x-cache": hit ? "hit" : "miss" } });
}
if (route === "POST fit") {
const anchors = parseAnchors(await readJson(req));
@@ -209,7 +267,7 @@ export async function startServer(opts: ServeOptions = {}): Promise<AlignServer>
if (route === "POST auto") {
const manual = parseAnchors(await readJson(req)).filter((a) => a.kind === "manual");
const s = await song(id);
const r = runAuto(s, manual);
const r = await runAuto(s, manual);
return Response.json({ ...state(s, r.anchors, r.fit), auto: autoInfo(r) });
}
if (route === "POST reset") return Response.json(state(await song(id), []));
@@ -276,6 +334,9 @@ export async function startServer(opts: ServeOptions = {}): Promise<AlignServer>
const origin = `http://${opts.urlHost ?? (host === "0.0.0.0" || host === "::" ? "localhost" : host)}:${outer.port}`;
return {
url: `${origin}/?key=${token}`, origin, token, port: outer.port!,
async stop() { await outer.stop(true); await inner.stop(true); },
async stop() {
await outer.stop(true); await inner.stop(true);
for (const w of [...renderWorkers, autoWorker]) w.terminate();
},
};
}
+88
View File
@@ -0,0 +1,88 @@
import { test, expect, describe } from "bun:test";
import { NO_MARKS, placeMark, midiMarkAt, markLaneSec, linkMarks, linkConflict, parseVolumes } from "../../../src/align/page/edit.ts";
import { makeWarp, type Anchor } from "../../../src/align/warp.ts";
import { parseTime, formatTime, buildPyramid, buildPyramidSliced } from "../../../src/align/page/timeline.ts";
const m = (midiSec: number, audioSec: number): Anchor => ({ midiSec, audioSec, kind: "manual" });
const a = (midiSec: number, audioSec: number): Anchor => ({ midiSec, audioSec, kind: "auto" });
const base = [m(10, 11), m(20, 22), m(30, 32)];
describe("pair marks", () => {
test("placing marks only changes the marks (anchors untouched)", () => {
const before = JSON.stringify(base);
let s = placeMark(NO_MARKS, "rec", 25);
s = placeMark(s, "midi", midiMarkAt(base, 26));
expect(s).toEqual({ rec: 25, midi: makeWarp(base).toMidi(26), active: "midi" });
expect(JSON.stringify(base)).toBe(before);
expect(placeMark(s, "rec", 27)).toEqual({ ...s, rec: 27, active: "rec" });
});
test("markLaneSec: rec mark is recording time, MIDI mark is drawn where the warp puts it", () => {
const s = placeMark(placeMark(NO_MARKS, "rec", 25), "midi", midiMarkAt(base, 26));
expect(markLaneSec(base, s, "rec")).toBe(25);
expect(markLaneSec(base, s, "midi")).toBeCloseTo(26, 9);
expect(markLaneSec(base, NO_MARKS, "midi")).toBeNull();
});
});
describe("linkMarks", () => {
test("creates a manual anchor {audioSec: rec mark, midiSec: warp.toMidi(MIDI-lane mark)}", () => {
const midi = midiMarkAt(base, 26);
const r = linkMarks(base, { rec: 25, midi, active: "midi" });
expect(r.anchors[r.index]).toEqual(m(makeWarp(base).toMidi(26), 25));
expect(r.anchors).toHaveLength(4);
expect(makeWarp(r.anchors).toAudio(midi)).toBeCloseTo(25, 9); // that MIDI moment now plays at the recording mark
});
test("needs both marks", () => {
expect(() => linkMarks(base, placeMark(NO_MARKS, "rec", 25))).toThrow(/both/);
});
test("refuses to cross a manual neighbour, with a clear message", () => {
// MIDI mark between anchors 20 and 30, recording mark before the anchor at 22 s
const s = { rec: 21, midi: 25, active: "rec" as const };
expect(linkConflict(base, 25, 21)).toMatch(/same order/);
expect(() => linkMarks(base, s)).toThrow(/Can't link/);
expect(linkConflict(base, 15, 33)).toMatch(/MIDI mark \(0:16\.50\) is before your anchor at 0:22\.00/);
expect(linkConflict(base, 20, 21.5)).toMatch(/already uses that MIDI moment/);
expect(linkConflict(base, 25, 27)).toBeNull();
});
test("automatic anchors never block a link (they are dropped)", () => {
const list = [m(0, 0), a(5, 5.2), a(10, 10.3), m(40, 40)];
expect(linkConflict(list, 10, 4)).toBeNull();
const r = linkMarks(list, { rec: 4, midi: 10, active: "rec" });
expect(r.anchors).toEqual([m(0, 0), m(10, 4), m(40, 40)]);
});
});
test("parseVolumes clamps to 0–150 and defaults to 100", () => {
expect(parseVolumes(null)).toEqual({ rec: 100, midi: 100 });
expect(parseVolumes("garbage")).toEqual({ rec: 100, midi: 100 });
expect(parseVolumes(JSON.stringify({ rec: 0, midi: 999 }))).toEqual({ rec: 0, midi: 150 });
expect(parseVolumes(JSON.stringify({ rec: 42.4, midi: "x" }))).toEqual({ rec: 42, midi: 100 });
});
describe("parseTime", () => {
test("m:ss.s, h:mm:ss, plain seconds, comma decimals", () => {
expect(parseTime("0:23.2")).toBeCloseTo(23.2, 9);
expect(parseTime(" 1:05 ")).toBe(65);
expect(parseTime("1:02:03.5")).toBeCloseTo(3723.5, 9);
expect(parseTime("24")).toBe(24);
expect(parseTime("24,5")).toBe(24.5);
expect(parseTime(".5")).toBe(0.5);
expect(parseTime(formatTime(83.25, 2))).toBeCloseTo(83.25, 9);
});
test("rejects junk, negatives and seconds >= 60 in m:ss", () => {
for (const s of ["", "abc", "-3", "1:60", "1::2", "1:2:3:4", "0:23.2.1", "12s"]) expect(parseTime(s)).toBeNull();
});
});
test("buildPyramidSliced matches buildPyramid and takes the min/max across channels", async () => {
const x = Float32Array.from({ length: 10_000 }, (_, i) => Math.sin(i / 13) * (i % 7) / 7);
let pauses = 0;
const a = await buildPyramidSliced([x], 64, async () => { pauses++; }, 1000);
const b = buildPyramid(x, 64);
expect(pauses).toBeGreaterThan(5);
expect(a.length).toBe(b.length);
a.forEach((l, k) => { expect(l.bucket).toBe(b[k]!.bucket); expect([...l.min]).toEqual([...b[k]!.min]); expect([...l.max]).toEqual([...b[k]!.max]); });
const y = x.map((v) => -v * 2);
const c = await buildPyramidSliced([x, y], 64, async () => {});
expect(c[0]!.max[3]).toBe(Math.max(...x.subarray(192, 256), ...y.subarray(192, 256)));
});
+54
View File
@@ -0,0 +1,54 @@
import { test, expect } from "bun:test";
import { planChunks, stitchChunks, MIN_CHUNK_SEC, PREROLL_SEC } from "../../src/align/preview.ts";
import { renderMidi, normalizePeak } from "../../src/align/render.ts";
import { makeWarp } from "../../src/align/warp.ts";
import type { RefNote, RefTrack } from "../../src/align/smf.ts";
test("planChunks covers [0, dur) contiguously, with pre-roll after the first chunk", () => {
const c = planChunks(280, 4);
expect(c).toHaveLength(4);
expect(c[0]!.from).toBe(0);
expect(c[3]!.to).toBe(280);
for (let k = 1; k < 4; k++) {
expect(c[k]!.from).toBe(c[k - 1]!.to);
expect(c[k]!.renderStart).toBeCloseTo(c[k]!.from - PREROLL_SEC, 9);
}
expect(planChunks(MIN_CHUNK_SEC * 1.5, 4)).toHaveLength(1);
expect(planChunks(5, 4)).toEqual([{ renderStart: 0, renderDur: 5, from: 0, to: 5 }]);
});
test("stitchChunks reproduces a signal from overlapping slices of it", () => {
const sr = 1000, dur = 90, n = dur * sr;
const x = Float32Array.from({ length: n }, (_, i) => Math.sin(i / 37) * 0.5);
const chunks = planChunks(dur, 3);
const pcm = chunks.map((c) => x.slice(Math.round(c.renderStart * sr), Math.round((c.renderStart + c.renderDur) * sr)));
const y = stitchChunks(chunks, pcm, sr, dur);
expect(y.length).toBe(n);
let maxErr = 0;
for (let i = 0; i < n; i++) maxErr = Math.max(maxErr, Math.abs(y[i]! - x[i]!));
expect(maxErr).toBeLessThan(1e-6);
});
test("a chunked render matches the whole render (onsets in place, small seam error)", async () => {
const SF2 = new Uint8Array(await Bun.file("node_modules/@coderline/alphatab/dist/soundfont/sonivox.sf2").arrayBuffer());
const notes: RefNote[] = [];
for (let t = 0.25; t < 58; t += 0.5) notes.push({ onset: t, offset: t + 0.4, pitch: 52 + ((t * 2) % 12), velocity: 100, channel: 0 });
notes.push({ onset: 18, offset: 23, pitch: 40, velocity: 100, channel: 0 }); // sustains across the first seam (20 s)
const tracks: RefTrack[] = [{ index: 0, name: "g", program: 25, isDrums: false, notes }];
const warp = makeWarp([{ midiSec: 0, audioSec: 0.3, kind: "manual" }, { midiSec: 50, audioSec: 51, kind: "manual" }]);
const sr = 22050, dur = 60;
const whole = renderMidi(tracks, warp, SF2, { sampleRate: sr, mono: true, durationSec: dur }).channels[0]!;
const chunks = planChunks(dur, 3);
const pcm = chunks.map((c) => renderMidi(tracks, warp, SF2, { sampleRate: sr, mono: true, startSec: c.renderStart, durationSec: c.renderDur, normalize: false }).channels[0]!);
const y = stitchChunks(chunks, pcm, sr, dur);
normalizePeak([y]);
// energy per 50 ms frame agrees closely everywhere (identical away from seams)
const frame = Math.floor(sr / 20);
let worst = 0;
for (let f = 0; f + frame <= y.length; f += frame) {
let a = 0, b = 0;
for (let i = f; i < f + frame; i++) { a += whole[i]! ** 2; b += y[i]! ** 2; }
worst = Math.max(worst, Math.abs(Math.sqrt(a / frame) - Math.sqrt(b / frame)));
}
expect(worst).toBeLessThan(0.05);
}, 60_000);
+15 -2
View File
@@ -151,10 +151,23 @@ describe("API", () => {
expect(String.fromCharCode(...b.subarray(0, 4))).toBe("OggS");
expect(await Bun.file(join(root, "fix", "audio.ogg")).exists()).toBe(true);
}, BIG);
test("render: RIFF WAV, mono 22.05 kHz, cached by anchors", async () => {
test("render: Ogg Opus by default (small), cached by anchors", async () => {
const body = json({ anchors: first.anchors });
const r = await api("/api/song/fix/render", body);
expect(r.status).toBe(200);
expect(r.headers.get("content-type")).toBe("audio/ogg");
const b = new Uint8Array(await r.arrayBuffer());
expect(String.fromCharCode(...b.subarray(0, 4))).toBe("OggS");
expect(new TextDecoder().decode(b.subarray(28, 36))).toBe("OpusHead");
expect(b.length).toBeLessThan(first.audioDur * 22050 * 2 / 4); // far below 16-bit PCM
const again = await api("/api/song/fix/render", body);
expect(again.headers.get("x-cache")).toBe("hit");
expect(again.headers.get("x-anchors-hash")).toBe(r.headers.get("x-anchors-hash"));
}, BIG);
test("render ?format=wav: RIFF WAV, mono 22.05 kHz, cached by anchors", async () => {
const body = json({ anchors: first.anchors });
const r = await api("/api/song/fix/render?format=wav", body);
expect(r.status).toBe(200);
expect(r.headers.get("content-type")).toBe("audio/wav");
const b = new Uint8Array(await r.arrayBuffer());
const dv = new DataView(b.buffer);
@@ -162,7 +175,7 @@ describe("API", () => {
expect(dv.getUint16(22, true)).toBe(1);
expect(dv.getUint32(24, true)).toBe(22050);
expect(b.length).toBeGreaterThan(first.audioDur * 22050 * 2);
const again = await api("/api/song/fix/render", body);
const again = await api("/api/song/fix/render?format=wav", body);
expect(again.headers.get("x-cache")).toBe("hit");
expect(again.headers.get("x-anchors-hash")).toBe(r.headers.get("x-anchors-hash"));
}, BIG);