From 8e0ecdaf85f8ee8bbab837069c33af3fed3ed0ca Mon Sep 17 00:00:00 2001 From: Lucas Winther Date: Thu, 29 Oct 2026 22:28:48 +0100 Subject: [PATCH] Add the synthetic meeting fixture and long-form measurement script Co-Authored-By: Claude Opus 5.5 --- scripts/measure-longform.ts | 167 +++++++++++++++++++++++++++++++++++ tests/meetingFixture.test.ts | 22 +++++ tests/meetingFixture.ts | 64 ++++++++++++++ 3 files changed, 253 insertions(+) create mode 100644 scripts/measure-longform.ts create mode 100644 tests/meetingFixture.test.ts create mode 100644 tests/meetingFixture.ts diff --git a/scripts/measure-longform.ts b/scripts/measure-longform.ts new file mode 100644 index 0000000..11d54b6 --- /dev/null +++ b/scripts/measure-longform.ts @@ -0,0 +1,167 @@ +/** + * Before/after measurement for the long-form pipeline (D-WS3a): wall-clock, + * WER against tests/audio_files/*.txt, generated tokens and decode calls, for + * the worker's current chunked path ("old") and long-form configurations. + * + * flock /tmp/whisper-heavy.lock bun scripts/measure-longform.ts \ + * [--clips en_aiproblem,da_festuge,da_festuge:auto,en_alien,meeting] \ + * [--configs old,lf,lf-ngram5] [--repeat 1] [--out dir] + * + * Runs on onnxruntime-node (CPU), Whisper Large v3 Turbo at its WASM-default + * dtype (q4) — the same weights and quantization as the browser's WASM + * engine. "meeting" is the synthetic stand-in (tests/meetingFixture.ts). + */ + +import { mkdir } from "node:fs/promises"; +import { join } from "node:path"; +import { parseArgs } from "node:util"; +import { pipeline, type AutomaticSpeechRecognitionPipeline } from "@huggingface/transformers"; +import { resolveDefaultDtype } from "../src/lib/models"; +import { + transcribeLongForm, + type LongFormOptions, +} from "../src/worker/longform"; +import { createWhisperDeps, loadSileroVad } from "../src/worker/longform/transformers"; +import { decodeClip, normalize, readReference } from "../tests/clips"; +import { wordErrorRate } from "../tests/dsp"; +import { buildMeetingFixture } from "../tests/meetingFixture"; + +const MODEL_ID = "onnx-community/whisper-large-v3-turbo"; + +/** Long-form configurations under test; "old" is the worker's chunked path. */ +const CONFIGS: Record | "old"> = { + old: "old", + lf: {}, + "lf-ngram3": { guard: { noRepeatNgramSize: 3, repetitionPenalty: 1 } }, + "lf-ngram5": { guard: { noRepeatNgramSize: 5, repetitionPenalty: 1 } }, + "lf-rep1.1": { guard: { noRepeatNgramSize: 0, repetitionPenalty: 1.1 } }, + "lf-fallback": { fallback: {} }, + "lf-sil160": { vad: { minSilenceDurationMs: 160 } }, +}; + +const { values } = parseArgs({ + options: { + clips: { type: "string", default: "en_aiproblem,da_festuge,da_festuge:auto,meeting,en_alien" }, + configs: { type: "string", default: "old,lf" }, + repeat: { type: "string", default: "1" }, + out: { type: "string" }, + }, +}); + +interface Clip { + name: string; + audio: Float32Array; + reference: string; + /** undefined = the UI's "Auto-detect". */ + language?: string; +} + +async function loadClip(spec: string): Promise { + const [base, mode] = spec.split(":"); + if (base === "meeting") { + const { audio, reference } = await buildMeetingFixture(); + return { name: "meeting (synthetic)", audio, reference, language: "en" }; + } + const file = (await Array.fromAsync(new Bun.Glob(`${base}.*`).scan(join(import.meta.dir, "../tests/audio_files")))) + .find((name) => !name.endsWith(".txt")); + if (!file) throw new Error(`no audio for ${base}`); + return { + name: spec, + audio: await decodeClip(file), + reference: await readReference(base!), + ...(mode !== "auto" && { language: base!.split("_")[0] }), + }; +} + +const asr = (await pipeline("automatic-speech-recognition", MODEL_ID, { + dtype: resolveDefaultDtype(MODEL_ID, "wasm"), +})) as AutomaticSpeechRecognitionPipeline; + +// Count generated tokens on every generate call, prompt prefix excluded +// (leading special tokens below the first timestamp token). +const model = asr.model as unknown as { generate(args: unknown): Promise }; +const originalGenerate = model.generate.bind(model); +let generated = 0; +let generateCalls = 0; +model.generate = async (args: unknown) => { + const out = (await originalGenerate(args)) as { tolist(): bigint[][] }; + const ids = out.tolist()[0]!.map(Number); + let prefix = 0; + while (prefix < ids.length && ids[prefix]! >= 50257 && ids[prefix]! <= 50364) prefix += 1; + generated += ids.length - prefix; + generateCalls += 1; + return out; +}; + +const vad = await loadSileroVad({ localOnly: false }); +const whisper = createWhisperDeps(asr, { multilingual: true }); + +// Warm-up so the first measured run doesn't pay session initialization. +await asr(new Float32Array(16_000), { max_new_tokens: 8, language: "en" }); + +const rows: string[] = []; +const outDir = values.out; +if (outDir) await mkdir(outDir, { recursive: true }); + +for (const spec of values.clips!.split(",")) { + const clip = await loadClip(spec); + for (let r = 0; r < Number(values.repeat); r++) { + for (const name of values.configs!.split(",")) { + const config = CONFIGS[name]; + if (config === undefined) throw new Error(`unknown config ${name}`); + generated = 0; + generateCalls = 0; + const audio = clip.audio.slice(); // the pipeline may keep views; give each run its own + const started = performance.now(); + let text: string; + let chunks: unknown; + let extra = ""; + if (config === "old") { + const output = (await asr(audio, { + chunk_length_s: 30, + stride_length_s: 5, + return_timestamps: true, + max_new_tokens: 440, + ...(clip.language !== undefined && { language: clip.language }), + task: "transcribe", + })) as { text: string; chunks: unknown }; + text = output.text; + chunks = output.chunks; + } else { + let vadMs = 0; + const result = await transcribeLongForm( + audio, + { + speechProbabilities: async (pcm) => { + const t = performance.now(); + const probs = await vad.speechProbabilities(pcm); + vadMs = performance.now() - t; + return probs; + }, + ...whisper, + }, + { ...config, ...(clip.language !== undefined && { language: clip.language }), task: "transcribe" }, + ); + text = result.text; + chunks = result.chunks; + extra = `vad ${(vadMs / 1000).toFixed(1)} s, speech ${result.speechSeconds.toFixed(0)} s, lang ${result.language}`; + } + const seconds = (performance.now() - started) / 1000; + const wer = wordErrorRate(normalize(clip.reference), normalize(text)); + const row = `| ${clip.name} | ${name} | ${seconds.toFixed(1)} | ${wer.toFixed(3)} | ${generated} | ${generateCalls} | ${extra} |`; + console.log(row); + rows.push(row); + if (outDir) { + await Bun.write( + join(outDir, `${spec.replace(":", "-")}.${name}.${r}.json`), + JSON.stringify({ text, chunks }, null, 1), + ); + } + } + } +} + +console.log("\n| clip | config | wall s | WER | gen tokens | decodes | notes |\n|---|---|---|---|---|---|---|"); +console.log(rows.join("\n")); +await asr.dispose(); +await vad.dispose(); diff --git a/tests/meetingFixture.test.ts b/tests/meetingFixture.test.ts new file mode 100644 index 0000000..5b5428f --- /dev/null +++ b/tests/meetingFixture.test.ts @@ -0,0 +1,22 @@ +import { describe, expect, test } from "bun:test"; +import { ALIEN_EXCERPTS, buildMeetingFixture } from "./meetingFixture"; + +describe("buildMeetingFixture (SYNTHETIC meeting stand-in)", () => { + test("joins en_aiproblem and the en_alien excerpts with pauses, deterministically", async () => { + const first = await buildMeetingFixture(); + const second = await buildMeetingFixture(); + const seconds = first.audio.length / 16_000; + const excerptSeconds = ALIEN_EXCERPTS.reduce((n, e) => n + e.end - e.start, 0); + // lead-in 2 + aiproblem ≈ 14.4 + excerpts + pauses 3+8+2+5 + expect(seconds).toBeCloseTo(2 + 14.4 + excerptSeconds + 18, 0); + expect(first.audio).toEqual(second.audio); + expect(first.reference).toBe(second.reference); + expect(first.reference).toStartWith("The problem I find with AI"); + expect(first.reference).toContain("Voices of the Void"); + expect(first.reference).toEndWith("Maybe no."); + // Pauses are room tone, not digital silence, and never louder than −50 dBFS. + const leadIn = first.audio.subarray(0, 2 * 16_000); + expect(leadIn.some((v) => v !== 0)).toBe(true); + expect(Math.max(...leadIn.map(Math.abs))).toBeLessThan(0.003); + }, 60_000); // decodes the 15-min en_alien twice +}); diff --git a/tests/meetingFixture.ts b/tests/meetingFixture.ts new file mode 100644 index 0000000..044a06a --- /dev/null +++ b/tests/meetingFixture.ts @@ -0,0 +1,64 @@ +/** + * SYNTHETIC meeting stand-in (spec 001 "Measurement fallback", coordination + * plan 001-0): no real meeting recording exists, so this joins the repo's + * own English clips with meeting-like pauses. Built at test time from + * tests/audio_files — nothing new is checked in. Every number measured on it + * is labelled synthetic. + * + * Layout (≈ 2 min 45 s): 2 s lead-in, en_aiproblem (whole), then three + * en_alien excerpts, each clip followed by a pause of 3, 8, 2 and 5 s. Pauses and lead-in carry + * deterministic room noise at about −60 dBFS (a real recording is never + * digital silence). Excerpts are cut on reference-line boundaries; their + * times were read once from a timestamped Large v3 Turbo transcript of + * en_alien (2026-10-29) and land in the pause before the line, so each + * excerpt's reference is exactly its lines of en_alien.txt. + */ + +import { decodeClip, readReference } from "./clips"; + +const RATE = 16_000; +const LEAD_IN_SECONDS = 2; +const PAUSES_SECONDS = [3, 8, 2, 5]; +const NOISE_AMPLITUDE = 0.001; // ≈ −60 dBFS + +/** en_alien excerpts: [start s, end s) and the 1-based reference lines they contain. */ +export const ALIEN_EXCERPTS: { start: number; end: number; firstLine: number; lastLine: number }[] = [ + { start: 0, end: 36.8, firstLine: 1, lastLine: 9 }, + { start: 36.8, end: 99.0, firstLine: 10, lastLine: 28 }, + { start: 165.4, end: 196.1, firstLine: 45, lastLine: 58 }, +]; + +function roomNoise(seconds: number, seed: number): Float32Array { + let state = seed; + return new Float32Array(Math.round(seconds * RATE)).map(() => { + state = (state * 1_103_515_245 + 12_345) % 2 ** 31; + return NOISE_AMPLITUDE * (state / 2 ** 30 - 1); + }); +} + +export async function buildMeetingFixture(): Promise<{ audio: Float32Array; reference: string }> { + const aiproblem = await decodeClip("en_aiproblem.m4a"); + const alien = await decodeClip("en_alien.opus"); + const alienLines = (await readReference("en_alien")).split("\n"); + + const speech: Float32Array[] = [aiproblem]; + const references: string[] = [(await readReference("en_aiproblem")).trim()]; + for (const excerpt of ALIEN_EXCERPTS) { + speech.push(alien.subarray(Math.round(excerpt.start * RATE), Math.round(excerpt.end * RATE))); + references.push(alienLines.slice(excerpt.firstLine - 1, excerpt.lastLine).join(" ").trim()); + } + + const parts: Float32Array[] = [roomNoise(LEAD_IN_SECONDS, 1)]; + // A pause after every clip — the last one is the meeting's trailing room tone. + speech.forEach((clip, i) => { + parts.push(clip); + parts.push(roomNoise(PAUSES_SECONDS[i % PAUSES_SECONDS.length]!, i + 2)); + }); + const audio = new Float32Array(parts.reduce((n, p) => n + p.length, 0)); + let at = 0; + for (const part of parts) { + audio.set(part, at); + at += part.length; + } + return { audio, reference: references.join(" ") }; +}