Add the synthetic meeting fixture and long-form measurement script
Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
Lucas Winther
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
f55ea6ecf8
commit
8e0ecdaf85
@@ -0,0 +1,167 @@
|
||||
/**
|
||||
* Before/after measurement for the long-form pipeline (D-WS3a): wall-clock,
|
||||
* WER against tests/audio_files/*.txt, generated tokens and decode calls, for
|
||||
* the worker's current chunked path ("old") and long-form configurations.
|
||||
*
|
||||
* flock /tmp/whisper-heavy.lock bun scripts/measure-longform.ts \
|
||||
* [--clips en_aiproblem,da_festuge,da_festuge:auto,en_alien,meeting] \
|
||||
* [--configs old,lf,lf-ngram5] [--repeat 1] [--out dir]
|
||||
*
|
||||
* Runs on onnxruntime-node (CPU), Whisper Large v3 Turbo at its WASM-default
|
||||
* dtype (q4) — the same weights and quantization as the browser's WASM
|
||||
* engine. "meeting" is the synthetic stand-in (tests/meetingFixture.ts).
|
||||
*/
|
||||
|
||||
import { mkdir } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { parseArgs } from "node:util";
|
||||
import { pipeline, type AutomaticSpeechRecognitionPipeline } from "@huggingface/transformers";
|
||||
import { resolveDefaultDtype } from "../src/lib/models";
|
||||
import {
|
||||
transcribeLongForm,
|
||||
type LongFormOptions,
|
||||
} from "../src/worker/longform";
|
||||
import { createWhisperDeps, loadSileroVad } from "../src/worker/longform/transformers";
|
||||
import { decodeClip, normalize, readReference } from "../tests/clips";
|
||||
import { wordErrorRate } from "../tests/dsp";
|
||||
import { buildMeetingFixture } from "../tests/meetingFixture";
|
||||
|
||||
const MODEL_ID = "onnx-community/whisper-large-v3-turbo";
|
||||
|
||||
/** Long-form configurations under test; "old" is the worker's chunked path. */
|
||||
const CONFIGS: Record<string, Omit<LongFormOptions, "language" | "task"> | "old"> = {
|
||||
old: "old",
|
||||
lf: {},
|
||||
"lf-ngram3": { guard: { noRepeatNgramSize: 3, repetitionPenalty: 1 } },
|
||||
"lf-ngram5": { guard: { noRepeatNgramSize: 5, repetitionPenalty: 1 } },
|
||||
"lf-rep1.1": { guard: { noRepeatNgramSize: 0, repetitionPenalty: 1.1 } },
|
||||
"lf-fallback": { fallback: {} },
|
||||
"lf-sil160": { vad: { minSilenceDurationMs: 160 } },
|
||||
};
|
||||
|
||||
const { values } = parseArgs({
|
||||
options: {
|
||||
clips: { type: "string", default: "en_aiproblem,da_festuge,da_festuge:auto,meeting,en_alien" },
|
||||
configs: { type: "string", default: "old,lf" },
|
||||
repeat: { type: "string", default: "1" },
|
||||
out: { type: "string" },
|
||||
},
|
||||
});
|
||||
|
||||
interface Clip {
|
||||
name: string;
|
||||
audio: Float32Array;
|
||||
reference: string;
|
||||
/** undefined = the UI's "Auto-detect". */
|
||||
language?: string;
|
||||
}
|
||||
|
||||
async function loadClip(spec: string): Promise<Clip> {
|
||||
const [base, mode] = spec.split(":");
|
||||
if (base === "meeting") {
|
||||
const { audio, reference } = await buildMeetingFixture();
|
||||
return { name: "meeting (synthetic)", audio, reference, language: "en" };
|
||||
}
|
||||
const file = (await Array.fromAsync(new Bun.Glob(`${base}.*`).scan(join(import.meta.dir, "../tests/audio_files"))))
|
||||
.find((name) => !name.endsWith(".txt"));
|
||||
if (!file) throw new Error(`no audio for ${base}`);
|
||||
return {
|
||||
name: spec,
|
||||
audio: await decodeClip(file),
|
||||
reference: await readReference(base!),
|
||||
...(mode !== "auto" && { language: base!.split("_")[0] }),
|
||||
};
|
||||
}
|
||||
|
||||
const asr = (await pipeline("automatic-speech-recognition", MODEL_ID, {
|
||||
dtype: resolveDefaultDtype(MODEL_ID, "wasm"),
|
||||
})) as AutomaticSpeechRecognitionPipeline;
|
||||
|
||||
// Count generated tokens on every generate call, prompt prefix excluded
|
||||
// (leading special tokens below the first timestamp token).
|
||||
const model = asr.model as unknown as { generate(args: unknown): Promise<unknown> };
|
||||
const originalGenerate = model.generate.bind(model);
|
||||
let generated = 0;
|
||||
let generateCalls = 0;
|
||||
model.generate = async (args: unknown) => {
|
||||
const out = (await originalGenerate(args)) as { tolist(): bigint[][] };
|
||||
const ids = out.tolist()[0]!.map(Number);
|
||||
let prefix = 0;
|
||||
while (prefix < ids.length && ids[prefix]! >= 50257 && ids[prefix]! <= 50364) prefix += 1;
|
||||
generated += ids.length - prefix;
|
||||
generateCalls += 1;
|
||||
return out;
|
||||
};
|
||||
|
||||
const vad = await loadSileroVad({ localOnly: false });
|
||||
const whisper = createWhisperDeps(asr, { multilingual: true });
|
||||
|
||||
// Warm-up so the first measured run doesn't pay session initialization.
|
||||
await asr(new Float32Array(16_000), { max_new_tokens: 8, language: "en" });
|
||||
|
||||
const rows: string[] = [];
|
||||
const outDir = values.out;
|
||||
if (outDir) await mkdir(outDir, { recursive: true });
|
||||
|
||||
for (const spec of values.clips!.split(",")) {
|
||||
const clip = await loadClip(spec);
|
||||
for (let r = 0; r < Number(values.repeat); r++) {
|
||||
for (const name of values.configs!.split(",")) {
|
||||
const config = CONFIGS[name];
|
||||
if (config === undefined) throw new Error(`unknown config ${name}`);
|
||||
generated = 0;
|
||||
generateCalls = 0;
|
||||
const audio = clip.audio.slice(); // the pipeline may keep views; give each run its own
|
||||
const started = performance.now();
|
||||
let text: string;
|
||||
let chunks: unknown;
|
||||
let extra = "";
|
||||
if (config === "old") {
|
||||
const output = (await asr(audio, {
|
||||
chunk_length_s: 30,
|
||||
stride_length_s: 5,
|
||||
return_timestamps: true,
|
||||
max_new_tokens: 440,
|
||||
...(clip.language !== undefined && { language: clip.language }),
|
||||
task: "transcribe",
|
||||
})) as { text: string; chunks: unknown };
|
||||
text = output.text;
|
||||
chunks = output.chunks;
|
||||
} else {
|
||||
let vadMs = 0;
|
||||
const result = await transcribeLongForm(
|
||||
audio,
|
||||
{
|
||||
speechProbabilities: async (pcm) => {
|
||||
const t = performance.now();
|
||||
const probs = await vad.speechProbabilities(pcm);
|
||||
vadMs = performance.now() - t;
|
||||
return probs;
|
||||
},
|
||||
...whisper,
|
||||
},
|
||||
{ ...config, ...(clip.language !== undefined && { language: clip.language }), task: "transcribe" },
|
||||
);
|
||||
text = result.text;
|
||||
chunks = result.chunks;
|
||||
extra = `vad ${(vadMs / 1000).toFixed(1)} s, speech ${result.speechSeconds.toFixed(0)} s, lang ${result.language}`;
|
||||
}
|
||||
const seconds = (performance.now() - started) / 1000;
|
||||
const wer = wordErrorRate(normalize(clip.reference), normalize(text));
|
||||
const row = `| ${clip.name} | ${name} | ${seconds.toFixed(1)} | ${wer.toFixed(3)} | ${generated} | ${generateCalls} | ${extra} |`;
|
||||
console.log(row);
|
||||
rows.push(row);
|
||||
if (outDir) {
|
||||
await Bun.write(
|
||||
join(outDir, `${spec.replace(":", "-")}.${name}.${r}.json`),
|
||||
JSON.stringify({ text, chunks }, null, 1),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
console.log("\n| clip | config | wall s | WER | gen tokens | decodes | notes |\n|---|---|---|---|---|---|---|");
|
||||
console.log(rows.join("\n"));
|
||||
await asr.dispose();
|
||||
await vad.dispose();
|
||||
@@ -0,0 +1,22 @@
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import { ALIEN_EXCERPTS, buildMeetingFixture } from "./meetingFixture";
|
||||
|
||||
describe("buildMeetingFixture (SYNTHETIC meeting stand-in)", () => {
|
||||
test("joins en_aiproblem and the en_alien excerpts with pauses, deterministically", async () => {
|
||||
const first = await buildMeetingFixture();
|
||||
const second = await buildMeetingFixture();
|
||||
const seconds = first.audio.length / 16_000;
|
||||
const excerptSeconds = ALIEN_EXCERPTS.reduce((n, e) => n + e.end - e.start, 0);
|
||||
// lead-in 2 + aiproblem ≈ 14.4 + excerpts + pauses 3+8+2+5
|
||||
expect(seconds).toBeCloseTo(2 + 14.4 + excerptSeconds + 18, 0);
|
||||
expect(first.audio).toEqual(second.audio);
|
||||
expect(first.reference).toBe(second.reference);
|
||||
expect(first.reference).toStartWith("The problem I find with AI");
|
||||
expect(first.reference).toContain("Voices of the Void");
|
||||
expect(first.reference).toEndWith("Maybe no.");
|
||||
// Pauses are room tone, not digital silence, and never louder than −50 dBFS.
|
||||
const leadIn = first.audio.subarray(0, 2 * 16_000);
|
||||
expect(leadIn.some((v) => v !== 0)).toBe(true);
|
||||
expect(Math.max(...leadIn.map(Math.abs))).toBeLessThan(0.003);
|
||||
}, 60_000); // decodes the 15-min en_alien twice
|
||||
});
|
||||
@@ -0,0 +1,64 @@
|
||||
/**
|
||||
* SYNTHETIC meeting stand-in (spec 001 "Measurement fallback", coordination
|
||||
* plan 001-0): no real meeting recording exists, so this joins the repo's
|
||||
* own English clips with meeting-like pauses. Built at test time from
|
||||
* tests/audio_files — nothing new is checked in. Every number measured on it
|
||||
* is labelled synthetic.
|
||||
*
|
||||
* Layout (≈ 2 min 45 s): 2 s lead-in, en_aiproblem (whole), then three
|
||||
* en_alien excerpts, each clip followed by a pause of 3, 8, 2 and 5 s. Pauses and lead-in carry
|
||||
* deterministic room noise at about −60 dBFS (a real recording is never
|
||||
* digital silence). Excerpts are cut on reference-line boundaries; their
|
||||
* times were read once from a timestamped Large v3 Turbo transcript of
|
||||
* en_alien (2026-10-29) and land in the pause before the line, so each
|
||||
* excerpt's reference is exactly its lines of en_alien.txt.
|
||||
*/
|
||||
|
||||
import { decodeClip, readReference } from "./clips";
|
||||
|
||||
const RATE = 16_000;
|
||||
const LEAD_IN_SECONDS = 2;
|
||||
const PAUSES_SECONDS = [3, 8, 2, 5];
|
||||
const NOISE_AMPLITUDE = 0.001; // ≈ −60 dBFS
|
||||
|
||||
/** en_alien excerpts: [start s, end s) and the 1-based reference lines they contain. */
|
||||
export const ALIEN_EXCERPTS: { start: number; end: number; firstLine: number; lastLine: number }[] = [
|
||||
{ start: 0, end: 36.8, firstLine: 1, lastLine: 9 },
|
||||
{ start: 36.8, end: 99.0, firstLine: 10, lastLine: 28 },
|
||||
{ start: 165.4, end: 196.1, firstLine: 45, lastLine: 58 },
|
||||
];
|
||||
|
||||
function roomNoise(seconds: number, seed: number): Float32Array {
|
||||
let state = seed;
|
||||
return new Float32Array(Math.round(seconds * RATE)).map(() => {
|
||||
state = (state * 1_103_515_245 + 12_345) % 2 ** 31;
|
||||
return NOISE_AMPLITUDE * (state / 2 ** 30 - 1);
|
||||
});
|
||||
}
|
||||
|
||||
export async function buildMeetingFixture(): Promise<{ audio: Float32Array; reference: string }> {
|
||||
const aiproblem = await decodeClip("en_aiproblem.m4a");
|
||||
const alien = await decodeClip("en_alien.opus");
|
||||
const alienLines = (await readReference("en_alien")).split("\n");
|
||||
|
||||
const speech: Float32Array[] = [aiproblem];
|
||||
const references: string[] = [(await readReference("en_aiproblem")).trim()];
|
||||
for (const excerpt of ALIEN_EXCERPTS) {
|
||||
speech.push(alien.subarray(Math.round(excerpt.start * RATE), Math.round(excerpt.end * RATE)));
|
||||
references.push(alienLines.slice(excerpt.firstLine - 1, excerpt.lastLine).join(" ").trim());
|
||||
}
|
||||
|
||||
const parts: Float32Array[] = [roomNoise(LEAD_IN_SECONDS, 1)];
|
||||
// A pause after every clip — the last one is the meeting's trailing room tone.
|
||||
speech.forEach((clip, i) => {
|
||||
parts.push(clip);
|
||||
parts.push(roomNoise(PAUSES_SECONDS[i % PAUSES_SECONDS.length]!, i + 2));
|
||||
});
|
||||
const audio = new Float32Array(parts.reduce((n, p) => n + p.length, 0));
|
||||
let at = 0;
|
||||
for (const part of parts) {
|
||||
audio.set(part, at);
|
||||
at += part.length;
|
||||
}
|
||||
return { audio, reference: references.join(" ") };
|
||||
}
|
||||
Reference in New Issue
Block a user