diff --git a/docs/Architecture.md b/docs/Architecture.md index 2725f62..8d1e582 100644 --- a/docs/Architecture.md +++ b/docs/Architecture.md @@ -84,6 +84,9 @@ src/ translate.worker.ts # transcript translation (Opus-MT), its own worker + ort instance (D-WS4b) translateProtocol.ts # typed messages for the translate worker opusMt.ts # Opus-MT loader shared by the worker, bench script and real-model test + longform/ # long-form file pipeline (D-WS3a): pure vad.ts / quality.ts / pipeline.ts + # behind transcribeLongForm(audio, deps, options); transformers.ts implements + # the deps (Silero VAD + Whisper decodes) on the same Transformers.js runtime pwa/ sw.ts # service worker (app-shell precache, /ort runtime cache) precache.ts # placeholder-injection helpers shared by build + dev serve @@ -161,6 +164,22 @@ environment). See `WorkBreakdown.md` for the itemized checklist.) to the decoder input of every 30 s window by a wrapper around the model's `generate()`, which also strips it from the output again (D-WS2a). Live passes (4.1b) carry the same `prompt`. +6. **Long-form branch (D-WS3a/D-WS3b, "Advanced → Skip silence", off by default).** + With the option on, every load posted for a file run or the Download button carries + `vad: true`; once the Whisper pipeline is ready the worker loads Silero VAD + (`onnx-community/silero-vad`, 2.2 MB, Cache API like the weights; offline it is + used only if already cached). A file pass with `longForm: true` then runs + `transcribeLongForm` (`worker/longform/`): VAD speech probabilities over the + whole buffer → speech spans (faster-whisper `vad.py` port) → chunks ≤ 30 s cut on + speech boundaries → one Whisper decode per chunk (language detected once on the + first chunk and pinned for multilingual models) → timestamps mapped back to the + file's timeline; progress follows the variable chunks + (`TranscribeProgressTracker.forWindows`). Same `partial` / + `transcribe-progress` / `result` messages as step 5. `choosePipeline` keeps the + chunked path whenever the VAD model isn't loaded, and live passes never take + this branch (live keeps its RMS segmenter, D-016/D-024). A finished transcript + with no words shows "No speech was found in this recording." The names-and-terms + hint (`prompt`) is applied to every chunk's decode the same way (D-WS2a). ### 4.1b Live microphone transcription (anonymous, browser engine — D-016) 1. `LiveControls` starts a session: `getUserMedia` → `AudioContext` at the @@ -224,7 +243,9 @@ environment). See `WorkBreakdown.md` for the itemized checklist.) large to precache eagerly. - Model weights live in the Cache API (`transformers-cache`) via Transformers.js — independent of the SW, which never touches that cache (D-005). `/api/*` is - network-only. + network-only. The Silero VAD model of the long-form branch lives there too + (D-WS3b); offline, the worker checks that cache before loading it and never + fetches, falling back to the chunked path if it is missing. - Updates: `pwa/register.ts` surfaces a waiting SW as a toast; accepting posts `SKIP_WAITING`, activation drops old `whisper-*` caches, and the page reloads. - Result: with the app installed and at least one model previously loaded, the full diff --git a/docs/Decisions.md b/docs/Decisions.md index 9ea8b51..ac96da0 100644 --- a/docs/Decisions.md +++ b/docs/Decisions.md @@ -1619,3 +1619,87 @@ itself and accepts only text, so the sidecar passes it the budget-cut ids decoded back to text (re-encoding gives the same ids, checked with the real tokenizer incl. "Århus, Ærø"); server-live passes keep the ids. Pinned by the stubbed-sidecar test "the vocabulary hint reaches the batched pipeline as text". + +## D-WS3a — Long-form file pipeline (Silero VAD chunks, language pinned once) ships as an Advanced option, off by default +**Status:** Accepted (2026-10-30) — spec 001 Goal 3 +Browser file transcription cuts audio into fixed 30 s windows with 5 s +overlap (Transformers.js's chunked pipeline), decoding silence and +overlaps, and with "Auto-detect" Transformers.js 4.2 silently decodes as +English. +**Decision:** `src/worker/longform/` adds `transcribeLongForm(audio, deps, +options)`: Silero VAD speech probabilities over the whole buffer → speech +spans (line-for-line port of faster-whisper 1.2.1 `get_speech_timestamps`, +parity-checked against the Python on 20 000 random windows) → chunks +≤ 30 s cut on speech boundaries (`collect_chunks`) → one Whisper decode per +chunk → timestamps mapped back to the file's timeline. For a multilingual +model without a language, it detects once on the first speech chunk (one +decoder step on that chunk's encoder output, which the decode then reuses) +and pins it for every chunk. Defaults: VAD `threshold 0.5`, +`min_silence 2000 ms`, `speech_pad 400 ms`, `max_speech 30 s`; repetition +guard **off** (`no_repeat_ngram_size 0`, `repetition_penalty 1`); +temperature fallback (compression ratio > 2.4 / avg log-prob < −1.0 → +0.2…1.0) implemented but **off**. Inference is injected (`LongFormDeps`), +so the 90-minute case is unit-tested: VAD gets the caller's buffer, decodes +get ≤ 30 s copies. Audio with no speech returns an empty transcript without +a decode; the UI shows "No speech was found in this recording." +**Measurements** (onnxruntime-node CPU, Large v3 Turbo q4 = the browser's +WASM weights; machine shared with six other workstreams, old/new run +back-to-back per clip; WER vs `tests/audio_files/*.txt`): + +| clip | old s | long-form s | old WER | long-form WER | old / lf tokens | +|---|---|---|---|---|---| +| en_aiproblem (14 s) | 3.6 | 3.3 | 0.000 | 0.000 | 47 / 47 | +| da_festuge, `da` (21 s) | 3.4 · 6.3 | 3.5 · 6.2 | 0.141 | 0.141 | 122 / 121 | +| da_festuge, auto | 6.7 | 4.7 | 0.938 (English) | 0.141 (`da` detected) | 88 / 121 | +| en_alien (15 min) | 210.2 | 115.0 | 0.255 | 0.140 | 3933 / 2258 | +| meeting (**synthetic**, 2 min 45 s) | 41.9 | 37.5 | 0.0317 | 0.0422 | 776 / 594 | + +(da_festuge with `da` was measured twice, in two load conditions. The +meeting is `tests/meetingFixture.ts`: en_aiproblem and three en_alien +excerpts joined by 3–8 s pauses of −60 dBFS noise — **synthetic**. Its +long-form errors are mostly spelling of casual speech: "gonna" for +"going to", "killin", "livestreamed".) +**Rollout (spec rule: default only if ≤ old WER + 0.01 and faster on every +clip):** not met. The synthetic meeting is +0.0106 WER (just over the +tolerance) and da_festuge with an explicit language is a tie within +noise (3.5 vs 3.4 s, then 6.2 vs 6.3 s). So the old chunked path stays +the default and long-form ships as **Advanced → "Skip silence in uploaded +files"**, off by default, remembered in localStorage (`whisper-long-form`). +It is the only path whose "Auto-detect" actually detects, and roughly +halves time and WER on the long, pause-heavy en_alien. +The repetition guard and fallback variants were not measured (heavy runs +kept to one baseline + one long-form run per clip); neither is on, so no +unmeasured setting ships as a default. +**Consequence:** live mode is untouched (its RMS segmenter, D-016/D-024). +`tests/live-real-model.test.ts` now asserts whole-file WER of the chunked +path and of long-form (explicit and auto language) against the per-clip +bars. Whisper's `generate()` drops kwargs not in `forward_params`, so +`createWhisperDeps` adds `encoder_outputs` to that list on its pipeline +instance; a Transformers.js upgrade must keep the "one encoder pass per +chunk" assertion green. WebGPU timings are unmeasured. + +**Integration note (2026-10-30, merge with D-WS2a):** the long-form adapter +(`worker/longform/transformers.ts`) applies the names-and-terms hint to every +chunk's decode with the same decoder-prompt prefix as the chunked path +(tokenized once per pass); before the merge the `prompt` option reached the +adapter but was ignored. Pinned by a unit test in +`tests/longform-transformers.test.ts`. + +## D-WS3b — Silero VAD model: onnx-community/silero-vad from the Hub, cached like Whisper weights, never fetched offline +**Status:** Accepted (2026-10-30) — spec 001 Goal 3 +**Decision:** the long-form pipeline's VAD is Silero VAD v5 as exported by +`onnx-community/silero-vad` (`onnx/model.onnx`, fp32, 2 243 022 bytes, MIT; +commit `e71cae9` on 2026-10-30). It loads through Transformers.js's +`AutoModel` as a bare ONNX graph (`model_type: "custom"`) on the same +onnxruntime as Whisper (WASM device — a 2 MB model gains nothing from a +GPU), fetched from the Hub like the weights (CORS, so COEP allows it) and +stored in the same Cache API store (D-005). Windows of 512 samples with 64 +samples of context and the `[2, 1, 128]` state, as in Silero's own +wrapper. It is loaded only with "Skip silence" on, after the Whisper +pipeline is ready, on the same user-initiated load (D-020). Offline +(`navigator.onLine === false`) it loads only if already cached and never +fetches; missing or failed, the worker logs it and files take the chunked +path (`choosePipeline`). Live mode keeps its energy gate (D-016). +**Rejected:** bundling the model in `dist/` (2 MB on every install for an +off-by-default option); running it on the main thread (D-013). +**Consequence:** the first "Skip silence" run online downloads 2.2 MB. diff --git a/docs/WorkBreakdown.md b/docs/WorkBreakdown.md index 40b0ecb..871f9d2 100644 --- a/docs/WorkBreakdown.md +++ b/docs/WorkBreakdown.md @@ -423,6 +423,16 @@ Found in spec 001 workstream 6 (D-WS6a), open: (D-WS6a). Judging them needs a longer, harder Danish reference clip than this repo has. +Found while building the long-form pipeline (spec 001 Goal 3, D-WS3a), 2026-10-29: +- [ ] "Auto-detect" on the default (chunked) file path does not detect: with no + `language`, Transformers.js 4.2 logs "No language specified - defaulting to + English" and decodes Danish audio as an English translation (da_festuge WER + 0.94 vs 0.14 with `da`). Long-form ("Skip silence") detects once and pins + the language; the chunked path is unchanged. +- [ ] Spec 001's measurement fallback mentions a hint list of the names in + `en_alien`'s reference; `tests/meetingFixture.ts` returns audio + reference + only. Add the list when the vocabulary hint (workstream 2) lands. + ## Recurring - [ ] Maintain `AGENTS.md`: - **Add a rule only in response to a real failure.** When an agent repeatedly diff --git a/scripts/measure-longform.ts b/scripts/measure-longform.ts new file mode 100644 index 0000000..11d54b6 --- /dev/null +++ b/scripts/measure-longform.ts @@ -0,0 +1,167 @@ +/** + * Before/after measurement for the long-form pipeline (D-WS3a): wall-clock, + * WER against tests/audio_files/*.txt, generated tokens and decode calls, for + * the worker's current chunked path ("old") and long-form configurations. + * + * flock /tmp/whisper-heavy.lock bun scripts/measure-longform.ts \ + * [--clips en_aiproblem,da_festuge,da_festuge:auto,en_alien,meeting] \ + * [--configs old,lf,lf-ngram5] [--repeat 1] [--out dir] + * + * Runs on onnxruntime-node (CPU), Whisper Large v3 Turbo at its WASM-default + * dtype (q4) — the same weights and quantization as the browser's WASM + * engine. "meeting" is the synthetic stand-in (tests/meetingFixture.ts). + */ + +import { mkdir } from "node:fs/promises"; +import { join } from "node:path"; +import { parseArgs } from "node:util"; +import { pipeline, type AutomaticSpeechRecognitionPipeline } from "@huggingface/transformers"; +import { resolveDefaultDtype } from "../src/lib/models"; +import { + transcribeLongForm, + type LongFormOptions, +} from "../src/worker/longform"; +import { createWhisperDeps, loadSileroVad } from "../src/worker/longform/transformers"; +import { decodeClip, normalize, readReference } from "../tests/clips"; +import { wordErrorRate } from "../tests/dsp"; +import { buildMeetingFixture } from "../tests/meetingFixture"; + +const MODEL_ID = "onnx-community/whisper-large-v3-turbo"; + +/** Long-form configurations under test; "old" is the worker's chunked path. */ +const CONFIGS: Record | "old"> = { + old: "old", + lf: {}, + "lf-ngram3": { guard: { noRepeatNgramSize: 3, repetitionPenalty: 1 } }, + "lf-ngram5": { guard: { noRepeatNgramSize: 5, repetitionPenalty: 1 } }, + "lf-rep1.1": { guard: { noRepeatNgramSize: 0, repetitionPenalty: 1.1 } }, + "lf-fallback": { fallback: {} }, + "lf-sil160": { vad: { minSilenceDurationMs: 160 } }, +}; + +const { values } = parseArgs({ + options: { + clips: { type: "string", default: "en_aiproblem,da_festuge,da_festuge:auto,meeting,en_alien" }, + configs: { type: "string", default: "old,lf" }, + repeat: { type: "string", default: "1" }, + out: { type: "string" }, + }, +}); + +interface Clip { + name: string; + audio: Float32Array; + reference: string; + /** undefined = the UI's "Auto-detect". */ + language?: string; +} + +async function loadClip(spec: string): Promise { + const [base, mode] = spec.split(":"); + if (base === "meeting") { + const { audio, reference } = await buildMeetingFixture(); + return { name: "meeting (synthetic)", audio, reference, language: "en" }; + } + const file = (await Array.fromAsync(new Bun.Glob(`${base}.*`).scan(join(import.meta.dir, "../tests/audio_files")))) + .find((name) => !name.endsWith(".txt")); + if (!file) throw new Error(`no audio for ${base}`); + return { + name: spec, + audio: await decodeClip(file), + reference: await readReference(base!), + ...(mode !== "auto" && { language: base!.split("_")[0] }), + }; +} + +const asr = (await pipeline("automatic-speech-recognition", MODEL_ID, { + dtype: resolveDefaultDtype(MODEL_ID, "wasm"), +})) as AutomaticSpeechRecognitionPipeline; + +// Count generated tokens on every generate call, prompt prefix excluded +// (leading special tokens below the first timestamp token). +const model = asr.model as unknown as { generate(args: unknown): Promise }; +const originalGenerate = model.generate.bind(model); +let generated = 0; +let generateCalls = 0; +model.generate = async (args: unknown) => { + const out = (await originalGenerate(args)) as { tolist(): bigint[][] }; + const ids = out.tolist()[0]!.map(Number); + let prefix = 0; + while (prefix < ids.length && ids[prefix]! >= 50257 && ids[prefix]! <= 50364) prefix += 1; + generated += ids.length - prefix; + generateCalls += 1; + return out; +}; + +const vad = await loadSileroVad({ localOnly: false }); +const whisper = createWhisperDeps(asr, { multilingual: true }); + +// Warm-up so the first measured run doesn't pay session initialization. +await asr(new Float32Array(16_000), { max_new_tokens: 8, language: "en" }); + +const rows: string[] = []; +const outDir = values.out; +if (outDir) await mkdir(outDir, { recursive: true }); + +for (const spec of values.clips!.split(",")) { + const clip = await loadClip(spec); + for (let r = 0; r < Number(values.repeat); r++) { + for (const name of values.configs!.split(",")) { + const config = CONFIGS[name]; + if (config === undefined) throw new Error(`unknown config ${name}`); + generated = 0; + generateCalls = 0; + const audio = clip.audio.slice(); // the pipeline may keep views; give each run its own + const started = performance.now(); + let text: string; + let chunks: unknown; + let extra = ""; + if (config === "old") { + const output = (await asr(audio, { + chunk_length_s: 30, + stride_length_s: 5, + return_timestamps: true, + max_new_tokens: 440, + ...(clip.language !== undefined && { language: clip.language }), + task: "transcribe", + })) as { text: string; chunks: unknown }; + text = output.text; + chunks = output.chunks; + } else { + let vadMs = 0; + const result = await transcribeLongForm( + audio, + { + speechProbabilities: async (pcm) => { + const t = performance.now(); + const probs = await vad.speechProbabilities(pcm); + vadMs = performance.now() - t; + return probs; + }, + ...whisper, + }, + { ...config, ...(clip.language !== undefined && { language: clip.language }), task: "transcribe" }, + ); + text = result.text; + chunks = result.chunks; + extra = `vad ${(vadMs / 1000).toFixed(1)} s, speech ${result.speechSeconds.toFixed(0)} s, lang ${result.language}`; + } + const seconds = (performance.now() - started) / 1000; + const wer = wordErrorRate(normalize(clip.reference), normalize(text)); + const row = `| ${clip.name} | ${name} | ${seconds.toFixed(1)} | ${wer.toFixed(3)} | ${generated} | ${generateCalls} | ${extra} |`; + console.log(row); + rows.push(row); + if (outDir) { + await Bun.write( + join(outDir, `${spec.replace(":", "-")}.${name}.${r}.json`), + JSON.stringify({ text, chunks }, null, 1), + ); + } + } + } +} + +console.log("\n| clip | config | wall s | WER | gen tokens | decodes | notes |\n|---|---|---|---|---|---|---|"); +console.log(rows.join("\n")); +await asr.dispose(); +await vad.dispose(); diff --git a/src/App.tsx b/src/App.tsx index 7e4f5fd..b3c4781 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -1,6 +1,7 @@ import { useEffect, useRef, useState } from "react"; import "./index.css"; +import { AdvancedOptions } from "./components/AdvancedOptions"; import { AuthForms } from "./components/AuthForms"; import { EngineToggle } from "./components/EngineToggle"; import { FileDropzone } from "./components/FileDropzone"; @@ -28,6 +29,7 @@ import { import { useAuth } from "./lib/useAuth"; import { LIVE_TUNING, LiveTranscriber, type LiveTranscribeResult } from "./lib/live"; import { startMicCapture, type MicCapture } from "./lib/mic"; +import { loadLongFormChoice, saveLongFormChoice } from "./lib/longForm"; import { getDownloadedModels } from "./lib/modelCache"; import { TRANSLATION_MODEL_IDS } from "./lib/translation"; import { registerServiceWorker } from "./pwa/register"; @@ -189,6 +191,11 @@ export function App() { const auth = useAuth(); const [engineChoice, setEngineChoice] = useState(() => loadEngineChoice()); const [serverStatus, setServerStatus] = useState(null); + // "Skip silence" (long-form pipeline, D-WS3a): remembered per browser. A + // ref mirrors it for the worker-message handler set up once per worker. + const [longForm, setLongForm] = useState(() => loadLongFormChoice()); + const longFormRef = useRef(longForm); + longFormRef.current = longForm; function refreshDownloaded() { // Downloaded badges (D-005): query Cache Storage for cached weights. @@ -442,6 +449,7 @@ export function App() { modelId: message.modelId, device: message.device, sequentialSessionLoad: isMobileLoadRisk(message.modelId, message.device), + vad: longFormRef.current, }); break; } @@ -502,6 +510,9 @@ export function App() { // use) only where the peak-memory crash risk is real; everywhere else // (desktop, low-risk models) the worker loads them in parallel. sequentialSessionLoad: isMobileLoadRisk(nextModelId, guessDevice(device)), + // D-WS3b: the speech-detection model rides along with the same + // user-initiated load when "Skip silence" is on. + vad: longFormRef.current, }); return worker; } @@ -634,6 +645,7 @@ export function App() { modelId, device, sequentialSessionLoad: isMobileLoadRisk(modelId, guessDevice(device)), + vad: longFormRef.current, }); } } @@ -690,6 +702,9 @@ export function App() { modelId, ...(multilingual && { language, task: taskForModel(modelId, task) }), prompt: hint, + // Browser runs only; the worker falls back to the chunked path when + // the VAD model isn't available (D-WS3a/D-WS3b). + ...(!useServer && { longForm }), onPartial: (fragment) => { if (fileRun.current === run) setPartialText((text) => text + fragment); }, @@ -1075,6 +1090,15 @@ export function App() { disabled={busy || liveActive} /> + { + setLongForm(enabled); + saveLongFormChoice(enabled); + }} + disabled={busy || liveActive} + /> + {/* Engine toggle (M5, D-010): signed-in users choose where uploaded files are transcribed. Live mode always stays on-device (D-016). */} {auth.user && ( diff --git a/src/components/AdvancedOptions.tsx b/src/components/AdvancedOptions.tsx new file mode 100644 index 0000000..870ae96 --- /dev/null +++ b/src/components/AdvancedOptions.tsx @@ -0,0 +1,37 @@ +interface AdvancedOptionsProps { + skipSilence: boolean; + onSkipSilenceChange: (enabled: boolean) => void; + disabled?: boolean; +} + +/** + * Technical file-transcription controls, collapsed by default (PRODUCT.md: + * every control stays available without crowding the primary flow). + * "Skip silence" turns on the long-form pipeline (D-WS3a), which needs a + * small speech-detection model downloaded once (D-WS3b). + */ +export function AdvancedOptions({ + skipSilence, + onSkipSilenceChange, + disabled = false, +}: AdvancedOptionsProps) { + return ( +
+ Advanced + +

+ Finds the speech first and transcribes only that, which can be faster for long recordings + with pauses. Downloads a small extra model (2 MB) the first time; after that it works + offline. Without it, files are transcribed the usual way. +

+
+ ); +} diff --git a/src/components/TranscriptView.tsx b/src/components/TranscriptView.tsx index d593f37..942fa48 100644 --- a/src/components/TranscriptView.tsx +++ b/src/components/TranscriptView.tsx @@ -1,5 +1,6 @@ import { useEffect, useRef, useState } from "react"; import { + emptyTranscriptNotice, exportFilename, transcriptToSrt, transcriptToTimestampedTxt, @@ -100,7 +101,11 @@ export function TranscriptView({ prevResultRef.current = result; }, [result]); - if (!transcribing && !text && !tentativeText) return null; + // A finished run that found no words (silence/music-only file) says so + // instead of vanishing (plan 001-0 Review Focus 4). + const notice = tentativeText ? null : emptyTranscriptNotice(result); + + if (!transcribing && !text && !tentativeText && !notice) return null; async function copy() { if (!copyText) return; @@ -191,7 +196,7 @@ export function TranscriptView({ )}

- {text || (tentativeText ? "" : placeholder)} + {text || notice || (tentativeText ? "" : placeholder)} {tentativeText && ( {tentativeText} )} diff --git a/src/index.css b/src/index.css index acf4ce1..a289f6a 100644 --- a/src/index.css +++ b/src/index.css @@ -573,6 +573,32 @@ body { color: var(--muted); } +/* --- AdvancedOptions (long-form "Skip silence", D-WS3a) --- */ +.advanced-options summary { + cursor: pointer; + color: var(--muted); + font-size: 0.85rem; +} + +.advanced-options label { + display: flex; + align-items: center; + gap: 0.5rem; + margin-top: 0.5rem; + font-size: 0.9rem; +} + +.advanced-options input[type="checkbox"] { + accent-color: var(--accent); +} + +.advanced-hint { + margin: 0.25rem 0 0; + color: var(--muted); + font-size: 0.85rem; + max-width: 60ch; +} + /* --- EngineToggle (M5, D-010) --- */ .engine-toggle { diff --git a/src/lib/engine.ts b/src/lib/engine.ts index 5a692c8..3614710 100644 --- a/src/lib/engine.ts +++ b/src/lib/engine.ts @@ -59,6 +59,11 @@ export interface EngineTranscribeOptions { * is worker termination via `BrowserEngine.failAll` (D-018). */ signal?: AbortSignal; + /** + * Browser engine only: use the long-form pipeline when the worker has the + * VAD model (D-WS3a). The server engine runs faster-whisper's own pipeline. + */ + longForm?: boolean; } export interface EngineTranscribeResult { @@ -146,6 +151,7 @@ export class BrowserEngine implements Engine { options.language !== "auto" && { language: options.language }), ...(options.task !== undefined && { task: options.task }), ...(options.prompt && { prompt: options.prompt }), + ...(options.longForm !== undefined && { longForm: options.longForm }), }; // Ownership of the audio moves to the worker — zero-copy transfer. this.worker.postMessage(message, [audio.buffer as ArrayBuffer]); diff --git a/src/lib/export.ts b/src/lib/export.ts index 23e0d4d..1d6ead5 100644 --- a/src/lib/export.ts +++ b/src/lib/export.ts @@ -14,6 +14,16 @@ import type { TranscriptChunk } from "../worker/protocol"; */ const FALLBACK_CUE_SECONDS = 2; +/** + * What to show instead of a blank transcript when a finished run found no + * words — silence- or music-only files (plan 001-0 Review Focus 4). Null + * while running (no result yet) or when there is text. + */ +export function emptyTranscriptNotice(result: { text: string } | null): string | null { + if (result === null || result.text.trim().length > 0) return null; + return "No speech was found in this recording."; +} + /** Final transcript as plain text: trimmed, single trailing newline. */ export function transcriptToTxt(result: { text: string }): string { return `${result.text.trim()}\n`; diff --git a/src/lib/longForm.ts b/src/lib/longForm.ts new file mode 100644 index 0000000..417e910 --- /dev/null +++ b/src/lib/longForm.ts @@ -0,0 +1,43 @@ +/** + * The "Skip silence" file-transcription setting (long-form pipeline, + * D-WS3a): remembered in localStorage like the engine choice. + */ + +import type { StorageLike } from "./engine"; + +export const LONG_FORM_KEY = "whisper-long-form"; + +/** + * Off by default: the spec's rollout rule makes the long-form pipeline the + * default only if it is faster and not less accurate on every measurement + * clip — see D-WS3a for the numbers that decided this. + */ +export const LONG_FORM_DEFAULT = false; + +function defaultStorage(): StorageLike | null { + try { + return (globalThis as { localStorage?: StorageLike }).localStorage ?? null; + } catch { + return null; // storage access can throw (privacy modes) + } +} + +export function loadLongFormChoice(storage: StorageLike | null = defaultStorage()): boolean { + try { + const value = storage?.getItem(LONG_FORM_KEY); + return value === "on" ? true : value === "off" ? false : LONG_FORM_DEFAULT; + } catch { + return LONG_FORM_DEFAULT; + } +} + +export function saveLongFormChoice( + enabled: boolean, + storage: StorageLike | null = defaultStorage(), +): void { + try { + storage?.setItem(LONG_FORM_KEY, enabled ? "on" : "off"); + } catch { + // Not persistable (private mode / quota) — the choice just won't stick. + } +} diff --git a/src/worker/inference.worker.ts b/src/worker/inference.worker.ts index 307871d..dc56d11 100644 --- a/src/worker/inference.worker.ts +++ b/src/worker/inference.worker.ts @@ -17,6 +17,8 @@ import { type ProgressInfo, } from "@huggingface/transformers"; import { resolveDefaultDtype } from "../lib/models"; +import { LONG_FORM_DEFAULTS, transcribeLongForm } from "./longform"; +import { createWhisperDeps, loadSileroVad, type SileroVad } from "./longform/transformers"; import { buildPromptPrefix, installPromptSupport, @@ -25,6 +27,7 @@ import { type PromptTokenizer, } from "./prompt"; import { + choosePipeline, deviceAttempts, dtypeRequiresF16, isUiToWorkerMessage, @@ -326,6 +329,36 @@ function load(message: LoadMessage): Promise return promise; } +/** + * Silero VAD for long-form file passes (D-WS3a/D-WS3b). Loaded on request + * (`load.vad`) once the Whisper pipeline is up — after it, so the Whisper + * attempt decides onnxruntime's thread pool (D-023/D-025). Offline it is + * used only if already cached: no network request (Review Focus 3). A failed + * load resolves to null (the old path runs) and is retried on a later load. + */ +let vad: Promise | null = null; + +function ensureVad(whisper: Promise): void { + if (vad !== null) return; + const attempt = (async () => { + await whisper.catch(() => {}); + try { + const model = await loadSileroVad({ localOnly: navigator.onLine === false, device: "wasm" }); + console.info("[whisper] speech-detection (VAD) model ready — long-form file pipeline available"); + return model; + } catch (error) { + console.info( + `[whisper] speech-detection (VAD) model unavailable, using the chunked file path: ${errorMessage(error)}`, + ); + return null; + } + })(); + vad = attempt; + void attempt.then((model) => { + if (model === null && vad === attempt) vad = null; + }); +} + // --------------------------------------------------------------------------- // Transcription // --------------------------------------------------------------------------- @@ -348,6 +381,17 @@ async function transcribe(message: TranscribeMessage): Promise { } const asr = await active.promise; + const streaming = message.stream ?? message.requestId === undefined; + // Live passes (not streaming) always keep the chunked path (D-016/D-024). + const vadModel = message.longForm && streaming && vad ? await vad : null; + if ( + choosePipeline({ longForm: message.longForm && streaming, vadReady: vadModel !== null }) === + "longform" + ) { + await transcribeLongFormPass(message, asr, vadModel!); + return; + } + const options: Record = { chunk_length_s: CHUNK_LENGTH_S, stride_length_s: STRIDE_LENGTH_S, @@ -366,7 +410,7 @@ async function transcribe(message: TranscribeMessage): Promise { // than any real 30 s window produces. max_new_tokens: 440, }; - if (message.stream ?? message.requestId === undefined) { + if (streaming) { // File-flow pass (id-less, or routed through BrowserEngine with an // explicit stream flag, D-010): stream partial tokens and report audio // progress. The streamer's chunk timestamps are window-relative; the @@ -443,6 +487,44 @@ async function transcribe(message: TranscribeMessage): Promise { post(result); } +/** + * A file pass through the long-form pipeline (D-WS3a): same partial / + * transcribe-progress / result messages as the chunked path. + */ +async function transcribeLongFormPass( + message: TranscribeMessage, + asr: AutomaticSpeechRecognitionPipeline, + vadModel: SileroVad, +): Promise { + const multilingual = Boolean( + (asr.model as unknown as { generation_config?: { is_multilingual?: boolean } }) + .generation_config?.is_multilingual, + ); + const result = await transcribeLongForm( + message.audio, + { + speechProbabilities: (audio) => vadModel.speechProbabilities(audio), + ...createWhisperDeps(asr, { multilingual }), + }, + { + ...LONG_FORM_DEFAULTS, + ...(multilingual && message.language !== undefined && { language: message.language }), + ...(multilingual && message.task !== undefined && { task: message.task }), + // Names-and-terms hint on every chunk, like the chunked path (D-WS2a). + ...(message.prompt && { prompt: message.prompt }), + onPartial: (text) => post({ type: "partial", text, requestId: message.requestId }), + onProgress: (processedSeconds, totalSeconds) => + post({ + type: "transcribe-progress", + processedSeconds, + totalSeconds, + requestId: message.requestId, + }), + }, + ); + post({ type: "result", text: result.text, chunks: result.chunks, requestId: message.requestId }); +} + scope.addEventListener("message", (event) => { const data = event.data; if (!isUiToWorkerMessage(data)) { @@ -454,14 +536,17 @@ scope.addEventListener("message", (event) => { return; } switch (data.type) { - case "load": - load(data).catch((error: unknown) => { + case "load": { + const whisper = load(data); + if (data.vad) ensureVad(whisper); + whisper.catch((error: unknown) => { // A restart-aborted load is not an error — the UI already got the // load-restart message and is respawning this worker (D-025). if (error instanceof RestartRequested) return; post({ type: "error", stage: "load", message: errorMessage(error) }); }); break; + } case "transcribe": transcribe(data).catch((error: unknown) => { if (error instanceof RestartRequested) return; diff --git a/src/worker/longform/index.ts b/src/worker/longform/index.ts new file mode 100644 index 0000000..8113b68 --- /dev/null +++ b/src/worker/longform/index.ts @@ -0,0 +1,12 @@ +/** Long-form file transcription pipeline (D-WS3a). See pipeline.ts. */ +export { DEFAULT_GUARD, LONG_FORM_DEFAULTS, transcribeLongForm } from "./pipeline"; +export { DEFAULT_FALLBACK, type FallbackOptions } from "./quality"; +export type { + DecodeRequest, + DecodeResult, + LongFormDeps, + LongFormOptions, + LongFormResult, + RepetitionGuard, +} from "./types"; +export { DEFAULT_VAD_OPTIONS, type VadOptions } from "./vad"; diff --git a/src/worker/longform/pipeline.ts b/src/worker/longform/pipeline.ts new file mode 100644 index 0000000..b3f43c6 --- /dev/null +++ b/src/worker/longform/pipeline.ts @@ -0,0 +1,172 @@ +/** + * Long-form file transcription (D-WS3a), mirroring faster-whisper's batched + * pipeline without the batching: Silero VAD drops silence and cuts the audio + * on speech boundaries into chunks ≤ 30 s; each chunk is one Whisper decode + * (no overlapping windows to stitch); the language is detected once on the + * first speech chunk and pinned; an optional quality check retries a bad + * decode at higher temperatures. + * + * Memory (plan 001-0 Review Focus 1): VAD reads the caller's buffer in + * place; only the chunk being decoded is copied (≤ 30 s), and the only state + * that grows with the file is the transcript itself. + */ + +import type { ProgressWindow, TranscriptChunk } from "../protocol"; +import { TranscribeProgressTracker } from "../protocol"; +import { compressionRatio, DEFAULT_FALLBACK, needsFallback, type FallbackOptions } from "./quality"; +import type { + DecodeRequest, + DecodeResult, + LongFormDeps, + LongFormOptions, + LongFormResult, + RepetitionGuard, +} from "./types"; +import { + chunkAudio, + chunkTimeToOriginal, + collectChunks, + getSpeechTimestamps, + SAMPLE_RATE, + type SpeechChunk, +} from "./vad"; + +/** Whisper's window: no chunk is longer. */ +const MAX_CHUNK_SECONDS = 30; + +/** No guard: Whisper's own decoding (the measured default, D-WS3a). */ +export const DEFAULT_GUARD: RepetitionGuard = { noRepeatNgramSize: 0, repetitionPenalty: 1 }; + +/** The tuning the worker runs with — chosen by measurement (D-WS3a). */ +export const LONG_FORM_DEFAULTS: Pick = { + guard: DEFAULT_GUARD, + fallback: false, +}; + +export async function transcribeLongForm( + audio: Float32Array, + deps: LongFormDeps, + options: LongFormOptions = {}, +): Promise { + const totalSeconds = audio.length / SAMPLE_RATE; + options.onProgress?.(0, totalSeconds); + + const probs = await deps.speechProbabilities(audio); + const spans = getSpeechTimestamps(probs, audio.length, { + ...options.vad, + maxSpeechDurationS: Math.min(options.vad?.maxSpeechDurationS ?? MAX_CHUNK_SECONDS, MAX_CHUNK_SECONDS), + }); + const chunks = collectChunks(spans, MAX_CHUNK_SECONDS * SAMPLE_RATE); + + const tracker = TranscribeProgressTracker.forWindows(totalSeconds, chunks.map(progressWindow(chunks, totalSeconds))); + const report = (processed: number | null) => { + if (processed !== null) options.onProgress?.(processed, totalSeconds); + }; + + const guard = options.guard ?? DEFAULT_GUARD; + const fallback: FallbackOptions | null = + options.fallback === false || options.fallback === undefined + ? null + : { ...DEFAULT_FALLBACK, ...options.fallback }; + let language = + options.language === undefined || options.language === "auto" ? undefined : options.language; + + const texts: string[] = []; + const outChunks: TranscriptChunk[] = []; + let decodeCalls = 0; + let generatedTokens = 0; + let speechSamples = 0; + + for (const chunk of chunks) { + const chunkPcm = chunkAudio(audio, chunk); + speechSamples += chunk.samples; + if (language === undefined && deps.detectLanguage) { + language = await deps.detectLanguage(chunkPcm); + } + const base: DecodeRequest = { + ...(language !== undefined && { language }), + ...(options.task !== undefined && { task: options.task }), + ...(options.prompt !== undefined && { prompt: options.prompt }), + temperature: 0, + noRepeatNgramSize: guard.noRepeatNgramSize, + repetitionPenalty: guard.repetitionPenalty, + ...(options.onPartial && { onPartial: options.onPartial }), + onTimestamp: (t) => report(tracker.timestamp(t)), + }; + const attempts = fallback?.temperatures ?? [0]; + const tried: { result: DecodeResult; ratio: number }[] = []; + let chosen: DecodeResult | null = null; + for (const temperature of attempts) { + const result = await deps.decode(chunkPcm, { ...base, temperature }); + decodeCalls += 1; + generatedTokens += result.generatedTokens; + if (!fallback) { + chosen = result; + break; + } + const ratio = await compressionRatio(result.text); + if (!needsFallback({ compressionRatio: ratio, avgLogProb: result.avgLogProb }, fallback)) { + chosen = result; + break; + } + tried.push({ result, ratio }); + } + chosen ??= bestAttempt(tried, fallback!); + + const text = chosen.text.trim(); + if (text.length > 0) texts.push(text); + for (const segment of chosen.chunks) { + const [start, end] = segment.timestamp; + outChunks.push({ + text: segment.text, + timestamp: [ + chunkTimeToOriginal(chunk, start), + end === null ? null : chunkTimeToOriginal(chunk, end), + ], + }); + } + report(tracker.finalizeWindow()); + } + if (chunks.length === 0) report(tracker.finalizeWindow()); + + return { + text: texts.join(" "), + chunks: outChunks, + ...(language !== undefined && { language }), + noSpeech: chunks.length === 0, + speechSeconds: speechSamples / SAMPLE_RATE, + decodeCalls, + generatedTokens, + }; +} + +/** + * Every temperature failed: like faster-whisper, prefer attempts under the + * compression threshold, then the highest average log-prob (unknown counts + * as lowest), then the lowest compression ratio. + */ +function bestAttempt( + tried: { result: DecodeResult; ratio: number }[], + fallback: FallbackOptions, +): DecodeResult { + const underThreshold = tried.filter((t) => t.ratio <= fallback.compressionRatioThreshold); + const pool = underThreshold.length > 0 ? underThreshold : tried; + return pool.reduce((best, t) => { + const a = best.result.avgLogProb ?? -Infinity; + const b = t.result.avgLogProb ?? -Infinity; + if (b > a || (b === a && t.ratio < best.ratio)) return t; + return best; + }).result; +} + +function progressWindow(chunks: SpeechChunk[], totalSeconds: number) { + return (chunk: SpeechChunk, index: number): ProgressWindow => { + const next = chunks[index + 1]; + const last = chunk.spans[chunk.spans.length - 1]!; + return { + toAbsolute: (t) => chunkTimeToOriginal(chunk, t), + endSeconds: last.end / SAMPLE_RATE, + nextStartSeconds: next ? next.spans[0]!.start / SAMPLE_RATE : totalSeconds, + }; + }; +} diff --git a/src/worker/longform/quality.ts b/src/worker/longform/quality.ts new file mode 100644 index 0000000..11aa2b8 --- /dev/null +++ b/src/worker/longform/quality.ts @@ -0,0 +1,42 @@ +/** + * Per-chunk decode quality checks for the long-form pipeline's temperature + * fallback (D-WS3a) — faster-whisper's `generate_with_fallback` criteria. + */ + +export interface FallbackOptions { + /** Decode temperatures in order; the first is the normal (greedy) pass. */ + temperatures: number[]; + /** Reject a decode whose text compresses better than this (a loop). */ + compressionRatioThreshold: number; + /** Reject a decode whose average token log-probability is below this. */ + logProbThreshold: number; +} + +/** faster-whisper / OpenAI Whisper defaults (spec Goal 3). */ +export const DEFAULT_FALLBACK: FallbackOptions = { + temperatures: [0, 0.2, 0.4, 0.6, 0.8, 1.0], + compressionRatioThreshold: 2.4, + logProbThreshold: -1.0, +}; + +/** + * UTF-8 length over zlib-deflated length — `get_compression_ratio` in + * faster-whisper (`zlib.compress`). `CompressionStream("deflate")` is the + * same zlib format and exists in workers and Bun alike. + */ +export async function compressionRatio(text: string): Promise { + const bytes = new TextEncoder().encode(text); + if (bytes.length === 0) return 0; + const stream = new Blob([bytes]).stream().pipeThrough(new CompressionStream("deflate")); + const compressed = await new Response(stream).arrayBuffer(); + return bytes.length / compressed.byteLength; +} + +/** Whether a decode should be retried at the next temperature. */ +export function needsFallback( + result: { compressionRatio: number; avgLogProb: number | null }, + thresholds: Pick = DEFAULT_FALLBACK, +): boolean { + if (result.compressionRatio > thresholds.compressionRatioThreshold) return true; + return result.avgLogProb !== null && result.avgLogProb < thresholds.logProbThreshold; +} diff --git a/src/worker/longform/transformers.ts b/src/worker/longform/transformers.ts new file mode 100644 index 0000000..f08108c --- /dev/null +++ b/src/worker/longform/transformers.ts @@ -0,0 +1,297 @@ +/** + * `LongFormDeps` over the existing Transformers.js runtime (D-WS3a/D-WS3b): + * Silero VAD runs as a plain ONNX model on the same onnxruntime instance the + * Whisper pipeline uses, and Whisper decodes go through the already-loaded + * ASR pipeline — no second inference runtime. Free of worker globals so + * `bun test` drives it on onnxruntime-node. + */ + +import { + AutoModel, + LogitsProcessor, + LogitsProcessorList, + Tensor, + WhisperTextStreamer, + type AutomaticSpeechRecognitionPipeline, + type PretrainedModelOptions, +} from "@huggingface/transformers"; +import { getDownloadedModels } from "../../lib/modelCache"; +import type { TranscriptChunk } from "../protocol"; +import { + buildPromptPrefix, + installPromptSupport, + PROMPT_PREFIX_KWARG, + type PromptableModel, + type PromptTokenizer, +} from "../prompt"; +import type { DecodeRequest, DecodeResult, LongFormDeps } from "./types"; +import { SAMPLE_RATE, VAD_WINDOW_SAMPLES } from "./vad"; + +/** + * Silero VAD v5, ONNX export by onnx-community (MIT; `onnx/model.onnx`, + * fp32, 2.2 MB). Fetched from the Hub like Whisper weights — CORS-enabled, + * so COEP allows it — and kept in the same Cache API store (D-005, D-WS3b). + */ +export const VAD_MODEL_ID = "onnx-community/silero-vad"; + +/** Samples of the previous window prepended to each one (Silero v5 wrapper). */ +const VAD_CONTEXT_SAMPLES = 64; + +export interface SileroVad { + speechProbabilities(audio: Float32Array): Promise; + dispose(): Promise; +} + +export interface LoadSileroVadOptions { + /** + * Never touch the network: load only if the model is already cached, + * otherwise reject (offline, or never downloaded — Review Focus 3). + */ + localOnly: boolean; + /** onnxruntime device; the worker passes "wasm" (a 2 MB model gains nothing from a GPU). */ + device?: PretrainedModelOptions["device"]; + /** Is the model in the cache? Defaults to the browser Cache API probe (modelCache.ts). */ + isCached?: () => Promise; +} + +async function vadInBrowserCache(): Promise { + return (await getDownloadedModels([VAD_MODEL_ID])).has(VAD_MODEL_ID); +} + +export async function loadSileroVad(options: LoadSileroVadOptions): Promise { + if (options.localOnly && !(await (options.isCached ?? vadInBrowserCache)())) { + throw new Error("The speech-detection model is not downloaded (D-WS3b)."); + } + const model = await AutoModel.from_pretrained(VAD_MODEL_ID, { + // The repo ships no config.json: the model is a bare ONNX graph. + config: { model_type: "custom" } as unknown as PretrainedModelOptions["config"], + dtype: "fp32", + ...(options.device !== undefined && { device: options.device }), + }); + return { + async speechProbabilities(audio) { + const windows = Math.ceil(audio.length / VAD_WINDOW_SAMPLES); + const probs = new Float32Array(windows); + const sr = new Tensor("int64", [BigInt(SAMPLE_RATE)], []); + let state = new Tensor("float32", new Float32Array(2 * 128), [2, 1, 128]); + const input = new Float32Array(VAD_CONTEXT_SAMPLES + VAD_WINDOW_SAMPLES); + for (let i = 0; i < windows; i++) { + // [64 samples of context | 512-sample window], zero-padded at the end. + input.copyWithin(0, VAD_WINDOW_SAMPLES); + const window = audio.subarray(i * VAD_WINDOW_SAMPLES, (i + 1) * VAD_WINDOW_SAMPLES); + input.set(window, VAD_CONTEXT_SAMPLES); + input.fill(0, VAD_CONTEXT_SAMPLES + window.length); + const out = (await model({ + input: new Tensor("float32", input, [1, input.length]), + sr, + state, + })) as { output: Tensor; stateN: Tensor }; + probs[i] = (out.output.data as Float32Array)[0]!; + state = out.stateN; + } + return probs; + }, + dispose: async () => { + await model.dispose(); + }, + }; +} + +// --------------------------------------------------------------------------- +// Whisper decodes +// --------------------------------------------------------------------------- + +/** + * Records the log-probability of each generated token and the token count. + * Kept last in the processor list (after Whisper's timestamp rules), so it + * sees the distribution the token is actually picked from. A step's token is + * known on the next call (it is the last input id), so the final token (EOS + * as a rule) is not averaged in — close enough for the −1.0 threshold. + */ +class LogProbRecorder extends LogitsProcessor { + steps = 0; + private sum = 0; + private count = 0; + private previous: { data: Float32Array; logSumExp: number } | null = null; + + override _call(inputIds: bigint[][], logits: Tensor): Tensor { + const ids = inputIds[0]!; + if (this.previous) { + const token = Number(ids[ids.length - 1]); + this.sum += this.previous.data[token]! - this.previous.logSumExp; + this.count += 1; + } + const data = (logits as unknown as Tensor[])[0]!.data as Float32Array; + let max = -Infinity; + for (const v of data) if (v > max) max = v; + let total = 0; + for (const v of data) total += Math.exp(v - max); + this.previous = { data, logSumExp: max + Math.log(total) }; + this.steps += 1; + return logits; + } + + average(): number | null { + return this.count > 0 ? this.sum / this.count : null; + } +} + +/** A processor list that keeps `last` at the end whatever Whisper pushes. */ +class ListWithLast extends LogitsProcessorList { + constructor(last: LogitsProcessor) { + super(); + this.processors.push(last); + } + + override push(item: LogitsProcessor): void { + this.processors.splice(this.processors.length - 1, 0, item); + } +} + +/** Only language tokens may be generated — one-step language detection. */ +class LanguageOnly extends LogitsProcessor { + constructor(private readonly allowed: Set) { + super(); + } + + override _call(_inputIds: bigint[][], logits: Tensor): Tensor { + const data = (logits as unknown as Tensor[])[0]!.data as Float32Array; + for (let i = 0; i < data.length; i++) if (!this.allowed.has(i)) data[i] = -Infinity; + return logits; + } +} + +interface WhisperGenerationConfig { + decoder_start_token_id: number; + lang_to_id?: Record; +} + +export function createWhisperDeps( + asr: AutomaticSpeechRecognitionPipeline, + options: { multilingual: boolean }, +): Pick { + // The hint is the same for every chunk of a pass: tokenize it once. + let cachedPrompt: { hint: string; prefix: number[] } | null = null; + const promptPrefixFor = (hint: string): number[] => { + if (cachedPrompt?.hint !== hint) { + const prefix = buildPromptPrefix(asr.tokenizer as unknown as PromptTokenizer, hint); + if (prefix.length > 0) installPromptSupport(asr.model as unknown as PromptableModel); + cachedPrompt = { hint, prefix }; + } + return cachedPrompt.prefix; + }; + + const decode = async ( + audio: Float32Array, + request: DecodeRequest, + encoderOutputs?: Tensor, + ): Promise => { + const recorder = new LogProbRecorder(); + const generate: Record = { + return_timestamps: true, + // One window, single-pass generate (D-021's seek-loop note applies). + max_new_tokens: 440, + logits_processor: new ListWithLast(recorder), + // generate() skips the encoder when handed its output. + ...(encoderOutputs && { encoder_outputs: encoderOutputs }), + }; + if (options.multilingual) { + if (request.language !== undefined) generate.language = request.language; + if (request.task !== undefined) generate.task = request.task; + } + if (request.prompt) { + // Same decoder-prompt mechanism as the chunked path (D-WS2a, + // src/worker/prompt.ts); the wrapper strips the prompt from the output. + const prefix = promptPrefixFor(request.prompt); + if (prefix.length > 0) generate[PROMPT_PREFIX_KWARG] = prefix; + } + if (request.noRepeatNgramSize > 0) generate.no_repeat_ngram_size = request.noRepeatNgramSize; + if (request.repetitionPenalty !== 1) generate.repetition_penalty = request.repetitionPenalty; + if (request.temperature > 0) { + generate.do_sample = true; + generate.temperature = request.temperature; + } + if (request.onPartial || request.onTimestamp) { + generate.streamer = new WhisperTextStreamer( + asr.tokenizer as ConstructorParameters[0], + { + skip_prompt: true, + callback_function: (text: string) => request.onPartial?.(text), + on_chunk_start: (t: number) => request.onTimestamp?.(t), + on_chunk_end: (t: number) => request.onTimestamp?.(t), + }, + ); + } + const output = await asr(audio, generate); + const single = (Array.isArray(output) ? output[0] : output) as { + text?: string; + chunks?: TranscriptChunk[]; + }; + return { + text: single.text ?? "", + chunks: single.chunks ?? [], + generatedTokens: recorder.steps, + avgLogProb: recorder.average(), + }; + }; + + // generate() keeps only the model's `forward_params` from its kwargs, and + // Whisper's list omits `encoder_outputs` — so a precomputed encoder output + // was silently dropped and the encoder ran again. Admitting it on this + // instance lets generate() skip the encoder (its own `"encoder_outputs" in + // model_inputs` check) when detection already encoded the chunk. + const params = (asr.model as unknown as { forward_params: string[] }).forward_params; + if (!params.includes("encoder_outputs")) params.push("encoder_outputs"); + + /** Encoder output of the chunk language detection just ran on. */ + let encoded: { audio: Float32Array; hidden: Tensor } | null = null; + const decodeReusingEncoder = (audio: Float32Array, request: DecodeRequest) => { + if (encoded?.audio !== audio) { + encoded = null; // a new chunk: drop the stale encoder output + return decode(audio, request); + } + return decode(audio, request, encoded.hidden); + }; + + if (!options.multilingual) return { decode: decodeReusingEncoder }; + + const detectLanguage = async (audio: Float32Array): Promise => { + const model = asr.model as unknown as { + generation_config: WhisperGenerationConfig; + generate(args: Record): Promise; + _prepare_encoder_decoder_kwargs_for_generation(args: { + inputs_tensor: Tensor; + model_inputs: Record; + model_input_name: string; + generation_config: unknown; + }): Promise<{ encoder_outputs: Tensor }>; + }; + const config = model.generation_config; + const languages = Object.entries(config.lang_to_id ?? {}); + const { input_features } = (await asr.processor(audio)) as { input_features: Tensor }; + // Encode once and keep it: the chunk's real decode reuses this encoder + // output, so detection costs one decoder step instead of a second encoder + // pass (the encoder dominates a 30 s window's cost). + const { encoder_outputs } = await model._prepare_encoder_decoder_kwargs_for_generation({ + inputs_tensor: input_features, + model_inputs: { input_features }, + model_input_name: "input_features", + generation_config: config, + }); + encoded = { audio, hidden: encoder_outputs }; + const list = new LogitsProcessorList(); + list.push(new LanguageOnly(new Set(languages.map(([, id]) => id)))); + const sequences = await model.generate({ + inputs: input_features, + encoder_outputs, + decoder_input_ids: [config.decoder_start_token_id], + max_new_tokens: 1, + logits_processor: list, + }); + const tokenId = Number((sequences.tolist() as bigint[][])[0]!.at(-1)); + const token = languages.find(([, id]) => id === tokenId)?.[0]; + if (!token) throw new Error(`Language detection produced a non-language token (${tokenId}).`); + return token.slice(2, -2); // "<|da|>" → "da" + }; + return { decode: decodeReusingEncoder, detectLanguage }; +} diff --git a/src/worker/longform/types.ts b/src/worker/longform/types.ts new file mode 100644 index 0000000..b052486 --- /dev/null +++ b/src/worker/longform/types.ts @@ -0,0 +1,83 @@ +/** + * The long-form pipeline's interface (D-WS3a, coordination plan 001-0). The + * pipeline is model-agnostic: everything that runs a model comes in through + * `LongFormDeps`, so a different implementation (a future Rust→WASM build, + * or batched decoding) can replace either side without touching the other. + */ + +import type { TranscriptChunk, WhisperTask } from "../protocol"; +import type { FallbackOptions } from "./quality"; +import type { VadOptions } from "./vad"; + +/** One Whisper decode of a ≤ 30 s chunk. */ +export interface DecodeRequest { + /** Pinned language code; undefined for English-only models. */ + language?: string; + task?: WhisperTask; + /** The user's names-and-terms hint (workstream 2); undefined = none. */ + prompt?: string; + /** 0 = greedy; > 0 = sampling (temperature fallback). */ + temperature: number; + /** 0 = off. */ + noRepeatNgramSize: number; + /** 1 = off. */ + repetitionPenalty: number; + /** Streamed text fragments. */ + onPartial?(text: string): void; + /** Whisper timestamp tokens as they are generated, chunk-relative seconds. */ + onTimestamp?(chunkSeconds: number): void; +} + +export interface DecodeResult { + text: string; + /** Segments with chunk-relative timestamps. */ + chunks: TranscriptChunk[]; + /** Tokens the decoder generated (prompt excluded). */ + generatedTokens: number; + /** Mean log-probability of the generated tokens; null when not measured. */ + avgLogProb: number | null; +} + +export interface LongFormDeps { + /** Silero VAD speech probability per 512-sample window of `audio`. */ + speechProbabilities(audio: Float32Array): Promise; + decode(audio: Float32Array, request: DecodeRequest): Promise; + /** Language code for `audio`; absent for English-only models. */ + detectLanguage?(audio: Float32Array): Promise; +} + +/** Decoder-side repetition guard (D-035's proposal; D-WS3a measured it). */ +export interface RepetitionGuard { + noRepeatNgramSize: number; + repetitionPenalty: number; +} + +export interface LongFormOptions { + /** Whisper language code; undefined/"auto" detects once, then pins it. */ + language?: string; + task?: WhisperTask; + /** Passed to every decode; a no-op when undefined. */ + prompt?: string; + vad?: Partial; + guard?: RepetitionGuard; + /** Temperature fallback; `false` disables it, `{}` uses the defaults. */ + fallback?: Partial | false; + onPartial?(text: string): void; + onProgress?(processedSeconds: number, totalSeconds: number): void; +} + +export interface LongFormResult { + text: string; + /** Segments with timestamps on the original file's timeline. */ + chunks: TranscriptChunk[]; + /** The language every chunk was decoded in (detected or given). */ + language?: string; + /** VAD found no speech at all — nothing was decoded. */ + noSpeech: boolean; + /** Seconds of audio actually decoded (speech + padding). */ + speechSeconds: number; + /** Decoder calls, fallback retries included. */ + decodeCalls: number; + /** Generated tokens, fallback retries included. */ + generatedTokens: number; +} diff --git a/src/worker/longform/vad.ts b/src/worker/longform/vad.ts new file mode 100644 index 0000000..ef18447 --- /dev/null +++ b/src/worker/longform/vad.ts @@ -0,0 +1,193 @@ +/** + * Speech segmentation for the long-form pipeline (D-WS3a). Pure: turns + * per-window Silero VAD speech probabilities into speech spans, and packs the + * spans into ≤ 30 s chunks cut on speech boundaries. A port of faster-whisper + * 1.2.1 `vad.py` (`get_speech_timestamps` after the model call, and the + * batched pipeline's `collect_chunks(…, max_duration=chunk_length)`), so the + * same thresholds mean the same thing on both engines. + * + * Nothing here copies audio except `chunkAudio`, which builds exactly one + * ≤ 30 s chunk when it is about to be decoded (a 90-minute file is never + * duplicated per stage — Review Focus 1 of plan 001-0). + */ + +/** Audio sample rate the whole app standardizes on (D-012). */ +export const SAMPLE_RATE = 16_000; +/** Silero VAD v5 window at 16 kHz. */ +export const VAD_WINDOW_SAMPLES = 512; + +export interface VadOptions { + /** Speech when the probability is at or above this. */ + threshold: number; + /** Silence when below this; defaults to max(threshold − 0.15, 0.01). */ + negThreshold?: number; + /** Speech spans shorter than this are dropped. */ + minSpeechDurationMs: number; + /** Spans longer than this are split (at a ≥ 98 ms pause when there is one). */ + maxSpeechDurationS: number; + /** A span ends only after this much silence. */ + minSilenceDurationMs: number; + /** Padding added on both sides of each span. */ + speechPadMs: number; +} + +/** + * faster-whisper's `VadOptions` defaults (the spec's starting point), with + * `max_speech_duration_s` at Whisper's 30 s window as in its batched pipeline. + */ +export const DEFAULT_VAD_OPTIONS: VadOptions = { + threshold: 0.5, + minSpeechDurationMs: 0, + maxSpeechDurationS: 30, + minSilenceDurationMs: 2000, + speechPadMs: 400, +}; + +/** A run of samples, `end` exclusive. */ +export interface SampleSpan { + start: number; + end: number; +} + +/** + * Speech spans (in samples) from per-window speech probabilities — + * `probs[i]` covers samples [i·512, (i+1)·512). + */ +export function getSpeechTimestamps( + probs: ArrayLike, + audioLength: number, + options: Partial = {}, +): SampleSpan[] { + const o = { ...DEFAULT_VAD_OPTIONS, ...options }; + const w = VAD_WINDOW_SAMPLES; + const minSpeechSamples = (SAMPLE_RATE * o.minSpeechDurationMs) / 1000; + const speechPadSamples = (SAMPLE_RATE * o.speechPadMs) / 1000; + const maxSpeechSamples = SAMPLE_RATE * o.maxSpeechDurationS - w - 2 * speechPadSamples; + const minSilenceSamples = (SAMPLE_RATE * o.minSilenceDurationMs) / 1000; + const minSilenceSamplesAtMaxSpeech = (SAMPLE_RATE * 98) / 1000; + const negThreshold = o.negThreshold ?? Math.max(o.threshold - 0.15, 0.01); + + let triggered = false; + const speeches: SampleSpan[] = []; + let current: Partial = {}; + let tempEnd = 0; + let prevEnd = 0; + let nextStart = 0; + + for (let i = 0; i < probs.length; i++) { + const prob = probs[i]!; + const at = w * i; + if (prob >= o.threshold && tempEnd) { + tempEnd = 0; + if (nextStart < prevEnd) nextStart = at; + } + if (prob >= o.threshold && !triggered) { + triggered = true; + current.start = at; + continue; + } + if (triggered && at - current.start! > maxSpeechSamples) { + if (prevEnd) { + speeches.push({ start: current.start!, end: prevEnd }); + current = {}; + if (nextStart < prevEnd) triggered = false; + else current.start = nextStart; + prevEnd = nextStart = tempEnd = 0; + } else { + speeches.push({ start: current.start!, end: at }); + current = {}; + prevEnd = nextStart = tempEnd = 0; + triggered = false; + continue; + } + } + if (prob < negThreshold && triggered) { + if (!tempEnd) tempEnd = at; + if (at - tempEnd > minSilenceSamplesAtMaxSpeech) prevEnd = tempEnd; + if (at - tempEnd < minSilenceSamples) continue; + if (tempEnd - current.start! > minSpeechSamples) { + speeches.push({ start: current.start!, end: tempEnd }); + } + current = {}; + prevEnd = nextStart = tempEnd = 0; + triggered = false; + continue; + } + } + if (current.start !== undefined && audioLength - current.start > minSpeechSamples) { + speeches.push({ start: current.start, end: audioLength }); + } + + for (let i = 0; i < speeches.length; i++) { + const speech = speeches[i]!; + if (i === 0) speech.start = Math.trunc(Math.max(0, speech.start - speechPadSamples)); + const next = speeches[i + 1]; + if (next) { + const silence = next.start - speech.end; + if (silence < 2 * speechPadSamples) { + speech.end += Math.floor(silence / 2); + next.start = Math.trunc(Math.max(0, next.start - Math.floor(silence / 2))); + } else { + speech.end = Math.trunc(Math.min(audioLength, speech.end + speechPadSamples)); + next.start = Math.trunc(Math.max(0, next.start - speechPadSamples)); + } + } else { + speech.end = Math.trunc(Math.min(audioLength, speech.end + speechPadSamples)); + } + } + return speeches; +} + +/** Speech spans decoded together as one Whisper window. */ +export interface SpeechChunk { + spans: SampleSpan[]; + /** Total samples across `spans` (the chunk's audio length). */ + samples: number; +} + +/** + * Greedily pack consecutive spans into chunks of at most `maxSamples` + * (faster-whisper `collect_chunks`). Metadata only — no audio is copied. + */ +export function collectChunks(spans: readonly SampleSpan[], maxSamples: number): SpeechChunk[] { + const chunks: SpeechChunk[] = []; + let current: SpeechChunk = { spans: [], samples: 0 }; + for (const span of spans) { + const length = span.end - span.start; + if (current.spans.length > 0 && current.samples + length > maxSamples) { + chunks.push(current); + current = { spans: [], samples: 0 }; + } + current.spans.push(span); + current.samples += length; + } + if (current.spans.length > 0) chunks.push(current); + return chunks; +} + +/** The chunk's audio: its spans concatenated (one ≤ 30 s copy). */ +export function chunkAudio(audio: Float32Array, chunk: SpeechChunk): Float32Array { + const out = new Float32Array(chunk.samples); + let at = 0; + for (const span of chunk.spans) { + out.set(audio.subarray(span.start, span.end), at); + at += span.end - span.start; + } + return out; +} + +/** + * Map a chunk-relative time (seconds into the concatenated chunk audio) to + * seconds on the original timeline. A time exactly on a splice maps to the + * earlier span's end; times past the chunk clamp to its last span's end. + */ +export function chunkTimeToOriginal(chunk: SpeechChunk, seconds: number): number { + let remaining = Math.max(0, seconds) * SAMPLE_RATE; + for (const span of chunk.spans) { + const length = span.end - span.start; + if (remaining <= length) return (span.start + remaining) / SAMPLE_RATE; + remaining -= length; + } + const last = chunk.spans[chunk.spans.length - 1]; + return last ? last.end / SAMPLE_RATE : 0; +} diff --git a/src/worker/protocol.ts b/src/worker/protocol.ts index b8f5bc2..b6d9feb 100644 --- a/src/worker/protocol.ts +++ b/src/worker/protocol.ts @@ -57,6 +57,13 @@ export interface LoadMessage { * in App.tsx); omitted/false loads in parallel. */ sequentialSessionLoad?: boolean; + /** + * Also make the Silero VAD model (2.2 MB) available for long-form file + * transcription (D-WS3a/D-WS3b): cache-first, fetched from the Hub only + * when online — offline it is used only if already cached. Sent with the + * same user-initiated loads as the Whisper weights (D-020). + */ + vad?: boolean; } export interface TranscribeMessage { @@ -94,6 +101,12 @@ export interface TranscribeMessage { * (src/worker/prompt.ts). Empty/omitted = no prompt. */ prompt?: string; + /** + * Run this file pass through the long-form pipeline (VAD-cut chunks, + * language pinned once, D-WS3a). Honoured only when the VAD model loaded; + * otherwise the classic 30 s/5 s chunked path runs (`choosePipeline`). + */ + longForm?: boolean; } export type UiToWorkerMessage = LoadMessage | TranscribeMessage; @@ -274,6 +287,19 @@ export function deviceAttempts( } } +/** + * Which file-transcription path a pass takes (D-WS3a/D-WS3b): long-form only + * when the pass asked for it and the VAD model is loaded. A missing VAD model + * (offline, never downloaded) silently keeps the classic chunked path — it is + * a fallback, never an error. + */ +export function choosePipeline(input: { + longForm: boolean | undefined; + vadReady: boolean; +}): "longform" | "chunked" { + return input.longForm === true && input.vadReady ? "longform" : "chunked"; +} + // --------------------------------------------------------------------------- // Runtime guards (postMessage data arrives untyped) // --------------------------------------------------------------------------- @@ -286,6 +312,10 @@ function isOptionalString(value: unknown): value is string | undefined { return value === undefined || typeof value === "string"; } +function isOptionalBoolean(value: unknown): value is boolean | undefined { + return value === undefined || typeof value === "boolean"; +} + const DEVICES: readonly string[] = ["webgpu", "wasm", "webnn"]; const DTYPES: readonly string[] = ["fp32", "fp16", "q8", "q4"]; @@ -315,7 +345,8 @@ export function isUiToWorkerMessage(value: unknown): value is UiToWorkerMessage return ( typeof value.modelId === "string" && isDevicePreference(value.device) && - (value.dtype === undefined || isDtypeSpec(value.dtype)) + (value.dtype === undefined || isDtypeSpec(value.dtype)) && + isOptionalBoolean(value.vad) ); case "transcribe": return ( @@ -325,8 +356,9 @@ export function isUiToWorkerMessage(value: unknown): value is UiToWorkerMessage value.task === "transcribe" || value.task === "translate") && isOptionalString(value.requestId) && - (value.stream === undefined || typeof value.stream === "boolean") && - isOptionalString(value.prompt) + isOptionalBoolean(value.stream) && + isOptionalString(value.prompt) && + isOptionalBoolean(value.longForm) ); default: return false; @@ -443,6 +475,22 @@ export function applyLoadProgress( // Transcription progress accumulation (pure; drives the in-progress UI) // --------------------------------------------------------------------------- +/** + * One generate pass's slice of the audio, for + * `TranscribeProgressTracker.forWindows` (variable-length chunks, D-WS3a). + */ +export interface ProgressWindow { + /** Window-relative seconds → absolute seconds on the file's timeline. */ + toAbsolute(seconds: number): number; + /** Absolute end of the audio this window covers; timestamps clamp here. */ + endSeconds: number; + /** + * Absolute progress once this window's pass finished: the next window's + * start, or the total for the last window. + */ + nextStartSeconds: number; +} + /** * Maps the streamer's window-relative timestamps to absolute audio progress. * @@ -454,25 +502,56 @@ export function applyLoadProgress( * ends. This tracker turns those into absolute seconds that are monotone * (windows overlap, so a later window's early timestamps can precede an * earlier window's late ones — regressions are swallowed) and clamped to - * [0, totalSeconds]. + * [0, totalSeconds]. The long-form pipeline (D-WS3a) uses `forWindows` + * instead: its windows are variable-length speech chunks with skipped + * silence between them. */ export class TranscribeProgressTracker { private windowIndex = 0; private reported = 0; + private readonly window: (index: number) => ProgressWindow | undefined; + /** + * Fixed-stride windows, as the chunked ASR pipeline produces: + * `windowSeconds` long, advancing by `jumpSeconds` (window − 2·stride). + */ constructor( private readonly totalSeconds: number, - private readonly windowSeconds: number, - private readonly jumpSeconds: number, - ) {} + windowSeconds: number, + jumpSeconds: number, + windows?: readonly ProgressWindow[], + ) { + this.window = windows + ? (index) => windows[index] + : (index) => ({ + toAbsolute: (t) => index * jumpSeconds + Math.min(t, windowSeconds), + endSeconds: index * jumpSeconds + windowSeconds, + nextStartSeconds: (index + 1) * jumpSeconds, + }); + } + + /** + * Variable-length windows — the long-form pipeline's speech chunks + * (D-WS3a): each maps its own relative time onto the file's timeline, and + * skipped silence between chunks counts as processed once a chunk ends. + */ + static forWindows( + totalSeconds: number, + windows: readonly ProgressWindow[], + ): TranscribeProgressTracker { + return new TranscribeProgressTracker(totalSeconds, 0, 0, windows); + } /** * Absolute progress for a window-relative timestamp, or null when it does * not advance past what was already reported. */ timestamp(withinWindowSeconds: number): number | null { - const clamped = Math.min(Math.max(withinWindowSeconds, 0), this.windowSeconds); - return this.advance(this.windowIndex * this.jumpSeconds + clamped); + const window = this.window(this.windowIndex); + if (!window) return null; + return this.advance( + Math.min(window.toAbsolute(Math.max(withinWindowSeconds, 0)), window.endSeconds), + ); } /** @@ -480,8 +559,9 @@ export class TranscribeProgressTracker { * of the next window (the amount of audio no later window revisits). */ finalizeWindow(): number | null { + const window = this.window(this.windowIndex); this.windowIndex += 1; - return this.advance(this.windowIndex * this.jumpSeconds); + return this.advance(window ? window.nextStartSeconds : this.totalSeconds); } private advance(processedSeconds: number): number | null { diff --git a/tests/clips.ts b/tests/clips.ts new file mode 100644 index 0000000..b52a0ca --- /dev/null +++ b/tests/clips.ts @@ -0,0 +1,49 @@ +/** + * Test-time access to the recordings in tests/audio_files: decode with + * ffmpeg-static (trusted devDependency) and the WER text normalization used + * by tests/live-real-model.test.ts. + */ +import { join } from "node:path"; +import ffmpegPath from "ffmpeg-static"; + +export const AUDIO_DIR = join(import.meta.dir, "audio_files"); + +/** Decode `tests/audio_files/` to mono float32 PCM at `rate` (16 kHz by default). */ +export async function decodeClip(name: string, rate = 16_000): Promise { + const path = join(AUDIO_DIR, name); + const proc = Bun.spawn( + [ + ffmpegPath as unknown as string, + ...["-hide_banner", "-loglevel", "error"], + ...["-i", path], + ...["-ac", "1", "-ar", String(rate), "-f", "f32le", "-"], + ], + { stdout: "pipe", stderr: "pipe" }, + ); + const [bytes, errors, exitCode] = await Promise.all([ + new Response(proc.stdout).arrayBuffer(), + new Response(proc.stderr).text(), + proc.exited, + ]); + if (exitCode !== 0) throw new Error(`ffmpeg failed on ${path}: ${errors}`); + return new Float32Array(bytes); +} + +/** Reference transcript of `tests/audio_files/.txt`. */ +export function readReference(base: string): Promise { + return Bun.file(join(AUDIO_DIR, `${base}.txt`)).text(); +} + +/** + * WER normalization: casing, punctuation, and apostrophes are not + * transcription accuracy; "Århus"/"Aarhus" unify (same as the real-model test). + */ +export function normalize(text: string): string { + return text + .toLowerCase() + .replaceAll("århus", "aarhus") + .replace(/['’]/g, "") + .replace(/[.,!?;:"“”‘‚«»()\-–—…]/g, " ") + .replace(/\s+/g, " ") + .trim(); +} diff --git a/tests/engine.test.ts b/tests/engine.test.ts index 23d8de2..c8201e6 100644 --- a/tests/engine.test.ts +++ b/tests/engine.test.ts @@ -88,6 +88,15 @@ describe("BrowserEngine", () => { expect("prompt" in worker.posted[2]!.message).toBe(false); }); + test("forwards the long-form request only when set (D-WS3a)", () => { + const worker = new FakeWorker(); + const engine = new BrowserEngine(worker); + void engine.transcribe(new Float32Array(4), { longForm: true }); + void engine.transcribe(new Float32Array(4)); + expect(worker.posted[0]!.message.longForm).toBe(true); + expect("longForm" in worker.posted[1]!.message).toBe(false); + }); + test("asks the worker to stream only when a consumer listens", () => { const worker = new FakeWorker(); const engine = new BrowserEngine(worker); diff --git a/tests/live-real-model.test.ts b/tests/live-real-model.test.ts index c4c5722..c572dcd 100644 --- a/tests/live-real-model.test.ts +++ b/tests/live-real-model.test.ts @@ -17,6 +17,9 @@ import { type PromptableModel, type PromptTokenizer, } from "../src/worker/prompt"; +import { LONG_FORM_DEFAULTS, transcribeLongForm } from "../src/worker/longform"; +import { createWhisperDeps, loadSileroVad } from "../src/worker/longform/transformers"; +import { decodeClip } from "./clips"; import { wordErrorRate } from "./dsp"; /** @@ -272,6 +275,9 @@ describe("live pipeline accuracy (real model, real recordings)", () => { normalize(reference), normalize((await transcribe(whole)).text), ); + // File-mode accuracy gate (spec 001): the chunked file path is held + // to the same per-clip bar as live. + expect(fileWer).toBeLessThanOrEqual(MAX_WER_BY_CLIP[clip.base] ?? MAX_WER); // File-mode accuracy gate (spec 001): the upload flow is held to the // same per-clip bar as live, so a file-mode regression fails here too. @@ -421,3 +427,76 @@ describe("vocabulary hint (real model, D-WS2a)", () => { DOWNLOAD_BUDGET_MS + 30 * MS_PER_AUDIO_SECOND, ); }); + +// --------------------------------------------------------------------------- +// File-mode accuracy of the long-form pipeline (D-WS3a): Silero VAD chunks, +// one decode per ≤ 30 s chunk — the path behind "Skip silence". Same +// per-clip bars as the live and chunked file paths above. +// --------------------------------------------------------------------------- + +describe("long-form file pipeline accuracy (real model, real recordings)", () => { + for (const clip of clips) { + test.skipIf(!available)( + `${clip.base} (${clip.language}): long-form whole-file transcript stays close to the reference`, + async () => { + const reference = await Bun.file(join(AUDIO_DIR, clip.transcriptFile)).text(); + const audio = await decodeClip(clip.audioFile); + const asr = await loadAsr(); + const vad = await loadSileroVad({ localOnly: false }); + try { + const result = await transcribeLongForm( + audio, + { + speechProbabilities: (pcm) => vad.speechProbabilities(pcm), + ...createWhisperDeps(asr, { multilingual: true }), + }, + { ...LONG_FORM_DEFAULTS, language: clip.language, task: "transcribe" }, + ); + const wer = wordErrorRate(normalize(reference), normalize(result.text)); + expect(wer).toBeLessThanOrEqual(MAX_WER_BY_CLIP[clip.base] ?? MAX_WER); + expect(result.noSpeech).toBe(false); + const last = result.chunks.at(-1)!; + expect(last.timestamp[1] ?? last.timestamp[0]).toBeLessThanOrEqual( + audio.length / RATE + 1, + ); + if (clip.language !== "en") { + // "Auto-detect" in the UI: the chunked path forces English here + // (Transformers.js 4.2 has no detection); long-form detects once + // on the first speech chunk and pins it (D-WS3a). + // Detection must not cost an extra encoder pass: the first + // chunk's decode reuses the encoder output detection computed. + const model = asr.model as unknown as Record; + const encode = model._prepare_encoder_decoder_kwargs_for_generation as ( + ...args: unknown[] + ) => Promise; + let encoderPasses = 0; + model._prepare_encoder_decoder_kwargs_for_generation = (...args: unknown[]) => { + encoderPasses += 1; + return encode.apply(model, args); + }; + const auto = await transcribeLongForm( + audio, + { + speechProbabilities: (pcm) => vad.speechProbabilities(pcm), + ...createWhisperDeps(asr, { multilingual: true }), + }, + { ...LONG_FORM_DEFAULTS, task: "transcribe" }, + ).finally(() => { + delete model._prepare_encoder_decoder_kwargs_for_generation; // back to the prototype's + }); + expect(auto.language).toBe(clip.language); + // One encoder pass per decoded chunk (fallback is off by default). + expect(encoderPasses).toBe(auto.decodeCalls); + expect( + wordErrorRate(normalize(reference), normalize(auto.text)), + ).toBeLessThanOrEqual(MAX_WER_BY_CLIP[clip.base] ?? MAX_WER); + } + } finally { + await vad.dispose(); + } + }, + DOWNLOAD_BUDGET_MS + + probeDurationSeconds(join(AUDIO_DIR, clip.audioFile)) * MS_PER_AUDIO_SECOND, + ); + } +}); diff --git a/tests/longform-pipeline.test.ts b/tests/longform-pipeline.test.ts new file mode 100644 index 0000000..c3a69ac --- /dev/null +++ b/tests/longform-pipeline.test.ts @@ -0,0 +1,230 @@ +import { describe, expect, test } from "bun:test"; +import { + transcribeLongForm, + type DecodeRequest, + type DecodeResult, + type LongFormDeps, +} from "../src/worker/longform"; +import { SAMPLE_RATE, VAD_WINDOW_SAMPLES } from "../src/worker/longform/vad"; + +const MAX_CHUNK_SAMPLES = 30 * SAMPLE_RATE; + +/** Speech probabilities for audio of `seconds` with speech in [start, end) ranges. */ +function maskProbs(seconds: number, speech: [number, number][]): Float32Array { + const windows = Math.ceil((seconds * SAMPLE_RATE) / VAD_WINDOW_SAMPLES); + const probs = new Float32Array(windows).fill(0.02); + for (const [start, end] of speech) { + const from = Math.floor((start * SAMPLE_RATE) / VAD_WINDOW_SAMPLES); + const to = Math.min(windows, Math.floor((end * SAMPLE_RATE) / VAD_WINDOW_SAMPLES)); + for (let i = from; i < to; i++) probs[i] = 0.95; + } + return probs; +} + +interface Recorder { + deps: LongFormDeps; + decodes: { audio: Float32Array; request: DecodeRequest }[]; + vadInputs: Float32Array[]; + detectCalls: Float32Array[]; +} + +function fakeDeps( + probs: Float32Array, + decode: (audio: Float32Array, request: DecodeRequest, call: number) => DecodeResult = ( + audio, + ) => ({ + text: ` words ${audio.length}`, + chunks: [{ text: ` words ${audio.length}`, timestamp: [0, audio.length / SAMPLE_RATE] }], + generatedTokens: 5, + avgLogProb: -0.2, + }), + detect: string | null = null, +): Recorder { + const recorder: Recorder = { decodes: [], vadInputs: [], detectCalls: [], deps: null as never }; + recorder.deps = { + speechProbabilities: async (audio) => { + recorder.vadInputs.push(audio); + return probs; + }, + decode: async (audio, request) => { + recorder.decodes.push({ audio, request }); + request.onPartial?.("x"); + request.onTimestamp?.(1); + return decode(audio, request, recorder.decodes.length); + }, + ...(detect !== null && { + detectLanguage: async (audio: Float32Array) => { + recorder.detectCalls.push(audio); + return detect; + }, + }), + }; + return recorder; +} + +describe("transcribeLongForm", () => { + test("no speech → empty result, decode never called", async () => { + const audio = new Float32Array(60 * SAMPLE_RATE); + const { deps, decodes } = fakeDeps(maskProbs(60, [])); + const progress: number[] = []; + const result = await transcribeLongForm(audio, deps, { + onProgress: (processed) => progress.push(processed), + }); + expect(decodes).toHaveLength(0); + expect(result.text).toBe(""); + expect(result.chunks).toEqual([]); + expect(result.noSpeech).toBe(true); + expect(progress.at(-1)).toBe(60); + }); + + test("speech regions are decoded per chunk and timestamps mapped to the file", async () => { + // Two 5 s regions 15 s apart → one ≤ 30 s chunk holding both spans. + const audio = new Float32Array(40 * SAMPLE_RATE); + const { deps, decodes } = fakeDeps(maskProbs(40, [[5, 10], [25, 30]]), (chunk) => ({ + text: " hello there", + chunks: [ + { text: " hello", timestamp: [0.5, 2] }, + { text: " there", timestamp: [6.5, 8] }, + ], + generatedTokens: 4, + avgLogProb: -0.1, + })); + const result = await transcribeLongForm(audio, deps, {}); + expect(decodes).toHaveLength(1); + // Window-aligned spans padded by 400 ms: [73 472, 166 144) and + // [393 472, 486 144) → 2 × 92 672 samples, silence between them dropped. + expect(decodes[0]!.audio.length).toBe(2 * 92_672); + expect(result.text).toBe("hello there"); + expect(result.noSpeech).toBe(false); + // The chunk starts at 4.592 s; 6.5 s into the chunk is past the first + // span (5.792 s long) → 0.708 s into the second, which starts at 24.592 s. + expect(result.chunks[0]!.timestamp[0]).toBeCloseTo(4.592 + 0.5, 3); + expect(result.chunks[1]!.timestamp[0]).toBeCloseTo(24.592 + 0.708, 3); + expect(result.chunks[1]!.timestamp[0]).toBeGreaterThan(24); + expect(result.chunks[1]!.timestamp[1]!).toBeLessThan(30.5); + }); + + test("auto language: detected once on the first speech chunk, then pinned", async () => { + const audio = new Float32Array(100 * SAMPLE_RATE); + const recorder = fakeDeps(maskProbs(100, [[0, 25], [40, 65], [75, 99]]), undefined, "da"); + const result = await transcribeLongForm(audio, recorder.deps, { language: "auto" }); + expect(recorder.detectCalls).toHaveLength(1); + expect(recorder.detectCalls[0]).toBe(recorder.decodes[0]!.audio); + expect(recorder.decodes.length).toBeGreaterThanOrEqual(3); + for (const { request } of recorder.decodes) expect(request.language).toBe("da"); + expect(result.language).toBe("da"); + }); + + test("an explicit language skips detection; English-only models send none", async () => { + const audio = new Float32Array(20 * SAMPLE_RATE); + const pinned = fakeDeps(maskProbs(20, [[2, 10]]), undefined, "de"); + await transcribeLongForm(audio, pinned.deps, { language: "en" }); + expect(pinned.detectCalls).toHaveLength(0); + expect(pinned.decodes[0]!.request.language).toBe("en"); + + const englishOnly = fakeDeps(maskProbs(20, [[2, 10]])); + await transcribeLongForm(audio, englishOnly.deps, {}); + expect(englishOnly.decodes[0]!.request.language).toBeUndefined(); + }); + + test("task, prompt and the repetition guard reach every decode", async () => { + const audio = new Float32Array(100 * SAMPLE_RATE); + const recorder = fakeDeps(maskProbs(100, [[0, 25], [40, 65], [75, 99]])); + await transcribeLongForm(audio, recorder.deps, { + task: "translate", + prompt: "Voices of the Void", + guard: { noRepeatNgramSize: 5, repetitionPenalty: 1.1 }, + }); + expect(recorder.decodes.length).toBeGreaterThanOrEqual(3); + for (const { request } of recorder.decodes) { + expect(request.task).toBe("translate"); + expect(request.prompt).toBe("Voices of the Void"); + expect(request.noRepeatNgramSize).toBe(5); + expect(request.repetitionPenalty).toBe(1.1); + expect(request.temperature).toBe(0); + } + }); + + test("fallback: a looping decode is retried at the next temperature", async () => { + const audio = new Float32Array(20 * SAMPLE_RATE); + const loop = " the main clar,".repeat(40); + const recorder = fakeDeps(maskProbs(20, [[2, 10]]), (_audio, request) => ({ + text: request.temperature === 0 ? loop : " a fine sentence about festivals", + chunks: [], + generatedTokens: request.temperature === 0 ? 400 : 8, + avgLogProb: -0.2, + })); + const result = await transcribeLongForm(audio, recorder.deps, { fallback: {} }); + expect(recorder.decodes.map((d) => d.request.temperature)).toEqual([0, 0.2]); + expect(result.text).toBe("a fine sentence about festivals"); + expect(result.generatedTokens).toBe(408); + }); + + test("fallback: when every temperature fails, the best attempt is kept", async () => { + const audio = new Float32Array(20 * SAMPLE_RATE); + const recorder = fakeDeps(maskProbs(20, [[2, 10]]), (_audio, request) => ({ + text: request.temperature === 0.6 ? " ok ok ok ok ok ok ok ok ok ok ok ok ok" : " la".repeat(80), + chunks: [], + generatedTokens: 10, + avgLogProb: null, + })); + const result = await transcribeLongForm(audio, recorder.deps, { fallback: {} }); + expect(recorder.decodes).toHaveLength(6); + expect(result.text).toBe("ok ok ok ok ok ok ok ok ok ok ok ok ok"); + }); + + test("fallback off → exactly one decode per chunk", async () => { + const audio = new Float32Array(20 * SAMPLE_RATE); + const recorder = fakeDeps(maskProbs(20, [[2, 10]]), () => ({ + text: " la".repeat(80), + chunks: [], + generatedTokens: 10, + avgLogProb: -3, + })); + await transcribeLongForm(audio, recorder.deps, { fallback: false }); + expect(recorder.decodes).toHaveLength(1); + }); + + test("progress is monotone and ends at the total; partials are forwarded", async () => { + const audio = new Float32Array(100 * SAMPLE_RATE); + const recorder = fakeDeps(maskProbs(100, [[0, 25], [40, 65], [75, 90]])); + const progress: number[] = []; + const partials: string[] = []; + await transcribeLongForm(audio, recorder.deps, { + onProgress: (processed, total) => { + expect(total).toBe(100); + progress.push(processed); + }, + onPartial: (text) => partials.push(text), + }); + expect(progress[0]).toBe(0); + expect(progress.at(-1)).toBe(100); + for (let i = 1; i < progress.length; i++) { + expect(progress[i]!).toBeGreaterThan(progress[i - 1]!); + } + expect(partials.length).toBe(recorder.decodes.length); + }); + + test("90-minute input: VAD sees the caller's buffer, decode sees ≤ 30 s chunks", async () => { + const seconds = 90 * 60; + const audio = new Float32Array(seconds * SAMPLE_RATE); + const speech: [number, number][] = []; + for (let t = 0; t < seconds; t += 25) speech.push([t, t + 20]); + const recorder = fakeDeps(maskProbs(seconds, speech)); + const started = performance.now(); + const result = await transcribeLongForm(audio, recorder.deps, {}); + expect(performance.now() - started).toBeLessThan(5_000); + expect(recorder.vadInputs).toHaveLength(1); + expect(recorder.vadInputs[0]).toBe(audio); // no copy before VAD + let decoded = 0; + for (const { audio: chunk } of recorder.decodes) { + expect(chunk.length).toBeLessThanOrEqual(MAX_CHUNK_SAMPLES); + expect(chunk.buffer).not.toBe(audio.buffer); + decoded += chunk.length; + } + // Every speech second is decoded once (padding included), silence never. + expect(decoded).toBeLessThan(audio.length); + expect(result.speechSeconds).toBeCloseTo(decoded / SAMPLE_RATE, 3); + expect(result.decodeCalls).toBe(recorder.decodes.length); + }); +}); diff --git a/tests/longform-quality.test.ts b/tests/longform-quality.test.ts new file mode 100644 index 0000000..7dad2d1 --- /dev/null +++ b/tests/longform-quality.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, test } from "bun:test"; +import { deflateSync } from "node:zlib"; +import { + compressionRatio, + DEFAULT_FALLBACK, + needsFallback, +} from "../src/worker/longform/quality"; + +describe("compressionRatio (faster-whisper get_compression_ratio)", () => { + test("matches zlib.compress on the same bytes", async () => { + const text = "The problem I find with AI is that it is probably a bit overhyped."; + const bytes = new TextEncoder().encode(text); + expect(await compressionRatio(text)).toBeCloseTo( + bytes.length / deflateSync(bytes).length, + 5, + ); + }); + + test("a repetition loop compresses far above 2.4, normal speech below", async () => { + expect(await compressionRatio("the main clar, ".repeat(40))).toBeGreaterThan(2.4); + expect( + await compressionRatio( + "På Aarhus Festuges sekretariat har vi løbet rigtig stærkt de sidste måneder.", + ), + ).toBeLessThan(2.4); + }); + + test("empty text is not a loop", async () => { + expect(await compressionRatio("")).toBe(0); + }); +}); + +describe("needsFallback", () => { + test("uses the spec thresholds by default", () => { + expect(DEFAULT_FALLBACK.compressionRatioThreshold).toBe(2.4); + expect(DEFAULT_FALLBACK.logProbThreshold).toBe(-1.0); + expect(DEFAULT_FALLBACK.temperatures).toEqual([0, 0.2, 0.4, 0.6, 0.8, 1.0]); + }); + + test("rejects a high compression ratio or a low average log-prob", () => { + expect(needsFallback({ compressionRatio: 3.1, avgLogProb: -0.2 })).toBe(true); + expect(needsFallback({ compressionRatio: 1.5, avgLogProb: -1.5 })).toBe(true); + expect(needsFallback({ compressionRatio: 1.5, avgLogProb: -0.3 })).toBe(false); + expect(needsFallback({ compressionRatio: 1.5, avgLogProb: null })).toBe(false); + }); +}); diff --git a/tests/longform-setting.test.ts b/tests/longform-setting.test.ts new file mode 100644 index 0000000..4c52374 --- /dev/null +++ b/tests/longform-setting.test.ts @@ -0,0 +1,55 @@ +import { describe, expect, test } from "bun:test"; +import { emptyTranscriptNotice } from "../src/lib/export"; +import { + LONG_FORM_DEFAULT, + LONG_FORM_KEY, + loadLongFormChoice, + saveLongFormChoice, +} from "../src/lib/longForm"; + +function memoryStorage(initial: Record = {}) { + const data = { ...initial }; + return { + data, + getItem: (key: string) => data[key] ?? null, + setItem: (key: string, value: string) => { + data[key] = value; + }, + }; +} + +describe("long-form setting persistence (D-WS3a)", () => { + test("defaults when nothing is stored or storage is unavailable", () => { + expect(loadLongFormChoice(memoryStorage())).toBe(LONG_FORM_DEFAULT); + expect(loadLongFormChoice(null)).toBe(LONG_FORM_DEFAULT); + const throwing = { + getItem: () => { + throw new Error("denied"); + }, + setItem: () => { + throw new Error("denied"); + }, + }; + expect(loadLongFormChoice(throwing)).toBe(LONG_FORM_DEFAULT); + expect(() => saveLongFormChoice(true, throwing)).not.toThrow(); + }); + + test("round-trips an explicit choice", () => { + const storage = memoryStorage(); + saveLongFormChoice(!LONG_FORM_DEFAULT, storage); + expect(storage.data[LONG_FORM_KEY]).toBeDefined(); + expect(loadLongFormChoice(storage)).toBe(!LONG_FORM_DEFAULT); + }); +}); + +describe("emptyTranscriptNotice (plan 001-0 Review Focus 4)", () => { + test("a finished transcript with no words says so plainly", () => { + expect(emptyTranscriptNotice({ text: "" })).toBe("No speech was found in this recording."); + expect(emptyTranscriptNotice({ text: " \n" })).toBe("No speech was found in this recording."); + }); + + test("no notice while running or when there is text", () => { + expect(emptyTranscriptNotice(null)).toBeNull(); + expect(emptyTranscriptNotice({ text: " Hello." })).toBeNull(); + }); +}); diff --git a/tests/longform-transformers.test.ts b/tests/longform-transformers.test.ts new file mode 100644 index 0000000..61ff9d0 --- /dev/null +++ b/tests/longform-transformers.test.ts @@ -0,0 +1,118 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { existsSync } from "node:fs"; +import { join } from "node:path"; +import { env } from "@huggingface/transformers"; +import { createWhisperDeps, loadSileroVad, VAD_MODEL_ID } from "../src/worker/longform/transformers"; +import { PROMPT_PREFIX_KWARG } from "../src/worker/prompt"; +import { getSpeechTimestamps, SAMPLE_RATE } from "../src/worker/longform/vad"; +import { decodeClip } from "./clips"; + +const realFetch = globalThis.fetch; +afterEach(() => { + globalThis.fetch = realFetch; +}); + +describe("loadSileroVad offline (plan 001-0 Review Focus 3)", () => { + test("localOnly load with nothing cached rejects without calling fetch", async () => { + let fetches = 0; + globalThis.fetch = (async () => { + fetches += 1; + throw new Error("network must not be touched"); + }) as unknown as typeof fetch; + await expect( + loadSileroVad({ localOnly: true, isCached: async () => false }), + ).rejects.toThrow(/not downloaded/); + expect(fetches).toBe(0); + }); +}); + +/** Cached locally, or downloadable right now — otherwise the real-VAD tests skip. */ +async function vadAvailable(): Promise { + if (env.cacheDir && existsSync(join(env.cacheDir, VAD_MODEL_ID))) return true; + try { + const response = await fetch( + `https://huggingface.co/${VAD_MODEL_ID}/resolve/main/onnx/model.onnx`, + { method: "HEAD", signal: AbortSignal.timeout(5_000) }, + ); + return response.ok; + } catch { + return false; + } +} +const available = await vadAvailable(); + +describe.skipIf(!available)("Silero VAD on the Transformers.js runtime (real model)", () => { + const seconds = 10; + const tone = new Float32Array(seconds * SAMPLE_RATE).map( + (_, i) => 0.3 * Math.sin((2 * Math.PI * 440 * i) / SAMPLE_RATE), + ); + let seed = 1; + const noise = new Float32Array(seconds * SAMPLE_RATE).map(() => { + seed = (seed * 1_103_515_245 + 12_345) % 2 ** 31; + return 0.2 * (seed / 2 ** 30 - 1); + }); + + for (const [name, audio] of [ + ["silence", new Float32Array(seconds * SAMPLE_RATE)], + ["a 440 Hz tone", tone], + ["white noise", noise], + ] as const) { + test(`${name} contains no speech`, async () => { + const vad = await loadSileroVad({ localOnly: false }); + const probs = await vad.speechProbabilities(audio); + expect(probs.length).toBe(Math.ceil(audio.length / 512)); + expect(getSpeechTimestamps(probs, audio.length)).toEqual([]); + }); + } + + test("en_aiproblem: speech is found and covers most of the clip", async () => { + const audio = await decodeClip("en_aiproblem.m4a"); + const vad = await loadSileroVad({ localOnly: false }); + const spans = getSpeechTimestamps(await vad.speechProbabilities(audio), audio.length); + expect(spans.length).toBeGreaterThan(0); + const covered = spans.reduce((n, s) => n + s.end - s.start, 0); + expect(covered / audio.length).toBeGreaterThan(0.8); + }); +}); + +describe("createWhisperDeps decode with the names-and-terms hint (D-WS2a × D-WS3a)", () => { + /** A fake ASR pipeline that records the generate options of each call. */ + function fakeAsr() { + const calls: Record[] = []; + const asr = Object.assign( + async (_audio: Float32Array, options: Record) => { + calls.push(options); + return { text: " hello", chunks: [] }; + }, + { + tokenizer: { + encode: (text: string) => [...text].map((char) => char.charCodeAt(0)), + convert_tokens_to_ids: (token: string) => (token === "<|startofprev|>" ? 50361 : -1), + }, + model: { + forward_params: ["input_features"], + generate: async () => ({}), + _prepare_generation_config: () => ({}), + _retrieve_init_tokens: () => [50258], + }, + }, + ); + return { asr, calls }; + } + + const baseRequest = { temperature: 0, noRepeatNgramSize: 0, repetitionPenalty: 1 }; + + test("passes the hint as a decoder-prompt prefix on every chunk, and nothing without one", async () => { + const { asr, calls } = fakeAsr(); + const deps = createWhisperDeps(asr as never, { multilingual: false }); + await deps.decode(new Float32Array(16), { ...baseRequest, prompt: "Maxwell" }); + await deps.decode(new Float32Array(16), { ...baseRequest, prompt: "Maxwell" }); + await deps.decode(new Float32Array(16), baseRequest); + + const prefix = calls[0]![PROMPT_PREFIX_KWARG] as number[]; + expect(prefix[0]).toBe(50361); // <|startofprev|> + expect(prefix.length).toBeGreaterThan(1); + expect(calls[1]![PROMPT_PREFIX_KWARG]).toEqual(prefix); + expect(PROMPT_PREFIX_KWARG in calls[2]!).toBe(false); + }); +}); diff --git a/tests/longform-vad.test.ts b/tests/longform-vad.test.ts new file mode 100644 index 0000000..200ca12 --- /dev/null +++ b/tests/longform-vad.test.ts @@ -0,0 +1,110 @@ +import { describe, expect, test } from "bun:test"; +import { + chunkAudio, + chunkTimeToOriginal, + collectChunks, + getSpeechTimestamps, + SAMPLE_RATE, + VAD_WINDOW_SAMPLES, +} from "../src/worker/longform/vad"; + +const W = VAD_WINDOW_SAMPLES; +const PAD = (400 / 1000) * SAMPLE_RATE; // speech_pad_ms 400 → 6400 samples + +/** Probabilities for `seconds` of audio with speech in the given [start, end) second ranges. */ +function probsFor(seconds: number, speech: [number, number][]): Float32Array { + const windows = Math.ceil((seconds * SAMPLE_RATE) / W); + const probs = new Float32Array(windows).fill(0.05); + for (const [start, end] of speech) { + const from = Math.floor((start * SAMPLE_RATE) / W); + const to = Math.floor((end * SAMPLE_RATE) / W); + for (let i = from; i < to; i++) probs[i] = 0.9; + } + return probs; +} + +describe("getSpeechTimestamps (port of faster-whisper vad.py)", () => { + test("no window above the threshold → no speech", () => { + const probs = new Float32Array(500).fill(0.3); + expect(getSpeechTimestamps(probs, 500 * W)).toEqual([]); + }); + + test("one burst is padded by speech_pad_ms on both sides", () => { + const probs = new Float32Array(200).fill(0.05); + for (let i = 50; i < 80; i++) probs[i] = 0.9; + expect(getSpeechTimestamps(probs, 200 * W)).toEqual([ + { start: 50 * W - PAD, end: 80 * W + PAD }, + ]); + }); + + test("padding is clamped to the audio", () => { + const probs = new Float32Array(40).fill(0.9); + expect(getSpeechTimestamps(probs, 40 * W)).toEqual([{ start: 0, end: 40 * W }]); + }); + + test("a pause shorter than min_silence_duration_ms does not split", () => { + const spans = getSpeechTimestamps(probsFor(20, [[2, 6], [7, 11]]), 20 * SAMPLE_RATE); + expect(spans).toHaveLength(1); + }); + + test("a pause longer than min_silence_duration_ms splits; close spans share the gap", () => { + const spans = getSpeechTimestamps(probsFor(30, [[2, 6], [9, 12]]), 30 * SAMPLE_RATE); + expect(spans).toHaveLength(2); + // 3 s gap ≥ 2·pad → each side padded by the full 400 ms. + expect(spans[0]!.end - Math.floor((6 * SAMPLE_RATE) / W) * W).toBe(PAD); + const narrow = getSpeechTimestamps(probsFor(30, [[2, 6], [8.5, 12]]), 30 * SAMPLE_RATE, { + minSilenceDurationMs: 2000, + speechPadMs: 1500, + }); + // 2.5 s gap < 2·1.5 s pad → the gap is split in half instead. + expect(narrow).toHaveLength(2); + expect(narrow[0]!.end).toBe(narrow[1]!.start); + }); + + test("continuous speech is cut at max_speech_duration_s", () => { + const spans = getSpeechTimestamps(probsFor(70, [[0, 70]]), 70 * SAMPLE_RATE, { + maxSpeechDurationS: 30, + }); + expect(spans.length).toBeGreaterThanOrEqual(3); + for (const span of spans) { + expect(span.end - span.start).toBeLessThanOrEqual(30 * SAMPLE_RATE); + } + }); +}); + +describe("collectChunks / chunk time mapping", () => { + const s = (sec: number) => sec * SAMPLE_RATE; + const spans = [ + { start: s(0), end: s(12) }, + { start: s(20), end: s(32) }, + { start: s(40), end: s(52) }, + ]; + + test("packs spans into chunks no longer than the maximum", () => { + const chunks = collectChunks(spans, s(30)); + expect(chunks.map((chunk) => chunk.samples)).toEqual([s(24), s(12)]); + expect(chunks[0]!.spans).toEqual([spans[0]!, spans[1]!]); + }); + + test("no spans → no chunks", () => { + expect(collectChunks([], s(30))).toEqual([]); + }); + + test("chunk-relative seconds map back to the original timeline", () => { + const [chunk] = collectChunks(spans, s(30)); + expect(chunkTimeToOriginal(chunk!, 0)).toBe(0); + expect(chunkTimeToOriginal(chunk!, 5)).toBe(5); + expect(chunkTimeToOriginal(chunk!, 12)).toBe(12); // splice → earlier span's end + expect(chunkTimeToOriginal(chunk!, 15)).toBe(23); + expect(chunkTimeToOriginal(chunk!, 99)).toBe(32); // clamps to the last span + }); + + test("chunkAudio copies exactly the chunk's spans", () => { + const audio = new Float32Array(s(60)).map((_, i) => i); + const [chunk] = collectChunks(spans, s(30)); + const out = chunkAudio(audio, chunk!); + expect(out.length).toBe(chunk!.samples); + expect(out[0]).toBe(0); + expect(out[s(12)]).toBe(s(20)); + }); +}); diff --git a/tests/meetingFixture.test.ts b/tests/meetingFixture.test.ts new file mode 100644 index 0000000..5b5428f --- /dev/null +++ b/tests/meetingFixture.test.ts @@ -0,0 +1,22 @@ +import { describe, expect, test } from "bun:test"; +import { ALIEN_EXCERPTS, buildMeetingFixture } from "./meetingFixture"; + +describe("buildMeetingFixture (SYNTHETIC meeting stand-in)", () => { + test("joins en_aiproblem and the en_alien excerpts with pauses, deterministically", async () => { + const first = await buildMeetingFixture(); + const second = await buildMeetingFixture(); + const seconds = first.audio.length / 16_000; + const excerptSeconds = ALIEN_EXCERPTS.reduce((n, e) => n + e.end - e.start, 0); + // lead-in 2 + aiproblem ≈ 14.4 + excerpts + pauses 3+8+2+5 + expect(seconds).toBeCloseTo(2 + 14.4 + excerptSeconds + 18, 0); + expect(first.audio).toEqual(second.audio); + expect(first.reference).toBe(second.reference); + expect(first.reference).toStartWith("The problem I find with AI"); + expect(first.reference).toContain("Voices of the Void"); + expect(first.reference).toEndWith("Maybe no."); + // Pauses are room tone, not digital silence, and never louder than −50 dBFS. + const leadIn = first.audio.subarray(0, 2 * 16_000); + expect(leadIn.some((v) => v !== 0)).toBe(true); + expect(Math.max(...leadIn.map(Math.abs))).toBeLessThan(0.003); + }, 60_000); // decodes the 15-min en_alien twice +}); diff --git a/tests/meetingFixture.ts b/tests/meetingFixture.ts new file mode 100644 index 0000000..044a06a --- /dev/null +++ b/tests/meetingFixture.ts @@ -0,0 +1,64 @@ +/** + * SYNTHETIC meeting stand-in (spec 001 "Measurement fallback", coordination + * plan 001-0): no real meeting recording exists, so this joins the repo's + * own English clips with meeting-like pauses. Built at test time from + * tests/audio_files — nothing new is checked in. Every number measured on it + * is labelled synthetic. + * + * Layout (≈ 2 min 45 s): 2 s lead-in, en_aiproblem (whole), then three + * en_alien excerpts, each clip followed by a pause of 3, 8, 2 and 5 s. Pauses and lead-in carry + * deterministic room noise at about −60 dBFS (a real recording is never + * digital silence). Excerpts are cut on reference-line boundaries; their + * times were read once from a timestamped Large v3 Turbo transcript of + * en_alien (2026-10-29) and land in the pause before the line, so each + * excerpt's reference is exactly its lines of en_alien.txt. + */ + +import { decodeClip, readReference } from "./clips"; + +const RATE = 16_000; +const LEAD_IN_SECONDS = 2; +const PAUSES_SECONDS = [3, 8, 2, 5]; +const NOISE_AMPLITUDE = 0.001; // ≈ −60 dBFS + +/** en_alien excerpts: [start s, end s) and the 1-based reference lines they contain. */ +export const ALIEN_EXCERPTS: { start: number; end: number; firstLine: number; lastLine: number }[] = [ + { start: 0, end: 36.8, firstLine: 1, lastLine: 9 }, + { start: 36.8, end: 99.0, firstLine: 10, lastLine: 28 }, + { start: 165.4, end: 196.1, firstLine: 45, lastLine: 58 }, +]; + +function roomNoise(seconds: number, seed: number): Float32Array { + let state = seed; + return new Float32Array(Math.round(seconds * RATE)).map(() => { + state = (state * 1_103_515_245 + 12_345) % 2 ** 31; + return NOISE_AMPLITUDE * (state / 2 ** 30 - 1); + }); +} + +export async function buildMeetingFixture(): Promise<{ audio: Float32Array; reference: string }> { + const aiproblem = await decodeClip("en_aiproblem.m4a"); + const alien = await decodeClip("en_alien.opus"); + const alienLines = (await readReference("en_alien")).split("\n"); + + const speech: Float32Array[] = [aiproblem]; + const references: string[] = [(await readReference("en_aiproblem")).trim()]; + for (const excerpt of ALIEN_EXCERPTS) { + speech.push(alien.subarray(Math.round(excerpt.start * RATE), Math.round(excerpt.end * RATE))); + references.push(alienLines.slice(excerpt.firstLine - 1, excerpt.lastLine).join(" ").trim()); + } + + const parts: Float32Array[] = [roomNoise(LEAD_IN_SECONDS, 1)]; + // A pause after every clip — the last one is the meeting's trailing room tone. + speech.forEach((clip, i) => { + parts.push(clip); + parts.push(roomNoise(PAUSES_SECONDS[i % PAUSES_SECONDS.length]!, i + 2)); + }); + const audio = new Float32Array(parts.reduce((n, p) => n + p.length, 0)); + let at = 0; + for (const part of parts) { + audio.set(part, at); + at += part.length; + } + return { audio, reference: references.join(" ") }; +} diff --git a/tests/offline.md b/tests/offline.md index d49bc87..edce92d 100644 --- a/tests/offline.md +++ b/tests/offline.md @@ -129,6 +129,25 @@ Result: Works Result: _pending (needs a real browser)_ +## 6. "Skip silence" (long-form pipeline, D-WS3a/D-WS3b) + +1. Online, open **Advanced**, tick **Skip silence in uploaded files**, and + transcribe a file. DevTools → Network shows one request for + `onnx-community/silero-vad/…/onnx/model.onnx` (2.2 MB); the console logs + "speech-detection (VAD) model ready". The transcript arrives with progress + moving in jumps over pauses. Repeat on WebGPU (the owner's backend) and note + wall-clock against the same file with the option off. +2. Go offline (DevTools "Offline"), reload, transcribe again: no network + request at all, same long-form path (console line as above). +3. Clear only the silero-vad entries from `transformers-cache` (Application → + Cache Storage), stay offline, transcribe: still no network request; the + console says the VAD model is unavailable and the file is transcribed the + usual way. +4. Transcribe a file of pure silence or music with the option on: the + transcript reads "No speech was found in this recording." + +Result: _pending (no real browser in the dev environment)_ + ## Recording the result When this passes in a real browser, flip the `[~]` offline-verification item diff --git a/tests/protocol.test.ts b/tests/protocol.test.ts index be42ff2..a08bdaf 100644 --- a/tests/protocol.test.ts +++ b/tests/protocol.test.ts @@ -4,6 +4,7 @@ import { deviceAttempts, dtypeRequiresF16, initialLoadProgress, + choosePipeline, isUiToWorkerMessage, isWorkerToUiMessage, TranscribeProgressTracker, @@ -36,7 +37,9 @@ describe("isUiToWorkerMessage", () => { { type: "transcribe", audio: new Float32Array(8), requestId: "engine-1-1", stream: true }, { type: "transcribe", audio: new Float32Array(8), stream: false }, { type: "transcribe", audio: new Float32Array(8), prompt: "Maxwell, Voices of the Void" }, + { type: "transcribe", audio: new Float32Array(8), stream: true, longForm: true }, ]; + loads.push({ type: "load", modelId: "onnx-community/whisper-base", device: "auto", vad: true }); for (const message of [...loads, ...transcribes]) { expect(isUiToWorkerMessage(message)).toBe(true); } @@ -69,10 +72,26 @@ describe("isUiToWorkerMessage", () => { expect( isUiToWorkerMessage({ type: "transcribe", audio: new Float32Array(4), prompt: 5 }), ).toBe(false); + expect( + isUiToWorkerMessage({ type: "transcribe", audio: new Float32Array(4), longForm: 1 }), + ).toBe(false); + expect(isUiToWorkerMessage({ type: "load", modelId: "x", device: "auto", vad: "yes" })).toBe( + false, + ); expect(isUiToWorkerMessage({ type: "result", text: "", chunks: [] })).toBe(false); }); }); +describe("choosePipeline (D-WS3a/D-WS3b)", () => { + test("long-form only when asked for and the VAD model loaded", () => { + expect(choosePipeline({ longForm: true, vadReady: true })).toBe("longform"); + // Offline / never downloaded: the old path, never an error (Review Focus 3). + expect(choosePipeline({ longForm: true, vadReady: false })).toBe("chunked"); + expect(choosePipeline({ longForm: false, vadReady: true })).toBe("chunked"); + expect(choosePipeline({ longForm: undefined, vadReady: true })).toBe("chunked"); + }); +}); + describe("isWorkerToUiMessage", () => { test("accepts every worker → UI message shape", () => { const messages: WorkerToUiMessage[] = [ @@ -278,6 +297,37 @@ describe("TranscribeProgressTracker", () => { expect(tracker.timestamp(8)).toBe(8); expect(tracker.finalizeWindow()).toBeNull(); // 20 clamps to 8, not an advance }); + + describe("forWindows (variable-length VAD chunks, D-WS3a)", () => { + // Two speech chunks with a skipped 15 s silence between them: [0–20 s] + // and [35–60 s] of a 70 s file. + const windows = [ + { toAbsolute: (t: number) => t, endSeconds: 20, nextStartSeconds: 35 }, + { toAbsolute: (t: number) => 35 + t, endSeconds: 60, nextStartSeconds: 70 }, + ]; + + test("maps chunk-relative times through each window", () => { + const tracker = TranscribeProgressTracker.forWindows(70, windows); + expect(tracker.timestamp(5)).toBe(5); + // Finishing a chunk counts the skipped silence up to the next one. + expect(tracker.finalizeWindow()).toBe(35); + expect(tracker.timestamp(10)).toBe(45); + expect(tracker.timestamp(2)).toBeNull(); // 37 < 45: no regression + expect(tracker.finalizeWindow()).toBe(70); + expect(tracker.finalizeWindow()).toBeNull(); // past the last window + }); + + test("clamps a stray timestamp to the window's end", () => { + const tracker = TranscribeProgressTracker.forWindows(70, windows); + expect(tracker.timestamp(400)).toBe(20); + }); + + test("no windows (no speech) → nothing to report but the end", () => { + const tracker = TranscribeProgressTracker.forWindows(70, []); + expect(tracker.timestamp(3)).toBeNull(); + expect(tracker.finalizeWindow()).toBe(70); + }); + }); }); describe("applyLoadProgress", () => {