diff --git a/docs/Architecture.md b/docs/Architecture.md index ea5fc28..7873131 100644 --- a/docs/Architecture.md +++ b/docs/Architecture.md @@ -74,6 +74,9 @@ src/ worker/ inference.worker.ts # Transformers.js pipeline, WebNN/WebGPU/WASM selection protocol.ts # typed postMessage messages (shared with UI) + longform/ # long-form file pipeline (D-WS3a): pure vad.ts / quality.ts / pipeline.ts + # behind transcribeLongForm(audio, deps, options); transformers.ts implements + # the deps (Silero VAD + Whisper decodes) on the same Transformers.js runtime pwa/ sw.ts # service worker (app-shell precache, /ort runtime cache) precache.ts # placeholder-injection helpers shared by build + dev serve @@ -144,6 +147,21 @@ environment). See `WorkBreakdown.md` for the itemized checklist.) (transcribe | translate). 5. Worker runs the pipeline (chunked, with timestamps), streaming partial results back via `{type:"partial"}` messages; final result as `{type:"result"}`. +6. **Long-form branch (D-WS3a/D-WS3b, "Advanced → Skip silence", off by default).** + With the option on, every load posted for a file run or the Download button carries + `vad: true`; once the Whisper pipeline is ready the worker loads Silero VAD + (`onnx-community/silero-vad`, 2.2 MB, Cache API like the weights; offline it is + used only if already cached). A file pass with `longForm: true` then runs + `transcribeLongForm` (`worker/longform/`): VAD speech probabilities over the + whole buffer → speech spans (faster-whisper `vad.py` port) → chunks ≤ 30 s cut on + speech boundaries → one Whisper decode per chunk (language detected once on the + first chunk and pinned for multilingual models) → timestamps mapped back to the + file's timeline; progress follows the variable chunks + (`TranscribeProgressTracker.forWindows`). Same `partial` / + `transcribe-progress` / `result` messages as step 5. `choosePipeline` keeps the + chunked path whenever the VAD model isn't loaded, and live passes never take + this branch (live keeps its RMS segmenter, D-016/D-024). A finished transcript + with no words shows "No speech was found in this recording." ### 4.1b Live microphone transcription (anonymous, browser engine — D-016) 1. `LiveControls` starts a session: `getUserMedia` → `AudioContext` at the @@ -189,7 +207,9 @@ environment). See `WorkBreakdown.md` for the itemized checklist.) large to precache eagerly. - Model weights live in the Cache API (`transformers-cache`) via Transformers.js — independent of the SW, which never touches that cache (D-005). `/api/*` is - network-only. + network-only. The Silero VAD model of the long-form branch lives there too + (D-WS3b); offline, the worker checks that cache before loading it and never + fetches, falling back to the chunked path if it is missing. - Updates: `pwa/register.ts` surfaces a waiting SW as a toast; accepting posts `SKIP_WAITING`, activation drops old `whisper-*` caches, and the page reloads. - Result: with the app installed and at least one model previously loaded, the full diff --git a/src/worker/longform/transformers.ts b/src/worker/longform/transformers.ts index 419dffe..f9552ba 100644 --- a/src/worker/longform/transformers.ts +++ b/src/worker/longform/transformers.ts @@ -163,13 +163,19 @@ export function createWhisperDeps( asr: AutomaticSpeechRecognitionPipeline, options: { multilingual: boolean }, ): Pick { - const decode = async (audio: Float32Array, request: DecodeRequest): Promise => { + const decode = async ( + audio: Float32Array, + request: DecodeRequest, + encoderOutputs?: Tensor, + ): Promise => { const recorder = new LogProbRecorder(); const generate: Record = { return_timestamps: true, // One window, single-pass generate (D-021's seek-loop note applies). max_new_tokens: 440, logits_processor: new ListWithLast(recorder), + // generate() skips the encoder when handed its output. + ...(encoderOutputs && { encoder_outputs: encoderOutputs }), }; if (options.multilingual) { if (request.language !== undefined) generate.language = request.language; @@ -205,20 +211,47 @@ export function createWhisperDeps( }; }; - if (!options.multilingual) return { decode }; + /** Encoder output of the chunk language detection just ran on. */ + let encoded: { audio: Float32Array; hidden: Tensor } | null = null; + const decodeReusingEncoder = (audio: Float32Array, request: DecodeRequest) => { + if (encoded?.audio !== audio) { + encoded = null; // a new chunk: drop the stale encoder output + return decode(audio, request); + } + return decode(audio, request, encoded.hidden); + }; + + if (!options.multilingual) return { decode: decodeReusingEncoder }; const detectLanguage = async (audio: Float32Array): Promise => { const model = asr.model as unknown as { generation_config: WhisperGenerationConfig; generate(args: Record): Promise; + _prepare_encoder_decoder_kwargs_for_generation(args: { + inputs_tensor: Tensor; + model_inputs: Record; + model_input_name: string; + generation_config: unknown; + }): Promise<{ encoder_outputs: Tensor }>; }; const config = model.generation_config; const languages = Object.entries(config.lang_to_id ?? {}); const { input_features } = (await asr.processor(audio)) as { input_features: Tensor }; + // Encode once and keep it: the chunk's real decode reuses this encoder + // output, so detection costs one decoder step instead of a second encoder + // pass (the encoder dominates a 30 s window's cost). + const { encoder_outputs } = await model._prepare_encoder_decoder_kwargs_for_generation({ + inputs_tensor: input_features, + model_inputs: { input_features }, + model_input_name: "input_features", + generation_config: config, + }); + encoded = { audio, hidden: encoder_outputs }; const list = new LogitsProcessorList(); list.push(new LanguageOnly(new Set(languages.map(([, id]) => id)))); const sequences = await model.generate({ inputs: input_features, + encoder_outputs, decoder_input_ids: [config.decoder_start_token_id], max_new_tokens: 1, logits_processor: list, @@ -228,5 +261,5 @@ export function createWhisperDeps( if (!token) throw new Error(`Language detection produced a non-language token (${tokenId}).`); return token.slice(2, -2); // "<|da|>" → "da" }; - return { decode, detectLanguage }; + return { decode: decodeReusingEncoder, detectLanguage }; } diff --git a/tests/live-real-model.test.ts b/tests/live-real-model.test.ts index 3003937..9c9ae12 100644 --- a/tests/live-real-model.test.ts +++ b/tests/live-real-model.test.ts @@ -10,6 +10,9 @@ import { type LiveTranscribeFn, } from "../src/lib/live"; import { resolveDefaultDtype } from "../src/lib/models"; +import { LONG_FORM_DEFAULTS, transcribeLongForm } from "../src/worker/longform"; +import { createWhisperDeps, loadSileroVad } from "../src/worker/longform/transformers"; +import { decodeClip } from "./clips"; import { wordErrorRate } from "./dsp"; /** @@ -265,6 +268,9 @@ describe("live pipeline accuracy (real model, real recordings)", () => { normalize(reference), normalize((await transcribe(whole)).text), ); + // File-mode accuracy gate (spec 001): the chunked file path is held + // to the same per-clip bar as live. + expect(fileWer).toBeLessThanOrEqual(MAX_WER_BY_CLIP[clip.base] ?? MAX_WER); // The live session, with the injected clock advancing with the audio // fed so pass scheduling behaves as in real time. @@ -310,3 +316,61 @@ describe("live pipeline accuracy (real model, real recordings)", () => { ); } }); + +// --------------------------------------------------------------------------- +// File-mode accuracy of the long-form pipeline (D-WS3a): Silero VAD chunks, +// one decode per ≤ 30 s chunk — the path behind "Skip silence". Same +// per-clip bars as the live and chunked file paths above. +// --------------------------------------------------------------------------- + +describe("long-form file pipeline accuracy (real model, real recordings)", () => { + for (const clip of clips) { + test.skipIf(!available)( + `${clip.base} (${clip.language}): long-form whole-file transcript stays close to the reference`, + async () => { + const reference = await Bun.file(join(AUDIO_DIR, clip.transcriptFile)).text(); + const audio = await decodeClip(clip.audioFile); + const asr = await loadAsr(); + const vad = await loadSileroVad({ localOnly: false }); + try { + const result = await transcribeLongForm( + audio, + { + speechProbabilities: (pcm) => vad.speechProbabilities(pcm), + ...createWhisperDeps(asr, { multilingual: true }), + }, + { ...LONG_FORM_DEFAULTS, language: clip.language, task: "transcribe" }, + ); + const wer = wordErrorRate(normalize(reference), normalize(result.text)); + expect(wer).toBeLessThanOrEqual(MAX_WER_BY_CLIP[clip.base] ?? MAX_WER); + expect(result.noSpeech).toBe(false); + const last = result.chunks.at(-1)!; + expect(last.timestamp[1] ?? last.timestamp[0]).toBeLessThanOrEqual( + audio.length / RATE + 1, + ); + if (clip.language !== "en") { + // "Auto-detect" in the UI: the chunked path forces English here + // (Transformers.js 4.2 has no detection); long-form detects once + // on the first speech chunk and pins it (D-WS3a). + const auto = await transcribeLongForm( + audio, + { + speechProbabilities: (pcm) => vad.speechProbabilities(pcm), + ...createWhisperDeps(asr, { multilingual: true }), + }, + { ...LONG_FORM_DEFAULTS, task: "transcribe" }, + ); + expect(auto.language).toBe(clip.language); + expect( + wordErrorRate(normalize(reference), normalize(auto.text)), + ).toBeLessThanOrEqual(MAX_WER_BY_CLIP[clip.base] ?? MAX_WER); + } + } finally { + await vad.dispose(); + } + }, + DOWNLOAD_BUDGET_MS + + probeDurationSeconds(join(AUDIO_DIR, clip.audioFile)) * MS_PER_AUDIO_SECOND, + ); + } +}); diff --git a/tests/offline.md b/tests/offline.md index c1f0912..aa7c45f 100644 --- a/tests/offline.md +++ b/tests/offline.md @@ -106,6 +106,25 @@ Result: Works. Result: Works +## 5. "Skip silence" (long-form pipeline, D-WS3a/D-WS3b) + +1. Online, open **Advanced**, tick **Skip silence in uploaded files**, and + transcribe a file. DevTools → Network shows one request for + `onnx-community/silero-vad/…/onnx/model.onnx` (2.2 MB); the console logs + "speech-detection (VAD) model ready". The transcript arrives with progress + moving in jumps over pauses. Repeat on WebGPU (the owner's backend) and note + wall-clock against the same file with the option off. +2. Go offline (DevTools "Offline"), reload, transcribe again: no network + request at all, same long-form path (console line as above). +3. Clear only the silero-vad entries from `transformers-cache` (Application → + Cache Storage), stay offline, transcribe: still no network request; the + console says the VAD model is unavailable and the file is transcribed the + usual way. +4. Transcribe a file of pure silence or music with the option on: the + transcript reads "No speech was found in this recording." + +Result: _pending (no real browser in the dev environment)_ + ## Recording the result When this passes in a real browser, flip the `[~]` offline-verification item