diff --git a/AGENTS.md b/AGENTS.md index de0e81b..f0b905b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -17,6 +17,8 @@ Test all: `bun test` — ~19 min: `tests/live-real-model.test.ts` downloads and weights whenever huggingface.co is reachable. Run it when touching the live pipeline or decoding. Build: `bun run build` (rewrites `dist/`) Opt-in real sidecar test (needs `pip install -r server-py/requirements.txt`): `FW_INTEGRATION=1 bun test tests/transcribe.test.ts` +Opt-in real Parakeet test (~670 MB download on first run; run when touching `src/lib/parakeet.ts` or +`src/worker/parakeetEngine.ts`): `PARAKEET_REAL=1 bun test tests/parakeet-real-model.test.ts` Typecheck: `bun run typecheck` (tsc 7, < 1 s). No linter configured. Browser-only behaviour (offline PWA, mic, WebGPU) has no automated coverage: follow `tests/offline.md`, @@ -36,7 +38,9 @@ step with the commands above, in the same commit. `Bun.$`. Bun API docs: `node_modules/bun-types/docs/`. - **Two engines, two libraries (D-013).** Browser inference = Transformers.js, inside `src/worker/inference.worker.ts` only — never on the main thread. Server inference = faster-whisper - via `server-py/sidecar.py` — never Transformers.js on the server. + via `server-py/sidecar.py` — never Transformers.js on the server. The one exception is Parakeet + (`family: "parakeet"`, D-WS7a): parakeet.js in the same worker, on the **same** onnxruntime-web + instance — never call its `fromUrls`/`fromHub`/`initOrt` (own ort copy, CDN wasm path). - **`@huggingface/transformers` is patched and a devDependency on purpose.** It is pinned by `patches/@huggingface%2Ftransformers@4.2.0.patch` (D-027/D-028) and kept out of runtime deps so the Docker image stays small (D-031). Upgrading or moving it means redoing the patch: ask first. diff --git a/README.md b/README.md index 94ea7fb..b1cf3e9 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,13 @@ users can optionally run inference on the **server** instead, backed by [faster-whisper](https://github.com/SYSTRAN/faster-whisper) (CTranslate2) through a small Python sidecar (`server-py/`, Decisions.md D-013). +On desktop browsers the on-device engine also offers **NVIDIA Parakeet TDT 0.6B v3** +(English only) via [parakeet.js](https://github.com/ysdede/parakeet.js) (MIT) on the +same onnxruntime-web. Model: [nvidia/parakeet-tdt-0.6b-v3](https://huggingface.co/nvidia/parakeet-tdt-0.6b-v3), +© NVIDIA, licensed [CC-BY-4.0](https://creativecommons.org/licenses/by/4.0/); ONNX +conversion [ysdede/parakeet-tdt-0.6b-v3-onnx](https://huggingface.co/ysdede/parakeet-tdt-0.6b-v3-onnx), +downloaded unmodified by the browser on first use (Decisions.md D-WS7a). + See `docs/Architecture.md` for the design, `docs/Decisions.md` for the decision log, and `PRODUCT.md` for the product spec. diff --git a/docs/Architecture.md b/docs/Architecture.md index ea5fc28..e62deee 100644 --- a/docs/Architecture.md +++ b/docs/Architecture.md @@ -68,11 +68,15 @@ src/ audio.ts # decode + resample to 16 kHz mono Float32Array (Web Audio API) live.ts # live-mode segment loop: RMS gate, LiveSegmenter, LiveTranscriber (pure, D-016) mic.ts # microphone capture: AudioWorklet chunker (Blob URL) + fallback (D-016) - models.ts # model catalog: ids, sizes, dtype defaults per device, languages + models.ts # model catalog: ids, family (whisper|parakeet), sizes, dtype defaults per device, languages + parakeet.ts # pure Parakeet logic: ≤30 s window plan, words→chunks, vocab tokenizer, pinned URLs modelCache.ts # "downloaded?" per model via Cache Storage (D-005; reused by M3 offline indicator) engine.ts # engine abstraction: BrowserEngine | ServerEngine worker/ - inference.worker.ts # Transformers.js pipeline, WebNN/WebGPU/WASM selection + inference.worker.ts # Transformers.js pipeline, WebNN/WebGPU/WASM selection; dispatches on catalog `family` + parakeetEngine.ts # Parakeet TDT path: parakeet.js ParakeetModel on the shared onnxruntime-web (D-WS7a) + modelFiles.ts # Cache-API weight download for non-Transformers.js paths (same cache, same progress msgs) + ortShim.ts # re-exports onnxruntime-web/webgpu; parakeet.js's own ort import is aliased here protocol.ts # typed postMessage messages (shared with UI) pwa/ sw.ts # service worker (app-shell precache, /ort runtime cache) @@ -217,11 +221,40 @@ environment). See `WorkBreakdown.md` for the itemized checklist.) - Anonymous users are never blocked from browser inference — auth only unlocks the server engine. +### 4.1c Parakeet (English-only, desktop-only browser model — D-WS7a) +Catalog entries carry `family?: "whisper" | "parakeet"` (absent ⇒ whisper). For +`family: "parakeet"` the worker skips the Transformers.js pipeline: +1. **Load:** `modelFiles.ts` fetches `vocab.txt`, the decoder_joint and the + encoder from the Hub (revision-pinned URLs, `lib/parakeet.ts`) into the same + `transformers-cache` Cache Storage cache Transformers.js uses, posting the same + `load-progress` messages — so badges, the storage panel, delete and + ModelLoadProgress work unchanged. A file is cached only once complete. + Devices: `parakeetDeviceAttempts` — WASM int8 unless WebGPU is chosen + explicitly (fp16 encoder on WebGPU + int8 decoder on WASM, needs + `shader-f16`); the D-025 restart rule applies. +2. **Runtime:** `parakeetEngine.ts` builds parakeet.js's `ParakeetModel` from our + own sessions on `onnxruntime-web/webgpu` — the module Transformers.js imports, + so there is one runtime and one `/ort/` wasm. parakeet.js's `initOrt` (own ort + copy, CDN wasm path) is never called; `scripts/ortAlias.ts` points its import + at `worker/ortShim.ts` so the bundle carries one ort (`tests/workerBundle.test.ts`). +3. **Transcribe:** audio is planned into ≤ 30 s windows cut at the quietest 100 ms + near each limit (`subarray` views, no copies); each window's word timestamps + become sentence-sized `TranscriptChunk`s with absolute times. File passes post + one `partial` + `transcribe-progress` per window; live passes resolve via + `result` like Whisper's, so `LiveTranscriber`, TranscriptView and export are + untouched. `language`/`task` are ignored (English only). +4. **UI:** the picker lists it under English-only with a one-line note, hides it on + phones/tablets (`modelsForDevice`), and the server-engine list never includes + it (no faster-whisper name). With it selected, a note replaces the language/task + controls. + ## 5. Invariants - **Privacy:** browser-engine audio and transcripts never touch the network. - **Anonymous-first:** every feature except the server engine works logged out. - **Offline-first:** app shell + previously-downloaded models must work offline. -- **One browser inference codebase:** Transformers.js in the browser; the server - engine runs faster-whisper (D-013). The model catalog stays shared across both. +- **One browser inference runtime:** onnxruntime-web, driven by Transformers.js for + Whisper and by parakeet.js for Parakeet (D-WS7a) — the same ort instance, never a + second runtime; the server engine runs faster-whisper (D-013). The model catalog + stays shared across both engines (browser-only entries have no faster-whisper name). - **Worker isolation:** inference never runs on the main thread. diff --git a/docs/Decisions.md b/docs/Decisions.md index 5dc8f08..b2d03b4 100644 --- a/docs/Decisions.md +++ b/docs/Decisions.md @@ -1224,3 +1224,93 @@ whole-file output is equally bad, which is exactly what the absolute bar exists to catch. **Consequence:** full `bun test` ≈ 19 min on CPU (en_alien ≈ 20 min of it on a cold run); the fast suite excludes this file. + +## D-WS7a — Parakeet TDT 0.6B v3 as an English-only, desktop-only browser model via parakeet.js on the shared onnxruntime-web +**Status:** Accepted (2026-10-29, spec 001 Goal 7) +**Runtime decision (evidence).** `@huggingface/transformers` 4.2.0 (ours) and +4.3.2 (npm `latest`) were inspected from their published tarballs: both ship +only `ParakeetForCTC` (`models/parakeet/modeling_parakeet.js`, identical in +both), routed by the ASR pipeline as `model_type: 'parakeet_ctc'` through the +wav2vec2 CTC path (argmax, no timestamps, no chunking). Neither contains a +TDT/transducer decoder. Parakeet TDT therefore runs on **parakeet.js 1.4.4** +(MIT; pre-approved by the spec for exactly this case), a devDependency +bundled into the worker only (like Transformers.js, D-031). +**How it runs on one runtime.** parakeet.js pins its own onnxruntime-web +1.24.1 and its `fromUrls`/`fromHub` → `initOrt` path imports it, sets a +jsdelivr `wasmPaths` (blocked by COEP, D-023; not offline) and fetches +weights outside our cache. We never call those: `src/worker/parakeetEngine.ts` +constructs `ParakeetModel` directly from sessions created on +`onnxruntime-web/webgpu` — the module Transformers.js imports — with our own +vocab tokenizer and parakeet.js's `JsPreprocessor` (`parakeet.js/mel`). A Bun +build plugin (`scripts/ortAlias.ts`, used by the build and the dev server) +resolves parakeet.js's bare `onnxruntime-web` import to `src/worker/ortShim.ts`, +so the bundle carries one ort (`tests/workerBundle.test.ts` asserts it, and +that without the plugin the 1.24.1 copy *would* be bundled). We rely on +`onnxruntime-web` being hoisted from Transformers.js rather than pinning it +ourselves, so the two can never drift apart (workstream 1's upgrade moves both). +**Model.** `ysdede/parakeet-tdt-0.6b-v3-onnx` (NVIDIA's CC-BY-4.0 weights, +NeMo→ONNX), URLs pinned to revision `e5356bf6…` — the one measured and +whose exact file sizes the catalog shows (int8 encoder 652.2 MB + int8 +decoder_joint 18.2 MB + vocab 0.1 MB = "~670 MB"; WebGPU fp16 encoder +1.24 GB). Attribution in README. The model knows 25 European languages; the +app offers it **English-only** (spec), so there is no language/task UI. +**Engine path.** Catalog entries gain `family?: "whisper" | "parakeet"` +(absent ⇒ whisper) and optional `exactBytes`; `fasterWhisperModel` becomes +optional (browser-only entries have none, so the server engine 400s them and +the picker's server list excludes them). The worker dispatches on the family. +Weights go through `src/worker/modelFiles.ts` into the `transformers-cache` +Cache Storage cache under their Hub URLs, posting the same `load-progress` +messages — badges, storage panel, delete, ModelLoadProgress unchanged +(D-005, D-033); a file is `put` only once complete. Download stays +user-initiated (D-020). Audio is cut into ≤ 30 s windows (spec: ≤ ~35 s per +encoder call) at the quietest 100 ms in the last 6 s before each limit, so no +overlap merging is needed; we don't use parakeet.js's `transcribeLongAudio` +(20–180 s windows, 90 s default). Word timestamps become sentence-sized +`TranscriptChunk`s (break after `.?!`, on ≥ 1 s pauses, or at 12 s). File +passes post one `partial` + absolute `transcribe-progress` per window; live +passes return `result` like Whisper, so live mode works unchanged. +**Devices.** WASM int8 is the baseline and the only path "auto" picks. An +explicit WebGPU choice uses the fp16 encoder on WebGPU + int8 decoder on WASM +(parakeet.js's "hybrid" split), only with `shader-f16`, falling back to WASM +through the D-025 restart. Not auto: WebGPU int8 is reported broken, the fp16 +encoder is a separate 1.24 GB download, and nothing WebGPU can be measured +here. WebNN has no path (runs WASM). WASM sessions use graph optimization +"all" (D-014's "basic" cap is about Whisper's 8-bit tied embeddings; measured +fine here). +**Mobile.** Load peak ≈ 2 × 652 MB = 1.3 GB (D-027 arithmetic) — above every +budget D-029 treats as safe (deviceMemory 8 ⇒ 800 MB). Instead of a confirm, +`modelsForDevice` hides it on phones/tablets (incl. iPadOS) and +`confirmRiskyMobileLoad` refuses it. +**Measurements (CPU only; this machine, 16 cores; synthetic nothing).** +`PARAKEET_REAL=1 bun test tests/parakeet-real-model.test.ts` (onnxruntime-web +WASM under Bun, 8 threads, the worker's own engine code): + +| clip | audio | file WER | file wall | live WER | live wall | +|---|---|---|---|---|---| +| en_aiproblem | 14.4 s | 0.030 | 0.9 s | 0.030 | 2.0 s | +| en_alien | 888.4 s | 0.096 | 36.4 s | 0.115 | 282 s | + +`scripts/bench-parakeet.ts` (whole-file, same normalization, 12 threads, +model load excluded, one run each under the shared lock): + +| engine (runtime) | en_aiproblem WER / wall | en_alien WER / wall | +|---|---|---| +| Parakeet int8 (onnxruntime-node CPU) | 0.030 / 0.3 s | 0.100 / 15.3 s | +| Parakeet int8 (onnxruntime-web WASM) | 0.030 / 0.9 s | 0.096 / 34.4 s | +| Large v3 Turbo q4 (onnxruntime-node, worker options) | 0.000 / 2.9 s | 0.255 / 183.7 s | + +(D-039 recorded Turbo en_alien whole-file 0.261 / 420 s on a busier run.) On +the long, noisy clip Parakeet more than halves Turbo's WER and is ~12× faster +on the same runtime; on the short clean clip Turbo is perfect and Parakeet +misses one word ("agentic" → "agency"). No WASM number for Turbo: the +Transformers.js node build always uses onnxruntime-node. +**Consequence.** For English, Parakeet is the accurate *and* fast choice on +desktop; Whisper stays the default and the only multilingual/translate +option. Live mode is supported (faster than real time on WASM here). +**Needs the owner's browser check:** real-Chrome WASM load + transcription +speed/memory (headless Chrome run here, see WorkBreakdown); explicit WebGPU +(fp16 encoder) on the owner's GPU — success, speed vs WASM, and whether the +fallback restart behaves; Firefox/Safari WASM. +**Revisit:** if Transformers.js gains a TDT decoder, move to it and drop +parakeet.js + the alias plugin; if WebGPU measures well on real hardware, +consider letting "auto" pick it. diff --git a/docs/WorkBreakdown.md b/docs/WorkBreakdown.md index 336b0b4..faca69a 100644 --- a/docs/WorkBreakdown.md +++ b/docs/WorkBreakdown.md @@ -341,6 +341,17 @@ cached. --- +## Spec 001 Goal 7 — Parakeet TDT 0.6B v3 (English, desktop, D-WS7a) +- [x] Runtime decision: Transformers.js 4.2.0/4.3.2 have CTC-only Parakeet → parakeet.js on the + shared onnxruntime-web (one runtime in the bundle, `tests/workerBundle.test.ts`). +- [x] Catalog `family` + exact sizes; worker engine path (≤ 30 s windows, word → chunk + timestamps, progress/partials); Cache-API downloader (`modelFiles.ts`). +- [x] File and live mode; WASM int8 baseline; explicit WebGPU = fp16 encoder (opt-in only). +- [x] Hidden on phones/tablets; English-only note instead of language/task controls. +- [x] Real-model WER/wall-clock vs Large v3 Turbo (CPU), opt-in `PARAKEET_REAL=1` test. +- [ ] Owner's browser check: real Chrome WASM load/transcribe (speed, memory), explicit + WebGPU on a real GPU (works? faster than WASM? fallback restart), Firefox/Safari WASM. + ## Deliberately out of scope (for now) Arbitrary Hub model ids, OAuth providers, multi-user transcript storage, whisper.cpp server backend, i18n of the UI. @@ -372,6 +383,11 @@ Found while bootstrapping agent docs (D-036); all fixed on 2026-10-29. — Light theme via tokens + self-hosted Inter (D-037). Also fixed: the Backend select overflowed its panel on 390 px-wide phones. +- [ ] Parakeet v3 also transcribes 24 other European languages, Danish included, but the app + offers it English-only (spec 001) — unmeasured on `da_festuge`. Candidate for a later spec. +- [ ] parakeet.js still logs `[Parakeet.js]` lines to the worker console on some paths; harmless, + not silenced (no logger hook in the library). + ## Recurring - [ ] Maintain `AGENTS.md`: - **Add a rule only in response to a real failure.** When an agent repeatedly diff --git a/scripts/bench-parakeet.ts b/scripts/bench-parakeet.ts new file mode 100644 index 0000000..7e645fd --- /dev/null +++ b/scripts/bench-parakeet.ts @@ -0,0 +1,102 @@ +/** + * Parakeet vs Whisper Large v3 Turbo benchmark (D-WS7a) — CPU only. + * + * flock /tmp/whisper-heavy.lock bun scripts/bench-parakeet.ts [parakeet-node|parakeet-web|turbo ...] + * + * Whole-file WER (same normalization as tests/live-real-model.test.ts) and + * wall-clock per English clip in tests/audio_files. Parakeet runs the + * worker's own `createParakeetEngine` on onnxruntime-node (CPU EP) and on + * onnxruntime-web (WASM, the browser engine's baseline); Turbo runs the + * Transformers.js pipeline with the worker's file-flow options on + * onnxruntime-node at its WASM-default q4 dtype. Weights come from / go to + * node_modules/@huggingface/transformers/.cache (Parakeet must already be + * there — `PARAKEET_REAL=1 bun test tests/parakeet-real-model.test.ts` + * downloads it). Wall-clock excludes model load. + */ + +import { join } from "node:path"; +import ffmpegPath from "ffmpeg-static"; +import { pipeline } from "@huggingface/transformers"; +import { PARAKEET_MODEL_ID, resolveDefaultDtype } from "../src/lib/models"; +import { PARAKEET_VOCAB_FILE, parakeetFileName } from "../src/lib/parakeet"; +import { createParakeetEngine, parakeetSessionOptions, type OrtLike } from "../src/worker/parakeetEngine"; +import { wordErrorRate } from "../tests/dsp"; + +const ROOT = join(import.meta.dir, ".."); +const AUDIO = join(ROOT, "tests/audio_files"); +const PARAKEET_DIR = join(ROOT, "node_modules/@huggingface/transformers/.cache", PARAKEET_MODEL_ID); +const CLIPS = ["en_aiproblem.m4a", "en_alien.opus"]; +const THREADS = 12; + +const normalize = (text: string) => + text.toLowerCase().replace(/['’]/g, "").replace(/[.,!?;:"“”‘‚«»()\-–—…]/g, " ").replace(/\s+/g, " ").trim(); + +async function decode(file: string): Promise { + const proc = Bun.spawn( + [ffmpegPath as unknown as string, "-hide_banner", "-loglevel", "error", "-i", join(AUDIO, file), + "-ac", "1", "-ar", "16000", "-f", "f32le", "-"], + { stdout: "pipe" }, + ); + return new Float32Array(await new Response(proc.stdout).arrayBuffer()); +} + +type Transcribe = (audio: Float32Array) => Promise; + +async function parakeet(runtime: "node" | "web"): Promise { + let ort: OrtLike; + let options = parakeetSessionOptions("wasm"); + if (runtime === "web") { + const web = await import("onnxruntime-web"); + web.env.wasm.numThreads = THREADS; + ort = web as unknown as OrtLike; + } else { + ort = (await import("onnxruntime-node")) as unknown as OrtLike; + const cpu = { executionProviders: ["cpu"], intraOpNumThreads: THREADS, graphOptimizationLevel: "all" }; + options = { encoder: cpu, decoder: cpu }; + } + const read = async (file: string) => new Uint8Array(await Bun.file(join(PARAKEET_DIR, file)).arrayBuffer()); + const engine = await createParakeetEngine( + ort, + { + encoder: await read(parakeetFileName("encoder", "q8")), + decoder: await read(parakeetFileName("decoder", "q8")), + vocab: await Bun.file(join(PARAKEET_DIR, PARAKEET_VOCAB_FILE)).text(), + }, + options, + ); + return async (audio) => (await engine.transcribe(audio)).text; +} + +async function turbo(): Promise { + const id = "onnx-community/whisper-large-v3-turbo"; + const asr = await pipeline("automatic-speech-recognition", id, { dtype: resolveDefaultDtype(id, "wasm") }); + return async (audio) => { + const output = await asr(audio, { + chunk_length_s: 30, stride_length_s: 5, return_timestamps: true, max_new_tokens: 440, + language: "en", task: "transcribe", + }); + return ((Array.isArray(output) ? output[0] : output) as { text?: string }).text ?? ""; + }; +} + +const engines: Record Promise> = { + "parakeet-node": () => parakeet("node"), + "parakeet-web": () => parakeet("web"), + turbo, +}; + +for (const name of process.argv.slice(2).length ? process.argv.slice(2) : Object.keys(engines)) { + const transcribe = await engines[name]!(); + for (const clip of CLIPS) { + const audio = await decode(clip); + const reference = await Bun.file(join(AUDIO, clip.replace(/\.[^.]+$/, ".txt"))).text(); + const start = performance.now(); + const text = await transcribe(audio); + const seconds = (performance.now() - start) / 1000; + const wer = wordErrorRate(normalize(reference), normalize(text)); + console.log( + `${name}\t${clip}\taudio ${(audio.length / 16000).toFixed(1)} s\twall ${seconds.toFixed(1)} s\tWER ${wer.toFixed(3)}`, + ); + } +} +process.exit(0);