diff --git a/Architecture.md b/Architecture.md index 51b9a77..d822182 100644 --- a/Architecture.md +++ b/Architecture.md @@ -24,7 +24,7 @@ Logged-in users may optionally route inference to the server. │ ▼ │ │ Inference Web Worker │ │ ├─ Transformers.js ASR pipeline (Whisper) │ -│ ├─ Device selection: WebGPU → WASM fallback │ +│ ├─ Device selection: WebNN → WebGPU → WASM fallback │ │ └─ Model weights cached via browser Cache API │ │ │ │ Service Worker (PWA) │ @@ -72,7 +72,7 @@ src/ modelCache.ts # "downloaded?" per model via Cache Storage (D-005; reused by M3 offline indicator) engine.ts # engine abstraction: BrowserEngine | ServerEngine worker/ - inference.worker.ts # Transformers.js pipeline, WebGPU/WASM selection + inference.worker.ts # Transformers.js pipeline, WebNN/WebGPU/WASM selection protocol.ts # typed postMessage messages (shared with UI) pwa/ sw.ts # service worker (app-shell precache, /ort runtime cache) @@ -89,7 +89,8 @@ tests/ # bun test ``` (Current state: M0–M3 built — browser transcription with the curated model -catalog, WebGPU→WASM auto-selection, model switching, language/task options, +catalog, WebNN→WebGPU→WASM auto-selection (D-017), model switching, +language/task options, and the installable offline PWA (`pwa/`, `scripts/build.ts` production build, `dist/` serving in `src/index.ts`; browser-level offline behavior pending the manual run in `tests/offline.md`). M4 accounts are built (`server/db.ts`, @@ -108,12 +109,15 @@ are the remaining M5 work; see `WorkBreakdown.md`.) 1. User picks a model in `ModelPicker` (curated catalog from `lib/models.ts`, with size estimates and a Cache-API-backed "downloaded" badge); UI posts `{type:"load", modelId, device}` where `device` is a preference - (`"auto" | "webgpu" | "wasm"`, default `"auto"`). An explicit `dtype` override - is expressible in the message but normally omitted. -2. Worker resolves the device: `navigator.gpu?.requestAdapter()` → `"webgpu"` if an - adapter exists (and pipeline creation succeeds — otherwise it retries on WASM), - else `"wasm"` (D-004). It then resolves the catalog's dtype default for that - device and creates the Transformers.js `automatic-speech-recognition` pipeline. + (`"auto" | "webnn" | "webgpu" | "wasm"`, default `"auto"`). An explicit `dtype` + override is expressible in the message but normally omitted. +2. Worker resolves the device (D-004, D-017): for `"auto"` it probes + `navigator.ml` (WebNN) first, then `navigator.gpu?.requestAdapter()` (WebGPU), + and picks in priority order WebNN → WebGPU → WASM; if pipeline creation on the + picked device throws, it retries down the same chain (WASM always last). An + explicit `"webnn"`/`"webgpu"` preference falls back straight to WASM. It then + resolves the catalog's dtype default for the device actually used and creates + the Transformers.js `automatic-speech-recognition` pipeline. Download progress events are forwarded to the UI; the terminal `{status:"ready", device}` message reports the backend actually selected, which the UI surfaces. Loading a different model/device disposes the old pipeline diff --git a/CLAUDE.md b/CLAUDE.md index 91433ea..8c6e422 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -12,7 +12,7 @@ A browser-first speech-to-text web app powered by OpenAI Whisper, running throug - Run Whisper models fully in the browser (no data leaves the device). - Choose which Whisper model to use (tiny → large, English-only and multilingual, quantized variants). - Keep working when the network — or our own server — is offline (installable PWA + cached model weights). -- Accelerate inference with WebGPU, falling back to WASM automatically. +- Accelerate inference with WebNN or WebGPU (auto-selected in that order, D-017), falling back to WASM automatically. - Transcribe from an uploaded audio/video file. **Logged-in users (Passport.js) get all of the above, plus:** diff --git a/Decisions.md b/Decisions.md index bf18d45..18275d7 100644 --- a/Decisions.md +++ b/Decisions.md @@ -31,7 +31,7 @@ thread would freeze the UI. Communication via a small typed postMessage protocol (`src/worker/protocol.ts`) shared between UI and worker. ## D-004 — WebGPU with automatic WASM fallback -**Status:** Accepted +**Status:** Accepted (amended by D-017 — "auto" now probes WebNN first) Detect via `navigator.gpu?.requestAdapter()` inside the worker; pass `device: "webgpu" | "wasm"` to the pipeline. The active backend is surfaced in the UI but users don't have to configure anything. Encoder-heavy Whisper benefits ~10-40× from @@ -222,3 +222,20 @@ latency); MediaRecorder + re-decoding compressed blobs (adds codec latency and device — the UI must keep tentative text visibly distinct from committed text. Server-engine live input (WebSocket streaming to faster-whisper) is a separate future decision; D-011's "WebSocket if needed" note is the hook. + +## D-017 — "Auto" device order: WebNN → WebGPU → WASM +**Status:** Accepted (2026-07-14; amends D-004, which only knew WebGPU → WASM) +With WebNN support added (commit "WebNN support initial"), the `"auto"` device +preference resolves in priority order **WebNN → WebGPU → WASM**: the worker +probes `navigator.ml` (WebNN) first, then `navigator.gpu.requestAdapter()` +(WebGPU), and always keeps WASM as the last resort. The probe/ordering policy +is pure and unit-tested (`deviceAttempts` in `src/worker/protocol.ts`, +`tests/protocol.test.ts`); the worker only supplies the capability probe. +Pipeline-creation failures walk the same chain — an auto-detected WebNN that +fails still tries WebGPU before WASM. An **explicit** `"webnn"` or `"webgpu"` +preference deliberately falls back straight to WASM (no detour through the +other accelerator — the user asked for a specific backend, so honor it or take +the universal fallback). The selected backend stays user-visible: the worker's +`{status:"ready", device}` message drives a "Running on …" badge next to the +Backend select (`ModelPicker`) and the engine line in the footer (D-004's +"surfaced in the UI" requirement). diff --git a/WorkBreakdown.md b/WorkBreakdown.md index 38907ff..1ba5d1c 100644 --- a/WorkBreakdown.md +++ b/WorkBreakdown.md @@ -8,6 +8,10 @@ Legend: `[ ]` todo · `[x]` done · `[~]` in progress --- +## WIP Tasks before usage limit cutoff: +- [~] Allow users to cancel current transcription. +- [~] Better user-feedback while transcription is in progress (better than "Listening for the first few words") + ## M0 — Scaffold cleanup & docs (small) Current state is the raw `bun init` React template. diff --git a/bun.lock b/bun.lock index 9e751c1..80bff18 100644 --- a/bun.lock +++ b/bun.lock @@ -17,10 +17,16 @@ "@types/passport-local": "^1.0.38", "@types/react": "^19", "@types/react-dom": "^19", + "ffmpeg-static": "^5.3.0", }, }, }, + "trustedDependencies": [ + "ffmpeg-static", + ], "packages": { + "@derhuerst/http-basic": ["@derhuerst/http-basic@8.2.4", "", { "dependencies": { "caseless": "^0.12.0", "concat-stream": "^2.0.0", "http-response-object": "^3.0.1", "parse-cache-control": "^1.0.1" } }, "sha512-F9rL9k9Xjf5blCz8HsJRO4diy111cayL2vkY2XE4r4t3n0yPXVYy3KD3nJ1qbrSn9743UWSXH4IwuCa/HWlGFw=="], + "@emnapi/runtime": ["@emnapi/runtime@1.11.2", "", { "dependencies": { "tslib": "^2.4.0" } }, "sha512-kyOl3X0DuTiT1h2ft8r2fYO8JYtU9a9Xis/zBSiGArNaagCOWx90N1k2wxp18czFDH+OgcWGb5ZP/XMt3dcyPA=="], "@huggingface/jinja": ["@huggingface/jinja@0.5.9", "", {}, "sha512-uWTG+l3VJRsl7EXxYizuL3P+cCPoc3cRqbWWRcQN0FhejRfbdq0RNhCmbY/YDtnTcz9icdLYuLDjsnz4d8JMuw=="], @@ -131,12 +137,22 @@ "adm-zip": ["adm-zip@0.5.18", "", {}, "sha512-ufJnssQGbxzLNS1Ho9bCtX4rQKCCvoVuDLHoJyc3F9dOGDB4BkWs2Ci0kv53lqocAEQ/Cbi+I2XCsNYGqVYqng=="], + "agent-base": ["agent-base@6.0.2", "", { "dependencies": { "debug": "4" } }, "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ=="], + "boolean": ["boolean@3.2.0", "", {}, "sha512-d0II/GO9uf9lfUHH2BQsjxzRJZBdsjgsBiW4BvhWk/3qoKwQFjIDVN19PfX8F2D/r9PCMTtLWjYVCFrpeYUzsw=="], + "buffer-from": ["buffer-from@1.1.2", "", {}, "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ=="], + "bun-types": ["bun-types@1.3.14", "", { "dependencies": { "@types/node": "*" } }, "sha512-4N0ig0fEomHt5R0KCFWjovxow98rIoRwKolrYdCcknNwMekCXRnWEUvgu5soYV8QXtVsrUD8B95MBOZGPvr6KQ=="], + "caseless": ["caseless@0.12.0", "", {}, "sha512-4tYFyifaFfGacoiObjJegolkwSU4xQNGbVgUiNYVUxbQ2x2lUsFvY4hVgVzGiIe6WLOPqycWXA40l+PWsxthUw=="], + + "concat-stream": ["concat-stream@2.0.0", "", { "dependencies": { "buffer-from": "^1.0.0", "inherits": "^2.0.3", "readable-stream": "^3.0.2", "typedarray": "^0.0.6" } }, "sha512-MWufYdFw53ccGjCA+Ol7XJYpAlW6/prSMzuPOTRnJGcGzuhLn4Scrz7qf6o8bROZ514ltazcIFJZevcfbo0x7A=="], + "csstype": ["csstype@3.2.3", "", {}, "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ=="], + "debug": ["debug@4.4.3", "", { "dependencies": { "ms": "^2.1.3" }, "peerDependencies": { "supports-color": "*" }, "optionalPeers": ["supports-color"] }, "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA=="], + "define-data-property": ["define-data-property@1.1.4", "", { "dependencies": { "es-define-property": "^1.0.0", "es-errors": "^1.3.0", "gopd": "^1.0.1" } }, "sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A=="], "define-properties": ["define-properties@1.2.1", "", { "dependencies": { "define-data-property": "^1.0.1", "has-property-descriptors": "^1.0.0", "object-keys": "^1.1.1" } }, "sha512-8QmQKqEASLd5nx0U1B1okLElbUuuttJ/AnYmRXbbbGDWh6uS208EjD4Xqq/I9wK7u0v6O08XhTWnt5XtEbR6Dg=="], @@ -145,6 +161,8 @@ "detect-node": ["detect-node@2.1.0", "", {}, "sha512-T0NIuQpnTvFDATNuHN5roPwSBG83rFsuO+MXXH9/3N1eFbn4wcPjttvjMLEPWJ0RGUYgQE7cGgS3tNxbqCGM7g=="], + "env-paths": ["env-paths@2.2.1", "", {}, "sha512-+h1lkLKhZMTYjog1VEpJNG7NZJWcuc2DDk/qsqSTRRCOXiLjeQ1d1/udrUGhqMxUgAlwKNZ0cf2uqan5GLuS2A=="], + "es-define-property": ["es-define-property@1.0.1", "", {}, "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g=="], "es-errors": ["es-errors@1.3.0", "", {}, "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw=="], @@ -153,6 +171,8 @@ "escape-string-regexp": ["escape-string-regexp@4.0.0", "", {}, "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA=="], + "ffmpeg-static": ["ffmpeg-static@5.3.0", "", { "dependencies": { "@derhuerst/http-basic": "^8.2.0", "env-paths": "^2.2.0", "https-proxy-agent": "^5.0.0", "progress": "^2.0.3" } }, "sha512-H+K6sW6TiIX6VGend0KQwthe+kaceeH/luE8dIZyOP35ik7ahYojDuqlTV1bOrtEwl01sy2HFNGQfi5IDJvotg=="], + "flatbuffers": ["flatbuffers@25.9.23", "", {}, "sha512-MI1qs7Lo4Syw0EOzUl0xjs2lsoeqFku44KpngfIduHBYvzm8h2+7K8YMQh1JtVVVrUvhLpNwqVi4DERegUJhPQ=="], "global-agent": ["global-agent@3.0.0", "", { "dependencies": { "boolean": "^3.0.1", "es6-error": "^4.1.1", "matcher": "^3.0.0", "roarr": "^2.15.3", "semver": "^7.3.2", "serialize-error": "^7.0.1" } }, "sha512-PT6XReJ+D07JvGoxQMkT6qji/jVNfX/h364XHZOWeRzy64sSFr+xJ5OX7LI3b4MPQzdL4H8Y8M0xzPpsVMwA8Q=="], @@ -165,12 +185,20 @@ "has-property-descriptors": ["has-property-descriptors@1.0.2", "", { "dependencies": { "es-define-property": "^1.0.0" } }, "sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg=="], + "http-response-object": ["http-response-object@3.0.2", "", { "dependencies": { "@types/node": "^10.0.3" } }, "sha512-bqX0XTF6fnXSQcEJ2Iuyr75yVakyjIDCqroJQ/aHfSdlM743Cwqoi2nDYMzLGWUcuTWGWy8AAvOKXTfiv6q9RA=="], + + "https-proxy-agent": ["https-proxy-agent@5.0.1", "", { "dependencies": { "agent-base": "6", "debug": "4" } }, "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA=="], + + "inherits": ["inherits@2.0.4", "", {}, "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="], + "json-stringify-safe": ["json-stringify-safe@5.0.1", "", {}, "sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA=="], "long": ["long@5.3.2", "", {}, "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA=="], "matcher": ["matcher@3.0.0", "", { "dependencies": { "escape-string-regexp": "^4.0.0" } }, "sha512-OkeDaAZ/bQCxeFAozM55PKcKU0yJMPGifLwV4Qgjitu+5MoAfSQN4lsLJeXZ1b8w0x+/Emda6MZgXS1jvsapng=="], + "ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="], + "object-keys": ["object-keys@1.1.1", "", {}, "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA=="], "onnxruntime-common": ["onnxruntime-common@1.24.3", "", {}, "sha512-GeuPZO6U/LBJXvwdaqHbuUmoXiEdeCjWi/EG7Y1HNnDwJYuk6WUbNXpF6luSUY8yASul3cmUlLGrCCL1ZgVXqA=="], @@ -179,6 +207,8 @@ "onnxruntime-web": ["onnxruntime-web@1.26.0-dev.20260416-b7804b056c", "", { "dependencies": { "flatbuffers": "^25.1.24", "guid-typescript": "^1.0.9", "long": "^5.2.3", "onnxruntime-common": "1.24.0-dev.20251116-b39e144322", "platform": "^1.3.6", "protobufjs": "^7.2.4" } }, "sha512-MD6Ss4GSpQBo6zqoJzyT9LRbKYs7x/JVN23FT24EcEvlqF4VuzPOeH6X38orZPKHQDbprn7K+SBpu0/mj2CQiw=="], + "parse-cache-control": ["parse-cache-control@1.0.1", "", {}, "sha512-60zvsJReQPX5/QP0Kzfd/VrpjScIQ7SHBW6bFCYfEP+fp0Eppr1SHhIO5nd1PjZtvclzSzES9D/p5nFJurwfWg=="], + "passport": ["passport@0.7.0", "", { "dependencies": { "passport-strategy": "1.x.x", "pause": "0.0.1", "utils-merge": "^1.0.1" } }, "sha512-cPLl+qZpSc+ireUvt+IzqbED1cHHkDoVYMo30jbJIdOOjQ1MQYZBPiNvmi8UM6lJuOpTPXJGZQk0DtC4y61MYQ=="], "passport-local": ["passport-local@1.0.0", "", { "dependencies": { "passport-strategy": "1.x.x" } }, "sha512-9wCE6qKznvf9mQYYbgJ3sVOHmCWoUNMVFoZzNoznmISbhnNNPhN9xfY3sLmScHMetEJeoY7CXwfhCe7argfQow=="], @@ -189,14 +219,20 @@ "platform": ["platform@1.3.6", "", {}, "sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg=="], + "progress": ["progress@2.0.3", "", {}, "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA=="], + "protobufjs": ["protobufjs@7.6.5", "", { "dependencies": { "@protobufjs/aspromise": "^1.1.2", "@protobufjs/base64": "^1.1.2", "@protobufjs/codegen": "^2.0.5", "@protobufjs/eventemitter": "^1.1.1", "@protobufjs/fetch": "^1.1.1", "@protobufjs/float": "^1.0.2", "@protobufjs/path": "^1.1.2", "@protobufjs/pool": "^1.1.0", "@protobufjs/utf8": "^1.1.1", "@types/node": ">=13.7.0", "long": "^5.3.2" } }, "sha512-/FPD0nUc9jH6rfFjji9IBqOz4pcSE3CsT1m7Ep6Mdb0LxSUMj8hgl6GomOvZzpNpAqqGaXA0P3VSrZLFzIhQrw=="], "react": ["react@19.2.7", "", {}, "sha512-HNe9WslTbXmFK8o8cmwgAeJFSBvt1bPdHCVKtaaV+WlAN36mpT4hcRpwbf3fY56ar2oIXzsBpOAiIRHAdY0OlQ=="], "react-dom": ["react-dom@19.2.7", "", { "dependencies": { "scheduler": "^0.27.0" }, "peerDependencies": { "react": "^19.2.7" } }, "sha512-t0BRVXvbiE/o20Hfw669rLbMCDWtYZLvmJigy2f0MxsXF+71pxhR3xOkspmsO8h3ZlNzyibAmtCa3l4lYKk6gQ=="], + "readable-stream": ["readable-stream@3.6.2", "", { "dependencies": { "inherits": "^2.0.3", "string_decoder": "^1.1.1", "util-deprecate": "^1.0.1" } }, "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA=="], + "roarr": ["roarr@2.15.4", "", { "dependencies": { "boolean": "^3.0.1", "detect-node": "^2.0.4", "globalthis": "^1.0.1", "json-stringify-safe": "^5.0.1", "semver-compare": "^1.0.0", "sprintf-js": "^1.1.2" } }, "sha512-CHhPh+UNHD2GTXNYhPWLnU8ONHdI+5DI+4EYIAOaiD63rHeYlZvyh8P+in5999TTSFgUYuKUAjzRI4mdh/p+2A=="], + "safe-buffer": ["safe-buffer@5.2.1", "", {}, "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ=="], + "scheduler": ["scheduler@0.27.0", "", {}, "sha512-eNv+WrVbKu1f3vbYJT/xtiF5syA5HPIMtf9IgY/nKg0sWqzAUEvqY/xm7OcZc/qafLx/iO9FgOmeSAp4v5ti/Q=="], "semver": ["semver@7.8.5", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA=="], @@ -209,14 +245,22 @@ "sprintf-js": ["sprintf-js@1.1.3", "", {}, "sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA=="], + "string_decoder": ["string_decoder@1.3.0", "", { "dependencies": { "safe-buffer": "~5.2.0" } }, "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA=="], + "tslib": ["tslib@2.8.1", "", {}, "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w=="], "type-fest": ["type-fest@0.13.1", "", {}, "sha512-34R7HTnG0XIJcBSn5XhDd7nNFPRcXYRZrBB2O2jdKqYODldSzBAqzsWoZYYvduky73toYS/ESqxPvkDf/F0XMg=="], + "typedarray": ["typedarray@0.0.6", "", {}, "sha512-/aCDEGatGvZ2BIk+HmLf4ifCJFwvKFNb9/JeZPMulfgFracn9QFcAf5GO8B/mweUjSoblS5In0cWhqpfs/5PQA=="], + "undici-types": ["undici-types@8.3.0", "", {}, "sha512-j375ScV60dom+YkPFIfTLcOiPxkN/buHz5GobjLhixFuANaNs3C9l4GmrWqejgXWJ7BbJcFYpTEUkS1Ge8bpZQ=="], + "util-deprecate": ["util-deprecate@1.0.2", "", {}, "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw=="], + "utils-merge": ["utils-merge@1.0.1", "", {}, "sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA=="], + "http-response-object/@types/node": ["@types/node@10.17.60", "", {}, "sha512-F0KIgDJfy2nA3zMLmWGKxcH2ZVEtCZXHHdOQs2gSaQ27+lNeEfGxzkIw90aXswATX7AZ33tahPbzy6KAfUreVw=="], + "onnxruntime-web/onnxruntime-common": ["onnxruntime-common@1.24.0-dev.20251116-b39e144322", "", {}, "sha512-BOoomdHYmNRL5r4iQ4bMvsl2t0/hzVQ3OM3PHD0gxeXu1PmggqBv3puZicEUVOA3AtHHYmqZtjMj9FOfGrATTw=="], } } diff --git a/package.json b/package.json index 41a6ab6..e08bfae 100644 --- a/package.json +++ b/package.json @@ -21,6 +21,10 @@ "@types/passport": "^1.0.17", "@types/passport-local": "^1.0.38", "@types/react": "^19", - "@types/react-dom": "^19" - } + "@types/react-dom": "^19", + "ffmpeg-static": "^5.3.0" + }, + "trustedDependencies": [ + "ffmpeg-static" + ] } diff --git a/src/App.tsx b/src/App.tsx index 1aeedef..3f50011 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -59,6 +59,11 @@ export function App() { ); const nextLiveRequest = useRef(0); + // File-flow run counter (D-018). Cancel bumps it, so a decode that was in + // flight when the user cancelled finds itself stale and never posts its + // audio to the (fresh) worker. Decoding itself cannot be aborted. + const fileRun = useRef(0); + const [busy, setBusy] = useState(false); const [decoding, setDecoding] = useState(false); const [loadStarted, setLoadStarted] = useState(false); @@ -213,8 +218,40 @@ export function App() { if (loadStarted) startLoad(modelId, preference); } + /** + * Cancel the in-flight file transcription (D-018). Transformers.js has no + * abort API for a running `asr()` call — and on the WASM backend the pass + * blocks the worker's event loop, so even a "cancel" message would sit + * unread until the pass finished. Terminating the worker is the only + * reliable preemption; it also aborts an in-flight model download. The next + * run spawns a fresh worker and reloads the pipeline — cheap, because the + * weights stay in the browser Cache API (D-005). + */ + function cancelTranscription() { + if (!busy) return; + fileRun.current++; // any decode still running is now stale + workerRef.current?.terminate(); + workerRef.current = null; + // The file flow never has live passes pending, but if anything is in the + // map (defensive), unblock it so nothing awaits a dead worker forever. + const cancelled = new Error("Transcription cancelled."); + for (const pending of pendingLive.current.values()) pending.reject(cancelled); + pendingLive.current.clear(); + setBusy(false); + setPartialText(""); + setResult(null); + setFileName(null); + setError(null); + // The worker (and any pipeline / partial download) is gone — reset the + // load state so the next transcription reloads from a clean slate. + setLoadStarted(false); + setLoadProgress(initialLoadProgress); + setActiveDevice(null); + } + async function handleFile(file: File) { if (busy || liveStatus !== "idle") return; + const run = ++fileRun.current; setBusy(true); setError(null); setPartialText(""); @@ -237,6 +274,7 @@ export function App() { // Decoded entirely on-device via Web Audio (D-012); the audio never // touches the network (Architecture.md §5). const audio = await decodeAudioFile(file); + if (fileRun.current !== run) return; // cancelled while decoding (D-018) // Transfer the buffer to the worker — zero copy. Language/task only // apply to multilingual models; `.en` variants take neither. postToWorker( @@ -247,6 +285,7 @@ export function App() { [audio.buffer as ArrayBuffer], ); } catch (decodeError) { + if (fileRun.current !== run) return; // this run was already cancelled setError(decodeError instanceof Error ? decodeError.message : String(decodeError)); setBusy(false); } finally { diff --git a/src/components/ModelPicker.tsx b/src/components/ModelPicker.tsx index 73d060f..a51db1f 100644 --- a/src/components/ModelPicker.tsx +++ b/src/components/ModelPicker.tsx @@ -22,12 +22,19 @@ interface ModelPickerProps { } const DEVICE_OPTIONS: readonly { value: DevicePreference; label: string }[] = [ - { value: "auto", label: "Auto (WebGPU if available)" }, + { value: "auto", label: "Auto (WebNN → WebGPU → WASM)" }, { value: "webgpu", label: "WebGPU" }, { value: "webnn", label: "WebNN" }, { value: "wasm", label: "WASM (CPU)" }, ]; +/** Human-readable backend names, shared with the active-device badge. */ +export const DEVICE_LABELS: Record = { + webnn: "WebNN", + webgpu: "WebGPU", + wasm: "WASM", +}; + /** * Curated model list (D-006): size estimate before download, English-only vs * multilingual grouping, downloaded badge from the Cache API (D-005), and a @@ -86,6 +93,17 @@ export function ModelPicker({ ))} + {/* Which backend the worker actually selected (D-004/D-017) — the + user's preference is only a preference, so show the outcome. */} + {activeDevice && ( + + Running on {DEVICE_LABELS[activeDevice]} + + )} {disabled && (

diff --git a/src/lib/models.ts b/src/lib/models.ts index 749eb7b..c9e5415 100644 --- a/src/lib/models.ts +++ b/src/lib/models.ts @@ -256,7 +256,7 @@ export function estimateDefaultDownloadBytes( export const DEFAULT_MODEL_ID = "onnx-community/whisper-tiny.en"; -/** UI default: let the worker probe WebGPU and fall back to WASM (D-004). */ +/** UI default: worker probes WebNN → WebGPU → WASM in order (D-004, D-017). */ export const DEFAULT_DEVICE_PREFERENCE: DevicePreference = "auto"; export const DEFAULT_TASK: WhisperTask = "transcribe"; diff --git a/src/worker/inference.worker.ts b/src/worker/inference.worker.ts index fb03b93..d5320b0 100644 --- a/src/worker/inference.worker.ts +++ b/src/worker/inference.worker.ts @@ -2,9 +2,10 @@ * Inference Web Worker (Decisions.md D-003 — inference never runs on the main * thread). Hosts the Transformers.js `automatic-speech-recognition` pipeline. * - * M2: WebGPU detection with automatic WASM fallback (D-004), catalog-driven - * dtype defaults (D-006), model switching with pipeline disposal, and - * language/task options for multilingual models. + * M2: hardware backend detection with automatic fallback — "auto" resolves + * WebNN → WebGPU → WASM (D-004, D-017) — catalog-driven dtype defaults + * (D-006), model switching with pipeline disposal, and language/task options + * for multilingual models. */ import { @@ -16,7 +17,9 @@ import { } from "@huggingface/transformers"; import { resolveDefaultDtype } from "../lib/models"; import { + deviceAttempts, isUiToWorkerMessage, + type DeviceCapabilities, type DevicePreference, type InferenceDevice, type LoadMessage, @@ -56,7 +59,7 @@ function errorMessage(error: unknown): string { * Forward Transformers.js progress events to the UI as load-progress messages. * The library's own "ready" event is swallowed — the worker posts its own * `ready` (with the selected device) once the pipeline actually resolved, - * which also covers the WebGPU → WASM fallback retry (D-004). + * which also covers the device fallback retries (D-004, D-017). */ function forwardProgress(info: ProgressInfo): void { switch (info.status) { @@ -83,7 +86,9 @@ function forwardProgress(info: ProgressInfo): void { } // --------------------------------------------------------------------------- -// Device selection (D-004): WebGPU when available, automatic WASM fallback +// Device selection (D-004, D-017): "auto" resolves WebNN → WebGPU → WASM. +// The probe only answers "does the backend exist?"; the attempt ordering and +// fallback policy live in the pure `deviceAttempts` (protocol.ts, unit-tested). // --------------------------------------------------------------------------- interface NavigatorWithHardware { @@ -91,21 +96,26 @@ interface NavigatorWithHardware { ml?: unknown; } -async function detectDevice(preference: DevicePreference): Promise { - if (preference === "wasm") return "wasm"; - if (preference === "webnn") { +async function probeCapabilities( + preference: DevicePreference, +): Promise { + const capabilities: DeviceCapabilities = { webnn: false, webgpu: false }; + const hardware = navigator as NavigatorWithHardware; + if (preference === "auto" || preference === "webnn") { try { - return (navigator as NavigatorWithHardware).ml ? "webnn" : "wasm"; + capabilities.webnn = Boolean(hardware.ml); } catch { - return "wasm"; + // Probing threw — treat WebNN as unavailable. } } - try { - const adapter = await (navigator as NavigatorWithHardware).gpu?.requestAdapter(); - return adapter ? "webgpu" : "wasm"; - } catch { - return "wasm"; + if (preference === "auto" || preference === "webgpu") { + try { + capabilities.webgpu = Boolean(await hardware.gpu?.requestAdapter()); + } catch { + // Probing threw — treat WebGPU as unavailable. + } } + return capabilities; } // --------------------------------------------------------------------------- @@ -131,12 +141,14 @@ function loadKey(message: LoadMessage): string { async function createPipeline( message: LoadMessage, ): Promise { - const detected = await detectDevice(message.device); - // If WebGPU was selected but pipeline creation throws (driver quirks, - // unsupported ops, out-of-memory), fall back to WASM automatically (D-004). - const attempts: InferenceDevice[] = - detected === "webgpu" ? ["webgpu", "wasm"] : - detected === "webnn" ? ["webnn", "wasm"] : ["wasm"]; + // Ordered devices to try (D-004/D-017): the first is the selected device; + // if pipeline creation throws there (driver quirks, unsupported ops, + // out-of-memory), the next one is tried automatically. For "auto" the chain + // is WebNN → WebGPU → WASM (filtered by the probe); WASM always terminates it. + const attempts: InferenceDevice[] = deviceAttempts( + message.device, + await probeCapabilities(message.device), + ); let lastError: unknown; for (const device of attempts) { diff --git a/src/worker/protocol.ts b/src/worker/protocol.ts index d577646..d8d5207 100644 --- a/src/worker/protocol.ts +++ b/src/worker/protocol.ts @@ -10,8 +10,10 @@ export type InferenceDevice = "webgpu" | "wasm" | "webnn"; /** - * What the UI may request. "auto" (the default) lets the worker probe - * `navigator.gpu` and pick WebGPU with automatic WASM fallback (D-004). + * What the UI may request. "auto" (the default) lets the worker probe the + * available backends and pick in priority order WebNN → WebGPU → WASM + * (D-004/D-017), with automatic fallback down the same chain if pipeline + * creation fails on the picked device. */ export type DevicePreference = InferenceDevice | "auto"; @@ -79,7 +81,7 @@ export type UiToWorkerMessage = LoadMessage | TranscribeMessage; * Model download / initialization progress. Mirrors the Transformers.js * `progress_callback` events we care about; `status: "ready"` marks the end * of a successful load and reports the backend that actually got selected - * (after WebGPU detection / fallback, D-004). + * (after device detection / fallback, D-004, D-017). */ export type LoadProgressMessage = | { type: "load-progress"; status: "initiate"; file: string } @@ -135,6 +137,53 @@ export type WorkerToUiMessage = | ResultMessage | ErrorMessage; +// --------------------------------------------------------------------------- +// Device selection policy (D-004, D-017) — pure so it is unit-testable; the +// worker probes the browser capabilities and this decides the attempt order. +// --------------------------------------------------------------------------- + +/** What the worker's capability probe found (see inference.worker.ts). */ +export interface DeviceCapabilities { + /** `navigator.ml` exists (WebNN). */ + webnn: boolean; + /** `navigator.gpu.requestAdapter()` returned an adapter (WebGPU). */ + webgpu: boolean; +} + +/** + * Resolve a device preference against probed capabilities into the ordered + * list of devices to attempt pipeline creation on. The first entry is the + * selected device; later entries are automatic fallbacks for when creation + * throws (driver quirks, unsupported ops, out-of-memory). Never empty — WASM + * is always the last resort. + * + * - "auto": WebNN → WebGPU → WASM (D-017), skipping backends the probe ruled + * out, so an auto-detected WebNN that fails still tries WebGPU before WASM. + * - Explicit "webnn"/"webgpu": that device (if available) → WASM. An explicit + * choice deliberately does not detour through the other accelerator. + * - "wasm": WASM only. + */ +export function deviceAttempts( + preference: DevicePreference, + capabilities: DeviceCapabilities, +): InferenceDevice[] { + switch (preference) { + case "wasm": + return ["wasm"]; + case "webgpu": + return capabilities.webgpu ? ["webgpu", "wasm"] : ["wasm"]; + case "webnn": + return capabilities.webnn ? ["webnn", "wasm"] : ["wasm"]; + case "auto": { + const attempts: InferenceDevice[] = []; + if (capabilities.webnn) attempts.push("webnn"); + if (capabilities.webgpu) attempts.push("webgpu"); + attempts.push("wasm"); + return attempts; + } + } +} + // --------------------------------------------------------------------------- // Runtime guards (postMessage data arrives untyped) // --------------------------------------------------------------------------- diff --git a/tests/audio_files/Festuge.48k.wav b/tests/audio_files/Festuge.48k.wav deleted file mode 100644 index e693d89..0000000 Binary files a/tests/audio_files/Festuge.48k.wav and /dev/null differ diff --git a/tests/audio_files/Festuge.m4a b/tests/audio_files/da_festuge.m4a similarity index 100% rename from tests/audio_files/Festuge.m4a rename to tests/audio_files/da_festuge.m4a diff --git a/tests/audio_files/Festuge.txt b/tests/audio_files/da_festuge.txt similarity index 100% rename from tests/audio_files/Festuge.txt rename to tests/audio_files/da_festuge.txt diff --git a/tests/audio_files/en_aiproblem.m4a b/tests/audio_files/en_aiproblem.m4a new file mode 100644 index 0000000..82ed5f1 Binary files /dev/null and b/tests/audio_files/en_aiproblem.m4a differ diff --git a/tests/audio_files/en_aiproblem.txt b/tests/audio_files/en_aiproblem.txt new file mode 100644 index 0000000..208a563 --- /dev/null +++ b/tests/audio_files/en_aiproblem.txt @@ -0,0 +1,2 @@ +The problem I find with AI is, other than its environmental impact, that it's probably a bit overhyped. +But lets see what this can do, with an agentic framework, and the whisper model. \ No newline at end of file diff --git a/tests/live-real-model.test.ts b/tests/live-real-model.test.ts index c6dd033..6dbd272 100644 --- a/tests/live-real-model.test.ts +++ b/tests/live-real-model.test.ts @@ -1,6 +1,7 @@ -import { describe, test, expect } from "bun:test"; -import { existsSync } from "node:fs"; +import { afterAll, describe, test, expect } from "bun:test"; +import { existsSync, readdirSync } from "node:fs"; import { join } from "node:path"; +import ffmpegPath from "ffmpeg-static"; import { env, pipeline } from "@huggingface/transformers"; import { StreamingResampler } from "../src/lib/audio"; import { @@ -13,88 +14,135 @@ import { wordErrorRate } from "./dsp"; /** * Real-model, real-speech accuracy of the live pipeline, measured as WER over - * a real Danish recording (tests/audio_files/Festuge.m4a, reference - * transcript Festuge.txt). tests/live-accuracy.test.ts proves the pipeline - * moves audio losslessly with a deterministic oracle; this test closes the - * remaining gap by running actual Whisper weights end to end over the exact - * mic path: 48 kHz PCM → StreamingResampler (as src/lib/mic.ts) → + * every recording in tests/audio_files. tests/live-accuracy.test.ts proves + * the pipeline moves audio losslessly with a deterministic oracle; this test + * closes the remaining gap by running actual Whisper weights end to end over + * the exact mic path: 48 kHz PCM → StreamingResampler (as src/lib/mic.ts) → * LiveTranscriber with production LIVE_TUNING → the same Transformers.js * pipeline options the inference worker sends (src/worker/inference.worker.ts). * + * Adding a clip: drop `_.` plus a reference + * transcript `_.txt` into tests/audio_files — it is discovered + * and held to the same bar automatically. `` is the Whisper language + * code the UI user would pick (e.g. `da`, `en`); it is pinned per clip + * because "auto" can misdetect (the Danish clip comes back as a rough English + * translation, in whole-file mode too — see tests/live.md). Audio is decoded + * at test time with ffmpeg-static (trusted devDependency), so any container + * ffmpeg reads is fine. + * * Model: the catalog's Whisper Large v3 Turbo at its WASM-default dtype (q4, - * D-006) — the model a Danish user should pick. Under bun test it runs on - * onnxruntime-node (CPU), i.e. the same weights and quantization as the - * browser WASM engine, just a different execution provider. `language` is - * pinned to "da" exactly as the worker does when the user selects Danish in - * the UI; "auto" misdetects this clip as English (whole-file mode too) and - * comes back as a rough English translation — see tests/live.md. - * - * The first run downloads ~600 MB into @huggingface/transformers' cache - * (node_modules/@huggingface/transformers/.cache/). When the model is neither - * cached nor reachable on the Hub, the test skips so offline development - * stays green (this repo is offline-first). - * - * Fixture: Festuge.48k.wav is Festuge.m4a decoded once, so the test needs no - * audio decoder: ffmpeg -i Festuge.m4a -ac 1 -ar 48000 -sample_fmt s16 \ - * Festuge.48k.wav + * D-006) — the model quality-sensitive users should pick. Under bun test it + * runs on onnxruntime-node (CPU): the same weights and quantization as the + * browser WASM engine, just a different execution provider. The first run + * downloads ~600 MB into node_modules/@huggingface/transformers/.cache/; + * when the model is neither cached nor reachable on the Hub the accuracy + * tests skip, so offline development stays green (this repo is offline-first). */ const MODEL_ID = "onnx-community/whisper-large-v3-turbo"; const MIC_RATE = 48_000; const RATE = 16_000; +/** + * "Close enough" for a quantized model on real-world speech: the Danish clip + * measures ≈ 0.11 live and the English one lower; the slack absorbs + * onnxruntime version-to-version drift, not real regressions — one mid-word + * cut or dropped segment on these short clips moves WER far more than this. + */ +const MAX_WER = 0.25; + +// --------------------------------------------------------------------------- +// Clip discovery: every reference transcript must have audio, and vice versa +// --------------------------------------------------------------------------- + +const AUDIO_DIR = join(import.meta.dir, "audio_files"); + +interface Clip { + /** `_`, e.g. "da_festuge". */ + base: string; + language: string; + audioFile: string; + transcriptFile: string; +} + +function discoverClips(): { clips: Clip[]; problems: string[] } { + const entries = readdirSync(AUDIO_DIR, { withFileTypes: true }) + .filter((entry) => entry.isFile()) + .map((entry) => entry.name); + const clips: Clip[] = []; + const problems: string[] = []; + const claimed = new Set(); + + for (const transcriptFile of entries.filter((name) => name.endsWith(".txt"))) { + const base = transcriptFile.slice(0, -".txt".length); + const language = /^([a-z]{2,3})_./.exec(base)?.[1]; + const audioFile = entries.find( + (name) => name !== transcriptFile && name.startsWith(`${base}.`), + ); + if (!language) { + problems.push( + `${transcriptFile}: name clips "_." (Whisper language code prefix, e.g. da_, en_) so the test knows which language to pin.`, + ); + continue; + } + if (!audioFile) { + problems.push(`${transcriptFile}: no matching audio file "${base}.".`); + continue; + } + claimed.add(transcriptFile); + claimed.add(audioFile); + clips.push({ base, language, audioFile, transcriptFile }); + } + + for (const name of entries) { + if (!claimed.has(name) && !name.endsWith(".txt")) { + problems.push( + `${name}: audio without a reference transcript — add "${name.replace(/\.[^.]+$/, "")}.txt" so accuracy is regression-tested.`, + ); + } + } + return { clips, problems }; +} + +const { clips, problems } = discoverClips(); + +/** Decode any audio container to 48 kHz mono float32 PCM (the mic rate). */ +async function decodeAudio(path: string): Promise { + const proc = Bun.spawn( + [ + ffmpegPath as unknown as string, + ...["-hide_banner", "-loglevel", "error"], + ...["-i", path], + ...["-ac", "1", "-ar", String(MIC_RATE), "-f", "f32le", "-"], + ], + { stdout: "pipe", stderr: "pipe" }, + ); + const [bytes, errors, exitCode] = await Promise.all([ + new Response(proc.stdout).arrayBuffer(), + new Response(proc.stderr).text(), + proc.exited, + ]); + if (exitCode !== 0) throw new Error(`ffmpeg failed on ${path}: ${errors}`); + return new Float32Array(bytes); +} /** - * WER normalization: casing and punctuation are not transcription accuracy. - * "Århus"/"Aarhus" are the same word in Danish orthography, so they unify. + * WER normalization: casing, punctuation, and apostrophes are not + * transcription accuracy. "Århus"/"Aarhus" are the same word in Danish + * orthography, so they unify. */ function normalize(text: string): string { return text .toLowerCase() .replaceAll("århus", "aarhus") - .replace(/[.,!?;:"“”‘’«»()\-–—…]/g, " ") + .replace(/['’]/g, "") + .replace(/[.,!?;:"“”‘‚«»()\-–—…]/g, " ") .replace(/\s+/g, " ") .trim(); } -/** Minimal RIFF/WAVE reader for the fixture (16-bit PCM mono). */ -function readWav(buffer: ArrayBuffer): { samples: Float32Array; rate: number } { - const view = new DataView(buffer); - const tag = (offset: number) => - String.fromCharCode( - view.getUint8(offset), - view.getUint8(offset + 1), - view.getUint8(offset + 2), - view.getUint8(offset + 3), - ); - if (tag(0) !== "RIFF" || tag(8) !== "WAVE") throw new Error("not a WAV file"); - - let rate = 0; - let offset = 12; - while (offset + 8 <= view.byteLength) { - const id = tag(offset); - const size = view.getUint32(offset + 4, true); - if (id === "fmt ") { - const format = view.getUint16(offset + 8, true); - const channels = view.getUint16(offset + 10, true); - const bits = view.getUint16(offset + 22, true); - if (format !== 1 || channels !== 1 || bits !== 16) { - throw new Error("fixture must be 16-bit PCM mono"); - } - rate = view.getUint32(offset + 12, true); - } else if (id === "data") { - const pcm = new Int16Array(buffer, offset + 8, size / 2); - const samples = new Float32Array(pcm.length); - for (let i = 0; i < pcm.length; i++) samples[i] = pcm[i]! / 32768; - return { samples, rate }; - } - offset += 8 + size + (size % 2); - } - throw new Error("WAV data chunk not found"); -} - -/** Cached locally, or downloadable right now — otherwise the test skips. */ +/** Cached locally, or downloadable right now — otherwise accuracy tests skip. */ async function modelAvailable(id: string): Promise { - if (existsSync(join(env.cacheDir, id))) return true; + if (env.cacheDir && existsSync(join(env.cacheDir, id))) return true; try { const response = await fetch( `https://huggingface.co/${id}/resolve/main/config.json`, @@ -108,27 +156,45 @@ async function modelAvailable(id: string): Promise { const available = await modelAvailable(MODEL_ID); -describe("live pipeline accuracy (real model, real Danish speech)", () => { - test.skipIf(!available)( - "Festuge.m4a over the mic path stays close to the reference transcript", - async () => { - const dir = join(import.meta.dir, "audio_files"); - const reference = await Bun.file(join(dir, "Festuge.txt")).text(); - const wav = readWav(await Bun.file(join(dir, "Festuge.48k.wav")).arrayBuffer()); - expect(wav.rate).toBe(MIC_RATE); +// The pipeline is loaded lazily by the first accuracy test and shared. +type Asr = Awaited>>; +let asrPromise: Promise | null = null; +function loadAsr(): Promise { + asrPromise ??= pipeline("automatic-speech-recognition", MODEL_ID, { + dtype: resolveDefaultDtype(MODEL_ID, "wasm"), + }); + return asrPromise; +} - const asr = await pipeline("automatic-speech-recognition", MODEL_ID, { - dtype: resolveDefaultDtype(MODEL_ID, "wasm"), - }); - try { +afterAll(async () => { + if (asrPromise) await (await asrPromise).dispose(); +}); + +describe("live pipeline accuracy (real model, real recordings)", () => { + test("every file in tests/audio_files forms a _ audio+transcript pair", () => { + expect(problems).toEqual([]); + expect(clips.length).toBeGreaterThan(0); + }); + + for (const clip of clips) { + test.skipIf(!available)( + `${clip.base} (${clip.language}): live mic-path transcript stays close to the reference`, + async () => { + const reference = await Bun.file( + join(AUDIO_DIR, clip.transcriptFile), + ).text(); + const micAudio = await decodeAudio(join(AUDIO_DIR, clip.audioFile)); + expect(micAudio.length).toBeGreaterThan(MIC_RATE); // ≥ 1 s of speech + + const asr = await loadAsr(); const transcribe: LiveTranscribeFn = async (audio) => { // Mirrors src/worker/inference.worker.ts `transcribe` options with - // the UI set to Danish. + // the UI language set to the clip's language. const output = await asr(audio, { chunk_length_s: 30, stride_length_s: 5, return_timestamps: true, - language: "da", + language: clip.language, task: "transcribe", }); const single = (Array.isArray(output) ? output[0] : output) as { @@ -142,9 +208,9 @@ describe("live pipeline accuracy (real model, real Danish speech)", () => { const resampler = new StreamingResampler(MIC_RATE, RATE); const blocks: Float32Array[] = []; const chunk = MIC_RATE / 10; - for (let i = 0; i < wav.samples.length; i += chunk) { + for (let i = 0; i < micAudio.length; i += chunk) { const out = resampler.process( - wav.samples.slice(i, Math.min(i + chunk, wav.samples.length)), + micAudio.slice(i, Math.min(i + chunk, micAudio.length)), ); if (out.length > 0) blocks.push(out); } @@ -186,32 +252,26 @@ describe("live pipeline accuracy (real model, real Danish speech)", () => { const result = await live.stop(); const liveWer = wordErrorRate(normalize(reference), normalize(result.text)); - // "Close enough": measured 0.11 (live) / 0.14 (whole-file) on this - // clip; the margin absorbs onnxruntime version-to-version drift, not - // real regressions — a single mid-word cut or dropped segment on a - // 64-word reference moves WER by far more than the slack here. - expect(liveWer).toBeLessThanOrEqual(0.25); + expect(liveWer).toBeLessThanOrEqual(MAX_WER); // Segmentation must not cost accuracy relative to whole-file mode. expect(liveWer).toBeLessThanOrEqual(fileWer + 0.1); // Live UX invariants, with a real model: tentative text streamed, and // the final chunks carry absolute session timestamps spanning the clip. expect(sawTentative).toBe(true); - expect(result.chunks.length).toBeGreaterThanOrEqual(3); + expect(result.chunks.length).toBeGreaterThanOrEqual(1); const first = result.chunks[0]!; const last = result.chunks[result.chunks.length - 1]!; - expect(first.timestamp[0]).toBeGreaterThan(0.5); // speech starts ~1.8 s in - expect(first.timestamp[0]).toBeLessThan(5); + expect(first.timestamp[0]).toBeGreaterThanOrEqual(0); + expect(first.timestamp[0]).toBeLessThan(durationSeconds / 2); expect(last.timestamp[1] ?? last.timestamp[0]).toBeGreaterThan( - durationSeconds - 5, + durationSeconds / 2, ); expect(last.timestamp[1] ?? last.timestamp[0]).toBeLessThanOrEqual( durationSeconds + 1, ); - } finally { - await asr.dispose(); - } - }, - 900_000, // first run downloads ~600 MB of weights - ); + }, + 900_000, // first run downloads ~600 MB of weights + ); + } }); diff --git a/tests/live.md b/tests/live.md index f1a89a9..55b2aca 100644 --- a/tests/live.md +++ b/tests/live.md @@ -5,11 +5,15 @@ unit tests, and transcript *accuracy* of the pipeline (segmentation, pass scheduling, mic-path resampling) is regression-tested as WER by `tests/live-accuracy.test.ts` + `tests/resampler.test.ts`. End-to-end quality with real Whisper weights and real speech is regression-tested too: -`tests/live-real-model.test.ts` runs the Danish clip -`tests/audio_files/Festuge.m4a` through the full mic path with -`whisper-large-v3-turbo` (q4) and holds live WER against the reference -transcript (≤ 0.25; measured ≈ 0.11) and against whole-file mode (it skips -itself when the model is neither cached nor downloadable). Microphone capture, +`tests/live-real-model.test.ts` runs **every** recording in +`tests/audio_files/` through the full mic path with `whisper-large-v3-turbo` +(q4) and holds live WER against the reference transcript (≤ 0.25; measured +≈ 0.11 for `da_festuge`, ≈ 0.06 for `en_aiproblem`) and against whole-file +mode (it skips itself when the model is neither cached nor downloadable). +To cover a new recording, drop `_.` plus its +reference transcript `_.txt` into `tests/audio_files/` — the test +discovers it, decodes it with ffmpeg-static, pins `` as the Whisper +language, and fails loudly on unpaired or unprefixed files. Microphone capture, the AudioWorklet path, and WebGPU/WASM execution in an actual browser can only be exercised manually. This is that procedure. diff --git a/tests/protocol.test.ts b/tests/protocol.test.ts index c6408bb..a55b6cc 100644 --- a/tests/protocol.test.ts +++ b/tests/protocol.test.ts @@ -1,6 +1,7 @@ import { describe, test, expect } from "bun:test"; import { applyLoadProgress, + deviceAttempts, initialLoadProgress, isUiToWorkerMessage, isWorkerToUiMessage, @@ -103,6 +104,57 @@ describe("isWorkerToUiMessage", () => { }); }); +describe("deviceAttempts (D-004, D-017)", () => { + test("auto resolves WebNN → WebGPU → WASM when everything is available", () => { + expect(deviceAttempts("auto", { webnn: true, webgpu: true })).toEqual([ + "webnn", + "webgpu", + "wasm", + ]); + }); + + test("auto skips backends the probe ruled out, in order", () => { + expect(deviceAttempts("auto", { webnn: false, webgpu: true })).toEqual([ + "webgpu", + "wasm", + ]); + expect(deviceAttempts("auto", { webnn: true, webgpu: false })).toEqual([ + "webnn", + "wasm", + ]); + expect(deviceAttempts("auto", { webnn: false, webgpu: false })).toEqual(["wasm"]); + }); + + test("explicit webgpu falls back to WASM only", () => { + expect(deviceAttempts("webgpu", { webnn: true, webgpu: true })).toEqual([ + "webgpu", + "wasm", + ]); + expect(deviceAttempts("webgpu", { webnn: true, webgpu: false })).toEqual(["wasm"]); + }); + + test("explicit webnn falls back to WASM only (no WebGPU detour)", () => { + expect(deviceAttempts("webnn", { webnn: true, webgpu: true })).toEqual([ + "webnn", + "wasm", + ]); + expect(deviceAttempts("webnn", { webnn: false, webgpu: true })).toEqual(["wasm"]); + }); + + test("wasm is honored as-is and every chain ends in WASM", () => { + expect(deviceAttempts("wasm", { webnn: true, webgpu: true })).toEqual(["wasm"]); + for (const preference of ["auto", "webnn", "webgpu", "wasm"] as const) { + for (const webnn of [true, false]) { + for (const webgpu of [true, false]) { + const attempts = deviceAttempts(preference, { webnn, webgpu }); + expect(attempts.length).toBeGreaterThan(0); + expect(attempts.at(-1)).toBe("wasm"); + } + } + } + }); +}); + describe("applyLoadProgress", () => { test("tracks per-file byte progress without mutating previous state", () => { const p1: LoadProgressMessage = { type: "load-progress", status: "initiate", file: "a.onnx" };