From 0084e0435a3c7450352f320c11f2441acb2294de Mon Sep 17 00:00:00 2001 From: Lucas Winther Date: Tue, 14 Jul 2026 19:32:10 +0200 Subject: [PATCH] Add cancel + progress UI, EngineToggle, and Docker deployment --- .dockerignore | 20 +++ .gitignore | 1 + Decisions.md | 27 +++ Dockerfile | 57 ++++++- README.md | 128 +++++++++++++++ WorkBreakdown.md | 45 ++++- src/App.tsx | 263 +++++++++++++++++++++++++----- src/components/AuthForms.tsx | 9 +- src/components/EngineToggle.tsx | 62 +++++++ src/components/TranscriptView.tsx | 36 +++- src/index.css | 78 +++++++++ src/lib/engine.ts | 26 ++- src/worker/inference.worker.ts | 44 ++++- src/worker/protocol.ts | 87 +++++++++- tests/engine.test.ts | 77 +++++++++ tests/protocol.test.ts | 72 ++++++++ 16 files changed, 972 insertions(+), 60 deletions(-) create mode 100644 README.md create mode 100644 src/components/EngineToggle.tsx diff --git a/.dockerignore b/.dockerignore index 973f830..cabc73c 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,3 +1,5 @@ +# Only what the Dockerfile COPYs matters: package.json/bun.lock/bunfig.toml, +# src/, scripts/, server-py/. Everything else is noise for the build context. .git node_modules dist @@ -7,3 +9,21 @@ tests .gitlab-ci.yml Dockerfile .dockerignore +docker-compose.yml + +# Python bytecode caches (server-py/ is COPY'd into the image) +__pycache__/ +*.pyc + +# Editor / agent state +.idea +.claude + +# Local env files — configuration is runtime `-e` flags (D-015), never baked in +.env +.env.* + +# Misc local output +coverage +logs +*.log diff --git a/.gitignore b/.gitignore index c4420eb..f12457f 100644 --- a/.gitignore +++ b/.gitignore @@ -28,6 +28,7 @@ report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json .env.local # caches +__pycache__/ .eslintcache .cache *.tsbuildinfo diff --git a/Decisions.md b/Decisions.md index 18275d7..260d818 100644 --- a/Decisions.md +++ b/Decisions.md @@ -239,3 +239,30 @@ the universal fallback). The selected backend stays user-visible: the worker's `{status:"ready", device}` message drives a "Running on …" badge next to the Backend select (`ModelPicker`) and the engine line in the footer (D-004's "surfaced in the UI" requirement). + +## D-018 — Cancel a file transcription by terminating the worker +**Status:** Accepted (2026-07-14) +Users can cancel an in-flight file run (decode, model download kicked off by +the drop, or the transcription pass itself) via a Cancel button next to the +file name. Transformers.js exposes no abort API for a running `asr()` call, +and on the WASM backend the pass blocks the worker's event loop — a "cancel" +postMessage would sit unread until the pass finished anyway. So cancellation +**terminates the inference worker** outright; it is the only reliable +preemption and also aborts an in-flight weight download. A `fileRun` counter +in `App.tsx` marks the decode that was running when the user cancelled as +stale (`decodeAudioData` itself cannot be aborted), so it never posts audio to +the fresh worker. The next run respawns the worker and reloads the pipeline — +cheap, because the weights stay in the browser Cache API (D-005); the UI +resets `loadStarted`/`loadProgress` so the reload starts from a clean slate. +Live mode is untouched: it has its own stop/drain semantics (D-016), and a +live session never coexists with a file run (mutually exclusive in the UI). +**Alternatives rejected:** a cancel message handled between chunks (dead on +WASM, where the event loop is blocked; and still seconds of latency +elsewhere); `AbortSignal` plumbed into Transformers.js (no upstream support). +**Consequence:** cancel throws away the loaded pipeline even when only the +transcription was unwanted — an accepted cost since reloading from cache takes +a few seconds. +**Amendment (2026-07-14, EngineToggle landing):** a server-engine run cancels +by aborting the `/api/transcribe` fetch instead (`AbortController`; the server +cancels the sidecar pass between segments, D-011/D-013) — the inference worker +and its loaded pipeline are left untouched in that path. diff --git a/Dockerfile b/Dockerfile index 78da403..ca147be 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,3 +1,16 @@ +# --------------------------------------------------------------------------- +# Multi-stage image with an OPTIONAL server-engine (faster-whisper) layer. +# +# docker build -t whisper-web . # Bun-only (default) +# docker build --target with-server-engine -t whisper-web:fw . # + Python/faster-whisper +# +# The default image stays Bun-only (Decisions.md D-013): no Python inside, and +# the server engine simply reports "unavailable" while the browser/anonymous +# path works in full. `--target with-server-engine` adds python3 + the +# server-py/requirements.txt packages so logged-in users can run inference on +# the server. See README.md ("Deployment") for run commands and env vars. +# --------------------------------------------------------------------------- + # Build stage: install all deps (build needs onnxruntime-web files from # node_modules) and emit the self-contained dist/ (scripts/build.ts). FROM oven/bun:1 AS build @@ -26,12 +39,54 @@ RUN bun install --frozen-lockfile --production COPY src ./src COPY scripts ./scripts +# The sidecar script ships in every image (it is one small .py file). Without +# a Python runtime it is never executed: the supervisor in +# src/server/transcribe.ts records "python3 is not available" once and +# GET /api/transcribe/status reports the server engine as unavailable (D-013). +COPY server-py ./server-py COPY --from=build /app/dist ./dist -# SQLite lives under /app/data — mount a volume there to persist users/sessions. +# SQLite lives under /app/data — mount a volume there to persist users/sessions +# (and, in the with-server-engine flavor, downloaded Whisper weights). RUN mkdir -p /app/data && chown -R bun:bun /app/data USER bun ENV PORT=3000 EXPOSE $PORT +HEALTHCHECK --interval=30s --timeout=5s --start-period=15s \ + CMD ["bun", "-e", "const r = await fetch(`http://127.0.0.1:${process.env.PORT || 3000}/`); if (!r.ok) throw new Error(String(r.status));"] CMD ["bun", "src/index.ts"] + +# --------------------------------------------------------------------------- +# Optional server-engine layer (build with `--target with-server-engine`). +# Adds Debian's python3 plus server-py/requirements.txt in a virtualenv +# (Debian pip is PEP 668 "externally managed", and a venv keeps the layer +# self-contained); SIDECAR_PYTHON points the supervisor at it. CPU-only by +# default — FASTER_WHISPER_DEVICE=auto resolves to CPU here, int8 compute. +# GPU (FASTER_WHISPER_DEVICE=cuda) needs a CUDA/cuDNN base image and +# `docker run --gpus`; that is deliberately not part of this Dockerfile. +# --------------------------------------------------------------------------- +FROM runtime AS with-server-engine +USER root + +RUN apt-get update \ + && apt-get install -y --no-install-recommends python3 python3-venv \ + && rm -rf /var/lib/apt/lists/* + +RUN python3 -m venv /opt/faster-whisper \ + && /opt/faster-whisper/bin/pip install --no-cache-dir -r server-py/requirements.txt + +ENV SIDECAR_PYTHON=/opt/faster-whisper/bin/python3 +# Cache Whisper weights inside the data volume so a container restart does not +# re-download them (first request per model downloads from the HF Hub). +ENV FASTER_WHISPER_DOWNLOAD_ROOT=/app/data/faster-whisper-models + +USER bun + +# --------------------------------------------------------------------------- +# Default target: a plain `docker build .` (what .gitlab-ci.yml runs) must +# produce the Bun-only image, but Docker builds the *last* stage by default +# and with-server-engine has to come after the stage it extends — so re-select +# the runtime stage here. Adds no layers. +# --------------------------------------------------------------------------- +FROM runtime AS default diff --git a/README.md b/README.md new file mode 100644 index 0000000..27f4975 --- /dev/null +++ b/README.md @@ -0,0 +1,128 @@ +# AU Whisper Web + +Browser-first speech-to-text powered by OpenAI Whisper. Anonymous users run +Whisper entirely in their browser via +[Transformers.js](https://github.com/huggingface/transformers) (WebNN → WebGPU +→ WASM, installable PWA, works offline once a model is cached). Logged-in +users can optionally run inference on the **server** instead, backed by +[faster-whisper](https://github.com/SYSTRAN/faster-whisper) (CTranslate2) +through a small Python sidecar (`server-py/`, Decisions.md D-013). + +See `Architecture.md` for the design, `Decisions.md` for the decision log, and +`CLAUDE.md` for the toolchain rules (Bun everywhere — no npm/node). + +## Development + +```sh +bun install +bun dev # dev server with HMR at http://localhost:3000 +bun test # unit tests +``` + +## Production build (bare metal) + +```sh +bun install +bun run build # emits self-contained dist/ (app shell, worker, sw.js, /ort/*) +bun start # NODE_ENV=production bun src/index.ts — serves dist/ + /api/* +``` + +To enable the server engine on bare metal, additionally install Python ≥ 3.9 +and the sidecar dependencies: `python3 -m pip install -r server-py/requirements.txt`. +Nothing else changes — the sidecar is spawned lazily on the first server-engine +request. + +## Deployment (Docker) + +The `Dockerfile` is multi-stage and produces **two image flavors**: + +| Flavor | Build command | Contents | +| --- | --- | --- | +| **Bun-only** (default) | `docker build -t whisper-web .` | Bun runtime + built `dist/`. No Python. | +| **With server engine** | `docker build --target with-server-engine -t whisper-web:server-engine .` | Everything above + Debian `python3` and `server-py/requirements.txt` (faster-whisper, CPU-only) in a virtualenv at `/opt/faster-whisper`. | + +`--target` was chosen over a build-arg because the optional layer is a strict +superset of the runtime stage: `with-server-engine` extends `runtime`, so the +Bun layers are shared byte-for-byte between both flavors and there is no +conditional logic inside the Dockerfile. A plain `docker build .` (what +`.gitlab-ci.yml` runs) intentionally resolves to the Bun-only flavor. + +Run either flavor the same way: + +```sh +docker run -d -p 3000:3000 -v whisper-data:/app/data whisper-web +# or, with the server engine: +docker run -d -p 3000:3000 -v whisper-data:/app/data whisper-web:server-engine +``` + +`docker compose up --build` builds and runs the Bun-only flavor with the +`whisper-data` volume already wired up (see `docker-compose.yml`). + +### Graceful degradation (why the Bun-only image is safe to deploy) + +The server engine is optional by design (D-013): + +- **Bun-only image:** everything anonymous users see works in full — browser + inference, model downloads, PWA/offline, and (if `AUTH_ENABLED` stays on) + accounts. The first server-engine probe finds no `python3`, the supervisor + records that as a permanent condition (no respawn storms), and + `GET /api/transcribe/status` answers `{"available": false, "reason": ...}`; + `POST /api/transcribe` answers `503` with the same reason. The UI simply + doesn't offer the server engine. +- **with-server-engine image:** `status` reports `available: true` and + logged-in users get the server/device toggle. The first request for a model + downloads its CTranslate2 weights from the Hugging Face Hub into + `FASTER_WHISPER_DOWNLOAD_ROOT` (inside the data volume, so restarts don't + re-download); the last-used model stays loaded between requests. + +### Runtime environment variables + +All configuration is runtime-only (D-015) — one image serves every +configuration; flip flags with `docker run -e` / compose `environment:`. + +| Variable | Default | Meaning | +| --- | --- | --- | +| `PORT` | `3000` | Port the Bun server listens on (also honored by the image's `EXPOSE`/healthcheck). | +| `NODE_ENV` | `production` (set in the image) | `production` serves the prebuilt `dist/`; anything else runs the dev server. Leave as-is in Docker. | +| `AUTH_ENABLED` | `true` | D-015 feature flag. `false`/`0`/`no`/`off` disables login/register: `/api/auth/*` **and** `/api/transcribe*` return 404 (the server engine is session-gated, so it disappears with auth) and the client hides the Sign in UI. | +| `DB_PATH` | `/app/data/app.db` (image) / `./data/app.db` (bare metal) | SQLite database (users + sessions, D-009). Keep it inside the mounted volume. | +| `TRANSCRIBE_MAX_UPLOAD_BYTES` | `536870912` (512 MiB ≈ 2¼ h of audio) | Upload cap for `POST /api/transcribe` (raw float32 mono 16 kHz PCM), enforced via Content-Length and again while streaming. | +| `SIDECAR_PYTHON` | `python3` (with-server-engine image: `/opt/faster-whisper/bin/python3`) | Python interpreter used to spawn `server-py/sidecar.py`. Only override for exotic setups. | +| `FASTER_WHISPER_DEVICE` | `auto` | CTranslate2 device for the sidecar (`auto`, `cpu`, `cuda`). In the stock image `auto` resolves to CPU. | +| `FASTER_WHISPER_COMPUTE` | `int8` | CTranslate2 compute type (`int8` is the fast, low-memory CPU choice; e.g. `float16` on GPU). | +| `FASTER_WHISPER_DOWNLOAD_ROOT` | unset (with-server-engine image: `/app/data/faster-whisper-models`) | Where the sidecar caches downloaded model weights. | + +There are no session-secret style variables: sessions are random 256-bit ids +stored server-side in SQLite; the cookie is `HttpOnly`, `SameSite=Lax`, and +`Secure` whenever `NODE_ENV=production`. + +### Persistence + +Mount a volume at **`/app/data`**. It holds: + +- `app.db` — the SQLite database (users + sessions). Without the volume every + redeploy logs everyone out and drops accounts. +- `faster-whisper-models/` — (server-engine flavor) downloaded Whisper + weights, from ~75 MB (`tiny`) to ~1.6 GB (`large-v3-turbo`). + +Browser-engine model weights never touch the server — they live in each +visitor's browser cache. + +### HTTPS / reverse proxy + +Run the container behind a TLS-terminating reverse proxy (Caddy, nginx, +Traefik…). HTTPS is **functionally required** in production, not just good +practice: service workers (PWA/offline), microphone capture +(`getUserMedia` for live transcription), WebGPU, and the `Secure` session +cookie all require a secure context. Plain HTTP only works on `localhost`. +The app uses relative URLs and also tolerates being served under a subpath. + +### GPU note + +The `with-server-engine` target is CPU-only on purpose (small-ish image, no +CUDA runtime). For GPU inference you would build your own image on an +`nvidia/cuda` base with cuDNN 9 + cuBLAS, install Bun and +`server-py/requirements.txt` the same way, run with `docker run --gpus all`, +and set `FASTER_WHISPER_DEVICE=cuda` (plus e.g. +`FASTER_WHISPER_COMPUTE=float16`). Not shipped here — CPU int8 already makes +the server engine several times faster than real time on modest hardware. diff --git a/WorkBreakdown.md b/WorkBreakdown.md index 1ba5d1c..1c517b4 100644 --- a/WorkBreakdown.md +++ b/WorkBreakdown.md @@ -9,8 +9,17 @@ Legend: `[ ]` todo · `[x]` done · `[~]` in progress --- ## WIP Tasks before usage limit cutoff: -- [~] Allow users to cancel current transcription. -- [~] Better user-feedback while transcription is in progress (better than "Listening for the first few words") +- [x] Allow users to cancel current transcription. — Cancel button on the + file line, wired to worker termination (D-018); covers decode, pending + model download, and the pass itself. Server-engine cancel is tracked in + M6 (needs the M5 EngineToggle first). +- [x] Better user-feedback while transcription is in progress (better than + "Listening for the first few words") — the worker now posts + `transcribe-progress` (seconds processed / total, from the streamer's + chunk timestamps via the pure `TranscribeProgressTracker`); the UI shows + a percent in the heading plus a progress bar with mm:ss, and the file + flow's placeholder reads "Processing audio…" (the live wording stays + live-only). ## M0 — Scaffold cleanup & docs (small) Current state is the raw `bun init` React template. @@ -143,7 +152,7 @@ Notes (M4): (availability probe for the toggle) - [x] `src/lib/engine.ts` — `Engine` interface; `BrowserEngine` (wraps worker) and `ServerEngine` (wraps fetch stream) (D-010) -- [ ] `EngineToggle` component — visible only with a session; persists choice +- [x] `EngineToggle` component — visible only with a session; persists choice (persistence helpers + status probe already in `lib/engine.ts`; remaining: the component, lifting `useAuth` out of `AuthForms` into `App` so both see one session state, and routing the file flow through the selected engine — @@ -154,7 +163,7 @@ Notes (M4): `Bun.spawn` against `tests/fixtures/fake_sidecar.py`); real inference is the opt-in `FW_INTEGRATION=1` test + manual run in `tests/server-engine.md` (no pip/browser in the dev environment; same caveat as M2/M3) -- [ ] Deployment: optional Python/faster-whisper layer for the Docker image + +- [x] Deployment: optional Python/faster-whisper layer for the Docker image + deployer note (base image stays Bun-only; install `server-py/requirements.txt` to enable the server engine) @@ -183,11 +192,37 @@ Notes (M5): - Smoke-tested live against the dev server: `/` + auth unchanged; status 401 → (register) → 200 `{available:false, reason:"faster-whisper is not installed…"}`; POST → 503 with the same reason, body untouched. +- EngineToggle: one `useAuth()` lives in `App` (passed into `AuthForms` and + gating the toggle); the status probe runs on sign-in and the server option + renders disabled-with-reason when unavailable. The file flow now goes + through the `Engine` interface for **both** paths (D-010): `BrowserEngine` + shares the worker with the live loop (distinct requestId namespaces) and a + new `stream` flag on the transcribe message keeps token partials + + transcribe-progress flowing for id-carrying engine passes (live passes stay + non-streaming). Server runs skip the browser model load, show segment + partials without a progress bar (faster-whisper's stream carries no window + boundaries), and cancel by aborting the fetch (D-018 amendment) — browser + runs keep worker-termination cancel. The choice persists in localStorage + but only takes effect while signed in with the engine available. +- Deployment: multi-stage `Dockerfile` — `runtime` (Bun-only, 244 MB) is the + default target so plain `docker build .` (existing CI) is unchanged; + `--target with-server-engine` (405 MB) adds python3 + faster-whisper in a + venv at `/opt/faster-whisper` (wired via `SIDECAR_PYTHON`), with + `FASTER_WHISPER_DOWNLOAD_ROOT` under the `/app/data` volume so weights + persist. README.md documents both flavors, all runtime env vars, the DB + volume, and the HTTPS/reverse-proxy requirement. Verified live in Docker: + Bun-only image degrades per D-013 (`available:false`, "python3 is not + available…"); server-engine image ran a real `POST /api/transcribe` of + `en_aiproblem.m4a` end-to-end, streamed segments matching the fixture + transcript — the server half of the M5 integration-test item is now + exercised for real (browser half still needs the manual run). ## M6 — Hardening & polish - [ ] Error taxonomy: unsupported file, out-of-memory (suggest smaller model), WebGPU device-lost, network loss mid-download (resume or clear retry) -- [ ] Cancel in-flight transcription (both engines) +- [x] Cancel in-flight transcription (both engines) — browser engine via + worker termination (D-018), server engine via aborting the + `/api/transcribe` fetch (D-018 amendment; landed with M5's EngineToggle) - [ ] Export transcript: .txt and .srt (timestamps already available) - [ ] Storage management UI: list cached models, per-model delete - [ ] Basic a11y pass (keyboard, focus, aria on progress/live transcript) diff --git a/src/App.tsx b/src/App.tsx index 3f50011..24725ce 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -2,6 +2,7 @@ import { useEffect, useRef, useState } from "react"; import "./index.css"; import { AuthForms } from "./components/AuthForms"; +import { EngineToggle } from "./components/EngineToggle"; import { FileDropzone } from "./components/FileDropzone"; import { LiveControls, type LiveStatus } from "./components/LiveControls"; import { ModelLoadProgress } from "./components/ModelLoadProgress"; @@ -9,6 +10,17 @@ import { ModelPicker } from "./components/ModelPicker"; import { TranscriptView } from "./components/TranscriptView"; import { TranscriptionOptions } from "./components/TranscriptionOptions"; import { decodeAudioFile } from "./lib/audio"; +import { + BrowserEngine, + ServerEngine, + fetchServerEngineStatus, + loadEngineChoice, + saveEngineChoice, + type Engine, + type EngineKind, + type ServerEngineStatus, +} from "./lib/engine"; +import { useAuth } from "./lib/useAuth"; import { LIVE_TUNING, LiveTranscriber, type LiveTranscribeResult } from "./lib/live"; import { startMicCapture, type MicCapture } from "./lib/mic"; import { getDownloadedModels } from "./lib/modelCache"; @@ -64,11 +76,25 @@ export function App() { // audio to the (fresh) worker. Decoding itself cannot be aborted. const fileRun = useRef(0); + // Engine plumbing (M5, D-010). The BrowserEngine is bound to the worker's + // lifetime (recreated together on cancel); the ServerEngine is stateless + // and lives for the app. A server run's fetch is cancelled via abortRef. + const browserEngineRef = useRef(null); + const serverEngineRef = useRef(null); + const abortRef = useRef(null); + const [busy, setBusy] = useState(false); + // Which engine the current file run uses; null while idle. Server runs + // don't need the browser model, so parts of the UI key off this. + const [runEngine, setRunEngine] = useState(null); const [decoding, setDecoding] = useState(false); const [loadStarted, setLoadStarted] = useState(false); const [loadProgress, setLoadProgress] = useState(initialLoadProgress); const [partialText, setPartialText] = useState(""); + const [transcribeProgress, setTranscribeProgress] = useState<{ + processedSeconds: number; + totalSeconds: number; + } | null>(null); const [result, setResult] = useState(null); const [error, setError] = useState(null); const [fileName, setFileName] = useState(null); @@ -91,6 +117,15 @@ export function App() { const [swControlled, setSwControlled] = useState(false); const [applyUpdate, setApplyUpdate] = useState<(() => void) | null>(null); + // Session + engine choice (M4/M5). One useAuth for the whole app — the + // header forms and the EngineToggle must agree on the session. The choice + // persists across reloads; it only takes effect while signed in AND the + // deployment reports the server engine available (anonymous-first: logged + // out, everything runs on-device exactly as before). + const auth = useAuth(); + const [engineChoice, setEngineChoice] = useState(() => loadEngineChoice()); + const [serverStatus, setServerStatus] = useState(null); + function refreshDownloaded() { // Downloaded badges (D-005): query Cache Storage for cached weights. getDownloadedModels(MODEL_IDS).then(setDownloaded, () => {}); @@ -109,9 +144,28 @@ export function App() { liveRef.current = null; workerRef.current?.terminate(); workerRef.current = null; + browserEngineRef.current = null; + abortRef.current?.abort(); + abortRef.current = null; }; }, []); + // Probe server-engine availability once a session exists (the endpoint is + // auth-guarded, so probing while anonymous is pointless). Never rejects. + useEffect(() => { + if (!auth.user) { + setServerStatus(null); + return; + } + let cancelled = false; + fetchServerEngineStatus().then((status) => { + if (!cancelled) setServerStatus(status); + }); + return () => { + cancelled = true; + }; + }, [auth.user]); + function takeLiveRequest(requestId: string | undefined) { if (requestId === undefined) return undefined; const pending = pendingLive.current.get(requestId); @@ -146,13 +200,24 @@ export function App() { setPartialText((text) => text + message.text); } break; + case "transcribe-progress": + // File flow only — live passes never carry progress (D-016). + if (message.requestId === undefined) { + setTranscribeProgress({ + processedSeconds: message.processedSeconds, + totalSeconds: message.totalSeconds, + }); + } + break; case "result": { const pending = takeLiveRequest(message.requestId); if (pending) { pending.resolve({ text: message.text, chunks: message.chunks }); } else if (message.requestId === undefined) { setResult({ text: message.text, chunks: message.chunks }); + setTranscribeProgress(null); setBusy(false); + setRunEngine(null); } break; } @@ -160,9 +225,13 @@ export function App() { const pending = takeLiveRequest(message.requestId); if (pending) { pending.reject(new Error(message.message)); - } else { + } else if (message.requestId === undefined) { + // Load-stage failures and legacy id-less passes. Engine-routed + // file passes carry an id and surface through the engine promise. setError(message.message); + setTranscribeProgress(null); setBusy(false); + setRunEngine(null); } break; } @@ -179,6 +248,9 @@ export function App() { setBusy(false); }); workerRef.current = worker; + // The BrowserEngine shares this worker and only sees its own request ids + // (engine-*), so it coexists with the live loop's live-* ids above. + browserEngineRef.current = new BrowserEngine(worker); return worker; } @@ -205,6 +277,7 @@ export function App() { setModelId(nextModelId); setActiveDevice(null); setPartialText(""); + setTranscribeProgress(null); setResult(null); startLoad(nextModelId, devicePreference); } @@ -219,78 +292,146 @@ export function App() { } /** - * Cancel the in-flight file transcription (D-018). Transformers.js has no - * abort API for a running `asr()` call — and on the WASM backend the pass + * Cancel the in-flight file transcription (D-018). Browser runs: no abort + * API exists for a running `asr()` call — and on the WASM backend the pass * blocks the worker's event loop, so even a "cancel" message would sit * unread until the pass finished. Terminating the worker is the only * reliable preemption; it also aborts an in-flight model download. The next * run spawns a fresh worker and reloads the pipeline — cheap, because the - * weights stay in the browser Cache API (D-005). + * weights stay in the browser Cache API (D-005). Server runs: abort the + * `/api/transcribe` fetch (the server cancels the sidecar pass, D-011) and + * leave the worker — and its loaded pipeline — untouched. */ function cancelTranscription() { if (!busy) return; fileRun.current++; // any decode still running is now stale - workerRef.current?.terminate(); - workerRef.current = null; + const cancelled = new Error("Transcription cancelled."); + if (runEngine === "server") { + abortRef.current?.abort(); + abortRef.current = null; + } else { + browserEngineRef.current?.failAll(cancelled); + browserEngineRef.current = null; + workerRef.current?.terminate(); + workerRef.current = null; + // The worker (and any pipeline / partial download) is gone — reset the + // load state so the next transcription reloads from a clean slate. + setLoadStarted(false); + setLoadProgress(initialLoadProgress); + setActiveDevice(null); + } // The file flow never has live passes pending, but if anything is in the // map (defensive), unblock it so nothing awaits a dead worker forever. - const cancelled = new Error("Transcription cancelled."); for (const pending of pendingLive.current.values()) pending.reject(cancelled); pendingLive.current.clear(); setBusy(false); + setRunEngine(null); setPartialText(""); + setTranscribeProgress(null); setResult(null); setFileName(null); setError(null); - // The worker (and any pipeline / partial download) is gone — reset the - // load state so the next transcription reloads from a clean slate. - setLoadStarted(false); - setLoadProgress(initialLoadProgress); - setActiveDevice(null); } async function handleFile(file: File) { if (busy || liveStatus !== "idle") return; const run = ++fileRun.current; + // The toggle's choice only takes effect with a session and an available + // server engine (D-010); everything else runs on-device as always. + const useServer = + engineChoice === "server" && + auth.user !== null && + serverStatus?.available === true; setBusy(true); + setRunEngine(useServer ? "server" : "browser"); setError(null); setPartialText(""); + setTranscribeProgress(null); setResult(null); setFileName(file.name); - // Kick off the (idempotent) model load immediately so the download runs - // while we decode the audio locally. - const worker = ensureWorker(); - if (!loadStarted) { - startLoad(modelId, devicePreference); + let engine: Engine; + if (useServer) { + engine = serverEngineRef.current ??= new ServerEngine(); } else { - postToWorker(worker, { type: "load", modelId, device: devicePreference }); + // Kick off the (idempotent) model load immediately so the download runs + // while we decode the audio locally. Server runs skip this — they don't + // need the browser pipeline. + ensureWorker(); + if (!loadStarted) { + startLoad(modelId, devicePreference); + } else { + postToWorker(workerRef.current!, { + type: "load", + modelId, + device: devicePreference, + }); + } + engine = browserEngineRef.current!; } const multilingual = getModel(modelId)?.multilingual ?? false; + // Decoded entirely on-device via Web Audio (D-012); for browser runs the + // audio never touches the network (Architecture.md §5) — server runs + // upload the decoded PCM, which is the point of the toggle. setDecoding(true); + let audio: Float32Array; try { - // Decoded entirely on-device via Web Audio (D-012); the audio never - // touches the network (Architecture.md §5). - const audio = await decodeAudioFile(file); - if (fileRun.current !== run) return; // cancelled while decoding (D-018) - // Transfer the buffer to the worker — zero copy. Language/task only - // apply to multilingual models; `.en` variants take neither. - postToWorker( - worker, - multilingual - ? { type: "transcribe", audio, language, task } - : { type: "transcribe", audio }, - [audio.buffer as ArrayBuffer], - ); + audio = await decodeAudioFile(file); } catch (decodeError) { - if (fileRun.current !== run) return; // this run was already cancelled - setError(decodeError instanceof Error ? decodeError.message : String(decodeError)); - setBusy(false); + if (fileRun.current === run) { + setError( + decodeError instanceof Error ? decodeError.message : String(decodeError), + ); + setBusy(false); + setRunEngine(null); + } + return; } finally { setDecoding(false); } + if (fileRun.current !== run) return; // cancelled while decoding (D-018) + + // Server runs cancel by aborting the fetch (D-018 amendment); browser + // runs cancel by terminating the worker, which rejects the promise via + // BrowserEngine.failAll. + const controller = useServer ? new AbortController() : null; + abortRef.current = controller; + try { + const outcome = await engine.transcribe(audio, { + // Language/task only apply to multilingual models; `.en` variants + // take neither. The browser engine ignores modelId (worker holds it). + modelId, + ...(multilingual && { language, task }), + onPartial: (fragment) => { + if (fileRun.current === run) setPartialText((text) => text + fragment); + }, + ...(!useServer && { + onProgress: (progress: { processedSeconds: number; totalSeconds: number }) => { + if (fileRun.current === run) setTranscribeProgress(progress); + }, + }), + ...(controller && { signal: controller.signal }), + }); + if (fileRun.current !== run) return; // cancelled mid-pass + setResult(outcome); + setTranscribeProgress(null); + setBusy(false); + setRunEngine(null); + } catch (transcribeError) { + if (fileRun.current !== run) return; // cancellation rejections are expected + setError( + transcribeError instanceof Error + ? transcribeError.message + : String(transcribeError), + ); + setTranscribeProgress(null); + setBusy(false); + setRunEngine(null); + } finally { + if (abortRef.current === controller) abortRef.current = null; + } } // ------------------------------------------------------------------------- @@ -381,8 +522,13 @@ export function App() { const model = getModel(modelId); const modelReady = loadProgress.ready; const showModelProgress = loadStarted && !modelReady && !error; - const transcribing = busy && !decoding && modelReady; + // Server runs never load the browser model, so they are "transcribing" the + // moment decoding ends. + const transcribing = busy && !decoding && (runEngine === "server" || modelReady); const liveActive = liveStatus !== "idle"; + // What the next file run will use (mirrors handleFile's decision). + const serverSelected = + engineChoice === "server" && auth.user !== null && serverStatus?.available === true; return (
@@ -398,7 +544,7 @@ export function App() { unobtrusive "Sign in" button — everything else is unchanged. Renders nothing until the /me probe confirms the server has auth enabled (runtime flag, D-015). */} - + )} + {/* Engine toggle (M5, D-010): signed-in users choose where uploaded + files are transcribed. Live mode always stays on-device (D-016). */} + {auth.user && ( + { + setEngineChoice(choice); + saveEngineChoice(choice); + }} + status={serverStatus} + disabled={busy || liveActive} + /> + )} + File: {fileName} {decoding && " — decoding audio…"} + {busy && ( + // Cancel the whole run (D-018): decode, pending model download, + // or the transcription pass itself. + + )}

)} @@ -454,6 +625,10 @@ export function App() { tentativeText={liveActive ? liveTentative : ""} result={result} transcribing={liveActive || transcribing} + progress={liveActive ? null : transcribeProgress} + placeholder={ + liveActive ? "Listening for the first words…" : "Processing audio…" + } activeLabel={ liveStatus === "stopping" ? "Finishing up…" @@ -482,13 +657,15 @@ export function App() {

Model: {modelId} · Engine:{" "} - {activeDevice === "webgpu" - ? "WebGPU (on-device)" - : activeDevice === "webnn" - ? "WebNN (on-device)" - : activeDevice === "wasm" - ? "WASM (on-device)" - : "on-device — backend selected on first load"} + {serverSelected + ? "server (faster-whisper) — files upload to this deployment" + : activeDevice === "webgpu" + ? "WebGPU (on-device)" + : activeDevice === "webnn" + ? "WebNN (on-device)" + : activeDevice === "wasm" + ? "WASM (on-device)" + : "on-device — backend selected on first load"}

diff --git a/src/components/AuthForms.tsx b/src/components/AuthForms.tsx index 07f9bab..424805c 100644 --- a/src/components/AuthForms.tsx +++ b/src/components/AuthForms.tsx @@ -1,6 +1,6 @@ import { useState, type FormEvent, type KeyboardEvent } from "react"; import { validatePassword, validateUsername } from "../lib/authClient"; -import { useAuth } from "../lib/useAuth"; +import type { AuthState } from "../lib/useAuth"; type Mode = "login" | "register"; @@ -10,11 +10,14 @@ type Mode = "login" | "register"; * that opens a compact login/register popover; once signed in, the username * and a "Sign out" button. * + * The session state comes from `App`'s single `useAuth()` — the EngineToggle + * (M5) reads the same state, so both always agree on who is signed in. + * * Validation mirrors the server rules for instant feedback, but the server * stays authoritative: its `{ error }` messages are shown verbatim on 4xx. */ -export function AuthForms() { - const { enabled, user, login, register, logout } = useAuth(); +export function AuthForms({ auth }: { auth: AuthState }) { + const { enabled, user, login, register, logout } = auth; const [open, setOpen] = useState(false); const [mode, setMode] = useState("login"); diff --git a/src/components/EngineToggle.tsx b/src/components/EngineToggle.tsx new file mode 100644 index 0000000..a4a96e2 --- /dev/null +++ b/src/components/EngineToggle.tsx @@ -0,0 +1,62 @@ +import type { EngineKind, ServerEngineStatus } from "../lib/engine"; + +interface EngineToggleProps { + /** The persisted choice (src/lib/engine.ts helpers). */ + choice: EngineKind; + onChoice: (choice: EngineKind) => void; + /** GET /api/transcribe/status result; null while the probe is in flight. */ + status: ServerEngineStatus | null; + /** Locked while a run or live session is active (like the other options). */ + disabled?: boolean; +} + +/** + * Where should file transcription run (M5, D-010)? Rendered by `App` only for + * a signed-in session — anonymous users never see it (anonymous-first, + * Architecture.md §5). The server option stays visible but disabled when the + * deployment reports the engine unavailable (e.g. no faster-whisper sidecar, + * D-013), showing the probe's reason — hiding it entirely would read as a + * broken account feature. Live mode always stays on-device (D-016). + */ +export function EngineToggle({ + choice, + onChoice, + status, + disabled = false, +}: EngineToggleProps) { + const serverAvailable = status?.available === true; + const effective: EngineKind = serverAvailable ? choice : "browser"; + + function option(kind: EngineKind, label: string, optionDisabled: boolean) { + return ( + + ); + } + + return ( +
+ Transcribe on + {option("browser", "This device", false)} + {option("server", "Server", !serverAvailable)} + + {status === null + ? "Checking server availability…" + : serverAvailable + ? effective === "server" + ? "Uploaded files are transcribed on the server (faster-whisper). Live mode always stays on this device." + : "Your audio stays on this device." + : (status.reason ?? "The server engine is unavailable.")} + +
+ ); +} diff --git a/src/components/TranscriptView.tsx b/src/components/TranscriptView.tsx index c7d3adc..10e122e 100644 --- a/src/components/TranscriptView.tsx +++ b/src/components/TranscriptView.tsx @@ -15,6 +15,14 @@ interface TranscriptViewProps { tentativeText?: string; /** Heading while running; defaults to "Transcribing…". */ activeLabel?: string; + /** + * How far the pass has progressed through the audio (file flow only) — + * renders a percent in the heading plus a thin progress bar. Live mode + * passes null; its feedback is the committed/tentative text itself. + */ + progress?: { processedSeconds: number; totalSeconds: number } | null; + /** Text shown before the first words arrive. */ + placeholder?: string; } function formatTimestamp(seconds: number): string { @@ -30,10 +38,17 @@ export function TranscriptView({ transcribing, tentativeText = "", activeLabel = "Transcribing…", + progress = null, + placeholder = "Listening for the first words…", }: TranscriptViewProps) { const [copied, setCopied] = useState(false); const text = result?.text ?? partialText; const copyText = `${text}${tentativeText}`.trim(); + const showProgress = + transcribing && !result && progress !== null && progress.totalSeconds > 0; + const percent = showProgress + ? Math.min(100, Math.round((progress.processedSeconds / progress.totalSeconds) * 100)) + : null; useEffect(() => { if (!copied) return; @@ -52,7 +67,12 @@ export function TranscriptView({ return (
-

{result ? "Transcript" : activeLabel}

+

+ {result ? "Transcript" : activeLabel} + {percent !== null && ( + {percent}% + )} +

+ {showProgress && ( +
+ + + {formatTimestamp(progress.processedSeconds)} /{" "} + {formatTimestamp(progress.totalSeconds)} + +
+ )}

- {text || (tentativeText ? "" : "Listening for the first words…")} + {text || (tentativeText ? "" : placeholder)} {tentativeText && ( {tentativeText} )} diff --git a/src/index.css b/src/index.css index a77e963..ea09faa 100644 --- a/src/index.css +++ b/src/index.css @@ -357,6 +357,38 @@ body { flex-wrap: wrap; } +/* --- EngineToggle (M5, D-010) --- */ + +.engine-toggle { + display: flex; + align-items: center; + gap: 1rem; + flex-wrap: wrap; + font-size: 0.9rem; +} + +.engine-toggle-label { + font-weight: 600; +} + +.engine-option { + display: inline-flex; + align-items: center; + gap: 0.35rem; + cursor: pointer; +} + +.engine-option:has(input:disabled) { + color: var(--muted); + cursor: default; +} + +.engine-toggle-hint { + flex-basis: 100%; + color: var(--muted); + font-size: 0.85rem; +} + /* --- FileDropzone --- */ .dropzone { @@ -458,6 +490,25 @@ body { font-size: 0.95rem; } +/* Cancel for the in-flight file run (D-018) — styled like .copy-button, with + a warning accent on hover. */ +.cancel-button { + margin-left: 0.75rem; + border: 1px solid var(--border); + background: transparent; + color: inherit; + border-radius: 8px; + padding: 0.3rem 0.9rem; + font-size: 0.85rem; + cursor: pointer; + transition: border-color 0.15s, color 0.15s; +} + +.cancel-button:hover { + border-color: #8b3a3a; + color: #f2b8b5; +} + .error { margin: 0; padding: 0.75rem 1rem; @@ -561,6 +612,33 @@ body { cursor: default; } +.transcript-percent { + color: var(--muted); + font-weight: normal; + font-variant-numeric: tabular-nums; +} + +/* Audio progress of the in-flight file pass (transcribe-progress messages). */ +.transcript-progress { + display: flex; + align-items: center; + gap: 0.75rem; + margin-top: 0.6rem; +} + +.transcript-progress progress { + flex: 1; + height: 6px; + accent-color: var(--accent); +} + +.transcript-progress-time { + color: var(--muted); + font-size: 0.85rem; + white-space: nowrap; + font-variant-numeric: tabular-nums; +} + .transcript-text { margin: 0.75rem 0 0; white-space: pre-wrap; diff --git a/src/lib/engine.ts b/src/lib/engine.ts index 5e6228f..5c60e6b 100644 --- a/src/lib/engine.ts +++ b/src/lib/engine.ts @@ -42,9 +42,16 @@ export interface EngineTranscribeOptions { * Browser engine: token fragments; server engine: whole segments. */ onPartial?: (fragment: string) => void; + /** + * How far the pass has progressed through the audio. Browser engine only — + * faster-whisper's stream carries no window boundaries, so the server + * engine never calls it (the UI degrades to segments-as-feedback). + */ + onProgress?: (progress: { processedSeconds: number; totalSeconds: number }) => void; /** * Abort the run. Honored by the server engine (the request is cancelled - * server-side too); the browser worker cannot abort a pass yet (M6). + * server-side too); the browser worker cannot abort a pass — cancel there + * is worker termination via `BrowserEngine.failAll` (D-018). */ signal?: AbortSignal; } @@ -83,6 +90,7 @@ interface PendingRequest { resolve: (result: EngineTranscribeResult) => void; reject: (error: Error) => void; onPartial?: (fragment: string) => void; + onProgress?: (progress: { processedSeconds: number; totalSeconds: number }) => void; } /** @@ -116,11 +124,19 @@ export class BrowserEngine implements Engine { ): Promise { const requestId = `engine-${this.instance}-${++this.nextRequestId}`; return new Promise((resolve, reject) => { - this.pending.set(requestId, { resolve, reject, onPartial: options.onPartial }); + this.pending.set(requestId, { + resolve, + reject, + onPartial: options.onPartial, + onProgress: options.onProgress, + }); const message: TranscribeMessage = { type: "transcribe", audio, requestId, + // Id-carrying requests default to no streaming in the worker (the + // live-pass rule) — ask for it explicitly when a consumer listens. + ...((options.onPartial || options.onProgress) && { stream: true }), ...(options.language !== undefined && options.language !== "auto" && { language: options.language }), ...(options.task !== undefined && { task: options.task }), @@ -146,6 +162,12 @@ export class BrowserEngine implements Engine { case "partial": request.onPartial?.(data.text); break; + case "transcribe-progress": + request.onProgress?.({ + processedSeconds: data.processedSeconds, + totalSeconds: data.totalSeconds, + }); + break; case "result": this.pending.delete(data.requestId); request.resolve({ text: data.text, chunks: data.chunks }); diff --git a/src/worker/inference.worker.ts b/src/worker/inference.worker.ts index d5320b0..1d6adfb 100644 --- a/src/worker/inference.worker.ts +++ b/src/worker/inference.worker.ts @@ -19,6 +19,7 @@ import { resolveDefaultDtype } from "../lib/models"; import { deviceAttempts, isUiToWorkerMessage, + TranscribeProgressTracker, type DeviceCapabilities, type DevicePreference, type InferenceDevice, @@ -209,6 +210,12 @@ function load(message: LoadMessage): Promise // Transcription // --------------------------------------------------------------------------- +/** Audio sample rate the whole app standardizes on (D-012). */ +const SAMPLE_RATE = 16_000; +/** Chunked-pipeline window / overlap, in seconds (Whisper's usual 30/5). */ +const CHUNK_LENGTH_S = 30; +const STRIDE_LENGTH_S = 5; + async function transcribe(message: TranscribeMessage): Promise { if (active === null) { post({ @@ -222,11 +229,39 @@ async function transcribe(message: TranscribeMessage): Promise { const asr = await active.promise; const options: Record = { - chunk_length_s: 30, - stride_length_s: 5, + chunk_length_s: CHUNK_LENGTH_S, + stride_length_s: STRIDE_LENGTH_S, return_timestamps: true, }; - if (message.requestId === undefined) { + if (message.stream ?? message.requestId === undefined) { + // File-flow pass (id-less, or routed through BrowserEngine with an + // explicit stream flag, D-010): stream partial tokens and report audio + // progress. The streamer's chunk timestamps are window-relative; the + // tracker maps them to monotone absolute seconds (see protocol.ts). + // Live passes skip both — they resolve wholesale via "result" (D-016). + const totalSeconds = message.audio.length / SAMPLE_RATE; + const tracker = new TranscribeProgressTracker( + totalSeconds, + CHUNK_LENGTH_S, + CHUNK_LENGTH_S - 2 * STRIDE_LENGTH_S, + ); + const postProgress = (processedSeconds: number | null) => { + if (processedSeconds === null) return; + post({ + type: "transcribe-progress", + processedSeconds, + totalSeconds, + requestId: message.requestId, + }); + }; + // Immediate 0/total so the UI shows the audio length before the first + // (possibly slow) token arrives. + post({ + type: "transcribe-progress", + processedSeconds: 0, + totalSeconds, + requestId: message.requestId, + }); options.streamer = new WhisperTextStreamer( // The ASR pipeline for Whisper models always carries a WhisperTokenizer. asr.tokenizer as ConstructorParameters[0], @@ -234,6 +269,9 @@ async function transcribe(message: TranscribeMessage): Promise { skip_prompt: true, callback_function: (text: string) => post({ type: "partial", text, requestId: message.requestId }), + on_chunk_start: (time: number) => postProgress(tracker.timestamp(time)), + on_chunk_end: (time: number) => postProgress(tracker.timestamp(time)), + on_finalize: () => postProgress(tracker.finalizeWindow()), }, ); } diff --git a/src/worker/protocol.ts b/src/worker/protocol.ts index d8d5207..61864f4 100644 --- a/src/worker/protocol.ts +++ b/src/worker/protocol.ts @@ -69,6 +69,14 @@ export interface TranscribeMessage { * responses to segment passes; the file flow omits it and is unchanged. */ requestId?: string; + /** + * Stream partial tokens and transcribe-progress for this request. Defaults + * to `requestId === undefined` (the historical rule: id-less file-flow + * passes stream, live segment passes resolve wholesale via "result"). + * `BrowserEngine` sets it explicitly so a file pass routed through the + * engine (which correlates via requestId, D-010) still streams. + */ + stream?: boolean; } export type UiToWorkerMessage = LoadMessage | TranscribeMessage; @@ -108,6 +116,22 @@ export interface PartialMessage { requestId?: string; } +/** + * How far a file-flow pass has progressed through the audio, in seconds. + * Posted once at pass start (0 / total) and then on every Whisper segment + * boundary, so slow models/backends show movement long before the first + * partial token. Values are monotone and clamped to [0, totalSeconds] + * (see TranscribeProgressTracker). Live passes (requestId set) don't report + * progress — the live UI has its own committed/tentative feedback (D-016). + */ +export interface TranscribeProgressMessage { + type: "transcribe-progress"; + processedSeconds: number; + totalSeconds: number; + /** Echo of the originating transcribe request's id, if it carried one. */ + requestId?: string; +} + /** One timestamped segment of the final transcript. */ export interface TranscriptChunk { text: string; @@ -134,6 +158,7 @@ export interface ErrorMessage { export type WorkerToUiMessage = | LoadProgressMessage | PartialMessage + | TranscribeProgressMessage | ResultMessage | ErrorMessage; @@ -234,7 +259,8 @@ export function isUiToWorkerMessage(value: unknown): value is UiToWorkerMessage (value.task === undefined || value.task === "transcribe" || value.task === "translate") && - isOptionalString(value.requestId) + isOptionalString(value.requestId) && + (value.stream === undefined || typeof value.stream === "boolean") ); default: return false; @@ -264,6 +290,12 @@ export function isWorkerToUiMessage(value: unknown): value is WorkerToUiMessage } case "partial": return typeof value.text === "string" && isOptionalString(value.requestId); + case "transcribe-progress": + return ( + typeof value.processedSeconds === "number" && + typeof value.totalSeconds === "number" && + isOptionalString(value.requestId) + ); case "result": return ( typeof value.text === "string" && @@ -331,3 +363,56 @@ export function applyLoadProgress( } return { ...state, files: { ...state.files, [message.file]: next } }; } + +// --------------------------------------------------------------------------- +// Transcription progress accumulation (pure; drives the in-progress UI) +// --------------------------------------------------------------------------- + +/** + * Maps the streamer's window-relative timestamps to absolute audio progress. + * + * The chunked ASR pipeline slices audio into windows of `windowSeconds`, + * advancing by `jumpSeconds` (window − 2·stride) per window, and runs one + * generate pass per window. Whisper's timestamp tokens — surfaced via + * `WhisperTextStreamer`'s `on_chunk_start`/`on_chunk_end` — are relative to + * the current window's start, and `on_finalize` fires when a window's pass + * ends. This tracker turns those into absolute seconds that are monotone + * (windows overlap, so a later window's early timestamps can precede an + * earlier window's late ones — regressions are swallowed) and clamped to + * [0, totalSeconds]. + */ +export class TranscribeProgressTracker { + private windowIndex = 0; + private reported = 0; + + constructor( + private readonly totalSeconds: number, + private readonly windowSeconds: number, + private readonly jumpSeconds: number, + ) {} + + /** + * Absolute progress for a window-relative timestamp, or null when it does + * not advance past what was already reported. + */ + timestamp(withinWindowSeconds: number): number | null { + const clamped = Math.min(Math.max(withinWindowSeconds, 0), this.windowSeconds); + return this.advance(this.windowIndex * this.jumpSeconds + clamped); + } + + /** + * The current window's generate pass finished; progress moves to the start + * of the next window (the amount of audio no later window revisits). + */ + finalizeWindow(): number | null { + this.windowIndex += 1; + return this.advance(this.windowIndex * this.jumpSeconds); + } + + private advance(processedSeconds: number): number | null { + const clamped = Math.min(processedSeconds, this.totalSeconds); + if (clamped <= this.reported) return null; + this.reported = clamped; + return clamped; + } +} diff --git a/tests/engine.test.ts b/tests/engine.test.ts index 2eb4f23..d174aa7 100644 --- a/tests/engine.test.ts +++ b/tests/engine.test.ts @@ -77,6 +77,44 @@ describe("BrowserEngine", () => { expect("modelId" in message).toBe(false); }); + test("asks the worker to stream only when a consumer listens", () => { + const worker = new FakeWorker(); + const engine = new BrowserEngine(worker); + void engine.transcribe(new Float32Array(4)); + void engine.transcribe(new Float32Array(4), { onPartial: () => {} }); + void engine.transcribe(new Float32Array(4), { onProgress: () => {} }); + expect("stream" in worker.posted[0]!.message).toBe(false); + expect(worker.posted[1]!.message.stream).toBe(true); + expect(worker.posted[2]!.message.stream).toBe(true); + }); + + test("routes transcribe-progress to onProgress, ignoring other ids", async () => { + const worker = new FakeWorker(); + const engine = new BrowserEngine(worker); + const updates: { processedSeconds: number; totalSeconds: number }[] = []; + const promise = engine.transcribe(new Float32Array(4), { + onProgress: (progress) => updates.push(progress), + }); + const requestId = worker.lastRequestId(); + + worker.emit({ type: "transcribe-progress", processedSeconds: 5, totalSeconds: 60 }); + worker.emit({ + type: "transcribe-progress", + processedSeconds: 7, + totalSeconds: 60, + requestId: "other", + }); + worker.emit({ type: "transcribe-progress", processedSeconds: 0, totalSeconds: 90, requestId }); + worker.emit({ type: "transcribe-progress", processedSeconds: 30, totalSeconds: 90, requestId }); + worker.emit({ type: "result", text: "done", chunks: [], requestId }); + + expect((await promise).text).toBe("done"); + expect(updates).toEqual([ + { processedSeconds: 0, totalSeconds: 90 }, + { processedSeconds: 30, totalSeconds: 90 }, + ]); + }); + test("resolves on the matching result and routes partials to onPartial", async () => { const worker = new FakeWorker(); const engine = new BrowserEngine(worker); @@ -265,6 +303,45 @@ describe("ServerEngine", () => { engine.transcribe(new Float32Array(4), { modelId: "onnx-community/whisper-tiny" }), ).rejects.toThrow(/could not reach the server/i); }); + + test("passes the abort signal to fetch and rethrows AbortError as-is", async () => { + // Cancel (D-018 amendment): the UI aborts the controller; the rejection + // must stay an AbortError, not be rewrapped as a network failure. + let seenSignal: AbortSignal | undefined; + const engine = new ServerEngine(async (_url, init) => { + seenSignal = init?.signal ?? undefined; + throw new DOMException("The user aborted a request.", "AbortError"); + }); + const controller = new AbortController(); + const promise = engine.transcribe(new Float32Array(4), { + modelId: "onnx-community/whisper-tiny", + signal: controller.signal, + }); + expect(promise).rejects.toMatchObject({ name: "AbortError" }); + await promise.catch(() => {}); + expect(seenSignal).toBe(controller.signal); + }); + + test("an abort mid-stream surfaces as the stream's rejection", () => { + // Once the 200 arrived, an abort tears down the body stream; whatever + // error the reader throws must propagate out of transcribe(). + const body = new ReadableStream({ + start(streamController) { + streamController.enqueue( + new TextEncoder().encode( + `${JSON.stringify({ type: "segment", text: " a", start: 0, end: 1 })}\n`, + ), + ); + streamController.error( + new DOMException("The user aborted a request.", "AbortError"), + ); + }, + }); + const engine = new ServerEngine(async () => new Response(body, { status: 200 })); + expect( + engine.transcribe(new Float32Array(4), { modelId: "onnx-community/whisper-tiny" }), + ).rejects.toMatchObject({ name: "AbortError" }); + }); }); // --------------------------------------------------------------------------- diff --git a/tests/protocol.test.ts b/tests/protocol.test.ts index a55b6cc..83d9d96 100644 --- a/tests/protocol.test.ts +++ b/tests/protocol.test.ts @@ -5,6 +5,7 @@ import { initialLoadProgress, isUiToWorkerMessage, isWorkerToUiMessage, + TranscribeProgressTracker, type LoadProgressMessage, type UiToWorkerMessage, type WorkerToUiMessage, @@ -31,6 +32,8 @@ describe("isUiToWorkerMessage", () => { { type: "transcribe", audio: new Float32Array(8), language: "da", task: "transcribe" }, { type: "transcribe", audio: new Float32Array(8), language: "auto", task: "translate" }, { type: "transcribe", audio: new Float32Array(8), requestId: "live-3" }, + { type: "transcribe", audio: new Float32Array(8), requestId: "engine-1-1", stream: true }, + { type: "transcribe", audio: new Float32Array(8), stream: false }, ]; for (const message of [...loads, ...transcribes]) { expect(isUiToWorkerMessage(message)).toBe(true); @@ -58,6 +61,9 @@ describe("isUiToWorkerMessage", () => { expect( isUiToWorkerMessage({ type: "transcribe", audio: new Float32Array(4), requestId: 7 }), ).toBe(false); + expect( + isUiToWorkerMessage({ type: "transcribe", audio: new Float32Array(4), stream: "yes" }), + ).toBe(false); expect(isUiToWorkerMessage({ type: "result", text: "", chunks: [] })).toBe(false); }); }); @@ -80,6 +86,8 @@ describe("isWorkerToUiMessage", () => { { type: "load-progress", status: "ready", device: "webgpu" }, { type: "partial", text: " hello" }, { type: "partial", text: " hello", requestId: "live-1" }, + { type: "transcribe-progress", processedSeconds: 0, totalSeconds: 90 }, + { type: "transcribe-progress", processedSeconds: 42.5, totalSeconds: 90, requestId: "live-1" }, { type: "result", text: "hello world", chunks: [{ text: "hello world", timestamp: [0, 1.2] }] }, { type: "result", text: "hello world", chunks: [], requestId: "live-1" }, { type: "error", stage: "load", message: "boom" }, @@ -98,6 +106,18 @@ describe("isWorkerToUiMessage", () => { expect(isWorkerToUiMessage({ type: "load-progress", status: "ready", device: "cuda" })).toBe(false); expect(isWorkerToUiMessage({ type: "partial" })).toBe(false); expect(isWorkerToUiMessage({ type: "partial", text: "x", requestId: 1 })).toBe(false); + expect(isWorkerToUiMessage({ type: "transcribe-progress", processedSeconds: 1 })).toBe(false); + expect( + isWorkerToUiMessage({ type: "transcribe-progress", processedSeconds: "1", totalSeconds: 2 }), + ).toBe(false); + expect( + isWorkerToUiMessage({ + type: "transcribe-progress", + processedSeconds: 1, + totalSeconds: 2, + requestId: 9, + }), + ).toBe(false); expect(isWorkerToUiMessage({ type: "result", text: "x", chunks: "nope" })).toBe(false); expect(isWorkerToUiMessage({ type: "error", stage: "decode", message: "m" })).toBe(false); expect(isWorkerToUiMessage({ type: "load", modelId: "x", device: "wasm" })).toBe(false); @@ -155,6 +175,58 @@ describe("deviceAttempts (D-004, D-017)", () => { }); }); +describe("TranscribeProgressTracker", () => { + // Production shape: 30 s windows advancing by 20 s (30 − 2·5, the worker's + // chunk/stride settings), like the chunked ASR pipeline. + const WINDOW = 30; + const JUMP = 20; + + test("maps window-relative timestamps to absolute seconds per window", () => { + const tracker = new TranscribeProgressTracker(90, WINDOW, JUMP); + expect(tracker.timestamp(0)).toBeNull(); // 0 does not advance past 0 + expect(tracker.timestamp(4.2)).toBe(4.2); + expect(tracker.timestamp(29.5)).toBe(29.5); + expect(tracker.finalizeWindow()).toBeNull(); // 1·20 < 29.5 already seen + // Second window: timestamps are relative again, offset by one jump. + expect(tracker.timestamp(3)).toBeNull(); // 20 + 3 < 29.5 + expect(tracker.timestamp(12)).toBe(32); + }); + + test("is monotone across overlapping windows", () => { + const tracker = new TranscribeProgressTracker(90, WINDOW, JUMP); + expect(tracker.timestamp(28)).toBe(28); + tracker.finalizeWindow(); + // The next window revisits overlapped audio — earlier absolute times + // than what was already reported are swallowed, never regressions. + expect(tracker.timestamp(5)).toBeNull(); // 20 + 5 = 25 < 28 + expect(tracker.timestamp(10)).toBe(30); + }); + + test("clamps within the window and to the total duration", () => { + const tracker = new TranscribeProgressTracker(35, WINDOW, JUMP); + // A stray timestamp beyond the window length clamps to the window end. + expect(tracker.timestamp(45)).toBe(30); + expect(tracker.timestamp(-1)).toBeNull(); // negative clamps to window start + tracker.finalizeWindow(); + // Second (final) window: 20 + 16 = 36 clamps to the 35 s total. + expect(tracker.timestamp(16)).toBe(35); + expect(tracker.timestamp(17)).toBeNull(); // already at the end + }); + + test("finalizeWindow advances to the next window start", () => { + const tracker = new TranscribeProgressTracker(90, WINDOW, JUMP); + expect(tracker.finalizeWindow()).toBe(20); + expect(tracker.finalizeWindow()).toBe(40); + }); + + test("single-window audio never reports past its duration", () => { + const tracker = new TranscribeProgressTracker(8, WINDOW, JUMP); + expect(tracker.timestamp(6)).toBe(6); + expect(tracker.timestamp(8)).toBe(8); + expect(tracker.finalizeWindow()).toBeNull(); // 20 clamps to 8, not an advance + }); +}); + describe("applyLoadProgress", () => { test("tracks per-file byte progress without mutating previous state", () => { const p1: LoadProgressMessage = { type: "load-progress", status: "initiate", file: "a.onnx" };