Add cancel + progress UI, EngineToggle, and Docker deployment
This commit is contained in:
@@ -1,3 +1,5 @@
|
|||||||
|
# Only what the Dockerfile COPYs matters: package.json/bun.lock/bunfig.toml,
|
||||||
|
# src/, scripts/, server-py/. Everything else is noise for the build context.
|
||||||
.git
|
.git
|
||||||
node_modules
|
node_modules
|
||||||
dist
|
dist
|
||||||
@@ -7,3 +9,21 @@ tests
|
|||||||
.gitlab-ci.yml
|
.gitlab-ci.yml
|
||||||
Dockerfile
|
Dockerfile
|
||||||
.dockerignore
|
.dockerignore
|
||||||
|
docker-compose.yml
|
||||||
|
|
||||||
|
# Python bytecode caches (server-py/ is COPY'd into the image)
|
||||||
|
__pycache__/
|
||||||
|
*.pyc
|
||||||
|
|
||||||
|
# Editor / agent state
|
||||||
|
.idea
|
||||||
|
.claude
|
||||||
|
|
||||||
|
# Local env files — configuration is runtime `-e` flags (D-015), never baked in
|
||||||
|
.env
|
||||||
|
.env.*
|
||||||
|
|
||||||
|
# Misc local output
|
||||||
|
coverage
|
||||||
|
logs
|
||||||
|
*.log
|
||||||
|
|||||||
@@ -28,6 +28,7 @@ report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json
|
|||||||
.env.local
|
.env.local
|
||||||
|
|
||||||
# caches
|
# caches
|
||||||
|
__pycache__/
|
||||||
.eslintcache
|
.eslintcache
|
||||||
.cache
|
.cache
|
||||||
*.tsbuildinfo
|
*.tsbuildinfo
|
||||||
|
|||||||
@@ -239,3 +239,30 @@ the universal fallback). The selected backend stays user-visible: the worker's
|
|||||||
`{status:"ready", device}` message drives a "Running on …" badge next to the
|
`{status:"ready", device}` message drives a "Running on …" badge next to the
|
||||||
Backend select (`ModelPicker`) and the engine line in the footer (D-004's
|
Backend select (`ModelPicker`) and the engine line in the footer (D-004's
|
||||||
"surfaced in the UI" requirement).
|
"surfaced in the UI" requirement).
|
||||||
|
|
||||||
|
## D-018 — Cancel a file transcription by terminating the worker
|
||||||
|
**Status:** Accepted (2026-07-14)
|
||||||
|
Users can cancel an in-flight file run (decode, model download kicked off by
|
||||||
|
the drop, or the transcription pass itself) via a Cancel button next to the
|
||||||
|
file name. Transformers.js exposes no abort API for a running `asr()` call,
|
||||||
|
and on the WASM backend the pass blocks the worker's event loop — a "cancel"
|
||||||
|
postMessage would sit unread until the pass finished anyway. So cancellation
|
||||||
|
**terminates the inference worker** outright; it is the only reliable
|
||||||
|
preemption and also aborts an in-flight weight download. A `fileRun` counter
|
||||||
|
in `App.tsx` marks the decode that was running when the user cancelled as
|
||||||
|
stale (`decodeAudioData` itself cannot be aborted), so it never posts audio to
|
||||||
|
the fresh worker. The next run respawns the worker and reloads the pipeline —
|
||||||
|
cheap, because the weights stay in the browser Cache API (D-005); the UI
|
||||||
|
resets `loadStarted`/`loadProgress` so the reload starts from a clean slate.
|
||||||
|
Live mode is untouched: it has its own stop/drain semantics (D-016), and a
|
||||||
|
live session never coexists with a file run (mutually exclusive in the UI).
|
||||||
|
**Alternatives rejected:** a cancel message handled between chunks (dead on
|
||||||
|
WASM, where the event loop is blocked; and still seconds of latency
|
||||||
|
elsewhere); `AbortSignal` plumbed into Transformers.js (no upstream support).
|
||||||
|
**Consequence:** cancel throws away the loaded pipeline even when only the
|
||||||
|
transcription was unwanted — an accepted cost since reloading from cache takes
|
||||||
|
a few seconds.
|
||||||
|
**Amendment (2026-07-14, EngineToggle landing):** a server-engine run cancels
|
||||||
|
by aborting the `/api/transcribe` fetch instead (`AbortController`; the server
|
||||||
|
cancels the sidecar pass between segments, D-011/D-013) — the inference worker
|
||||||
|
and its loaded pipeline are left untouched in that path.
|
||||||
|
|||||||
+56
-1
@@ -1,3 +1,16 @@
|
|||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Multi-stage image with an OPTIONAL server-engine (faster-whisper) layer.
|
||||||
|
#
|
||||||
|
# docker build -t whisper-web . # Bun-only (default)
|
||||||
|
# docker build --target with-server-engine -t whisper-web:fw . # + Python/faster-whisper
|
||||||
|
#
|
||||||
|
# The default image stays Bun-only (Decisions.md D-013): no Python inside, and
|
||||||
|
# the server engine simply reports "unavailable" while the browser/anonymous
|
||||||
|
# path works in full. `--target with-server-engine` adds python3 + the
|
||||||
|
# server-py/requirements.txt packages so logged-in users can run inference on
|
||||||
|
# the server. See README.md ("Deployment") for run commands and env vars.
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
# Build stage: install all deps (build needs onnxruntime-web files from
|
# Build stage: install all deps (build needs onnxruntime-web files from
|
||||||
# node_modules) and emit the self-contained dist/ (scripts/build.ts).
|
# node_modules) and emit the self-contained dist/ (scripts/build.ts).
|
||||||
FROM oven/bun:1 AS build
|
FROM oven/bun:1 AS build
|
||||||
@@ -26,12 +39,54 @@ RUN bun install --frozen-lockfile --production
|
|||||||
|
|
||||||
COPY src ./src
|
COPY src ./src
|
||||||
COPY scripts ./scripts
|
COPY scripts ./scripts
|
||||||
|
# The sidecar script ships in every image (it is one small .py file). Without
|
||||||
|
# a Python runtime it is never executed: the supervisor in
|
||||||
|
# src/server/transcribe.ts records "python3 is not available" once and
|
||||||
|
# GET /api/transcribe/status reports the server engine as unavailable (D-013).
|
||||||
|
COPY server-py ./server-py
|
||||||
COPY --from=build /app/dist ./dist
|
COPY --from=build /app/dist ./dist
|
||||||
|
|
||||||
# SQLite lives under /app/data — mount a volume there to persist users/sessions.
|
# SQLite lives under /app/data — mount a volume there to persist users/sessions
|
||||||
|
# (and, in the with-server-engine flavor, downloaded Whisper weights).
|
||||||
RUN mkdir -p /app/data && chown -R bun:bun /app/data
|
RUN mkdir -p /app/data && chown -R bun:bun /app/data
|
||||||
USER bun
|
USER bun
|
||||||
|
|
||||||
ENV PORT=3000
|
ENV PORT=3000
|
||||||
EXPOSE $PORT
|
EXPOSE $PORT
|
||||||
|
HEALTHCHECK --interval=30s --timeout=5s --start-period=15s \
|
||||||
|
CMD ["bun", "-e", "const r = await fetch(`http://127.0.0.1:${process.env.PORT || 3000}/`); if (!r.ok) throw new Error(String(r.status));"]
|
||||||
CMD ["bun", "src/index.ts"]
|
CMD ["bun", "src/index.ts"]
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Optional server-engine layer (build with `--target with-server-engine`).
|
||||||
|
# Adds Debian's python3 plus server-py/requirements.txt in a virtualenv
|
||||||
|
# (Debian pip is PEP 668 "externally managed", and a venv keeps the layer
|
||||||
|
# self-contained); SIDECAR_PYTHON points the supervisor at it. CPU-only by
|
||||||
|
# default — FASTER_WHISPER_DEVICE=auto resolves to CPU here, int8 compute.
|
||||||
|
# GPU (FASTER_WHISPER_DEVICE=cuda) needs a CUDA/cuDNN base image and
|
||||||
|
# `docker run --gpus`; that is deliberately not part of this Dockerfile.
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
FROM runtime AS with-server-engine
|
||||||
|
USER root
|
||||||
|
|
||||||
|
RUN apt-get update \
|
||||||
|
&& apt-get install -y --no-install-recommends python3 python3-venv \
|
||||||
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
RUN python3 -m venv /opt/faster-whisper \
|
||||||
|
&& /opt/faster-whisper/bin/pip install --no-cache-dir -r server-py/requirements.txt
|
||||||
|
|
||||||
|
ENV SIDECAR_PYTHON=/opt/faster-whisper/bin/python3
|
||||||
|
# Cache Whisper weights inside the data volume so a container restart does not
|
||||||
|
# re-download them (first request per model downloads from the HF Hub).
|
||||||
|
ENV FASTER_WHISPER_DOWNLOAD_ROOT=/app/data/faster-whisper-models
|
||||||
|
|
||||||
|
USER bun
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Default target: a plain `docker build .` (what .gitlab-ci.yml runs) must
|
||||||
|
# produce the Bun-only image, but Docker builds the *last* stage by default
|
||||||
|
# and with-server-engine has to come after the stage it extends — so re-select
|
||||||
|
# the runtime stage here. Adds no layers.
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
FROM runtime AS default
|
||||||
|
|||||||
@@ -0,0 +1,128 @@
|
|||||||
|
# AU Whisper Web
|
||||||
|
|
||||||
|
Browser-first speech-to-text powered by OpenAI Whisper. Anonymous users run
|
||||||
|
Whisper entirely in their browser via
|
||||||
|
[Transformers.js](https://github.com/huggingface/transformers) (WebNN → WebGPU
|
||||||
|
→ WASM, installable PWA, works offline once a model is cached). Logged-in
|
||||||
|
users can optionally run inference on the **server** instead, backed by
|
||||||
|
[faster-whisper](https://github.com/SYSTRAN/faster-whisper) (CTranslate2)
|
||||||
|
through a small Python sidecar (`server-py/`, Decisions.md D-013).
|
||||||
|
|
||||||
|
See `Architecture.md` for the design, `Decisions.md` for the decision log, and
|
||||||
|
`CLAUDE.md` for the toolchain rules (Bun everywhere — no npm/node).
|
||||||
|
|
||||||
|
## Development
|
||||||
|
|
||||||
|
```sh
|
||||||
|
bun install
|
||||||
|
bun dev # dev server with HMR at http://localhost:3000
|
||||||
|
bun test # unit tests
|
||||||
|
```
|
||||||
|
|
||||||
|
## Production build (bare metal)
|
||||||
|
|
||||||
|
```sh
|
||||||
|
bun install
|
||||||
|
bun run build # emits self-contained dist/ (app shell, worker, sw.js, /ort/*)
|
||||||
|
bun start # NODE_ENV=production bun src/index.ts — serves dist/ + /api/*
|
||||||
|
```
|
||||||
|
|
||||||
|
To enable the server engine on bare metal, additionally install Python ≥ 3.9
|
||||||
|
and the sidecar dependencies: `python3 -m pip install -r server-py/requirements.txt`.
|
||||||
|
Nothing else changes — the sidecar is spawned lazily on the first server-engine
|
||||||
|
request.
|
||||||
|
|
||||||
|
## Deployment (Docker)
|
||||||
|
|
||||||
|
The `Dockerfile` is multi-stage and produces **two image flavors**:
|
||||||
|
|
||||||
|
| Flavor | Build command | Contents |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| **Bun-only** (default) | `docker build -t whisper-web .` | Bun runtime + built `dist/`. No Python. |
|
||||||
|
| **With server engine** | `docker build --target with-server-engine -t whisper-web:server-engine .` | Everything above + Debian `python3` and `server-py/requirements.txt` (faster-whisper, CPU-only) in a virtualenv at `/opt/faster-whisper`. |
|
||||||
|
|
||||||
|
`--target` was chosen over a build-arg because the optional layer is a strict
|
||||||
|
superset of the runtime stage: `with-server-engine` extends `runtime`, so the
|
||||||
|
Bun layers are shared byte-for-byte between both flavors and there is no
|
||||||
|
conditional logic inside the Dockerfile. A plain `docker build .` (what
|
||||||
|
`.gitlab-ci.yml` runs) intentionally resolves to the Bun-only flavor.
|
||||||
|
|
||||||
|
Run either flavor the same way:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
docker run -d -p 3000:3000 -v whisper-data:/app/data whisper-web
|
||||||
|
# or, with the server engine:
|
||||||
|
docker run -d -p 3000:3000 -v whisper-data:/app/data whisper-web:server-engine
|
||||||
|
```
|
||||||
|
|
||||||
|
`docker compose up --build` builds and runs the Bun-only flavor with the
|
||||||
|
`whisper-data` volume already wired up (see `docker-compose.yml`).
|
||||||
|
|
||||||
|
### Graceful degradation (why the Bun-only image is safe to deploy)
|
||||||
|
|
||||||
|
The server engine is optional by design (D-013):
|
||||||
|
|
||||||
|
- **Bun-only image:** everything anonymous users see works in full — browser
|
||||||
|
inference, model downloads, PWA/offline, and (if `AUTH_ENABLED` stays on)
|
||||||
|
accounts. The first server-engine probe finds no `python3`, the supervisor
|
||||||
|
records that as a permanent condition (no respawn storms), and
|
||||||
|
`GET /api/transcribe/status` answers `{"available": false, "reason": ...}`;
|
||||||
|
`POST /api/transcribe` answers `503` with the same reason. The UI simply
|
||||||
|
doesn't offer the server engine.
|
||||||
|
- **with-server-engine image:** `status` reports `available: true` and
|
||||||
|
logged-in users get the server/device toggle. The first request for a model
|
||||||
|
downloads its CTranslate2 weights from the Hugging Face Hub into
|
||||||
|
`FASTER_WHISPER_DOWNLOAD_ROOT` (inside the data volume, so restarts don't
|
||||||
|
re-download); the last-used model stays loaded between requests.
|
||||||
|
|
||||||
|
### Runtime environment variables
|
||||||
|
|
||||||
|
All configuration is runtime-only (D-015) — one image serves every
|
||||||
|
configuration; flip flags with `docker run -e` / compose `environment:`.
|
||||||
|
|
||||||
|
| Variable | Default | Meaning |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| `PORT` | `3000` | Port the Bun server listens on (also honored by the image's `EXPOSE`/healthcheck). |
|
||||||
|
| `NODE_ENV` | `production` (set in the image) | `production` serves the prebuilt `dist/`; anything else runs the dev server. Leave as-is in Docker. |
|
||||||
|
| `AUTH_ENABLED` | `true` | D-015 feature flag. `false`/`0`/`no`/`off` disables login/register: `/api/auth/*` **and** `/api/transcribe*` return 404 (the server engine is session-gated, so it disappears with auth) and the client hides the Sign in UI. |
|
||||||
|
| `DB_PATH` | `/app/data/app.db` (image) / `./data/app.db` (bare metal) | SQLite database (users + sessions, D-009). Keep it inside the mounted volume. |
|
||||||
|
| `TRANSCRIBE_MAX_UPLOAD_BYTES` | `536870912` (512 MiB ≈ 2¼ h of audio) | Upload cap for `POST /api/transcribe` (raw float32 mono 16 kHz PCM), enforced via Content-Length and again while streaming. |
|
||||||
|
| `SIDECAR_PYTHON` | `python3` (with-server-engine image: `/opt/faster-whisper/bin/python3`) | Python interpreter used to spawn `server-py/sidecar.py`. Only override for exotic setups. |
|
||||||
|
| `FASTER_WHISPER_DEVICE` | `auto` | CTranslate2 device for the sidecar (`auto`, `cpu`, `cuda`). In the stock image `auto` resolves to CPU. |
|
||||||
|
| `FASTER_WHISPER_COMPUTE` | `int8` | CTranslate2 compute type (`int8` is the fast, low-memory CPU choice; e.g. `float16` on GPU). |
|
||||||
|
| `FASTER_WHISPER_DOWNLOAD_ROOT` | unset (with-server-engine image: `/app/data/faster-whisper-models`) | Where the sidecar caches downloaded model weights. |
|
||||||
|
|
||||||
|
There are no session-secret style variables: sessions are random 256-bit ids
|
||||||
|
stored server-side in SQLite; the cookie is `HttpOnly`, `SameSite=Lax`, and
|
||||||
|
`Secure` whenever `NODE_ENV=production`.
|
||||||
|
|
||||||
|
### Persistence
|
||||||
|
|
||||||
|
Mount a volume at **`/app/data`**. It holds:
|
||||||
|
|
||||||
|
- `app.db` — the SQLite database (users + sessions). Without the volume every
|
||||||
|
redeploy logs everyone out and drops accounts.
|
||||||
|
- `faster-whisper-models/` — (server-engine flavor) downloaded Whisper
|
||||||
|
weights, from ~75 MB (`tiny`) to ~1.6 GB (`large-v3-turbo`).
|
||||||
|
|
||||||
|
Browser-engine model weights never touch the server — they live in each
|
||||||
|
visitor's browser cache.
|
||||||
|
|
||||||
|
### HTTPS / reverse proxy
|
||||||
|
|
||||||
|
Run the container behind a TLS-terminating reverse proxy (Caddy, nginx,
|
||||||
|
Traefik…). HTTPS is **functionally required** in production, not just good
|
||||||
|
practice: service workers (PWA/offline), microphone capture
|
||||||
|
(`getUserMedia` for live transcription), WebGPU, and the `Secure` session
|
||||||
|
cookie all require a secure context. Plain HTTP only works on `localhost`.
|
||||||
|
The app uses relative URLs and also tolerates being served under a subpath.
|
||||||
|
|
||||||
|
### GPU note
|
||||||
|
|
||||||
|
The `with-server-engine` target is CPU-only on purpose (small-ish image, no
|
||||||
|
CUDA runtime). For GPU inference you would build your own image on an
|
||||||
|
`nvidia/cuda` base with cuDNN 9 + cuBLAS, install Bun and
|
||||||
|
`server-py/requirements.txt` the same way, run with `docker run --gpus all`,
|
||||||
|
and set `FASTER_WHISPER_DEVICE=cuda` (plus e.g.
|
||||||
|
`FASTER_WHISPER_COMPUTE=float16`). Not shipped here — CPU int8 already makes
|
||||||
|
the server engine several times faster than real time on modest hardware.
|
||||||
+40
-5
@@ -9,8 +9,17 @@ Legend: `[ ]` todo · `[x]` done · `[~]` in progress
|
|||||||
---
|
---
|
||||||
|
|
||||||
## WIP Tasks before usage limit cutoff:
|
## WIP Tasks before usage limit cutoff:
|
||||||
- [~] Allow users to cancel current transcription.
|
- [x] Allow users to cancel current transcription. — Cancel button on the
|
||||||
- [~] Better user-feedback while transcription is in progress (better than "Listening for the first few words")
|
file line, wired to worker termination (D-018); covers decode, pending
|
||||||
|
model download, and the pass itself. Server-engine cancel is tracked in
|
||||||
|
M6 (needs the M5 EngineToggle first).
|
||||||
|
- [x] Better user-feedback while transcription is in progress (better than
|
||||||
|
"Listening for the first few words") — the worker now posts
|
||||||
|
`transcribe-progress` (seconds processed / total, from the streamer's
|
||||||
|
chunk timestamps via the pure `TranscribeProgressTracker`); the UI shows
|
||||||
|
a percent in the heading plus a progress bar with mm:ss, and the file
|
||||||
|
flow's placeholder reads "Processing audio…" (the live wording stays
|
||||||
|
live-only).
|
||||||
|
|
||||||
## M0 — Scaffold cleanup & docs (small)
|
## M0 — Scaffold cleanup & docs (small)
|
||||||
Current state is the raw `bun init` React template.
|
Current state is the raw `bun init` React template.
|
||||||
@@ -143,7 +152,7 @@ Notes (M4):
|
|||||||
(availability probe for the toggle)
|
(availability probe for the toggle)
|
||||||
- [x] `src/lib/engine.ts` — `Engine` interface; `BrowserEngine` (wraps worker) and
|
- [x] `src/lib/engine.ts` — `Engine` interface; `BrowserEngine` (wraps worker) and
|
||||||
`ServerEngine` (wraps fetch stream) (D-010)
|
`ServerEngine` (wraps fetch stream) (D-010)
|
||||||
- [ ] `EngineToggle` component — visible only with a session; persists choice
|
- [x] `EngineToggle` component — visible only with a session; persists choice
|
||||||
(persistence helpers + status probe already in `lib/engine.ts`; remaining:
|
(persistence helpers + status probe already in `lib/engine.ts`; remaining:
|
||||||
the component, lifting `useAuth` out of `AuthForms` into `App` so both see
|
the component, lifting `useAuth` out of `AuthForms` into `App` so both see
|
||||||
one session state, and routing the file flow through the selected engine —
|
one session state, and routing the file flow through the selected engine —
|
||||||
@@ -154,7 +163,7 @@ Notes (M4):
|
|||||||
`Bun.spawn` against `tests/fixtures/fake_sidecar.py`); real inference is
|
`Bun.spawn` against `tests/fixtures/fake_sidecar.py`); real inference is
|
||||||
the opt-in `FW_INTEGRATION=1` test + manual run in `tests/server-engine.md`
|
the opt-in `FW_INTEGRATION=1` test + manual run in `tests/server-engine.md`
|
||||||
(no pip/browser in the dev environment; same caveat as M2/M3)
|
(no pip/browser in the dev environment; same caveat as M2/M3)
|
||||||
- [ ] Deployment: optional Python/faster-whisper layer for the Docker image +
|
- [x] Deployment: optional Python/faster-whisper layer for the Docker image +
|
||||||
deployer note (base image stays Bun-only; install
|
deployer note (base image stays Bun-only; install
|
||||||
`server-py/requirements.txt` to enable the server engine)
|
`server-py/requirements.txt` to enable the server engine)
|
||||||
|
|
||||||
@@ -183,11 +192,37 @@ Notes (M5):
|
|||||||
- Smoke-tested live against the dev server: `/` + auth unchanged; status
|
- Smoke-tested live against the dev server: `/` + auth unchanged; status
|
||||||
401 → (register) → 200 `{available:false, reason:"faster-whisper is not
|
401 → (register) → 200 `{available:false, reason:"faster-whisper is not
|
||||||
installed…"}`; POST → 503 with the same reason, body untouched.
|
installed…"}`; POST → 503 with the same reason, body untouched.
|
||||||
|
- EngineToggle: one `useAuth()` lives in `App` (passed into `AuthForms` and
|
||||||
|
gating the toggle); the status probe runs on sign-in and the server option
|
||||||
|
renders disabled-with-reason when unavailable. The file flow now goes
|
||||||
|
through the `Engine` interface for **both** paths (D-010): `BrowserEngine`
|
||||||
|
shares the worker with the live loop (distinct requestId namespaces) and a
|
||||||
|
new `stream` flag on the transcribe message keeps token partials +
|
||||||
|
transcribe-progress flowing for id-carrying engine passes (live passes stay
|
||||||
|
non-streaming). Server runs skip the browser model load, show segment
|
||||||
|
partials without a progress bar (faster-whisper's stream carries no window
|
||||||
|
boundaries), and cancel by aborting the fetch (D-018 amendment) — browser
|
||||||
|
runs keep worker-termination cancel. The choice persists in localStorage
|
||||||
|
but only takes effect while signed in with the engine available.
|
||||||
|
- Deployment: multi-stage `Dockerfile` — `runtime` (Bun-only, 244 MB) is the
|
||||||
|
default target so plain `docker build .` (existing CI) is unchanged;
|
||||||
|
`--target with-server-engine` (405 MB) adds python3 + faster-whisper in a
|
||||||
|
venv at `/opt/faster-whisper` (wired via `SIDECAR_PYTHON`), with
|
||||||
|
`FASTER_WHISPER_DOWNLOAD_ROOT` under the `/app/data` volume so weights
|
||||||
|
persist. README.md documents both flavors, all runtime env vars, the DB
|
||||||
|
volume, and the HTTPS/reverse-proxy requirement. Verified live in Docker:
|
||||||
|
Bun-only image degrades per D-013 (`available:false`, "python3 is not
|
||||||
|
available…"); server-engine image ran a real `POST /api/transcribe` of
|
||||||
|
`en_aiproblem.m4a` end-to-end, streamed segments matching the fixture
|
||||||
|
transcript — the server half of the M5 integration-test item is now
|
||||||
|
exercised for real (browser half still needs the manual run).
|
||||||
|
|
||||||
## M6 — Hardening & polish
|
## M6 — Hardening & polish
|
||||||
- [ ] Error taxonomy: unsupported file, out-of-memory (suggest smaller model),
|
- [ ] Error taxonomy: unsupported file, out-of-memory (suggest smaller model),
|
||||||
WebGPU device-lost, network loss mid-download (resume or clear retry)
|
WebGPU device-lost, network loss mid-download (resume or clear retry)
|
||||||
- [ ] Cancel in-flight transcription (both engines)
|
- [x] Cancel in-flight transcription (both engines) — browser engine via
|
||||||
|
worker termination (D-018), server engine via aborting the
|
||||||
|
`/api/transcribe` fetch (D-018 amendment; landed with M5's EngineToggle)
|
||||||
- [ ] Export transcript: .txt and .srt (timestamps already available)
|
- [ ] Export transcript: .txt and .srt (timestamps already available)
|
||||||
- [ ] Storage management UI: list cached models, per-model delete
|
- [ ] Storage management UI: list cached models, per-model delete
|
||||||
- [ ] Basic a11y pass (keyboard, focus, aria on progress/live transcript)
|
- [ ] Basic a11y pass (keyboard, focus, aria on progress/live transcript)
|
||||||
|
|||||||
+220
-43
@@ -2,6 +2,7 @@ import { useEffect, useRef, useState } from "react";
|
|||||||
import "./index.css";
|
import "./index.css";
|
||||||
|
|
||||||
import { AuthForms } from "./components/AuthForms";
|
import { AuthForms } from "./components/AuthForms";
|
||||||
|
import { EngineToggle } from "./components/EngineToggle";
|
||||||
import { FileDropzone } from "./components/FileDropzone";
|
import { FileDropzone } from "./components/FileDropzone";
|
||||||
import { LiveControls, type LiveStatus } from "./components/LiveControls";
|
import { LiveControls, type LiveStatus } from "./components/LiveControls";
|
||||||
import { ModelLoadProgress } from "./components/ModelLoadProgress";
|
import { ModelLoadProgress } from "./components/ModelLoadProgress";
|
||||||
@@ -9,6 +10,17 @@ import { ModelPicker } from "./components/ModelPicker";
|
|||||||
import { TranscriptView } from "./components/TranscriptView";
|
import { TranscriptView } from "./components/TranscriptView";
|
||||||
import { TranscriptionOptions } from "./components/TranscriptionOptions";
|
import { TranscriptionOptions } from "./components/TranscriptionOptions";
|
||||||
import { decodeAudioFile } from "./lib/audio";
|
import { decodeAudioFile } from "./lib/audio";
|
||||||
|
import {
|
||||||
|
BrowserEngine,
|
||||||
|
ServerEngine,
|
||||||
|
fetchServerEngineStatus,
|
||||||
|
loadEngineChoice,
|
||||||
|
saveEngineChoice,
|
||||||
|
type Engine,
|
||||||
|
type EngineKind,
|
||||||
|
type ServerEngineStatus,
|
||||||
|
} from "./lib/engine";
|
||||||
|
import { useAuth } from "./lib/useAuth";
|
||||||
import { LIVE_TUNING, LiveTranscriber, type LiveTranscribeResult } from "./lib/live";
|
import { LIVE_TUNING, LiveTranscriber, type LiveTranscribeResult } from "./lib/live";
|
||||||
import { startMicCapture, type MicCapture } from "./lib/mic";
|
import { startMicCapture, type MicCapture } from "./lib/mic";
|
||||||
import { getDownloadedModels } from "./lib/modelCache";
|
import { getDownloadedModels } from "./lib/modelCache";
|
||||||
@@ -64,11 +76,25 @@ export function App() {
|
|||||||
// audio to the (fresh) worker. Decoding itself cannot be aborted.
|
// audio to the (fresh) worker. Decoding itself cannot be aborted.
|
||||||
const fileRun = useRef(0);
|
const fileRun = useRef(0);
|
||||||
|
|
||||||
|
// Engine plumbing (M5, D-010). The BrowserEngine is bound to the worker's
|
||||||
|
// lifetime (recreated together on cancel); the ServerEngine is stateless
|
||||||
|
// and lives for the app. A server run's fetch is cancelled via abortRef.
|
||||||
|
const browserEngineRef = useRef<BrowserEngine | null>(null);
|
||||||
|
const serverEngineRef = useRef<ServerEngine | null>(null);
|
||||||
|
const abortRef = useRef<AbortController | null>(null);
|
||||||
|
|
||||||
const [busy, setBusy] = useState(false);
|
const [busy, setBusy] = useState(false);
|
||||||
|
// Which engine the current file run uses; null while idle. Server runs
|
||||||
|
// don't need the browser model, so parts of the UI key off this.
|
||||||
|
const [runEngine, setRunEngine] = useState<EngineKind | null>(null);
|
||||||
const [decoding, setDecoding] = useState(false);
|
const [decoding, setDecoding] = useState(false);
|
||||||
const [loadStarted, setLoadStarted] = useState(false);
|
const [loadStarted, setLoadStarted] = useState(false);
|
||||||
const [loadProgress, setLoadProgress] = useState(initialLoadProgress);
|
const [loadProgress, setLoadProgress] = useState(initialLoadProgress);
|
||||||
const [partialText, setPartialText] = useState("");
|
const [partialText, setPartialText] = useState("");
|
||||||
|
const [transcribeProgress, setTranscribeProgress] = useState<{
|
||||||
|
processedSeconds: number;
|
||||||
|
totalSeconds: number;
|
||||||
|
} | null>(null);
|
||||||
const [result, setResult] = useState<TranscriptResult | null>(null);
|
const [result, setResult] = useState<TranscriptResult | null>(null);
|
||||||
const [error, setError] = useState<string | null>(null);
|
const [error, setError] = useState<string | null>(null);
|
||||||
const [fileName, setFileName] = useState<string | null>(null);
|
const [fileName, setFileName] = useState<string | null>(null);
|
||||||
@@ -91,6 +117,15 @@ export function App() {
|
|||||||
const [swControlled, setSwControlled] = useState(false);
|
const [swControlled, setSwControlled] = useState(false);
|
||||||
const [applyUpdate, setApplyUpdate] = useState<(() => void) | null>(null);
|
const [applyUpdate, setApplyUpdate] = useState<(() => void) | null>(null);
|
||||||
|
|
||||||
|
// Session + engine choice (M4/M5). One useAuth for the whole app — the
|
||||||
|
// header forms and the EngineToggle must agree on the session. The choice
|
||||||
|
// persists across reloads; it only takes effect while signed in AND the
|
||||||
|
// deployment reports the server engine available (anonymous-first: logged
|
||||||
|
// out, everything runs on-device exactly as before).
|
||||||
|
const auth = useAuth();
|
||||||
|
const [engineChoice, setEngineChoice] = useState<EngineKind>(() => loadEngineChoice());
|
||||||
|
const [serverStatus, setServerStatus] = useState<ServerEngineStatus | null>(null);
|
||||||
|
|
||||||
function refreshDownloaded() {
|
function refreshDownloaded() {
|
||||||
// Downloaded badges (D-005): query Cache Storage for cached weights.
|
// Downloaded badges (D-005): query Cache Storage for cached weights.
|
||||||
getDownloadedModels(MODEL_IDS).then(setDownloaded, () => {});
|
getDownloadedModels(MODEL_IDS).then(setDownloaded, () => {});
|
||||||
@@ -109,9 +144,28 @@ export function App() {
|
|||||||
liveRef.current = null;
|
liveRef.current = null;
|
||||||
workerRef.current?.terminate();
|
workerRef.current?.terminate();
|
||||||
workerRef.current = null;
|
workerRef.current = null;
|
||||||
|
browserEngineRef.current = null;
|
||||||
|
abortRef.current?.abort();
|
||||||
|
abortRef.current = null;
|
||||||
};
|
};
|
||||||
}, []);
|
}, []);
|
||||||
|
|
||||||
|
// Probe server-engine availability once a session exists (the endpoint is
|
||||||
|
// auth-guarded, so probing while anonymous is pointless). Never rejects.
|
||||||
|
useEffect(() => {
|
||||||
|
if (!auth.user) {
|
||||||
|
setServerStatus(null);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let cancelled = false;
|
||||||
|
fetchServerEngineStatus().then((status) => {
|
||||||
|
if (!cancelled) setServerStatus(status);
|
||||||
|
});
|
||||||
|
return () => {
|
||||||
|
cancelled = true;
|
||||||
|
};
|
||||||
|
}, [auth.user]);
|
||||||
|
|
||||||
function takeLiveRequest(requestId: string | undefined) {
|
function takeLiveRequest(requestId: string | undefined) {
|
||||||
if (requestId === undefined) return undefined;
|
if (requestId === undefined) return undefined;
|
||||||
const pending = pendingLive.current.get(requestId);
|
const pending = pendingLive.current.get(requestId);
|
||||||
@@ -146,13 +200,24 @@ export function App() {
|
|||||||
setPartialText((text) => text + message.text);
|
setPartialText((text) => text + message.text);
|
||||||
}
|
}
|
||||||
break;
|
break;
|
||||||
|
case "transcribe-progress":
|
||||||
|
// File flow only — live passes never carry progress (D-016).
|
||||||
|
if (message.requestId === undefined) {
|
||||||
|
setTranscribeProgress({
|
||||||
|
processedSeconds: message.processedSeconds,
|
||||||
|
totalSeconds: message.totalSeconds,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
break;
|
||||||
case "result": {
|
case "result": {
|
||||||
const pending = takeLiveRequest(message.requestId);
|
const pending = takeLiveRequest(message.requestId);
|
||||||
if (pending) {
|
if (pending) {
|
||||||
pending.resolve({ text: message.text, chunks: message.chunks });
|
pending.resolve({ text: message.text, chunks: message.chunks });
|
||||||
} else if (message.requestId === undefined) {
|
} else if (message.requestId === undefined) {
|
||||||
setResult({ text: message.text, chunks: message.chunks });
|
setResult({ text: message.text, chunks: message.chunks });
|
||||||
|
setTranscribeProgress(null);
|
||||||
setBusy(false);
|
setBusy(false);
|
||||||
|
setRunEngine(null);
|
||||||
}
|
}
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
@@ -160,9 +225,13 @@ export function App() {
|
|||||||
const pending = takeLiveRequest(message.requestId);
|
const pending = takeLiveRequest(message.requestId);
|
||||||
if (pending) {
|
if (pending) {
|
||||||
pending.reject(new Error(message.message));
|
pending.reject(new Error(message.message));
|
||||||
} else {
|
} else if (message.requestId === undefined) {
|
||||||
|
// Load-stage failures and legacy id-less passes. Engine-routed
|
||||||
|
// file passes carry an id and surface through the engine promise.
|
||||||
setError(message.message);
|
setError(message.message);
|
||||||
|
setTranscribeProgress(null);
|
||||||
setBusy(false);
|
setBusy(false);
|
||||||
|
setRunEngine(null);
|
||||||
}
|
}
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
@@ -179,6 +248,9 @@ export function App() {
|
|||||||
setBusy(false);
|
setBusy(false);
|
||||||
});
|
});
|
||||||
workerRef.current = worker;
|
workerRef.current = worker;
|
||||||
|
// The BrowserEngine shares this worker and only sees its own request ids
|
||||||
|
// (engine-*), so it coexists with the live loop's live-* ids above.
|
||||||
|
browserEngineRef.current = new BrowserEngine(worker);
|
||||||
return worker;
|
return worker;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -205,6 +277,7 @@ export function App() {
|
|||||||
setModelId(nextModelId);
|
setModelId(nextModelId);
|
||||||
setActiveDevice(null);
|
setActiveDevice(null);
|
||||||
setPartialText("");
|
setPartialText("");
|
||||||
|
setTranscribeProgress(null);
|
||||||
setResult(null);
|
setResult(null);
|
||||||
startLoad(nextModelId, devicePreference);
|
startLoad(nextModelId, devicePreference);
|
||||||
}
|
}
|
||||||
@@ -219,78 +292,146 @@ export function App() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Cancel the in-flight file transcription (D-018). Transformers.js has no
|
* Cancel the in-flight file transcription (D-018). Browser runs: no abort
|
||||||
* abort API for a running `asr()` call — and on the WASM backend the pass
|
* API exists for a running `asr()` call — and on the WASM backend the pass
|
||||||
* blocks the worker's event loop, so even a "cancel" message would sit
|
* blocks the worker's event loop, so even a "cancel" message would sit
|
||||||
* unread until the pass finished. Terminating the worker is the only
|
* unread until the pass finished. Terminating the worker is the only
|
||||||
* reliable preemption; it also aborts an in-flight model download. The next
|
* reliable preemption; it also aborts an in-flight model download. The next
|
||||||
* run spawns a fresh worker and reloads the pipeline — cheap, because the
|
* run spawns a fresh worker and reloads the pipeline — cheap, because the
|
||||||
* weights stay in the browser Cache API (D-005).
|
* weights stay in the browser Cache API (D-005). Server runs: abort the
|
||||||
|
* `/api/transcribe` fetch (the server cancels the sidecar pass, D-011) and
|
||||||
|
* leave the worker — and its loaded pipeline — untouched.
|
||||||
*/
|
*/
|
||||||
function cancelTranscription() {
|
function cancelTranscription() {
|
||||||
if (!busy) return;
|
if (!busy) return;
|
||||||
fileRun.current++; // any decode still running is now stale
|
fileRun.current++; // any decode still running is now stale
|
||||||
workerRef.current?.terminate();
|
const cancelled = new Error("Transcription cancelled.");
|
||||||
workerRef.current = null;
|
if (runEngine === "server") {
|
||||||
|
abortRef.current?.abort();
|
||||||
|
abortRef.current = null;
|
||||||
|
} else {
|
||||||
|
browserEngineRef.current?.failAll(cancelled);
|
||||||
|
browserEngineRef.current = null;
|
||||||
|
workerRef.current?.terminate();
|
||||||
|
workerRef.current = null;
|
||||||
|
// The worker (and any pipeline / partial download) is gone — reset the
|
||||||
|
// load state so the next transcription reloads from a clean slate.
|
||||||
|
setLoadStarted(false);
|
||||||
|
setLoadProgress(initialLoadProgress);
|
||||||
|
setActiveDevice(null);
|
||||||
|
}
|
||||||
// The file flow never has live passes pending, but if anything is in the
|
// The file flow never has live passes pending, but if anything is in the
|
||||||
// map (defensive), unblock it so nothing awaits a dead worker forever.
|
// map (defensive), unblock it so nothing awaits a dead worker forever.
|
||||||
const cancelled = new Error("Transcription cancelled.");
|
|
||||||
for (const pending of pendingLive.current.values()) pending.reject(cancelled);
|
for (const pending of pendingLive.current.values()) pending.reject(cancelled);
|
||||||
pendingLive.current.clear();
|
pendingLive.current.clear();
|
||||||
setBusy(false);
|
setBusy(false);
|
||||||
|
setRunEngine(null);
|
||||||
setPartialText("");
|
setPartialText("");
|
||||||
|
setTranscribeProgress(null);
|
||||||
setResult(null);
|
setResult(null);
|
||||||
setFileName(null);
|
setFileName(null);
|
||||||
setError(null);
|
setError(null);
|
||||||
// The worker (and any pipeline / partial download) is gone — reset the
|
|
||||||
// load state so the next transcription reloads from a clean slate.
|
|
||||||
setLoadStarted(false);
|
|
||||||
setLoadProgress(initialLoadProgress);
|
|
||||||
setActiveDevice(null);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async function handleFile(file: File) {
|
async function handleFile(file: File) {
|
||||||
if (busy || liveStatus !== "idle") return;
|
if (busy || liveStatus !== "idle") return;
|
||||||
const run = ++fileRun.current;
|
const run = ++fileRun.current;
|
||||||
|
// The toggle's choice only takes effect with a session and an available
|
||||||
|
// server engine (D-010); everything else runs on-device as always.
|
||||||
|
const useServer =
|
||||||
|
engineChoice === "server" &&
|
||||||
|
auth.user !== null &&
|
||||||
|
serverStatus?.available === true;
|
||||||
setBusy(true);
|
setBusy(true);
|
||||||
|
setRunEngine(useServer ? "server" : "browser");
|
||||||
setError(null);
|
setError(null);
|
||||||
setPartialText("");
|
setPartialText("");
|
||||||
|
setTranscribeProgress(null);
|
||||||
setResult(null);
|
setResult(null);
|
||||||
setFileName(file.name);
|
setFileName(file.name);
|
||||||
|
|
||||||
// Kick off the (idempotent) model load immediately so the download runs
|
let engine: Engine;
|
||||||
// while we decode the audio locally.
|
if (useServer) {
|
||||||
const worker = ensureWorker();
|
engine = serverEngineRef.current ??= new ServerEngine();
|
||||||
if (!loadStarted) {
|
|
||||||
startLoad(modelId, devicePreference);
|
|
||||||
} else {
|
} else {
|
||||||
postToWorker(worker, { type: "load", modelId, device: devicePreference });
|
// Kick off the (idempotent) model load immediately so the download runs
|
||||||
|
// while we decode the audio locally. Server runs skip this — they don't
|
||||||
|
// need the browser pipeline.
|
||||||
|
ensureWorker();
|
||||||
|
if (!loadStarted) {
|
||||||
|
startLoad(modelId, devicePreference);
|
||||||
|
} else {
|
||||||
|
postToWorker(workerRef.current!, {
|
||||||
|
type: "load",
|
||||||
|
modelId,
|
||||||
|
device: devicePreference,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
engine = browserEngineRef.current!;
|
||||||
}
|
}
|
||||||
|
|
||||||
const multilingual = getModel(modelId)?.multilingual ?? false;
|
const multilingual = getModel(modelId)?.multilingual ?? false;
|
||||||
|
|
||||||
|
// Decoded entirely on-device via Web Audio (D-012); for browser runs the
|
||||||
|
// audio never touches the network (Architecture.md §5) — server runs
|
||||||
|
// upload the decoded PCM, which is the point of the toggle.
|
||||||
setDecoding(true);
|
setDecoding(true);
|
||||||
|
let audio: Float32Array;
|
||||||
try {
|
try {
|
||||||
// Decoded entirely on-device via Web Audio (D-012); the audio never
|
audio = await decodeAudioFile(file);
|
||||||
// touches the network (Architecture.md §5).
|
|
||||||
const audio = await decodeAudioFile(file);
|
|
||||||
if (fileRun.current !== run) return; // cancelled while decoding (D-018)
|
|
||||||
// Transfer the buffer to the worker — zero copy. Language/task only
|
|
||||||
// apply to multilingual models; `.en` variants take neither.
|
|
||||||
postToWorker(
|
|
||||||
worker,
|
|
||||||
multilingual
|
|
||||||
? { type: "transcribe", audio, language, task }
|
|
||||||
: { type: "transcribe", audio },
|
|
||||||
[audio.buffer as ArrayBuffer],
|
|
||||||
);
|
|
||||||
} catch (decodeError) {
|
} catch (decodeError) {
|
||||||
if (fileRun.current !== run) return; // this run was already cancelled
|
if (fileRun.current === run) {
|
||||||
setError(decodeError instanceof Error ? decodeError.message : String(decodeError));
|
setError(
|
||||||
setBusy(false);
|
decodeError instanceof Error ? decodeError.message : String(decodeError),
|
||||||
|
);
|
||||||
|
setBusy(false);
|
||||||
|
setRunEngine(null);
|
||||||
|
}
|
||||||
|
return;
|
||||||
} finally {
|
} finally {
|
||||||
setDecoding(false);
|
setDecoding(false);
|
||||||
}
|
}
|
||||||
|
if (fileRun.current !== run) return; // cancelled while decoding (D-018)
|
||||||
|
|
||||||
|
// Server runs cancel by aborting the fetch (D-018 amendment); browser
|
||||||
|
// runs cancel by terminating the worker, which rejects the promise via
|
||||||
|
// BrowserEngine.failAll.
|
||||||
|
const controller = useServer ? new AbortController() : null;
|
||||||
|
abortRef.current = controller;
|
||||||
|
try {
|
||||||
|
const outcome = await engine.transcribe(audio, {
|
||||||
|
// Language/task only apply to multilingual models; `.en` variants
|
||||||
|
// take neither. The browser engine ignores modelId (worker holds it).
|
||||||
|
modelId,
|
||||||
|
...(multilingual && { language, task }),
|
||||||
|
onPartial: (fragment) => {
|
||||||
|
if (fileRun.current === run) setPartialText((text) => text + fragment);
|
||||||
|
},
|
||||||
|
...(!useServer && {
|
||||||
|
onProgress: (progress: { processedSeconds: number; totalSeconds: number }) => {
|
||||||
|
if (fileRun.current === run) setTranscribeProgress(progress);
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
...(controller && { signal: controller.signal }),
|
||||||
|
});
|
||||||
|
if (fileRun.current !== run) return; // cancelled mid-pass
|
||||||
|
setResult(outcome);
|
||||||
|
setTranscribeProgress(null);
|
||||||
|
setBusy(false);
|
||||||
|
setRunEngine(null);
|
||||||
|
} catch (transcribeError) {
|
||||||
|
if (fileRun.current !== run) return; // cancellation rejections are expected
|
||||||
|
setError(
|
||||||
|
transcribeError instanceof Error
|
||||||
|
? transcribeError.message
|
||||||
|
: String(transcribeError),
|
||||||
|
);
|
||||||
|
setTranscribeProgress(null);
|
||||||
|
setBusy(false);
|
||||||
|
setRunEngine(null);
|
||||||
|
} finally {
|
||||||
|
if (abortRef.current === controller) abortRef.current = null;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// -------------------------------------------------------------------------
|
// -------------------------------------------------------------------------
|
||||||
@@ -381,8 +522,13 @@ export function App() {
|
|||||||
const model = getModel(modelId);
|
const model = getModel(modelId);
|
||||||
const modelReady = loadProgress.ready;
|
const modelReady = loadProgress.ready;
|
||||||
const showModelProgress = loadStarted && !modelReady && !error;
|
const showModelProgress = loadStarted && !modelReady && !error;
|
||||||
const transcribing = busy && !decoding && modelReady;
|
// Server runs never load the browser model, so they are "transcribing" the
|
||||||
|
// moment decoding ends.
|
||||||
|
const transcribing = busy && !decoding && (runEngine === "server" || modelReady);
|
||||||
const liveActive = liveStatus !== "idle";
|
const liveActive = liveStatus !== "idle";
|
||||||
|
// What the next file run will use (mirrors handleFile's decision).
|
||||||
|
const serverSelected =
|
||||||
|
engineChoice === "server" && auth.user !== null && serverStatus?.available === true;
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<main className="app">
|
<main className="app">
|
||||||
@@ -398,7 +544,7 @@ export function App() {
|
|||||||
unobtrusive "Sign in" button — everything else is unchanged.
|
unobtrusive "Sign in" button — everything else is unchanged.
|
||||||
Renders nothing until the /me probe confirms the server has auth
|
Renders nothing until the /me probe confirms the server has auth
|
||||||
enabled (runtime flag, D-015). */}
|
enabled (runtime flag, D-015). */}
|
||||||
<AuthForms />
|
<AuthForms auth={auth} />
|
||||||
</header>
|
</header>
|
||||||
|
|
||||||
<ModelPicker
|
<ModelPicker
|
||||||
@@ -423,6 +569,20 @@ export function App() {
|
|||||||
/>
|
/>
|
||||||
)}
|
)}
|
||||||
|
|
||||||
|
{/* Engine toggle (M5, D-010): signed-in users choose where uploaded
|
||||||
|
files are transcribed. Live mode always stays on-device (D-016). */}
|
||||||
|
{auth.user && (
|
||||||
|
<EngineToggle
|
||||||
|
choice={engineChoice}
|
||||||
|
onChoice={(choice) => {
|
||||||
|
setEngineChoice(choice);
|
||||||
|
saveEngineChoice(choice);
|
||||||
|
}}
|
||||||
|
status={serverStatus}
|
||||||
|
disabled={busy || liveActive}
|
||||||
|
/>
|
||||||
|
)}
|
||||||
|
|
||||||
<FileDropzone onFile={handleFile} disabled={busy || liveActive} />
|
<FileDropzone onFile={handleFile} disabled={busy || liveActive} />
|
||||||
|
|
||||||
<LiveControls
|
<LiveControls
|
||||||
@@ -436,6 +596,17 @@ export function App() {
|
|||||||
<p className="file-name">
|
<p className="file-name">
|
||||||
File: <strong>{fileName}</strong>
|
File: <strong>{fileName}</strong>
|
||||||
{decoding && " — decoding audio…"}
|
{decoding && " — decoding audio…"}
|
||||||
|
{busy && (
|
||||||
|
// Cancel the whole run (D-018): decode, pending model download,
|
||||||
|
// or the transcription pass itself.
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
className="cancel-button"
|
||||||
|
onClick={cancelTranscription}
|
||||||
|
>
|
||||||
|
Cancel
|
||||||
|
</button>
|
||||||
|
)}
|
||||||
</p>
|
</p>
|
||||||
)}
|
)}
|
||||||
|
|
||||||
@@ -454,6 +625,10 @@ export function App() {
|
|||||||
tentativeText={liveActive ? liveTentative : ""}
|
tentativeText={liveActive ? liveTentative : ""}
|
||||||
result={result}
|
result={result}
|
||||||
transcribing={liveActive || transcribing}
|
transcribing={liveActive || transcribing}
|
||||||
|
progress={liveActive ? null : transcribeProgress}
|
||||||
|
placeholder={
|
||||||
|
liveActive ? "Listening for the first words…" : "Processing audio…"
|
||||||
|
}
|
||||||
activeLabel={
|
activeLabel={
|
||||||
liveStatus === "stopping"
|
liveStatus === "stopping"
|
||||||
? "Finishing up…"
|
? "Finishing up…"
|
||||||
@@ -482,13 +657,15 @@ export function App() {
|
|||||||
<footer className="app-footer">
|
<footer className="app-footer">
|
||||||
<p>
|
<p>
|
||||||
Model: <code>{modelId}</code> · Engine:{" "}
|
Model: <code>{modelId}</code> · Engine:{" "}
|
||||||
{activeDevice === "webgpu"
|
{serverSelected
|
||||||
? "WebGPU (on-device)"
|
? "server (faster-whisper) — files upload to this deployment"
|
||||||
: activeDevice === "webnn"
|
: activeDevice === "webgpu"
|
||||||
? "WebNN (on-device)"
|
? "WebGPU (on-device)"
|
||||||
: activeDevice === "wasm"
|
: activeDevice === "webnn"
|
||||||
? "WASM (on-device)"
|
? "WebNN (on-device)"
|
||||||
: "on-device — backend selected on first load"}
|
: activeDevice === "wasm"
|
||||||
|
? "WASM (on-device)"
|
||||||
|
: "on-device — backend selected on first load"}
|
||||||
</p>
|
</p>
|
||||||
</footer>
|
</footer>
|
||||||
</main>
|
</main>
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
import { useState, type FormEvent, type KeyboardEvent } from "react";
|
import { useState, type FormEvent, type KeyboardEvent } from "react";
|
||||||
import { validatePassword, validateUsername } from "../lib/authClient";
|
import { validatePassword, validateUsername } from "../lib/authClient";
|
||||||
import { useAuth } from "../lib/useAuth";
|
import type { AuthState } from "../lib/useAuth";
|
||||||
|
|
||||||
type Mode = "login" | "register";
|
type Mode = "login" | "register";
|
||||||
|
|
||||||
@@ -10,11 +10,14 @@ type Mode = "login" | "register";
|
|||||||
* that opens a compact login/register popover; once signed in, the username
|
* that opens a compact login/register popover; once signed in, the username
|
||||||
* and a "Sign out" button.
|
* and a "Sign out" button.
|
||||||
*
|
*
|
||||||
|
* The session state comes from `App`'s single `useAuth()` — the EngineToggle
|
||||||
|
* (M5) reads the same state, so both always agree on who is signed in.
|
||||||
|
*
|
||||||
* Validation mirrors the server rules for instant feedback, but the server
|
* Validation mirrors the server rules for instant feedback, but the server
|
||||||
* stays authoritative: its `{ error }` messages are shown verbatim on 4xx.
|
* stays authoritative: its `{ error }` messages are shown verbatim on 4xx.
|
||||||
*/
|
*/
|
||||||
export function AuthForms() {
|
export function AuthForms({ auth }: { auth: AuthState }) {
|
||||||
const { enabled, user, login, register, logout } = useAuth();
|
const { enabled, user, login, register, logout } = auth;
|
||||||
|
|
||||||
const [open, setOpen] = useState(false);
|
const [open, setOpen] = useState(false);
|
||||||
const [mode, setMode] = useState<Mode>("login");
|
const [mode, setMode] = useState<Mode>("login");
|
||||||
|
|||||||
@@ -0,0 +1,62 @@
|
|||||||
|
import type { EngineKind, ServerEngineStatus } from "../lib/engine";
|
||||||
|
|
||||||
|
interface EngineToggleProps {
|
||||||
|
/** The persisted choice (src/lib/engine.ts helpers). */
|
||||||
|
choice: EngineKind;
|
||||||
|
onChoice: (choice: EngineKind) => void;
|
||||||
|
/** GET /api/transcribe/status result; null while the probe is in flight. */
|
||||||
|
status: ServerEngineStatus | null;
|
||||||
|
/** Locked while a run or live session is active (like the other options). */
|
||||||
|
disabled?: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Where should file transcription run (M5, D-010)? Rendered by `App` only for
|
||||||
|
* a signed-in session — anonymous users never see it (anonymous-first,
|
||||||
|
* Architecture.md §5). The server option stays visible but disabled when the
|
||||||
|
* deployment reports the engine unavailable (e.g. no faster-whisper sidecar,
|
||||||
|
* D-013), showing the probe's reason — hiding it entirely would read as a
|
||||||
|
* broken account feature. Live mode always stays on-device (D-016).
|
||||||
|
*/
|
||||||
|
export function EngineToggle({
|
||||||
|
choice,
|
||||||
|
onChoice,
|
||||||
|
status,
|
||||||
|
disabled = false,
|
||||||
|
}: EngineToggleProps) {
|
||||||
|
const serverAvailable = status?.available === true;
|
||||||
|
const effective: EngineKind = serverAvailable ? choice : "browser";
|
||||||
|
|
||||||
|
function option(kind: EngineKind, label: string, optionDisabled: boolean) {
|
||||||
|
return (
|
||||||
|
<label className="engine-option">
|
||||||
|
<input
|
||||||
|
type="radio"
|
||||||
|
name="engine"
|
||||||
|
value={kind}
|
||||||
|
checked={effective === kind}
|
||||||
|
disabled={disabled || optionDisabled}
|
||||||
|
onChange={() => onChoice(kind)}
|
||||||
|
/>
|
||||||
|
{label}
|
||||||
|
</label>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
return (
|
||||||
|
<section className="engine-toggle" aria-label="Transcription engine">
|
||||||
|
<span className="engine-toggle-label">Transcribe on</span>
|
||||||
|
{option("browser", "This device", false)}
|
||||||
|
{option("server", "Server", !serverAvailable)}
|
||||||
|
<span className="engine-toggle-hint">
|
||||||
|
{status === null
|
||||||
|
? "Checking server availability…"
|
||||||
|
: serverAvailable
|
||||||
|
? effective === "server"
|
||||||
|
? "Uploaded files are transcribed on the server (faster-whisper). Live mode always stays on this device."
|
||||||
|
: "Your audio stays on this device."
|
||||||
|
: (status.reason ?? "The server engine is unavailable.")}
|
||||||
|
</span>
|
||||||
|
</section>
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -15,6 +15,14 @@ interface TranscriptViewProps {
|
|||||||
tentativeText?: string;
|
tentativeText?: string;
|
||||||
/** Heading while running; defaults to "Transcribing…". */
|
/** Heading while running; defaults to "Transcribing…". */
|
||||||
activeLabel?: string;
|
activeLabel?: string;
|
||||||
|
/**
|
||||||
|
* How far the pass has progressed through the audio (file flow only) —
|
||||||
|
* renders a percent in the heading plus a thin progress bar. Live mode
|
||||||
|
* passes null; its feedback is the committed/tentative text itself.
|
||||||
|
*/
|
||||||
|
progress?: { processedSeconds: number; totalSeconds: number } | null;
|
||||||
|
/** Text shown before the first words arrive. */
|
||||||
|
placeholder?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
function formatTimestamp(seconds: number): string {
|
function formatTimestamp(seconds: number): string {
|
||||||
@@ -30,10 +38,17 @@ export function TranscriptView({
|
|||||||
transcribing,
|
transcribing,
|
||||||
tentativeText = "",
|
tentativeText = "",
|
||||||
activeLabel = "Transcribing…",
|
activeLabel = "Transcribing…",
|
||||||
|
progress = null,
|
||||||
|
placeholder = "Listening for the first words…",
|
||||||
}: TranscriptViewProps) {
|
}: TranscriptViewProps) {
|
||||||
const [copied, setCopied] = useState(false);
|
const [copied, setCopied] = useState(false);
|
||||||
const text = result?.text ?? partialText;
|
const text = result?.text ?? partialText;
|
||||||
const copyText = `${text}${tentativeText}`.trim();
|
const copyText = `${text}${tentativeText}`.trim();
|
||||||
|
const showProgress =
|
||||||
|
transcribing && !result && progress !== null && progress.totalSeconds > 0;
|
||||||
|
const percent = showProgress
|
||||||
|
? Math.min(100, Math.round((progress.processedSeconds / progress.totalSeconds) * 100))
|
||||||
|
: null;
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
if (!copied) return;
|
if (!copied) return;
|
||||||
@@ -52,7 +67,12 @@ export function TranscriptView({
|
|||||||
return (
|
return (
|
||||||
<section className="transcript">
|
<section className="transcript">
|
||||||
<div className="transcript-header">
|
<div className="transcript-header">
|
||||||
<h2>{result ? "Transcript" : activeLabel}</h2>
|
<h2>
|
||||||
|
{result ? "Transcript" : activeLabel}
|
||||||
|
{percent !== null && (
|
||||||
|
<span className="transcript-percent"> {percent}%</span>
|
||||||
|
)}
|
||||||
|
</h2>
|
||||||
<button
|
<button
|
||||||
type="button"
|
type="button"
|
||||||
className="copy-button"
|
className="copy-button"
|
||||||
@@ -62,8 +82,20 @@ export function TranscriptView({
|
|||||||
{copied ? "Copied!" : "Copy"}
|
{copied ? "Copied!" : "Copy"}
|
||||||
</button>
|
</button>
|
||||||
</div>
|
</div>
|
||||||
|
{showProgress && (
|
||||||
|
<div className="transcript-progress">
|
||||||
|
<progress
|
||||||
|
max={progress.totalSeconds}
|
||||||
|
value={Math.min(progress.processedSeconds, progress.totalSeconds)}
|
||||||
|
/>
|
||||||
|
<span className="transcript-progress-time">
|
||||||
|
{formatTimestamp(progress.processedSeconds)} /{" "}
|
||||||
|
{formatTimestamp(progress.totalSeconds)}
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
<p className="transcript-text" aria-live="polite">
|
<p className="transcript-text" aria-live="polite">
|
||||||
{text || (tentativeText ? "" : "Listening for the first words…")}
|
{text || (tentativeText ? "" : placeholder)}
|
||||||
{tentativeText && (
|
{tentativeText && (
|
||||||
<span className="transcript-tentative">{tentativeText}</span>
|
<span className="transcript-tentative">{tentativeText}</span>
|
||||||
)}
|
)}
|
||||||
|
|||||||
@@ -357,6 +357,38 @@ body {
|
|||||||
flex-wrap: wrap;
|
flex-wrap: wrap;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* --- EngineToggle (M5, D-010) --- */
|
||||||
|
|
||||||
|
.engine-toggle {
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
gap: 1rem;
|
||||||
|
flex-wrap: wrap;
|
||||||
|
font-size: 0.9rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.engine-toggle-label {
|
||||||
|
font-weight: 600;
|
||||||
|
}
|
||||||
|
|
||||||
|
.engine-option {
|
||||||
|
display: inline-flex;
|
||||||
|
align-items: center;
|
||||||
|
gap: 0.35rem;
|
||||||
|
cursor: pointer;
|
||||||
|
}
|
||||||
|
|
||||||
|
.engine-option:has(input:disabled) {
|
||||||
|
color: var(--muted);
|
||||||
|
cursor: default;
|
||||||
|
}
|
||||||
|
|
||||||
|
.engine-toggle-hint {
|
||||||
|
flex-basis: 100%;
|
||||||
|
color: var(--muted);
|
||||||
|
font-size: 0.85rem;
|
||||||
|
}
|
||||||
|
|
||||||
/* --- FileDropzone --- */
|
/* --- FileDropzone --- */
|
||||||
|
|
||||||
.dropzone {
|
.dropzone {
|
||||||
@@ -458,6 +490,25 @@ body {
|
|||||||
font-size: 0.95rem;
|
font-size: 0.95rem;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Cancel for the in-flight file run (D-018) — styled like .copy-button, with
|
||||||
|
a warning accent on hover. */
|
||||||
|
.cancel-button {
|
||||||
|
margin-left: 0.75rem;
|
||||||
|
border: 1px solid var(--border);
|
||||||
|
background: transparent;
|
||||||
|
color: inherit;
|
||||||
|
border-radius: 8px;
|
||||||
|
padding: 0.3rem 0.9rem;
|
||||||
|
font-size: 0.85rem;
|
||||||
|
cursor: pointer;
|
||||||
|
transition: border-color 0.15s, color 0.15s;
|
||||||
|
}
|
||||||
|
|
||||||
|
.cancel-button:hover {
|
||||||
|
border-color: #8b3a3a;
|
||||||
|
color: #f2b8b5;
|
||||||
|
}
|
||||||
|
|
||||||
.error {
|
.error {
|
||||||
margin: 0;
|
margin: 0;
|
||||||
padding: 0.75rem 1rem;
|
padding: 0.75rem 1rem;
|
||||||
@@ -561,6 +612,33 @@ body {
|
|||||||
cursor: default;
|
cursor: default;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.transcript-percent {
|
||||||
|
color: var(--muted);
|
||||||
|
font-weight: normal;
|
||||||
|
font-variant-numeric: tabular-nums;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Audio progress of the in-flight file pass (transcribe-progress messages). */
|
||||||
|
.transcript-progress {
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
gap: 0.75rem;
|
||||||
|
margin-top: 0.6rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.transcript-progress progress {
|
||||||
|
flex: 1;
|
||||||
|
height: 6px;
|
||||||
|
accent-color: var(--accent);
|
||||||
|
}
|
||||||
|
|
||||||
|
.transcript-progress-time {
|
||||||
|
color: var(--muted);
|
||||||
|
font-size: 0.85rem;
|
||||||
|
white-space: nowrap;
|
||||||
|
font-variant-numeric: tabular-nums;
|
||||||
|
}
|
||||||
|
|
||||||
.transcript-text {
|
.transcript-text {
|
||||||
margin: 0.75rem 0 0;
|
margin: 0.75rem 0 0;
|
||||||
white-space: pre-wrap;
|
white-space: pre-wrap;
|
||||||
|
|||||||
+24
-2
@@ -42,9 +42,16 @@ export interface EngineTranscribeOptions {
|
|||||||
* Browser engine: token fragments; server engine: whole segments.
|
* Browser engine: token fragments; server engine: whole segments.
|
||||||
*/
|
*/
|
||||||
onPartial?: (fragment: string) => void;
|
onPartial?: (fragment: string) => void;
|
||||||
|
/**
|
||||||
|
* How far the pass has progressed through the audio. Browser engine only —
|
||||||
|
* faster-whisper's stream carries no window boundaries, so the server
|
||||||
|
* engine never calls it (the UI degrades to segments-as-feedback).
|
||||||
|
*/
|
||||||
|
onProgress?: (progress: { processedSeconds: number; totalSeconds: number }) => void;
|
||||||
/**
|
/**
|
||||||
* Abort the run. Honored by the server engine (the request is cancelled
|
* Abort the run. Honored by the server engine (the request is cancelled
|
||||||
* server-side too); the browser worker cannot abort a pass yet (M6).
|
* server-side too); the browser worker cannot abort a pass — cancel there
|
||||||
|
* is worker termination via `BrowserEngine.failAll` (D-018).
|
||||||
*/
|
*/
|
||||||
signal?: AbortSignal;
|
signal?: AbortSignal;
|
||||||
}
|
}
|
||||||
@@ -83,6 +90,7 @@ interface PendingRequest {
|
|||||||
resolve: (result: EngineTranscribeResult) => void;
|
resolve: (result: EngineTranscribeResult) => void;
|
||||||
reject: (error: Error) => void;
|
reject: (error: Error) => void;
|
||||||
onPartial?: (fragment: string) => void;
|
onPartial?: (fragment: string) => void;
|
||||||
|
onProgress?: (progress: { processedSeconds: number; totalSeconds: number }) => void;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -116,11 +124,19 @@ export class BrowserEngine implements Engine {
|
|||||||
): Promise<EngineTranscribeResult> {
|
): Promise<EngineTranscribeResult> {
|
||||||
const requestId = `engine-${this.instance}-${++this.nextRequestId}`;
|
const requestId = `engine-${this.instance}-${++this.nextRequestId}`;
|
||||||
return new Promise((resolve, reject) => {
|
return new Promise((resolve, reject) => {
|
||||||
this.pending.set(requestId, { resolve, reject, onPartial: options.onPartial });
|
this.pending.set(requestId, {
|
||||||
|
resolve,
|
||||||
|
reject,
|
||||||
|
onPartial: options.onPartial,
|
||||||
|
onProgress: options.onProgress,
|
||||||
|
});
|
||||||
const message: TranscribeMessage = {
|
const message: TranscribeMessage = {
|
||||||
type: "transcribe",
|
type: "transcribe",
|
||||||
audio,
|
audio,
|
||||||
requestId,
|
requestId,
|
||||||
|
// Id-carrying requests default to no streaming in the worker (the
|
||||||
|
// live-pass rule) — ask for it explicitly when a consumer listens.
|
||||||
|
...((options.onPartial || options.onProgress) && { stream: true }),
|
||||||
...(options.language !== undefined &&
|
...(options.language !== undefined &&
|
||||||
options.language !== "auto" && { language: options.language }),
|
options.language !== "auto" && { language: options.language }),
|
||||||
...(options.task !== undefined && { task: options.task }),
|
...(options.task !== undefined && { task: options.task }),
|
||||||
@@ -146,6 +162,12 @@ export class BrowserEngine implements Engine {
|
|||||||
case "partial":
|
case "partial":
|
||||||
request.onPartial?.(data.text);
|
request.onPartial?.(data.text);
|
||||||
break;
|
break;
|
||||||
|
case "transcribe-progress":
|
||||||
|
request.onProgress?.({
|
||||||
|
processedSeconds: data.processedSeconds,
|
||||||
|
totalSeconds: data.totalSeconds,
|
||||||
|
});
|
||||||
|
break;
|
||||||
case "result":
|
case "result":
|
||||||
this.pending.delete(data.requestId);
|
this.pending.delete(data.requestId);
|
||||||
request.resolve({ text: data.text, chunks: data.chunks });
|
request.resolve({ text: data.text, chunks: data.chunks });
|
||||||
|
|||||||
@@ -19,6 +19,7 @@ import { resolveDefaultDtype } from "../lib/models";
|
|||||||
import {
|
import {
|
||||||
deviceAttempts,
|
deviceAttempts,
|
||||||
isUiToWorkerMessage,
|
isUiToWorkerMessage,
|
||||||
|
TranscribeProgressTracker,
|
||||||
type DeviceCapabilities,
|
type DeviceCapabilities,
|
||||||
type DevicePreference,
|
type DevicePreference,
|
||||||
type InferenceDevice,
|
type InferenceDevice,
|
||||||
@@ -209,6 +210,12 @@ function load(message: LoadMessage): Promise<AutomaticSpeechRecognitionPipeline>
|
|||||||
// Transcription
|
// Transcription
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
/** Audio sample rate the whole app standardizes on (D-012). */
|
||||||
|
const SAMPLE_RATE = 16_000;
|
||||||
|
/** Chunked-pipeline window / overlap, in seconds (Whisper's usual 30/5). */
|
||||||
|
const CHUNK_LENGTH_S = 30;
|
||||||
|
const STRIDE_LENGTH_S = 5;
|
||||||
|
|
||||||
async function transcribe(message: TranscribeMessage): Promise<void> {
|
async function transcribe(message: TranscribeMessage): Promise<void> {
|
||||||
if (active === null) {
|
if (active === null) {
|
||||||
post({
|
post({
|
||||||
@@ -222,11 +229,39 @@ async function transcribe(message: TranscribeMessage): Promise<void> {
|
|||||||
const asr = await active.promise;
|
const asr = await active.promise;
|
||||||
|
|
||||||
const options: Record<string, unknown> = {
|
const options: Record<string, unknown> = {
|
||||||
chunk_length_s: 30,
|
chunk_length_s: CHUNK_LENGTH_S,
|
||||||
stride_length_s: 5,
|
stride_length_s: STRIDE_LENGTH_S,
|
||||||
return_timestamps: true,
|
return_timestamps: true,
|
||||||
};
|
};
|
||||||
if (message.requestId === undefined) {
|
if (message.stream ?? message.requestId === undefined) {
|
||||||
|
// File-flow pass (id-less, or routed through BrowserEngine with an
|
||||||
|
// explicit stream flag, D-010): stream partial tokens and report audio
|
||||||
|
// progress. The streamer's chunk timestamps are window-relative; the
|
||||||
|
// tracker maps them to monotone absolute seconds (see protocol.ts).
|
||||||
|
// Live passes skip both — they resolve wholesale via "result" (D-016).
|
||||||
|
const totalSeconds = message.audio.length / SAMPLE_RATE;
|
||||||
|
const tracker = new TranscribeProgressTracker(
|
||||||
|
totalSeconds,
|
||||||
|
CHUNK_LENGTH_S,
|
||||||
|
CHUNK_LENGTH_S - 2 * STRIDE_LENGTH_S,
|
||||||
|
);
|
||||||
|
const postProgress = (processedSeconds: number | null) => {
|
||||||
|
if (processedSeconds === null) return;
|
||||||
|
post({
|
||||||
|
type: "transcribe-progress",
|
||||||
|
processedSeconds,
|
||||||
|
totalSeconds,
|
||||||
|
requestId: message.requestId,
|
||||||
|
});
|
||||||
|
};
|
||||||
|
// Immediate 0/total so the UI shows the audio length before the first
|
||||||
|
// (possibly slow) token arrives.
|
||||||
|
post({
|
||||||
|
type: "transcribe-progress",
|
||||||
|
processedSeconds: 0,
|
||||||
|
totalSeconds,
|
||||||
|
requestId: message.requestId,
|
||||||
|
});
|
||||||
options.streamer = new WhisperTextStreamer(
|
options.streamer = new WhisperTextStreamer(
|
||||||
// The ASR pipeline for Whisper models always carries a WhisperTokenizer.
|
// The ASR pipeline for Whisper models always carries a WhisperTokenizer.
|
||||||
asr.tokenizer as ConstructorParameters<typeof WhisperTextStreamer>[0],
|
asr.tokenizer as ConstructorParameters<typeof WhisperTextStreamer>[0],
|
||||||
@@ -234,6 +269,9 @@ async function transcribe(message: TranscribeMessage): Promise<void> {
|
|||||||
skip_prompt: true,
|
skip_prompt: true,
|
||||||
callback_function: (text: string) =>
|
callback_function: (text: string) =>
|
||||||
post({ type: "partial", text, requestId: message.requestId }),
|
post({ type: "partial", text, requestId: message.requestId }),
|
||||||
|
on_chunk_start: (time: number) => postProgress(tracker.timestamp(time)),
|
||||||
|
on_chunk_end: (time: number) => postProgress(tracker.timestamp(time)),
|
||||||
|
on_finalize: () => postProgress(tracker.finalizeWindow()),
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|||||||
+86
-1
@@ -69,6 +69,14 @@ export interface TranscribeMessage {
|
|||||||
* responses to segment passes; the file flow omits it and is unchanged.
|
* responses to segment passes; the file flow omits it and is unchanged.
|
||||||
*/
|
*/
|
||||||
requestId?: string;
|
requestId?: string;
|
||||||
|
/**
|
||||||
|
* Stream partial tokens and transcribe-progress for this request. Defaults
|
||||||
|
* to `requestId === undefined` (the historical rule: id-less file-flow
|
||||||
|
* passes stream, live segment passes resolve wholesale via "result").
|
||||||
|
* `BrowserEngine` sets it explicitly so a file pass routed through the
|
||||||
|
* engine (which correlates via requestId, D-010) still streams.
|
||||||
|
*/
|
||||||
|
stream?: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
export type UiToWorkerMessage = LoadMessage | TranscribeMessage;
|
export type UiToWorkerMessage = LoadMessage | TranscribeMessage;
|
||||||
@@ -108,6 +116,22 @@ export interface PartialMessage {
|
|||||||
requestId?: string;
|
requestId?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* How far a file-flow pass has progressed through the audio, in seconds.
|
||||||
|
* Posted once at pass start (0 / total) and then on every Whisper segment
|
||||||
|
* boundary, so slow models/backends show movement long before the first
|
||||||
|
* partial token. Values are monotone and clamped to [0, totalSeconds]
|
||||||
|
* (see TranscribeProgressTracker). Live passes (requestId set) don't report
|
||||||
|
* progress — the live UI has its own committed/tentative feedback (D-016).
|
||||||
|
*/
|
||||||
|
export interface TranscribeProgressMessage {
|
||||||
|
type: "transcribe-progress";
|
||||||
|
processedSeconds: number;
|
||||||
|
totalSeconds: number;
|
||||||
|
/** Echo of the originating transcribe request's id, if it carried one. */
|
||||||
|
requestId?: string;
|
||||||
|
}
|
||||||
|
|
||||||
/** One timestamped segment of the final transcript. */
|
/** One timestamped segment of the final transcript. */
|
||||||
export interface TranscriptChunk {
|
export interface TranscriptChunk {
|
||||||
text: string;
|
text: string;
|
||||||
@@ -134,6 +158,7 @@ export interface ErrorMessage {
|
|||||||
export type WorkerToUiMessage =
|
export type WorkerToUiMessage =
|
||||||
| LoadProgressMessage
|
| LoadProgressMessage
|
||||||
| PartialMessage
|
| PartialMessage
|
||||||
|
| TranscribeProgressMessage
|
||||||
| ResultMessage
|
| ResultMessage
|
||||||
| ErrorMessage;
|
| ErrorMessage;
|
||||||
|
|
||||||
@@ -234,7 +259,8 @@ export function isUiToWorkerMessage(value: unknown): value is UiToWorkerMessage
|
|||||||
(value.task === undefined ||
|
(value.task === undefined ||
|
||||||
value.task === "transcribe" ||
|
value.task === "transcribe" ||
|
||||||
value.task === "translate") &&
|
value.task === "translate") &&
|
||||||
isOptionalString(value.requestId)
|
isOptionalString(value.requestId) &&
|
||||||
|
(value.stream === undefined || typeof value.stream === "boolean")
|
||||||
);
|
);
|
||||||
default:
|
default:
|
||||||
return false;
|
return false;
|
||||||
@@ -264,6 +290,12 @@ export function isWorkerToUiMessage(value: unknown): value is WorkerToUiMessage
|
|||||||
}
|
}
|
||||||
case "partial":
|
case "partial":
|
||||||
return typeof value.text === "string" && isOptionalString(value.requestId);
|
return typeof value.text === "string" && isOptionalString(value.requestId);
|
||||||
|
case "transcribe-progress":
|
||||||
|
return (
|
||||||
|
typeof value.processedSeconds === "number" &&
|
||||||
|
typeof value.totalSeconds === "number" &&
|
||||||
|
isOptionalString(value.requestId)
|
||||||
|
);
|
||||||
case "result":
|
case "result":
|
||||||
return (
|
return (
|
||||||
typeof value.text === "string" &&
|
typeof value.text === "string" &&
|
||||||
@@ -331,3 +363,56 @@ export function applyLoadProgress(
|
|||||||
}
|
}
|
||||||
return { ...state, files: { ...state.files, [message.file]: next } };
|
return { ...state, files: { ...state.files, [message.file]: next } };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Transcription progress accumulation (pure; drives the in-progress UI)
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Maps the streamer's window-relative timestamps to absolute audio progress.
|
||||||
|
*
|
||||||
|
* The chunked ASR pipeline slices audio into windows of `windowSeconds`,
|
||||||
|
* advancing by `jumpSeconds` (window − 2·stride) per window, and runs one
|
||||||
|
* generate pass per window. Whisper's timestamp tokens — surfaced via
|
||||||
|
* `WhisperTextStreamer`'s `on_chunk_start`/`on_chunk_end` — are relative to
|
||||||
|
* the current window's start, and `on_finalize` fires when a window's pass
|
||||||
|
* ends. This tracker turns those into absolute seconds that are monotone
|
||||||
|
* (windows overlap, so a later window's early timestamps can precede an
|
||||||
|
* earlier window's late ones — regressions are swallowed) and clamped to
|
||||||
|
* [0, totalSeconds].
|
||||||
|
*/
|
||||||
|
export class TranscribeProgressTracker {
|
||||||
|
private windowIndex = 0;
|
||||||
|
private reported = 0;
|
||||||
|
|
||||||
|
constructor(
|
||||||
|
private readonly totalSeconds: number,
|
||||||
|
private readonly windowSeconds: number,
|
||||||
|
private readonly jumpSeconds: number,
|
||||||
|
) {}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Absolute progress for a window-relative timestamp, or null when it does
|
||||||
|
* not advance past what was already reported.
|
||||||
|
*/
|
||||||
|
timestamp(withinWindowSeconds: number): number | null {
|
||||||
|
const clamped = Math.min(Math.max(withinWindowSeconds, 0), this.windowSeconds);
|
||||||
|
return this.advance(this.windowIndex * this.jumpSeconds + clamped);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The current window's generate pass finished; progress moves to the start
|
||||||
|
* of the next window (the amount of audio no later window revisits).
|
||||||
|
*/
|
||||||
|
finalizeWindow(): number | null {
|
||||||
|
this.windowIndex += 1;
|
||||||
|
return this.advance(this.windowIndex * this.jumpSeconds);
|
||||||
|
}
|
||||||
|
|
||||||
|
private advance(processedSeconds: number): number | null {
|
||||||
|
const clamped = Math.min(processedSeconds, this.totalSeconds);
|
||||||
|
if (clamped <= this.reported) return null;
|
||||||
|
this.reported = clamped;
|
||||||
|
return clamped;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -77,6 +77,44 @@ describe("BrowserEngine", () => {
|
|||||||
expect("modelId" in message).toBe(false);
|
expect("modelId" in message).toBe(false);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("asks the worker to stream only when a consumer listens", () => {
|
||||||
|
const worker = new FakeWorker();
|
||||||
|
const engine = new BrowserEngine(worker);
|
||||||
|
void engine.transcribe(new Float32Array(4));
|
||||||
|
void engine.transcribe(new Float32Array(4), { onPartial: () => {} });
|
||||||
|
void engine.transcribe(new Float32Array(4), { onProgress: () => {} });
|
||||||
|
expect("stream" in worker.posted[0]!.message).toBe(false);
|
||||||
|
expect(worker.posted[1]!.message.stream).toBe(true);
|
||||||
|
expect(worker.posted[2]!.message.stream).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("routes transcribe-progress to onProgress, ignoring other ids", async () => {
|
||||||
|
const worker = new FakeWorker();
|
||||||
|
const engine = new BrowserEngine(worker);
|
||||||
|
const updates: { processedSeconds: number; totalSeconds: number }[] = [];
|
||||||
|
const promise = engine.transcribe(new Float32Array(4), {
|
||||||
|
onProgress: (progress) => updates.push(progress),
|
||||||
|
});
|
||||||
|
const requestId = worker.lastRequestId();
|
||||||
|
|
||||||
|
worker.emit({ type: "transcribe-progress", processedSeconds: 5, totalSeconds: 60 });
|
||||||
|
worker.emit({
|
||||||
|
type: "transcribe-progress",
|
||||||
|
processedSeconds: 7,
|
||||||
|
totalSeconds: 60,
|
||||||
|
requestId: "other",
|
||||||
|
});
|
||||||
|
worker.emit({ type: "transcribe-progress", processedSeconds: 0, totalSeconds: 90, requestId });
|
||||||
|
worker.emit({ type: "transcribe-progress", processedSeconds: 30, totalSeconds: 90, requestId });
|
||||||
|
worker.emit({ type: "result", text: "done", chunks: [], requestId });
|
||||||
|
|
||||||
|
expect((await promise).text).toBe("done");
|
||||||
|
expect(updates).toEqual([
|
||||||
|
{ processedSeconds: 0, totalSeconds: 90 },
|
||||||
|
{ processedSeconds: 30, totalSeconds: 90 },
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
test("resolves on the matching result and routes partials to onPartial", async () => {
|
test("resolves on the matching result and routes partials to onPartial", async () => {
|
||||||
const worker = new FakeWorker();
|
const worker = new FakeWorker();
|
||||||
const engine = new BrowserEngine(worker);
|
const engine = new BrowserEngine(worker);
|
||||||
@@ -265,6 +303,45 @@ describe("ServerEngine", () => {
|
|||||||
engine.transcribe(new Float32Array(4), { modelId: "onnx-community/whisper-tiny" }),
|
engine.transcribe(new Float32Array(4), { modelId: "onnx-community/whisper-tiny" }),
|
||||||
).rejects.toThrow(/could not reach the server/i);
|
).rejects.toThrow(/could not reach the server/i);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("passes the abort signal to fetch and rethrows AbortError as-is", async () => {
|
||||||
|
// Cancel (D-018 amendment): the UI aborts the controller; the rejection
|
||||||
|
// must stay an AbortError, not be rewrapped as a network failure.
|
||||||
|
let seenSignal: AbortSignal | undefined;
|
||||||
|
const engine = new ServerEngine(async (_url, init) => {
|
||||||
|
seenSignal = init?.signal ?? undefined;
|
||||||
|
throw new DOMException("The user aborted a request.", "AbortError");
|
||||||
|
});
|
||||||
|
const controller = new AbortController();
|
||||||
|
const promise = engine.transcribe(new Float32Array(4), {
|
||||||
|
modelId: "onnx-community/whisper-tiny",
|
||||||
|
signal: controller.signal,
|
||||||
|
});
|
||||||
|
expect(promise).rejects.toMatchObject({ name: "AbortError" });
|
||||||
|
await promise.catch(() => {});
|
||||||
|
expect(seenSignal).toBe(controller.signal);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("an abort mid-stream surfaces as the stream's rejection", () => {
|
||||||
|
// Once the 200 arrived, an abort tears down the body stream; whatever
|
||||||
|
// error the reader throws must propagate out of transcribe().
|
||||||
|
const body = new ReadableStream<Uint8Array>({
|
||||||
|
start(streamController) {
|
||||||
|
streamController.enqueue(
|
||||||
|
new TextEncoder().encode(
|
||||||
|
`${JSON.stringify({ type: "segment", text: " a", start: 0, end: 1 })}\n`,
|
||||||
|
),
|
||||||
|
);
|
||||||
|
streamController.error(
|
||||||
|
new DOMException("The user aborted a request.", "AbortError"),
|
||||||
|
);
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const engine = new ServerEngine(async () => new Response(body, { status: 200 }));
|
||||||
|
expect(
|
||||||
|
engine.transcribe(new Float32Array(4), { modelId: "onnx-community/whisper-tiny" }),
|
||||||
|
).rejects.toMatchObject({ name: "AbortError" });
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import {
|
|||||||
initialLoadProgress,
|
initialLoadProgress,
|
||||||
isUiToWorkerMessage,
|
isUiToWorkerMessage,
|
||||||
isWorkerToUiMessage,
|
isWorkerToUiMessage,
|
||||||
|
TranscribeProgressTracker,
|
||||||
type LoadProgressMessage,
|
type LoadProgressMessage,
|
||||||
type UiToWorkerMessage,
|
type UiToWorkerMessage,
|
||||||
type WorkerToUiMessage,
|
type WorkerToUiMessage,
|
||||||
@@ -31,6 +32,8 @@ describe("isUiToWorkerMessage", () => {
|
|||||||
{ type: "transcribe", audio: new Float32Array(8), language: "da", task: "transcribe" },
|
{ type: "transcribe", audio: new Float32Array(8), language: "da", task: "transcribe" },
|
||||||
{ type: "transcribe", audio: new Float32Array(8), language: "auto", task: "translate" },
|
{ type: "transcribe", audio: new Float32Array(8), language: "auto", task: "translate" },
|
||||||
{ type: "transcribe", audio: new Float32Array(8), requestId: "live-3" },
|
{ type: "transcribe", audio: new Float32Array(8), requestId: "live-3" },
|
||||||
|
{ type: "transcribe", audio: new Float32Array(8), requestId: "engine-1-1", stream: true },
|
||||||
|
{ type: "transcribe", audio: new Float32Array(8), stream: false },
|
||||||
];
|
];
|
||||||
for (const message of [...loads, ...transcribes]) {
|
for (const message of [...loads, ...transcribes]) {
|
||||||
expect(isUiToWorkerMessage(message)).toBe(true);
|
expect(isUiToWorkerMessage(message)).toBe(true);
|
||||||
@@ -58,6 +61,9 @@ describe("isUiToWorkerMessage", () => {
|
|||||||
expect(
|
expect(
|
||||||
isUiToWorkerMessage({ type: "transcribe", audio: new Float32Array(4), requestId: 7 }),
|
isUiToWorkerMessage({ type: "transcribe", audio: new Float32Array(4), requestId: 7 }),
|
||||||
).toBe(false);
|
).toBe(false);
|
||||||
|
expect(
|
||||||
|
isUiToWorkerMessage({ type: "transcribe", audio: new Float32Array(4), stream: "yes" }),
|
||||||
|
).toBe(false);
|
||||||
expect(isUiToWorkerMessage({ type: "result", text: "", chunks: [] })).toBe(false);
|
expect(isUiToWorkerMessage({ type: "result", text: "", chunks: [] })).toBe(false);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
@@ -80,6 +86,8 @@ describe("isWorkerToUiMessage", () => {
|
|||||||
{ type: "load-progress", status: "ready", device: "webgpu" },
|
{ type: "load-progress", status: "ready", device: "webgpu" },
|
||||||
{ type: "partial", text: " hello" },
|
{ type: "partial", text: " hello" },
|
||||||
{ type: "partial", text: " hello", requestId: "live-1" },
|
{ type: "partial", text: " hello", requestId: "live-1" },
|
||||||
|
{ type: "transcribe-progress", processedSeconds: 0, totalSeconds: 90 },
|
||||||
|
{ type: "transcribe-progress", processedSeconds: 42.5, totalSeconds: 90, requestId: "live-1" },
|
||||||
{ type: "result", text: "hello world", chunks: [{ text: "hello world", timestamp: [0, 1.2] }] },
|
{ type: "result", text: "hello world", chunks: [{ text: "hello world", timestamp: [0, 1.2] }] },
|
||||||
{ type: "result", text: "hello world", chunks: [], requestId: "live-1" },
|
{ type: "result", text: "hello world", chunks: [], requestId: "live-1" },
|
||||||
{ type: "error", stage: "load", message: "boom" },
|
{ type: "error", stage: "load", message: "boom" },
|
||||||
@@ -98,6 +106,18 @@ describe("isWorkerToUiMessage", () => {
|
|||||||
expect(isWorkerToUiMessage({ type: "load-progress", status: "ready", device: "cuda" })).toBe(false);
|
expect(isWorkerToUiMessage({ type: "load-progress", status: "ready", device: "cuda" })).toBe(false);
|
||||||
expect(isWorkerToUiMessage({ type: "partial" })).toBe(false);
|
expect(isWorkerToUiMessage({ type: "partial" })).toBe(false);
|
||||||
expect(isWorkerToUiMessage({ type: "partial", text: "x", requestId: 1 })).toBe(false);
|
expect(isWorkerToUiMessage({ type: "partial", text: "x", requestId: 1 })).toBe(false);
|
||||||
|
expect(isWorkerToUiMessage({ type: "transcribe-progress", processedSeconds: 1 })).toBe(false);
|
||||||
|
expect(
|
||||||
|
isWorkerToUiMessage({ type: "transcribe-progress", processedSeconds: "1", totalSeconds: 2 }),
|
||||||
|
).toBe(false);
|
||||||
|
expect(
|
||||||
|
isWorkerToUiMessage({
|
||||||
|
type: "transcribe-progress",
|
||||||
|
processedSeconds: 1,
|
||||||
|
totalSeconds: 2,
|
||||||
|
requestId: 9,
|
||||||
|
}),
|
||||||
|
).toBe(false);
|
||||||
expect(isWorkerToUiMessage({ type: "result", text: "x", chunks: "nope" })).toBe(false);
|
expect(isWorkerToUiMessage({ type: "result", text: "x", chunks: "nope" })).toBe(false);
|
||||||
expect(isWorkerToUiMessage({ type: "error", stage: "decode", message: "m" })).toBe(false);
|
expect(isWorkerToUiMessage({ type: "error", stage: "decode", message: "m" })).toBe(false);
|
||||||
expect(isWorkerToUiMessage({ type: "load", modelId: "x", device: "wasm" })).toBe(false);
|
expect(isWorkerToUiMessage({ type: "load", modelId: "x", device: "wasm" })).toBe(false);
|
||||||
@@ -155,6 +175,58 @@ describe("deviceAttempts (D-004, D-017)", () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe("TranscribeProgressTracker", () => {
|
||||||
|
// Production shape: 30 s windows advancing by 20 s (30 − 2·5, the worker's
|
||||||
|
// chunk/stride settings), like the chunked ASR pipeline.
|
||||||
|
const WINDOW = 30;
|
||||||
|
const JUMP = 20;
|
||||||
|
|
||||||
|
test("maps window-relative timestamps to absolute seconds per window", () => {
|
||||||
|
const tracker = new TranscribeProgressTracker(90, WINDOW, JUMP);
|
||||||
|
expect(tracker.timestamp(0)).toBeNull(); // 0 does not advance past 0
|
||||||
|
expect(tracker.timestamp(4.2)).toBe(4.2);
|
||||||
|
expect(tracker.timestamp(29.5)).toBe(29.5);
|
||||||
|
expect(tracker.finalizeWindow()).toBeNull(); // 1·20 < 29.5 already seen
|
||||||
|
// Second window: timestamps are relative again, offset by one jump.
|
||||||
|
expect(tracker.timestamp(3)).toBeNull(); // 20 + 3 < 29.5
|
||||||
|
expect(tracker.timestamp(12)).toBe(32);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("is monotone across overlapping windows", () => {
|
||||||
|
const tracker = new TranscribeProgressTracker(90, WINDOW, JUMP);
|
||||||
|
expect(tracker.timestamp(28)).toBe(28);
|
||||||
|
tracker.finalizeWindow();
|
||||||
|
// The next window revisits overlapped audio — earlier absolute times
|
||||||
|
// than what was already reported are swallowed, never regressions.
|
||||||
|
expect(tracker.timestamp(5)).toBeNull(); // 20 + 5 = 25 < 28
|
||||||
|
expect(tracker.timestamp(10)).toBe(30);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("clamps within the window and to the total duration", () => {
|
||||||
|
const tracker = new TranscribeProgressTracker(35, WINDOW, JUMP);
|
||||||
|
// A stray timestamp beyond the window length clamps to the window end.
|
||||||
|
expect(tracker.timestamp(45)).toBe(30);
|
||||||
|
expect(tracker.timestamp(-1)).toBeNull(); // negative clamps to window start
|
||||||
|
tracker.finalizeWindow();
|
||||||
|
// Second (final) window: 20 + 16 = 36 clamps to the 35 s total.
|
||||||
|
expect(tracker.timestamp(16)).toBe(35);
|
||||||
|
expect(tracker.timestamp(17)).toBeNull(); // already at the end
|
||||||
|
});
|
||||||
|
|
||||||
|
test("finalizeWindow advances to the next window start", () => {
|
||||||
|
const tracker = new TranscribeProgressTracker(90, WINDOW, JUMP);
|
||||||
|
expect(tracker.finalizeWindow()).toBe(20);
|
||||||
|
expect(tracker.finalizeWindow()).toBe(40);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("single-window audio never reports past its duration", () => {
|
||||||
|
const tracker = new TranscribeProgressTracker(8, WINDOW, JUMP);
|
||||||
|
expect(tracker.timestamp(6)).toBe(6);
|
||||||
|
expect(tracker.timestamp(8)).toBe(8);
|
||||||
|
expect(tracker.finalizeWindow()).toBeNull(); // 20 clamps to 8, not an advance
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
describe("applyLoadProgress", () => {
|
describe("applyLoadProgress", () => {
|
||||||
test("tracks per-file byte progress without mutating previous state", () => {
|
test("tracks per-file byte progress without mutating previous state", () => {
|
||||||
const p1: LoadProgressMessage = { type: "load-progress", status: "initiate", file: "a.onnx" };
|
const p1: LoadProgressMessage = { type: "load-progress", status: "initiate", file: "a.onnx" };
|
||||||
|
|||||||
Reference in New Issue
Block a user