From 033ea3296817044b001be408b8f760a765de4f4d Mon Sep 17 00:00:00 2001 From: Lucas Winther Date: Fri, 10 Jul 2026 09:28:42 +0200 Subject: [PATCH] Start on live transcription and python sidecar (faster-whisper) --- Architecture.md | 33 +- Decisions.md | 52 +++- WorkBreakdown.md | 89 +++++- src/App.tsx | 187 +++++++++++- src/components/TranscriptView.tsx | 33 +- src/index.css | 62 ++++ src/index.ts | 16 +- src/lib/engine.ts | 392 +++++++++++++++++++++++- src/lib/models.ts | 26 ++ src/server/transcribe.ts | 479 +++++++++++++++++++++++++++++- src/worker/inference.worker.ts | 12 +- src/worker/protocol.ts | 32 +- tests/models.test.ts | 16 + tests/protocol.test.ts | 8 + 14 files changed, 1376 insertions(+), 61 deletions(-) diff --git a/Architecture.md b/Architecture.md index 5416e60..51b9a77 100644 --- a/Architecture.md +++ b/Architecture.md @@ -17,6 +17,7 @@ Logged-in users may optionally route inference to the server. │ React UI (App shell) │ │ ├─ ModelPicker (tiny → large, .en, quantized variants) │ │ ├─ FileDropzone (audio/video upload) │ +│ ├─ LiveControls (microphone start/stop — D-016) │ │ ├─ TranscriptView (streaming partial results) │ │ └─ EngineToggle (device / server — logged-in only) │ │ │ postMessage protocol │ @@ -62,9 +63,11 @@ src/ index.html # HTML entry, imports frontend.tsx frontend.tsx # React root App.tsx # top-level UI composition - components/ # ModelPicker, TranscriptionOptions, FileDropzone, TranscriptView, EngineToggle, AuthForms + components/ # ModelPicker, TranscriptionOptions, FileDropzone, LiveControls, TranscriptView, EngineToggle, AuthForms lib/ audio.ts # decode + resample to 16 kHz mono Float32Array (Web Audio API) + live.ts # live-mode segment loop: RMS gate, LiveSegmenter, LiveTranscriber (pure, D-016) + mic.ts # microphone capture: AudioWorklet chunker (Blob URL) + fallback (D-016) models.ts # model catalog: ids, sizes, dtype defaults per device, languages modelCache.ts # "downloaded?" per model via Cache Storage (D-005; reused by M3 offline indicator) engine.ts # engine abstraction: BrowserEngine | ServerEngine @@ -90,9 +93,14 @@ catalog, WebGPU→WASM auto-selection, model switching, language/task options, and the installable offline PWA (`pwa/`, `scripts/build.ts` production build, `dist/` serving in `src/index.ts`; browser-level offline behavior pending the manual run in `tests/offline.md`). M4 accounts are built (`server/db.ts`, -`server/auth.ts`, `/api/auth/*`, `AuthForms`). `lib/engine.ts`, -`server/transcribe.ts`, and `server-py/` are TODO for M5; see -`WorkBreakdown.md`.) +`server/auth.ts`, `/api/auth/*`, `AuthForms`). M7 live microphone +transcription is built (`lib/live.ts`, `lib/mic.ts`, `LiveControls`, D-016; +in-browser run pending per `tests/live.md`). M5's server side is built — +`server-py/sidecar.py`, `server/transcribe.ts` (supervisor + +`/api/transcribe` + `/status`), `lib/engine.ts` (both engines, D-010), and +the catalog mapping in `lib/models.ts` — with graceful degradation when +Python/faster-whisper is absent; the `EngineToggle` component and App wiring +are the remaining M5 work; see `WorkBreakdown.md`.) ## 4. Key flows @@ -121,6 +129,23 @@ manual run in `tests/offline.md`). M4 accounts are built (`server/db.ts`, 5. Worker runs the pipeline (chunked, with timestamps), streaming partial results back via `{type:"partial"}` messages; final result as `{type:"result"}`. +### 4.1b Live microphone transcription (anonymous, browser engine — D-016) +1. `LiveControls` starts a session: `getUserMedia` → `AudioContext` at the + device's native rate → AudioWorklet chunker (module from a Blob URL; + `ScriptProcessorNode` fallback) posts ~100 ms Float32 frames; the main + thread resamples each to 16 kHz (`lib/mic.ts`). +2. `LiveTranscriber` (`lib/live.ts`, pure) feeds frames through an RMS energy + gate: segments start at speech onset (short pre-roll), commit on trailing + silence or at ~25 s; pure silence is never transcribed. +3. The current segment is re-transcribed through the normal inference worker + on a rolling basis (throttled, single-flight, latest-snapshot-wins) as + *tentative* text; a committed segment gets one final pass and freezes. + `requestId` on the protocol messages correlates responses. +4. Stopping flushes the segmenter, drains pending final passes, and yields a + normal transcript (text + absolute-timestamped chunks) in `TranscriptView`. + Everything runs on-device; works offline like the file flow. Server-engine + live input is a recorded future want (WorkBreakdown M7, D-016). + ### 4.2 Offline (PWA) - Service worker (`pwa/sw.ts`, hand-rolled — D-007) precaches the built app shell (hashed JS/CSS, `/`, worker bundle, manifest, icons) on install into diff --git a/Decisions.md b/Decisions.md index 174f9ea..2d8c8d8 100644 --- a/Decisions.md +++ b/Decisions.md @@ -83,11 +83,17 @@ The UI never branches on engine type beyond the toggle. Keeps the server engine (milestone M5) from leaking into UI code built in M1–M4. ## D-011 — Server engine transport: streaming HTTP response first, WebSocket if needed -**Status:** Proposed +**Status:** Accepted (implemented in M5, 2026-07-10) `POST /api/transcribe` returns a streamed (chunked JSON-lines) response with progress and partial transcripts. Simpler than WebSockets (no connection lifecycle, works with plain fetch + ReadableStream). Switch to `Bun.serve` websockets only if we need bidirectional control (e.g. cancel mid-run) that abort signals can't cover. +**As implemented:** `application/x-ndjson` of `segment`* then `result` | `error` +lines (wire types + guards shared client/server via `src/lib/engine.ts`); the +request body is raw float32 mono 16 kHz PCM — the decoded form the browser +already has — so the server never decodes containers. Cancel-mid-run needed no +WebSocket: `fetch`'s AbortSignal reaches `request.signal`, which forwards a +`cancel` to the sidecar (it checks between segments). ## D-012 — Audio decoding on the main thread via Web Audio API **Status:** Accepted @@ -157,3 +163,47 @@ anyway), preserving the anonymous-first invariant (no error surfaced). **Consequence:** any future flag that must reach the browser needs a server-derived signal (an endpoint the client can probe, like /me here) — `BUN_PUBLIC_*` build-time inlining is not used for feature flags. + +## D-016 — Live microphone transcription via segment re-transcription (browser engine) +**Status:** Accepted (2026-07-10; promotes "live transcription" out of the +WorkBreakdown out-of-scope list → milestone M7) +Live (microphone) transcription runs on the **browser engine only** — same +privacy story as file uploads: audio never touches the network, works for +anonymous users, works offline. Whisper is not a streaming model (it consumes +up-to-30 s windows), so "live" is approximated by a **segment loop**: + +- **Capture (main thread):** `getUserMedia` → `AudioContext` at the device's + native rate → an `AudioWorklet` chunker posts ~100 ms Float32 frames to the + main thread, which resamples them to 16 kHz with the existing pure + `resampleLinear` (D-012's documented fallback; adequate for speech). + The worklet module is loaded from a **Blob URL** built from an inline code + string — no extra served file, no build/service-worker changes, and it keeps + working offline. Where `audioWorklet`/Blob modules are unavailable + (older Safari), fall back to a `ScriptProcessorNode`. The context is *not* + forced to 16 kHz because Firefox cannot connect a `MediaStreamAudioSourceNode` + across sample rates. +- **Segmentation (pure, unit-tested):** a simple RMS energy gate + (`src/lib/live.ts`) splits speech into segments: a segment starts at speech + onset (with a short pre-roll), commits after a run of trailing silence or at + a max length (~25 s, under Whisper's 30 s window), and pure-silence audio is + dropped entirely — Whisper hallucinates on silence, so never transcribe it. + No ML VAD (e.g. Silero) until the energy gate proves insufficient in real use. +- **Inference (existing worker, serialized):** the current segment is + re-transcribed through the **existing** inference worker/pipeline on a + rolling basis (tentative text, replaced each pass); a committed segment gets + one final pass and its text freezes. At most one inference is in flight; + final passes take priority over tentative refreshes, which are throttled and + always use the latest audio snapshot (intermediate snapshots are skipped, so + the loop degrades gracefully when inference is slower than real time). +- **Protocol:** UI↔worker messages gain an optional `requestId` echoed on + `partial`/`result`/`error`, so the live controller can correlate responses + and coexist with the file flow. The file flow keeps working without ids. + +**Alternatives rejected:** streaming into the worker with a custom +chunk-stitching decoder (large fork of Transformers.js internals for marginal +latency); MediaRecorder + re-decoding compressed blobs (adds codec latency and +`decodeAudioData` round-trips for no benefit over raw PCM taps). +**Consequence:** live quality/latency is bounded by model speed on the user's +device — the UI must keep tentative text visibly distinct from committed text. +Server-engine live input (WebSocket streaming to faster-whisper) is a separate +future decision; D-011's "WebSocket if needed" note is the hook. diff --git a/WorkBreakdown.md b/WorkBreakdown.md index 451933d..38907ff 100644 --- a/WorkBreakdown.md +++ b/WorkBreakdown.md @@ -128,19 +128,57 @@ Notes (M4): offline) fail silent to anonymous — the app is offline-first. ## M5 — Server engine + toggle -- [ ] Server-side pipeline in `src/server/transcribe.ts` — **faster-whisper** +- [x] Server-side pipeline in `src/server/transcribe.ts` — **faster-whisper** (CTranslate2) via a Python sidecar in `server-py/`, spawned/supervised by Bun (D-013 supersedes the Transformers.js-under-Bun plan); map the shared - model catalog to `Systran/faster-whisper-*` model ids in `src/lib/models.ts` -- [ ] Degrade gracefully when Python/the sidecar is absent (server engine + model catalog to faster-whisper model ids in `src/lib/models.ts` +- [x] Degrade gracefully when Python/the sidecar is absent (server engine unavailable; browser engine untouched — anonymous path deploys with Bun alone) -- [ ] `POST /api/transcribe` — auth-guarded, streamed JSON-lines response (D-011), - upload size limit, request abort handling -- [ ] `src/lib/engine.ts` — `Engine` interface; `BrowserEngine` (wraps worker) and +- [x] `POST /api/transcribe` — auth-guarded, streamed JSON-lines response (D-011), + upload size limit, request abort handling; plus `GET /api/transcribe/status` + (availability probe for the toggle) +- [x] `src/lib/engine.ts` — `Engine` interface; `BrowserEngine` (wraps worker) and `ServerEngine` (wraps fetch stream) (D-010) - [ ] `EngineToggle` component — visible only with a session; persists choice -- [ ] Server model caching/warm-up strategy (don't reload weights per request) -- [ ] Integration test: same fixture audio through both engines, transcripts agree + (persistence helpers + status probe already in `lib/engine.ts`; remaining: + the component, lifting `useAuth` out of `AuthForms` into `App` so both see + one session state, and routing the file flow through the selected engine — + live stays on `BrowserEngine` per D-016) +- [x] Server model caching/warm-up strategy (don't reload weights per request) +- [~] Integration test: same fixture audio through both engines, transcripts agree + — supervisor/HTTP layers covered in `tests/transcribe.test.ts` (real + `Bun.spawn` against `tests/fixtures/fake_sidecar.py`); real inference is + the opt-in `FW_INTEGRATION=1` test + manual run in `tests/server-engine.md` + (no pip/browser in the dev environment; same caveat as M2/M3) +- [ ] Deployment: optional Python/faster-whisper layer for the Docker image + + deployer note (base image stays Bun-only; install + `server-py/requirements.txt` to enable the server engine) + +Notes (M5): +- Sidecar protocol (`server-py/sidecar.py`): JSON-lines over stdio — + `ready`/`fatal` handshake, `transcribe` → `segment`* → `done`, `cancel` + between segments, `shutdown`/EOF exits. Audio is handed over as a temp file + of raw float32 mono 16 kHz PCM (exactly what the browser engine consumes), + read with `np.fromfile` — no container decoding server-side. The last-used + model stays loaded (replaced on change, so memory is bounded to one model); + device/compute via `FASTER_WHISPER_DEVICE`/`FASTER_WHISPER_COMPUTE` + (default auto/int8). +- Supervision (`createSidecar`): lazy spawn on first use; a crash rejects + in-flight requests and the next request respawns; a missing runtime + (no python3 / `fatal` handshake) is remembered as a *permanent* unavailable + reason — surfaced verbatim by `GET /api/transcribe/status` and 503s — so a + Bun-only deployment never respawn-storms. Tests force this path with + `python3 -S -I` (site-packages off) against the real sidecar. +- Upload limit: `TRANSCRIBE_MAX_UPLOAD_BYTES` (default 512 MiB ≈ 2¼ h of raw + f32 PCM), enforced pre-read via Content-Length *and* while streaming the + body to the temp file. Errors after the 200 header travel in-stream as an + `{"type":"error"}` line (wire format shared with the client via + `src/lib/engine.ts`, so server and `ServerEngine` cannot drift). +- `AUTH_ENABLED=false` (D-015) 404s `/api/transcribe*` along with the auth + routes — the server engine is session-gated, so it disappears with auth. +- Smoke-tested live against the dev server: `/` + auth unchanged; status + 401 → (register) → 200 `{available:false, reason:"faster-whisper is not + installed…"}`; POST → 503 with the same reason, body untouched. ## M6 — Hardening & polish - [ ] Error taxonomy: unsupported file, out-of-memory (suggest smaller model), @@ -151,9 +189,40 @@ Notes (M4): - [ ] Basic a11y pass (keyboard, focus, aria on progress/live transcript) - [ ] Production build + `bun start` deployment check; README for deployers +## M7 — Live microphone transcription (browser engine, D-016) +Promoted out of the out-of-scope list via D-016. Browser engine only: audio +never touches the network, anonymous-first, works offline once a model is +cached. + +- [x] Protocol: optional `requestId` on `transcribe`, echoed on + `partial`/`result`/`error` so the live loop can correlate responses + (file flow unchanged without ids) +- [x] `src/lib/live.ts` — pure, unit-tested segment loop: RMS energy gate, + `LiveSegmenter` (speech onset + pre-roll, silence/max-length commits, + pure silence dropped), `LiveTranscriber` (single-flight scheduling, + final passes before throttled tentative refreshes, stop/drain, + absolute-timestamped chunks) +- [x] `src/lib/mic.ts` — `getUserMedia` + AudioWorklet chunker from a Blob URL + (ScriptProcessorNode fallback), native-rate capture resampled to 16 kHz + on the main thread (D-016) +- [x] `LiveControls` component — start/stop button + recording indicator; + mutually exclusive with the file flow +- [x] Live transcript UI: committed text + visually-distinct tentative tail in + `TranscriptView`; final stop() result reuses the normal transcript view + (copy button, timestamps) +- [x] Unit tests: segmenter behavior, transcriber scheduling with a fake + engine and injected clock, protocol `requestId` guards +- [~] Manual in-browser verification (mic permission, live text, offline run) + — procedure in `tests/live.md`, pending a real browser (same dev-env + caveat as M2/M3) +- [ ] **Future want:** live transcription on the **server engine** — stream + mic audio over a WebSocket (D-011's escape hatch) to faster-whisper + (D-013) for logged-in users on weak devices. Needs its own decision + entry; blocked on M5 landing first. + --- ## Deliberately out of scope (for now) -Microphone/live transcription, arbitrary Hub model ids, OAuth providers, -multi-user transcript storage, whisper.cpp server backend, i18n of the UI. +Arbitrary Hub model ids, OAuth providers, multi-user transcript storage, +whisper.cpp server backend, i18n of the UI. Promote to a milestone via a new entry in `Decisions.md` first. diff --git a/src/App.tsx b/src/App.tsx index 5fd6131..70a47fc 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -3,11 +3,14 @@ import "./index.css"; import { AuthForms } from "./components/AuthForms"; import { FileDropzone } from "./components/FileDropzone"; +import { LiveControls, type LiveStatus } from "./components/LiveControls"; import { ModelLoadProgress } from "./components/ModelLoadProgress"; import { ModelPicker } from "./components/ModelPicker"; import { TranscriptView } from "./components/TranscriptView"; import { TranscriptionOptions } from "./components/TranscriptionOptions"; import { decodeAudioFile } from "./lib/audio"; +import { LiveTranscriber, type LiveTranscribeResult } from "./lib/live"; +import { startMicCapture, type MicCapture } from "./lib/mic"; import { getDownloadedModels } from "./lib/modelCache"; import { registerServiceWorker } from "./pwa/register"; import { @@ -39,6 +42,23 @@ const MODEL_IDS = MODELS.map((model) => model.id); export function App() { const workerRef = useRef(null); + // Live-mode session (M7, D-016). The mic and transcriber live outside React + // state; only display text goes through setState. Segment passes are + // correlated with worker responses via requestId. + const liveRef = useRef<{ mic: MicCapture; transcriber: LiveTranscriber } | null>( + null, + ); + const pendingLive = useRef( + new Map< + string, + { + resolve: (result: LiveTranscribeResult) => void; + reject: (error: Error) => void; + } + >(), + ); + const nextLiveRequest = useRef(0); + const [busy, setBusy] = useState(false); const [decoding, setDecoding] = useState(false); const [loadStarted, setLoadStarted] = useState(false); @@ -48,6 +68,10 @@ export function App() { const [error, setError] = useState(null); const [fileName, setFileName] = useState(null); + const [liveStatus, setLiveStatus] = useState("idle"); + const [liveCommitted, setLiveCommitted] = useState(""); + const [liveTentative, setLiveTentative] = useState(""); + const [modelId, setModelId] = useState(DEFAULT_MODEL_ID); const [devicePreference, setDevicePreference] = useState( DEFAULT_DEVICE_PREFERENCE, @@ -76,11 +100,20 @@ export function App() { onUpdateReady: (apply) => setApplyUpdate(() => apply), }); return () => { + void liveRef.current?.mic.stop().catch(() => {}); + liveRef.current = null; workerRef.current?.terminate(); workerRef.current = null; }; }, []); + function takeLiveRequest(requestId: string | undefined) { + if (requestId === undefined) return undefined; + const pending = pendingLive.current.get(requestId); + if (pending) pendingLive.current.delete(requestId); + return pending; + } + function ensureWorker(): Worker { if (workerRef.current) return workerRef.current; // Inference always runs in a dedicated Web Worker (D-003), never on the @@ -102,20 +135,42 @@ export function App() { } break; case "partial": - setPartialText((text) => text + message.text); + // Live segment passes (requestId set) resolve wholesale via + // "result"; their token stream is not rendered incrementally. + if (message.requestId === undefined) { + setPartialText((text) => text + message.text); + } break; - case "result": - setResult({ text: message.text, chunks: message.chunks }); - setBusy(false); + case "result": { + const pending = takeLiveRequest(message.requestId); + if (pending) { + pending.resolve({ text: message.text, chunks: message.chunks }); + } else if (message.requestId === undefined) { + setResult({ text: message.text, chunks: message.chunks }); + setBusy(false); + } break; - case "error": - setError(message.message); - setBusy(false); + } + case "error": { + const pending = takeLiveRequest(message.requestId); + if (pending) { + pending.reject(new Error(message.message)); + } else { + setError(message.message); + setBusy(false); + } break; + } } }); worker.addEventListener("error", (event) => { - setError(event.message || "The inference worker crashed unexpectedly."); + const failure = new Error( + event.message || "The inference worker crashed unexpectedly.", + ); + // Unblock any live passes so a live session can drain and stop. + for (const pending of pendingLive.current.values()) pending.reject(failure); + pendingLive.current.clear(); + setError(failure.message); setBusy(false); }); workerRef.current = worker; @@ -141,7 +196,7 @@ export function App() { } function handleModelSelect(nextModelId: string) { - if (busy || nextModelId === modelId) return; + if (busy || liveStatus !== "idle" || nextModelId === modelId) return; setModelId(nextModelId); setActiveDevice(null); setPartialText(""); @@ -150,7 +205,7 @@ export function App() { } function handleDevicePreference(preference: DevicePreference) { - if (busy || preference === devicePreference) return; + if (busy || liveStatus !== "idle" || preference === devicePreference) return; setDevicePreference(preference); setActiveDevice(null); // Only reload eagerly if a load was already in motion; otherwise the next @@ -159,7 +214,7 @@ export function App() { } async function handleFile(file: File) { - if (busy) return; + if (busy || liveStatus !== "idle") return; setBusy(true); setError(null); setPartialText(""); @@ -199,10 +254,95 @@ export function App() { } } + // ------------------------------------------------------------------------- + // Live microphone transcription (M7, D-016) + // ------------------------------------------------------------------------- + + /** One inference pass over a live segment, correlated via requestId. */ + function transcribeLiveSegment( + worker: Worker, + audio: Float32Array, + options: { language?: string; task?: WhisperTask }, + ): Promise { + const requestId = `live-${++nextLiveRequest.current}`; + return new Promise((resolve, reject) => { + pendingLive.current.set(requestId, { resolve, reject }); + // The controller hands over ownership, so the buffer can transfer. + postToWorker(worker, { type: "transcribe", audio, requestId, ...options }, [ + audio.buffer as ArrayBuffer, + ]); + }); + } + + async function startLive() { + if (busy || liveStatus !== "idle") return; + setError(null); + setResult(null); + setPartialText(""); + setFileName(null); + setLiveCommitted(""); + setLiveTentative(""); + setLiveStatus("starting"); + + // Idempotent model load, like the file flow — a pending download runs + // while the mic opens; segment passes queue in the worker until ready. + const worker = ensureWorker(); + if (!loadStarted) { + startLoad(modelId, devicePreference); + } else { + postToWorker(worker, { type: "load", modelId, device: devicePreference }); + } + + // Language/task are fixed for the session; the options are disabled while + // live. `.en` models take neither (same rule as the file flow). + const options = (getModel(modelId)?.multilingual ?? false) ? { language, task } : {}; + const transcriber = new LiveTranscriber({ + transcribe: (audio) => transcribeLiveSegment(worker, audio, options), + onUpdate: (update) => { + setLiveCommitted(update.committedText); + setLiveTentative(update.tentativeText); + if (update.error) setError(update.error); + }, + }); + + try { + const mic = await startMicCapture((chunk) => transcriber.push(chunk)); + liveRef.current = { mic, transcriber }; + setLiveStatus("listening"); + } catch (micError) { + setLiveStatus("idle"); + setError( + micError instanceof DOMException && micError.name === "NotAllowedError" + ? "Microphone access was denied. Allow it in your browser's site settings and try again." + : micError instanceof Error + ? micError.message + : String(micError), + ); + } + } + + async function stopLive() { + const session = liveRef.current; + if (!session || liveStatus !== "listening") return; + setLiveStatus("stopping"); + // Stop feeding audio first, then drain the remaining segment passes. + await session.mic.stop(); + try { + const final = await session.transcriber.stop(); + setResult(final); + setLiveCommitted(""); + setLiveTentative(""); + } finally { + liveRef.current = null; + setLiveStatus("idle"); + } + } + const model = getModel(modelId); const modelReady = loadProgress.ready; const showModelProgress = loadStarted && !modelReady && !error; const transcribing = busy && !decoding && modelReady; + const liveActive = liveStatus !== "idle"; return (
@@ -230,7 +370,7 @@ export function App() { activeDevice={activeDevice} downloaded={downloaded} offlineReady={swControlled} - disabled={busy} + disabled={busy || liveActive} /> {model?.multilingual && ( @@ -239,11 +379,18 @@ export function App() { onLanguageChange={setLanguage} task={task} onTaskChange={setTask} - disabled={busy} + disabled={busy || liveActive} /> )} - + + + void startLive()} + onStop={() => void stopLive()} + disabled={busy} + /> {fileName && (

@@ -263,9 +410,17 @@ export function App() { )} {applyUpdate && ( diff --git a/src/components/TranscriptView.tsx b/src/components/TranscriptView.tsx index a3dc009..c7d3adc 100644 --- a/src/components/TranscriptView.tsx +++ b/src/components/TranscriptView.tsx @@ -2,12 +2,19 @@ import { useEffect, useState } from "react"; import type { TranscriptChunk } from "../worker/protocol"; interface TranscriptViewProps { - /** Text streamed so far (accumulated partial messages). */ + /** Text streamed so far (accumulated partials / committed live segments). */ partialText: string; /** Final transcript, once available. */ result: { text: string; chunks: TranscriptChunk[] } | null; /** True while a transcription is in flight. */ transcribing: boolean; + /** + * Rolling live-mode text that may still change (D-016) — rendered visually + * distinct from the committed text it follows. + */ + tentativeText?: string; + /** Heading while running; defaults to "Transcribing…". */ + activeLabel?: string; } function formatTimestamp(seconds: number): string { @@ -17,9 +24,16 @@ function formatTimestamp(seconds: number): string { } /** Streaming transcript: live partials while running, final text + copy button after. */ -export function TranscriptView({ partialText, result, transcribing }: TranscriptViewProps) { +export function TranscriptView({ + partialText, + result, + transcribing, + tentativeText = "", + activeLabel = "Transcribing…", +}: TranscriptViewProps) { const [copied, setCopied] = useState(false); const text = result?.text ?? partialText; + const copyText = `${text}${tentativeText}`.trim(); useEffect(() => { if (!copied) return; @@ -27,29 +41,32 @@ export function TranscriptView({ partialText, result, transcribing }: Transcript return () => clearTimeout(timer); }, [copied]); - if (!transcribing && !text) return null; + if (!transcribing && !text && !tentativeText) return null; async function copy() { - if (!text) return; - await navigator.clipboard.writeText(text); + if (!copyText) return; + await navigator.clipboard.writeText(copyText); setCopied(true); } return (

-

{result ? "Transcript" : "Transcribing…"}

+

{result ? "Transcript" : activeLabel}

- {text || "Listening for the first words…"} + {text || (tentativeText ? "" : "Listening for the first words…")} + {tentativeText && ( + {tentativeText} + )} {transcribing &&

{result && result.chunks.length > 0 && ( diff --git a/src/index.css b/src/index.css index a852e8c..a77e963 100644 --- a/src/index.css +++ b/src/index.css @@ -397,6 +397,61 @@ body { font-size: 0.9rem; } +/* --- LiveControls (M7, D-016) --- */ + +.live-controls { + display: flex; + align-items: center; + gap: 0.75rem; +} + +.live-button { + border: 1px solid var(--border); + background: transparent; + color: inherit; + border-radius: 8px; + padding: 0.3rem 0.9rem; + font-size: 0.85rem; + cursor: pointer; + transition: border-color 0.15s; + white-space: nowrap; +} + +.live-button:hover:enabled { + border-color: var(--accent); +} + +.live-button:disabled { + opacity: 0.5; + cursor: default; +} + +.live-button-active, +.live-button-active:hover:enabled { + border-color: #e0605e; +} + +.live-dot { + flex-shrink: 0; + width: 0.6rem; + height: 0.6rem; + border-radius: 999px; + background: #e0605e; + animation: pulse 1.2s ease-in-out infinite; +} + +@keyframes pulse { + 50% { + opacity: 0.25; + } +} + +.live-hint { + margin: 0; + color: var(--muted); + font-size: 0.9rem; +} + .file-name { margin: 0; color: var(--muted); @@ -512,6 +567,13 @@ body { overflow-wrap: anywhere; } +/* Live-mode text that may still change (D-016) — visually distinct from the + committed text it follows. */ +.transcript-tentative { + color: var(--muted); + font-style: italic; +} + .cursor { display: inline-block; width: 0.55em; diff --git a/src/index.ts b/src/index.ts index 9280082..000d9f8 100644 --- a/src/index.ts +++ b/src/index.ts @@ -4,6 +4,7 @@ import { ensureIcons, ICONS_DIR } from "../scripts/generate-icons.ts"; import { injectPrecacheManifest } from "./pwa/precache.ts"; import { auth } from "./server/auth.ts"; import { AUTH_ENABLED } from "./server/flags.ts"; +import { transcribe } from "./server/transcribe.ts"; const ROOT = join(import.meta.dir, ".."); const IS_PRODUCTION = process.env.NODE_ENV === "production"; @@ -58,8 +59,14 @@ async function serveOrtFile(dir: string, name: string): Promise { async function handleApiRequest(request: Request): Promise { const { pathname } = new URL(request.url); // Runtime feature flag (D-015): with auth disabled, the endpoints don't - // exist — the client's /me probe reads the 404 as "feature off". - if (!AUTH_ENABLED && pathname.startsWith("/api/auth/")) return NOT_FOUND(); + // exist — the client's /me probe reads the 404 as "feature off". The + // server engine (M5) is session-gated, so it disappears along with auth. + if ( + !AUTH_ENABLED && + (pathname.startsWith("/api/auth/") || pathname.startsWith("/api/transcribe")) + ) { + return NOT_FOUND(); + } switch (`${request.method} ${pathname}`) { case "POST /api/auth/register": return auth.register(request); @@ -69,6 +76,11 @@ async function handleApiRequest(request: Request): Promise { return auth.logout(request); case "GET /api/auth/me": return auth.me(request); + // Server engine (M5, D-011/D-013): availability probe + streamed run. + case "GET /api/transcribe/status": + return transcribe.status(request); + case "POST /api/transcribe": + return transcribe.post(request); default: return NOT_FOUND(); } diff --git a/src/lib/engine.ts b/src/lib/engine.ts index 34f1a3e..79c387d 100644 --- a/src/lib/engine.ts +++ b/src/lib/engine.ts @@ -1,11 +1,389 @@ /** - * Engine abstraction (Decisions.md D-010). + * Engine abstraction (Decisions.md D-010): the UI transcribes through one + * `Engine` interface and never branches on where inference runs beyond the + * EngineToggle. `BrowserEngine` wraps the inference Web Worker (D-003); + * `ServerEngine` wraps the streamed `POST /api/transcribe` fetch (D-011, + * faster-whisper sidecar per D-013). * - * TODO(M5): define the `Engine` interface — `{ load(model), transcribe(audio), - * onProgress }` — plus `BrowserEngine` (wraps the inference worker) and - * `ServerEngine` (wraps the streamed `POST /api/transcribe` fetch, D-011). - * - * Until then the UI talks to the worker directly via src/worker/protocol.ts. + * Deliberately framework-free like authClient.ts: the worker and fetch are + * injectable, so `bun test` exercises everything without a DOM or a server + * (tests/engine.test.ts). This module is also where the HTTP stream's wire + * format lives — src/server/transcribe.ts imports it so client and server + * can never drift apart. */ -export {}; +import { + isWorkerToUiMessage, + type TranscribeMessage, + type TranscriptChunk, + type WhisperTask, +} from "../worker/protocol"; +import type { FetchLike } from "./authClient"; + +// --------------------------------------------------------------------------- +// The Engine interface (D-010) +// --------------------------------------------------------------------------- + +export type EngineKind = "browser" | "server"; + +export interface EngineTranscribeOptions { + /** + * Catalog model id (src/lib/models.ts). Required by the server engine — + * each request names its model since the server holds no per-client state. + * The browser engine ignores it: its model is whatever the worker loaded. + */ + modelId?: string; + /** Whisper language code; "auto"/undefined lets Whisper detect. */ + language?: string; + /** transcribe | translate (multilingual models only). */ + task?: WhisperTask; + /** + * Streaming transcript fragments — append each to the text shown so far. + * Browser engine: token fragments; server engine: whole segments. + */ + onPartial?: (fragment: string) => void; + /** + * Abort the run. Honored by the server engine (the request is cancelled + * server-side too); the browser worker cannot abort a pass yet (M6). + */ + signal?: AbortSignal; +} + +export interface EngineTranscribeResult { + text: string; + chunks: TranscriptChunk[]; +} + +export interface Engine { + readonly kind: EngineKind; + /** + * Transcribe 16 kHz mono PCM (src/lib/audio.ts). Takes ownership of + * `audio` — the buffer may be transferred or consumed. + */ + transcribe( + audio: Float32Array, + options?: EngineTranscribeOptions, + ): Promise; +} + +// --------------------------------------------------------------------------- +// Browser engine — wraps the inference worker (D-003) +// --------------------------------------------------------------------------- + +/** The slice of `Worker` the engine touches, injectable for tests. */ +export interface WorkerLike { + postMessage(message: unknown, transfer?: Transferable[]): void; + addEventListener( + type: "message" | "error", + listener: (event: { data?: unknown; message?: string }) => void, + ): void; +} + +interface PendingRequest { + resolve: (result: EngineTranscribeResult) => void; + reject: (error: Error) => void; + onPartial?: (fragment: string) => void; +} + +/** + * Correlates transcribe requests with worker responses via the protocol's + * `requestId` (D-016). Model loading stays outside: the UI drives `load` + * messages and the progress panel directly against the worker, and both can + * share the worker because the engine ignores messages without one of its + * request ids. + */ +let nextEngineInstance = 0; + +export class BrowserEngine implements Engine { + readonly kind = "browser"; + private readonly pending = new Map(); + /** Ids carry an instance discriminator so engines can share one worker. */ + private readonly instance = ++nextEngineInstance; + private nextRequestId = 0; + + constructor(private readonly worker: WorkerLike) { + worker.addEventListener("message", (event) => this.handleMessage(event.data)); + worker.addEventListener("error", (event) => { + this.failAll( + new Error(event.message || "The inference worker crashed unexpectedly."), + ); + }); + } + + transcribe( + audio: Float32Array, + options: EngineTranscribeOptions = {}, + ): Promise { + const requestId = `engine-${this.instance}-${++this.nextRequestId}`; + return new Promise((resolve, reject) => { + this.pending.set(requestId, { resolve, reject, onPartial: options.onPartial }); + const message: TranscribeMessage = { + type: "transcribe", + audio, + requestId, + ...(options.language !== undefined && + options.language !== "auto" && { language: options.language }), + ...(options.task !== undefined && { task: options.task }), + }; + // Ownership of the audio moves to the worker — zero-copy transfer. + this.worker.postMessage(message, [audio.buffer as ArrayBuffer]); + }); + } + + /** Reject every in-flight request (worker crash / teardown). */ + failAll(error: Error): void { + const requests = [...this.pending.values()]; + this.pending.clear(); + for (const request of requests) request.reject(error); + } + + private handleMessage(data: unknown): void { + if (!isWorkerToUiMessage(data)) return; + if (!("requestId" in data) || data.requestId === undefined) return; + const request = this.pending.get(data.requestId); + if (!request) return; + switch (data.type) { + case "partial": + request.onPartial?.(data.text); + break; + case "result": + this.pending.delete(data.requestId); + request.resolve({ text: data.text, chunks: data.chunks }); + break; + case "error": + this.pending.delete(data.requestId); + request.reject(new Error(data.message)); + break; + } + } +} + +// --------------------------------------------------------------------------- +// Server transcription wire format (D-011: streamed JSON lines) +// --------------------------------------------------------------------------- + +/** + * One line of POST /api/transcribe's `application/x-ndjson` response body. + * Produced by src/server/transcribe.ts, consumed by ServerEngine. Errors + * after the 200 status line travel in-stream as an "error" line. + */ +export type TranscribeStreamLine = + | { type: "segment"; text: string; start: number; end: number } + | { type: "result"; text: string; chunks: TranscriptChunk[] } + | { type: "error"; message: string }; + +export function isTranscribeStreamLine( + value: unknown, +): value is TranscribeStreamLine { + if (typeof value !== "object" || value === null) return false; + const line = value as Record; + switch (line.type) { + case "segment": + return ( + typeof line.text === "string" && + typeof line.start === "number" && + typeof line.end === "number" + ); + case "result": + return typeof line.text === "string" && Array.isArray(line.chunks); + case "error": + return typeof line.message === "string"; + default: + return false; + } +} + +/** Split a byte stream into complete lines (trailing partial line included). */ +async function* streamLines( + stream: ReadableStream, +): AsyncGenerator { + const decoder = new TextDecoder(); + let buffer = ""; + for await (const chunk of stream) { + buffer += decoder.decode(chunk, { stream: true }); + let newline: number; + while ((newline = buffer.indexOf("\n")) !== -1) { + yield buffer.slice(0, newline); + buffer = buffer.slice(newline + 1); + } + } + buffer += decoder.decode(); + if (buffer.trim().length > 0) yield buffer; +} + +// --------------------------------------------------------------------------- +// Server engine — wraps POST /api/transcribe (D-011, D-013) +// --------------------------------------------------------------------------- + +const defaultFetch: FetchLike = (input, init) => globalThis.fetch(input, init); + +/** UI-safe message for a non-OK /api/transcribe response. */ +async function serverErrorMessage(response: Response): Promise { + let detail: string | undefined; + try { + const body: unknown = await response.json(); + const error = (body as { error?: unknown } | null)?.error; + if (typeof error === "string") detail = error; + } catch { + // Non-JSON body — fall through to the status-based message. + } + switch (response.status) { + case 401: + return "Sign in to use the server engine."; + case 404: + return "The server engine is not available on this deployment."; + case 413: + return detail ?? "The audio is too large for the server engine."; + case 503: + return detail + ? `The server engine is unavailable: ${detail}` + : "The server engine is unavailable right now."; + default: + return detail ?? `Server transcription failed (${response.status}).`; + } +} + +export class ServerEngine implements Engine { + readonly kind = "server"; + + constructor(private readonly fetchFn: FetchLike = defaultFetch) {} + + async transcribe( + audio: Float32Array, + options: EngineTranscribeOptions = {}, + ): Promise { + if (!options.modelId) { + throw new Error("The server engine needs a model id per request."); + } + const params = new URLSearchParams({ model: options.modelId }); + if (options.language !== undefined && options.language !== "auto") { + params.set("language", options.language); + } + if (options.task !== undefined) params.set("task", options.task); + + let response: Response; + try { + response = await this.fetchFn(`/api/transcribe?${params}`, { + method: "POST", + credentials: "same-origin", + headers: { "Content-Type": "application/octet-stream" }, + // Raw float32 PCM — exactly what the browser engine consumes, so the + // server never has to decode containers (server-py/sidecar.py). The + // byte view narrows the buffer typing to what BodyInit accepts. + body: new Uint8Array(audio.buffer as ArrayBuffer, audio.byteOffset, audio.byteLength), + signal: options.signal, + }); + } catch (error) { + if (error instanceof DOMException && error.name === "AbortError") throw error; + throw new Error("Could not reach the server. Check your connection."); + } + if (!response.ok) throw new Error(await serverErrorMessage(response)); + if (!response.body) { + throw new Error("The server returned an empty transcription stream."); + } + + for await (const rawLine of streamLines(response.body)) { + if (rawLine.trim().length === 0) continue; + let parsed: unknown; + try { + parsed = JSON.parse(rawLine); + } catch { + throw new Error("The server returned a malformed transcription stream."); + } + if (!isTranscribeStreamLine(parsed)) { + throw new Error("The server returned a malformed transcription stream."); + } + switch (parsed.type) { + case "segment": + options.onPartial?.(parsed.text); + break; + case "result": + return { text: parsed.text, chunks: parsed.chunks }; + case "error": + throw new Error(parsed.message); + } + } + // The stream closed without a result line — the connection was cut. + throw new Error("The connection was lost before the transcript finished."); + } +} + +// --------------------------------------------------------------------------- +// Server engine availability (EngineToggle probes this after sign-in) +// --------------------------------------------------------------------------- + +export interface ServerEngineStatus { + available: boolean; + /** Human-readable explanation when unavailable (e.g. sidecar missing). */ + reason?: string; +} + +/** + * GET /api/transcribe/status → availability. Never rejects: any failure + * (401, 404 when the deployment has no server engine, offline) resolves to + * unavailable, mirroring fetchAuthStatus's offline-first behavior. + */ +export async function fetchServerEngineStatus( + fetchFn: FetchLike = defaultFetch, +): Promise { + try { + const response = await fetchFn("/api/transcribe/status", { + method: "GET", + credentials: "same-origin", + }); + if (!response.ok) { + return { available: false, reason: await serverErrorMessage(response) }; + } + const body: unknown = await response.json().catch(() => null); + const status = body as { available?: unknown; reason?: unknown } | null; + if (typeof status?.available !== "boolean") return { available: false }; + return { + available: status.available, + ...(typeof status.reason === "string" && { reason: status.reason }), + }; + } catch { + return { available: false, reason: "The server cannot be reached." }; + } +} + +// --------------------------------------------------------------------------- +// Engine choice persistence (EngineToggle, survives reloads) +// --------------------------------------------------------------------------- + +export const ENGINE_CHOICE_KEY = "whisper-engine-choice"; + +export interface StorageLike { + getItem(key: string): string | null; + setItem(key: string, value: string): void; +} + +function defaultStorage(): StorageLike | null { + try { + return (globalThis as { localStorage?: StorageLike }).localStorage ?? null; + } catch { + return null; // storage access can throw (privacy modes) + } +} + +/** Persisted toggle choice; anything unreadable falls back to "browser". */ +export function loadEngineChoice( + storage: StorageLike | null = defaultStorage(), +): EngineKind { + try { + const value = storage?.getItem(ENGINE_CHOICE_KEY); + return value === "server" ? "server" : "browser"; + } catch { + return "browser"; + } +} + +export function saveEngineChoice( + choice: EngineKind, + storage: StorageLike | null = defaultStorage(), +): void { + try { + storage?.setItem(ENGINE_CHOICE_KEY, choice); + } catch { + // Not persistable (private mode / quota) — the choice just won't stick. + } +} diff --git a/src/lib/models.ts b/src/lib/models.ts index a4d1c14..f8731b5 100644 --- a/src/lib/models.ts +++ b/src/lib/models.ts @@ -48,6 +48,14 @@ export interface WhisperModel { decoderParamsM: number; /** Per-device dtype defaults (D-004 / D-006). */ defaults: Record; + /** + * Model name the server engine passes to faster-whisper (D-013). These are + * faster-whisper's own aliases, which it resolves to the official + * CTranslate2 conversions on the Hub (`Systran/faster-whisper-*`; + * large-v3-turbo → `mobiuslabsgmbh/faster-whisper-large-v3-turbo`), so the + * pinned library version stays the single authority on the exact repo. + */ + fasterWhisperModel: string; } // --------------------------------------------------------------------------- @@ -121,6 +129,7 @@ const WEBGPU_FP16_Q4: DtypeDefault = { export const MODELS: readonly WhisperModel[] = [ { id: "onnx-community/whisper-tiny.en", + fasterWhisperModel: "tiny.en", displayName: "Whisper Tiny (English)", multilingual: false, encoderParamsM: 8, @@ -133,6 +142,7 @@ export const MODELS: readonly WhisperModel[] = [ }, { id: "onnx-community/whisper-tiny", + fasterWhisperModel: "tiny", displayName: "Whisper Tiny", multilingual: true, encoderParamsM: 8, @@ -141,6 +151,7 @@ export const MODELS: readonly WhisperModel[] = [ }, { id: "onnx-community/whisper-base.en", + fasterWhisperModel: "base.en", displayName: "Whisper Base (English)", multilingual: false, encoderParamsM: 20, @@ -149,6 +160,7 @@ export const MODELS: readonly WhisperModel[] = [ }, { id: "onnx-community/whisper-base", + fasterWhisperModel: "base", displayName: "Whisper Base", multilingual: true, encoderParamsM: 20, @@ -157,6 +169,7 @@ export const MODELS: readonly WhisperModel[] = [ }, { id: "onnx-community/whisper-small.en", + fasterWhisperModel: "small.en", displayName: "Whisper Small (English)", multilingual: false, encoderParamsM: 88, @@ -165,6 +178,7 @@ export const MODELS: readonly WhisperModel[] = [ }, { id: "onnx-community/whisper-small", + fasterWhisperModel: "small", displayName: "Whisper Small", multilingual: true, encoderParamsM: 88, @@ -173,6 +187,7 @@ export const MODELS: readonly WhisperModel[] = [ }, { id: "onnx-community/whisper-medium.en", + fasterWhisperModel: "medium.en", displayName: "Whisper Medium (English)", multilingual: false, encoderParamsM: 306, @@ -181,6 +196,7 @@ export const MODELS: readonly WhisperModel[] = [ }, { id: "onnx-community/whisper-medium", + fasterWhisperModel: "medium", displayName: "Whisper Medium", multilingual: true, encoderParamsM: 306, @@ -189,6 +205,7 @@ export const MODELS: readonly WhisperModel[] = [ }, { id: "onnx-community/whisper-large-v3-turbo", + fasterWhisperModel: "large-v3-turbo", displayName: "Whisper Large v3 Turbo", multilingual: true, encoderParamsM: 635, @@ -201,6 +218,15 @@ export function getModel(id: string): WhisperModel | undefined { return MODELS.find((model) => model.id === id); } +/** + * faster-whisper model name for a catalog id (server engine, D-013), or + * undefined for ids outside the catalog — the transcribe endpoint answers 400 + * rather than forwarding arbitrary strings to the sidecar. + */ +export function getFasterWhisperModel(id: string): string | undefined { + return getModel(id)?.fasterWhisperModel; +} + /** Default dtype for a model on a given device. Unknown ids fall back to q8. */ export function resolveDefaultDtype( modelId: string, diff --git a/src/server/transcribe.ts b/src/server/transcribe.ts index 7812464..9c0fef6 100644 --- a/src/server/transcribe.ts +++ b/src/server/transcribe.ts @@ -1,10 +1,477 @@ /** - * Server-side transcription engine — TODO(M5). + * Server-side transcription engine (M5, Decisions.md D-013): bridges the + * faster-whisper Python sidecar (server-py/sidecar.py) to the streamed + * JSON-lines HTTP contract (D-011, wire format in src/lib/engine.ts). * - * POST /api/transcribe: auth-guarded, runs the same Transformers.js pipeline - * under Bun (ONNX Runtime, CPU), streams progress + partial transcripts as a - * chunked JSON-lines response (Decisions.md D-011). Reuses the shared model - * catalog in src/lib/models.ts; keep pipelines warm across requests. + * Routes (wired in src/index.ts, auth-guarded — the server engine is the one + * logged-in-only feature, Architecture.md §5): + * GET /api/transcribe/status → 200 {available, reason?} | 401 + * POST /api/transcribe?model=[&language=..][&task=..] + * body: raw float32 mono 16 kHz PCM (application/octet-stream) + * → 200 application/x-ndjson (segment* then result | error lines) + * | 400 | 401 | 413 | 503 {error} + * + * The sidecar is spawned lazily on first use and supervised: a crash rejects + * the requests it was carrying and the next request respawns it, while a + * missing runtime (no python3, faster-whisper not installed) is remembered as + * a permanent "unavailable" reason — fixing that requires a deployment + * change, so we never busy-respawn. Either way the browser/anonymous path is + * untouched: this module holds the only Python dependency in the app. + * + * Warm-up strategy (WorkBreakdown M5): the sidecar keeps the last-used model + * loaded between requests, so only the first request for a model (or a model + * switch) pays the load; weights download once into the sidecar's cache. */ -export {}; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { TranscribeStreamLine } from "../lib/engine"; +import { getFasterWhisperModel } from "../lib/models"; +import type { TranscriptChunk, WhisperTask } from "../worker/protocol"; +import { getSessionUser as appGetSessionUser, type SessionUser } from "./auth.ts"; + +// --------------------------------------------------------------------------- +// Sidecar interface (injectable — tests use a fake, see tests/transcribe.test.ts) +// --------------------------------------------------------------------------- + +export interface SidecarSegment { + start: number; + end: number; + text: string; +} + +export interface SidecarTranscribeRequest { + /** faster-whisper model name (catalog mapping, src/lib/models.ts). */ + model: string; + /** Path to a file of raw float32 mono 16 kHz PCM. */ + audioPath: string; + language?: string; + task?: WhisperTask; +} + +export interface SidecarStatus { + available: boolean; + reason?: string; +} + +export interface Sidecar { + /** Spawns the process if needed; reports whether inference can run. */ + status(): Promise; + /** + * Run one transcription, streaming segments as they decode. Resolves after + * the sidecar's "done" (cancelled: true when `signal` fired first); + * rejects on sidecar errors or crashes. + */ + transcribe( + request: SidecarTranscribeRequest, + onSegment: (segment: SidecarSegment) => void, + signal?: AbortSignal, + ): Promise<{ cancelled: boolean }>; +} + +// --------------------------------------------------------------------------- +// The real sidecar: Bun.spawn + JSON-lines over stdio (D-013) +// --------------------------------------------------------------------------- + +const SIDECAR_SCRIPT = join(import.meta.dir, "../../server-py/sidecar.py"); + +/** How long the spawned process gets to import faster-whisper and report. */ +const READY_TIMEOUT_MS = 60_000; + +interface SidecarProcess { + proc: ReturnType; + ready: Promise; +} + +interface PendingTranscription { + onSegment: (segment: SidecarSegment) => void; + resolve: (outcome: { cancelled: boolean }) => void; + reject: (error: Error) => void; +} + +export function createSidecar(command?: readonly string[]): Sidecar { + // "python3" resolved per spawn — SIDECAR_PYTHON overrides for exotic setups. + const cmd = command ?? [process.env.SIDECAR_PYTHON || "python3", SIDECAR_SCRIPT]; + + let current: SidecarProcess | null = null; + /** Deployment-level failure (no python, deps missing): never respawn. */ + let permanentFailure: string | null = null; + const pending = new Map(); + let nextRequestId = 0; + + function failAllPending(error: Error): void { + const requests = [...pending.values()]; + pending.clear(); + for (const request of requests) request.reject(error); + } + + function routeLine(rawLine: string): void { + let message: Record; + try { + message = JSON.parse(rawLine) as Record; + } catch { + return; // stray non-JSON output must not kill the bridge + } + const request = pending.get(String(message.id)); + if (!request) return; + switch (message.type) { + case "segment": + if ( + typeof message.start === "number" && + typeof message.end === "number" && + typeof message.text === "string" + ) { + request.onSegment({ + start: message.start, + end: message.end, + text: message.text, + }); + } + break; + case "done": + pending.delete(String(message.id)); + request.resolve({ cancelled: message.cancelled === true }); + break; + case "error": + pending.delete(String(message.id)); + request.reject(new Error(String(message.message ?? "sidecar error"))); + break; + } + } + + function spawn(): SidecarProcess { + const proc = Bun.spawn({ + cmd: [...cmd], + stdin: "pipe", + stdout: "pipe", + stderr: "inherit", // model download progress / tracebacks → server log + }); + + const ready = new Promise((resolve, reject) => { + let settled = false; + const timeout = setTimeout(() => { + if (settled) return; + settled = true; + proc.kill(); + reject(new Error("the transcription sidecar did not start in time")); + }, READY_TIMEOUT_MS); + + void (async () => { + // First line decides readiness; everything after is request traffic. + const decoder = new TextDecoder(); + let buffer = ""; + for await (const chunk of proc.stdout as ReadableStream) { + buffer += decoder.decode(chunk, { stream: true }); + let newline: number; + while ((newline = buffer.indexOf("\n")) !== -1) { + const line = buffer.slice(0, newline); + buffer = buffer.slice(newline + 1); + if (!settled) { + settled = true; + clearTimeout(timeout); + let handshake: Record = {}; + try { + handshake = JSON.parse(line) as Record; + } catch { + // fall through — treated as a bad handshake below + } + if (handshake.type === "ready") { + resolve(); + } else { + // "fatal": faster-whisper missing — a deployment problem. + permanentFailure = String( + handshake.message ?? "the transcription sidecar failed to start", + ); + reject(new Error(permanentFailure)); + } + } else { + routeLine(line); + } + } + } + // stdout closed: the process exited (crash or shutdown). + if (!settled) { + settled = true; + clearTimeout(timeout); + reject(new Error("the transcription sidecar exited during startup")); + } + failAllPending(new Error("the transcription sidecar exited unexpectedly")); + if (current?.proc === proc) current = null; // next request respawns + })(); + }); + // A failed startup must not surface as an unhandled rejection when nobody + // is awaiting yet (e.g. crash between requests). + ready.catch(() => {}); + return { proc, ready }; + } + + async function ensure(): Promise { + if (permanentFailure) throw new Error(permanentFailure); + if (current) return current; + try { + current = spawn(); + } catch (error) { + // Bun.spawn throws synchronously when the binary is missing. + permanentFailure = `python3 is not available on this server (${ + error instanceof Error ? error.message : String(error) + })`; + throw new Error(permanentFailure); + } + return current; + } + + function send(proc: ReturnType, message: unknown): void { + const stdin = proc.stdin as { write(data: string): unknown; flush(): unknown }; + stdin.write(`${JSON.stringify(message)}\n`); + stdin.flush(); + } + + return { + async status() { + try { + const sidecar = await ensure(); + await sidecar.ready; + return { available: true }; + } catch (error) { + return { + available: false, + reason: error instanceof Error ? error.message : String(error), + }; + } + }, + + async transcribe(request, onSegment, signal) { + const sidecar = await ensure(); + await sidecar.ready; + const id = `req-${++nextRequestId}`; + return new Promise<{ cancelled: boolean }>((resolve, reject) => { + pending.set(id, { onSegment, resolve, reject }); + signal?.addEventListener( + "abort", + () => { + // The sidecar checks between segments and still answers "done" + // (cancelled: true), which resolves this promise normally. + if (pending.has(id) && current?.proc === sidecar.proc) { + send(sidecar.proc, { type: "cancel", id }); + } + }, + { once: true }, + ); + try { + send(sidecar.proc, { + type: "transcribe", + id, + model: request.model, + audio_path: request.audioPath, + language: request.language ?? null, + task: request.task ?? "transcribe", + }); + } catch (error) { + pending.delete(id); + reject(error instanceof Error ? error : new Error(String(error))); + } + }); + }, + }; +} + +// --------------------------------------------------------------------------- +// HTTP handlers +// --------------------------------------------------------------------------- + +/** Default cap ≈ 2¼ hours of raw float32 16 kHz mono (64 kB/s). */ +const DEFAULT_MAX_UPLOAD_BYTES = 512 * 1024 * 1024; + +const LANGUAGE_RE = /^[a-z]{2,3}$/; + +export interface TranscribeHandlers { + status(request: Request): Promise; + post(request: Request): Promise; +} + +export interface TranscribeDeps { + sidecar: Sidecar; + getSessionUser(request: Request): SessionUser | null; + maxUploadBytes?: number; + /** Directory for uploaded-PCM temp files. Default: the OS temp dir. */ + tempDir?: string; +} + +function json(body: unknown, status: number): Response { + return new Response(JSON.stringify(body), { + status, + headers: { "Content-Type": "application/json" }, + }); +} + +interface TranscribeParams { + modelId: string; + fasterWhisperModel: string; + language?: string; + task: WhisperTask; +} + +/** Parse + validate query params; a string is the 400 error text. */ +function parseParams(url: URL): TranscribeParams | string { + const modelId = url.searchParams.get("model"); + if (!modelId) return "missing model parameter"; + const fasterWhisperModel = getFasterWhisperModel(modelId); + // Catalog-only (D-006/D-013): never forward arbitrary ids to the sidecar. + if (!fasterWhisperModel) return `unknown model: ${modelId}`; + + const rawLanguage = url.searchParams.get("language"); + let language: string | undefined; + if (rawLanguage !== null && rawLanguage !== "" && rawLanguage !== "auto") { + if (!LANGUAGE_RE.test(rawLanguage)) return `invalid language: ${rawLanguage}`; + language = rawLanguage; + } + + const rawTask = url.searchParams.get("task") ?? "transcribe"; + if (rawTask !== "transcribe" && rawTask !== "translate") { + return `invalid task: ${rawTask}`; + } + + return { modelId, fasterWhisperModel, language, task: rawTask }; +} + +/** + * Stream the request body to `path`, enforcing the size limit even for + * chunked bodies. Returns bytes written, or a Response to answer with. + */ +async function writeBodyToFile( + request: Request, + path: string, + maxBytes: number, +): Promise { + const declared = Number(request.headers.get("content-length")); + if (Number.isFinite(declared) && declared > maxBytes) { + return json({ error: `audio exceeds the ${maxBytes}-byte upload limit` }, 413); + } + if (!request.body) return json({ error: "missing request body" }, 400); + + const sink = Bun.file(path).writer(); + let written = 0; + try { + for await (const chunk of request.body) { + written += chunk.byteLength; + if (written > maxBytes) { + return json({ error: `audio exceeds the ${maxBytes}-byte upload limit` }, 413); + } + sink.write(chunk); + } + } finally { + await sink.end(); + } + if (written === 0) return json({ error: "missing request body" }, 400); + if (written % 4 !== 0) { + return json({ error: "body must be raw float32 PCM (length % 4 == 0)" }, 400); + } + return written; +} + +export function createTranscribeHandlers(deps: TranscribeDeps): TranscribeHandlers { + const maxUploadBytes = deps.maxUploadBytes ?? DEFAULT_MAX_UPLOAD_BYTES; + const tempDir = deps.tempDir ?? tmpdir(); + + return { + async status(request) { + if (!deps.getSessionUser(request)) { + return json({ error: "not authenticated" }, 401); + } + const status = await deps.sidecar.status(); + return json(status, 200); + }, + + async post(request) { + if (!deps.getSessionUser(request)) { + return json({ error: "not authenticated" }, 401); + } + const params = parseParams(new URL(request.url)); + if (typeof params === "string") return json({ error: params }, 400); + + // Refuse before consuming a potentially huge body when inference can't + // run anyway (degrade gracefully — D-013). + const status = await deps.sidecar.status(); + if (!status.available) { + return json({ error: status.reason ?? "server engine unavailable" }, 503); + } + + const audioPath = join(tempDir, `whisper-upload-${crypto.randomUUID()}.f32`); + const written = await writeBodyToFile(request, audioPath, maxUploadBytes); + if (written instanceof Response) { + await Bun.file(audioPath).unlink().catch(() => {}); + return written; + } + + // 200 with an ndjson stream (D-011); failures beyond this point travel + // in-stream as an "error" line since the status is already committed. + const encoder = new TextEncoder(); + const stream = new ReadableStream({ + async start(controller) { + const enqueue = (line: TranscribeStreamLine) => + controller.enqueue(encoder.encode(`${JSON.stringify(line)}\n`)); + const chunks: TranscriptChunk[] = []; + try { + await deps.sidecar.transcribe( + { + model: params.fasterWhisperModel, + audioPath, + language: params.language, + task: params.task, + }, + (segment) => { + chunks.push({ + text: segment.text, + timestamp: [segment.start, segment.end], + }); + enqueue({ + type: "segment", + text: segment.text, + start: segment.start, + end: segment.end, + }); + }, + // Client gone (abort/refresh) → cancel inference between segments. + request.signal, + ); + enqueue({ + type: "result", + text: chunks.map((chunk) => chunk.text).join("").trim(), + chunks, + }); + } catch (error) { + enqueue({ + type: "error", + message: error instanceof Error ? error.message : String(error), + }); + } finally { + await Bun.file(audioPath).unlink().catch(() => {}); + try { + controller.close(); + } catch { + // Already errored/cancelled — the temp file cleanup above is + // what actually matters here. + } + } + }, + }); + + return new Response(stream, { + headers: { + "Content-Type": "application/x-ndjson; charset=utf-8", + "Cache-Control": "no-store", + }, + }); + }, + }; +} + +// --------------------------------------------------------------------------- +// Default instance (real sidecar + app sessions), wired in src/index.ts +// --------------------------------------------------------------------------- + +export const transcribe: TranscribeHandlers = createTranscribeHandlers({ + sidecar: createSidecar(), + getSessionUser: appGetSessionUser, + maxUploadBytes: Number(process.env.TRANSCRIBE_MAX_UPLOAD_BYTES) || undefined, +}); diff --git a/src/worker/inference.worker.ts b/src/worker/inference.worker.ts index 405268b..4bf6eb1 100644 --- a/src/worker/inference.worker.ts +++ b/src/worker/inference.worker.ts @@ -194,6 +194,7 @@ async function transcribe(message: TranscribeMessage): Promise { type: "error", stage: "transcribe", message: "No model loaded — send a \"load\" message first.", + requestId: message.requestId, }); return; } @@ -204,7 +205,8 @@ async function transcribe(message: TranscribeMessage): Promise { asr.tokenizer as ConstructorParameters[0], { skip_prompt: true, - callback_function: (text: string) => post({ type: "partial", text }), + callback_function: (text: string) => + post({ type: "partial", text, requestId: message.requestId }), }, ); @@ -233,6 +235,7 @@ async function transcribe(message: TranscribeMessage): Promise { type: "result", text: (single.text ?? "").trim(), chunks: single.chunks ?? [], + requestId: message.requestId, }; post(result); } @@ -255,7 +258,12 @@ scope.addEventListener("message", (event) => { break; case "transcribe": transcribe(data).catch((error: unknown) => { - post({ type: "error", stage: "transcribe", message: errorMessage(error) }); + post({ + type: "error", + stage: "transcribe", + message: errorMessage(error), + requestId: data.requestId, + }); }); break; } diff --git a/src/worker/protocol.ts b/src/worker/protocol.ts index 9756e46..3c48644 100644 --- a/src/worker/protocol.ts +++ b/src/worker/protocol.ts @@ -61,6 +61,12 @@ export interface TranscribeMessage { language?: string; /** Only meaningful for multilingual models. Defaults to "transcribe". */ task?: WhisperTask; + /** + * Optional correlation id, echoed on the partial/result/error messages this + * request produces (D-016). The live-transcription loop uses it to match + * responses to segment passes; the file flow omits it and is unchanged. + */ + requestId?: string; } export type UiToWorkerMessage = LoadMessage | TranscribeMessage; @@ -96,6 +102,8 @@ export type LoadProgressMessage = export interface PartialMessage { type: "partial"; text: string; + /** Echo of the originating transcribe request's id, if it carried one. */ + requestId?: string; } /** One timestamped segment of the final transcript. */ @@ -109,12 +117,16 @@ export interface ResultMessage { type: "result"; text: string; chunks: TranscriptChunk[]; + /** Echo of the originating transcribe request's id, if it carried one. */ + requestId?: string; } export interface ErrorMessage { type: "error"; stage: "load" | "transcribe"; message: string; + /** Echo of the failing transcribe request's id, if it carried one. */ + requestId?: string; } export type WorkerToUiMessage = @@ -131,6 +143,10 @@ function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null; } +function isOptionalString(value: unknown): value is string | undefined { + return value === undefined || typeof value === "string"; +} + const DEVICES: readonly string[] = ["webgpu", "wasm"]; const DTYPES: readonly string[] = ["fp32", "fp16", "q8", "q4"]; @@ -165,10 +181,11 @@ export function isUiToWorkerMessage(value: unknown): value is UiToWorkerMessage case "transcribe": return ( value.audio instanceof Float32Array && - (value.language === undefined || typeof value.language === "string") && + isOptionalString(value.language) && (value.task === undefined || value.task === "transcribe" || - value.task === "translate") + value.task === "translate") && + isOptionalString(value.requestId) ); default: return false; @@ -197,13 +214,18 @@ export function isWorkerToUiMessage(value: unknown): value is WorkerToUiMessage return false; } case "partial": - return typeof value.text === "string"; + return typeof value.text === "string" && isOptionalString(value.requestId); case "result": - return typeof value.text === "string" && Array.isArray(value.chunks); + return ( + typeof value.text === "string" && + Array.isArray(value.chunks) && + isOptionalString(value.requestId) + ); case "error": return ( typeof value.message === "string" && - (value.stage === "load" || value.stage === "transcribe") + (value.stage === "load" || value.stage === "transcribe") && + isOptionalString(value.requestId) ); default: return false; diff --git a/tests/models.test.ts b/tests/models.test.ts index b71078a..dfafcf2 100644 --- a/tests/models.test.ts +++ b/tests/models.test.ts @@ -6,6 +6,7 @@ import { estimateDefaultDownloadBytes, estimateDownloadBytes, formatModelSize, + getFasterWhisperModel, getModel, resolveDefaultDtype, } from "../src/lib/models"; @@ -59,6 +60,21 @@ describe("MODELS catalog", () => { expect(getModel(DEFAULT_MODEL_ID)).toBeDefined(); expect(getModel("onnx-community/whisper-nonexistent")).toBeUndefined(); }); + + test("every model maps to a faster-whisper name for the server engine (D-013)", () => { + for (const model of MODELS) { + // The onnx-community ids and faster-whisper's aliases share the same + // size suffix ("tiny.en", …, "large-v3-turbo"); keeping them aligned is + // what lets one catalog drive both engines. + expect(model.fasterWhisperModel).toBe( + model.id.replace("onnx-community/whisper-", ""), + ); + } + expect(getFasterWhisperModel(DEFAULT_MODEL_ID)).toBe("tiny.en"); + // Unknown ids stay undefined so /api/transcribe can answer 400 instead of + // forwarding arbitrary strings to the sidecar. + expect(getFasterWhisperModel("someone/some-model")).toBeUndefined(); + }); }); describe("resolveDefaultDtype", () => { diff --git a/tests/protocol.test.ts b/tests/protocol.test.ts index 0435626..c6408bb 100644 --- a/tests/protocol.test.ts +++ b/tests/protocol.test.ts @@ -29,6 +29,7 @@ describe("isUiToWorkerMessage", () => { { type: "transcribe", audio: new Float32Array(8) }, { type: "transcribe", audio: new Float32Array(8), language: "da", task: "transcribe" }, { type: "transcribe", audio: new Float32Array(8), language: "auto", task: "translate" }, + { type: "transcribe", audio: new Float32Array(8), requestId: "live-3" }, ]; for (const message of [...loads, ...transcribes]) { expect(isUiToWorkerMessage(message)).toBe(true); @@ -53,6 +54,9 @@ describe("isUiToWorkerMessage", () => { expect( isUiToWorkerMessage({ type: "transcribe", audio: new Float32Array(4), task: "detect" }), ).toBe(false); + expect( + isUiToWorkerMessage({ type: "transcribe", audio: new Float32Array(4), requestId: 7 }), + ).toBe(false); expect(isUiToWorkerMessage({ type: "result", text: "", chunks: [] })).toBe(false); }); }); @@ -74,8 +78,11 @@ describe("isWorkerToUiMessage", () => { { type: "load-progress", status: "ready", device: "wasm" }, { type: "load-progress", status: "ready", device: "webgpu" }, { type: "partial", text: " hello" }, + { type: "partial", text: " hello", requestId: "live-1" }, { type: "result", text: "hello world", chunks: [{ text: "hello world", timestamp: [0, 1.2] }] }, + { type: "result", text: "hello world", chunks: [], requestId: "live-1" }, { type: "error", stage: "load", message: "boom" }, + { type: "error", stage: "transcribe", message: "boom", requestId: "live-2" }, ]; for (const message of messages) { expect(isWorkerToUiMessage(message)).toBe(true); @@ -89,6 +96,7 @@ describe("isWorkerToUiMessage", () => { expect(isWorkerToUiMessage({ type: "load-progress", status: "ready" })).toBe(false); expect(isWorkerToUiMessage({ type: "load-progress", status: "ready", device: "cuda" })).toBe(false); expect(isWorkerToUiMessage({ type: "partial" })).toBe(false); + expect(isWorkerToUiMessage({ type: "partial", text: "x", requestId: 1 })).toBe(false); expect(isWorkerToUiMessage({ type: "result", text: "x", chunks: "nope" })).toBe(false); expect(isWorkerToUiMessage({ type: "error", stage: "decode", message: "m" })).toBe(false); expect(isWorkerToUiMessage({ type: "load", modelId: "x", device: "wasm" })).toBe(false);