import { useEffect, useRef, useState } from "react"; import "./index.css"; import { AuthForms } from "./components/AuthForms"; import { EngineToggle } from "./components/EngineToggle"; import { FileDropzone } from "./components/FileDropzone"; import { LiveControls, type LiveStatus } from "./components/LiveControls"; import { ModelLoadProgress } from "./components/ModelLoadProgress"; import { ModelPicker } from "./components/ModelPicker"; import { StorageManager } from "./components/StorageManager"; import { TranscriptView } from "./components/TranscriptView"; import { TranscriptionOptions } from "./components/TranscriptionOptions"; import { TranslatePanel } from "./components/TranslatePanel"; import { decodeAudioFile } from "./lib/audio"; import { classifyError, type ClassifiedError } from "./lib/errors"; import { BrowserEngine, ServerEngine, fetchServerEngineStatus, loadEngineChoice, saveEngineChoice, type Engine, type EngineKind, type ServerEngineStatus, } from "./lib/engine"; import { useAuth } from "./lib/useAuth"; import { LIVE_TUNING, LiveTranscriber, type LiveTranscribeResult } from "./lib/live"; import { startMicCapture, type MicCapture } from "./lib/mic"; import { getDownloadedModels } from "./lib/modelCache"; import { TRANSLATION_MODEL_IDS } from "./lib/translation"; import { registerServiceWorker } from "./pwa/register"; import { AUTO_LANGUAGE, DEFAULT_DEVICE_PREFERENCE, DEFAULT_MODEL_ID, DEFAULT_TASK, MODELS, estimatePeakLoadBytes, formatModelSize, getModel, resolveDefaultDtype, taskForModel, } from "./lib/models"; import { shouldWarnBeforeLoad, type MobileMemoryHint } from "./lib/mobileMemoryGuard"; import { applyLoadProgress, initialLoadProgress, isWorkerToUiMessage, type DevicePreference, type InferenceDevice, type TranscriptChunk, type UiToWorkerMessage, type WhisperTask, } from "./worker/protocol"; interface TranscriptResult { text: string; chunks: TranscriptChunk[]; } /** * Displayed error state (M6 error taxonomy, `src/lib/errors.ts`). `retryKind` * says which retry affordance (if any) the error box should offer — computed * at render time against current component state, not baked into a closure * captured when the worker's message handler was first registered. */ interface AppErrorState { classified: ClassifiedError; retryKind?: "load" | "file"; } const MODEL_IDS = MODELS.map((model) => model.id); // Downloaded badges cover the Opus-MT models too (transcript translation, D-WS4b). const CACHED_MODEL_IDS = [...MODEL_IDS, ...TRANSLATION_MODEL_IDS]; export function App() { const workerRef = useRef(null); // Live-mode session (M7, D-016). The mic and transcriber live outside React // state; only display text goes through setState. Segment passes are // correlated with worker responses via requestId. const liveRef = useRef<{ mic: MicCapture; transcriber: LiveTranscriber | null } | null>( null, ); const liveWsRef = useRef(null); const pendingLive = useRef( new Map< string, { resolve: (result: LiveTranscribeResult) => void; reject: (error: Error) => void; } >(), ); const nextLiveRequest = useRef(0); // File-flow run counter (D-018). Cancel bumps it, so a decode that was in // flight when the user cancelled finds itself stale and never posts its // audio to the (fresh) worker. Decoding itself cannot be aborted. const fileRun = useRef(0); // Last file handed to handleFile, so a retryable transcribe-stage error // (network failure talking to the server engine) can offer a Retry button // without holding onto File objects in state. const lastFileRef = useRef(null); // Accelerator failures learned this session (D-025), keyed // "modelId::preference" → the explicit device to load with instead, so // repeat loads skip the doomed accelerator attempt (and its restart) // entirely. Session-only on purpose: a browser update may fix the backend. const deviceOverrides = useRef(new Map()); // Mobile memory-crash warnings the user already accepted this session // (D-027), keyed "modelId::device" — once confirmed, repeat loads of the // same combination (e.g. the lazy auto-load after an explicit Download) // don't ask again. const mobileWarningAcknowledged = useRef(new Set()); // Resolves when the current browser-engine load reports ready (rejects on // load failure/cancel). Transcribe posts await it, so no request is ever // queued inside a worker that a load-restart (D-025) is about to terminate // — its audio buffer would die with the worker. const loadGate = useRef<{ promise: Promise; resolve: () => void; reject: (error: Error) => void; } | null>(null); // Engine plumbing (M5, D-010). The BrowserEngine is bound to the worker's // lifetime (recreated together on cancel); the ServerEngine is stateless // and lives for the app. A server run's fetch is cancelled via abortRef. const browserEngineRef = useRef(null); const serverEngineRef = useRef(null); const abortRef = useRef(null); const [busy, setBusy] = useState(false); // Which engine the current file run uses; null while idle. Server runs // don't need the browser model, so parts of the UI key off this. const [runEngine, setRunEngine] = useState(null); const [decoding, setDecoding] = useState(false); const [loadStarted, setLoadStarted] = useState(false); const [loadProgress, setLoadProgress] = useState(initialLoadProgress); // Was the model already fully cached (D-005/`modelCache.ts`) when this load // started? Threaded into ModelLoadProgress so its heading/hint don't say // "Downloading" for a load that never touches the network (D-033). const [loadedFromCache, setLoadedFromCache] = useState(false); const [partialText, setPartialText] = useState(""); const [transcribeProgress, setTranscribeProgress] = useState<{ processedSeconds: number; totalSeconds: number; } | null>(null); const [result, setResult] = useState(null); const [error, setError] = useState(null); const [fileName, setFileName] = useState(null); const [liveStatus, setLiveStatus] = useState("idle"); const [liveCommitted, setLiveCommitted] = useState(""); const [liveTentative, setLiveTentative] = useState(""); // Committed segments still waiting for their final pass — surfaced while // stopping so a long drain shows movement instead of a frozen label. const [livePending, setLivePending] = useState(0); const [modelId, setModelId] = useState(DEFAULT_MODEL_ID); const [devicePreference, setDevicePreference] = useState( DEFAULT_DEVICE_PREFERENCE, ); const [activeDevice, setActiveDevice] = useState(null); // WASM thread count ort actually got (D-025 observability); null until ready. const [activeThreads, setActiveThreads] = useState(null); const [language, setLanguage] = useState(AUTO_LANGUAGE); const [task, setTask] = useState(DEFAULT_TASK); const [downloaded, setDownloaded] = useState>(new Set()); // PWA state (M3, D-007): is a service worker controlling this page (app // shell available offline), and is an updated version waiting? const [swControlled, setSwControlled] = useState(false); const [applyUpdate, setApplyUpdate] = useState<(() => void) | null>(null); // Session + engine choice (M4/M5). One useAuth for the whole app — the // header forms and the EngineToggle must agree on the session. The choice // persists across reloads; it only takes effect while signed in AND the // deployment reports the server engine available (anonymous-first: logged // out, everything runs on-device exactly as before). const auth = useAuth(); const [engineChoice, setEngineChoice] = useState(() => loadEngineChoice()); const [serverStatus, setServerStatus] = useState(null); function refreshDownloaded() { // Downloaded badges (D-005): query Cache Storage for cached weights. getDownloadedModels(CACHED_MODEL_IDS).then(setDownloaded, () => {}); } // Terminate the worker when the app unmounts (also keeps HMR tidy). useEffect(() => { refreshDownloaded(); // Registration is idempotent, so the StrictMode double-invoke is fine. registerServiceWorker({ onControlled: () => setSwControlled(true), onUpdateReady: (apply) => setApplyUpdate(() => apply), }); return () => { void liveRef.current?.mic.stop().catch(() => {}); liveRef.current = null; liveWsRef.current?.close(); liveWsRef.current = null; workerRef.current?.terminate(); workerRef.current = null; browserEngineRef.current = null; abortRef.current?.abort(); abortRef.current = null; }; }, []); // Probe server-engine availability once a session exists (the endpoint is // auth-guarded, so probing while anonymous is pointless). Never rejects. useEffect(() => { if (!auth.user) { setServerStatus(null); return; } let cancelled = false; fetchServerEngineStatus().then((status) => { if (!cancelled) setServerStatus(status); }); return () => { cancelled = true; }; }, [auth.user]); /** `navigator.onLine`, guarded for non-browser test environments. */ function isOffline(): boolean { return typeof navigator !== "undefined" && !navigator.onLine; } function overrideKey(forModelId: string, preference: DevicePreference): string { return `${forModelId}::${preference}`; } /** Device to actually load with — the preference unless D-025 learned better. */ function effectiveDevice( forModelId: string, preference: DevicePreference, ): DevicePreference { return deviceOverrides.current.get(overrideKey(forModelId, preference)) ?? preference; } /** Mirrors ModelPicker's size-estimate fallback: until the worker reports an * active device, assume WASM unless the user asked for an accelerator. */ function guessDevice(preference: DevicePreference): InferenceDevice { return ( activeDevice ?? (preference === "webgpu" ? "webgpu" : preference === "webnn" ? "webnn" : "wasm") ); } /** * Whether loading `forModelId` on `device` risks crashing a mobile tab * (D-027) — independent of whether the user has already been asked. Used * both to decide whether to show the confirm dialog and, regardless of the * user's answer, to decide whether the worker should load sub-model * sessions sequentially (memory-safe but slower) or in parallel (fast, * safe everywhere this isn't true). */ function isMobileLoadRisk(forModelId: string, device: InferenceDevice): boolean { const targetModel = getModel(forModelId); if (!targetModel) return false; const peakBytes = estimatePeakLoadBytes(targetModel, resolveDefaultDtype(forModelId, device)); const hint: MobileMemoryHint = { userAgent: typeof navigator !== "undefined" ? navigator.userAgent : undefined, maxTouchPoints: typeof navigator !== "undefined" ? navigator.maxTouchPoints : undefined, deviceMemoryGB: typeof navigator !== "undefined" ? (navigator as Navigator & { deviceMemory?: number }).deviceMemory : undefined, }; return shouldWarnBeforeLoad(hint, peakBytes); } /** * Confirm before a browser-engine load that risks crashing the tab on * mobile (D-027). Returns true if the load should proceed — either it * isn't risky, or the user already accepted the risk (this load, or an * identical one earlier this session). A declined confirm returns false * with no side effects, so callers can bail out before touching any state. */ function confirmRiskyMobileLoad(forModelId: string, preference: DevicePreference): boolean { const targetModel = getModel(forModelId); if (!targetModel) return true; const device = guessDevice(preference); const key = overrideKey(forModelId, device); if (mobileWarningAcknowledged.current.has(key)) return true; if (!isMobileLoadRisk(forModelId, device)) return true; const peakBytes = estimatePeakLoadBytes(targetModel, resolveDefaultDtype(forModelId, device)); const proceed = window.confirm( `${targetModel.displayName} needs roughly ${formatModelSize(peakBytes)} of memory at ` + "once to load — more than many phones and tablets allow a browser tab to use " + "before force-closing it. This can crash your browser (and on some phones, the " + "whole screen briefly goes back to the home screen). A smaller model (Tiny, Base " + "or Small) is much safer on mobile.\n\nContinue anyway?", ); if (proceed) mobileWarningAcknowledged.current.add(key); return proceed; } function newLoadGate(): void { let resolve!: () => void; let reject!: (error: Error) => void; const promise = new Promise((res, rej) => { resolve = res; reject = rej; }); // Waiters attach their own handlers; this swallows the "unhandled // rejection" a gate rejected with nobody waiting would otherwise raise. promise.catch(() => {}); loadGate.current = { promise, resolve, reject }; } /** Wait until the browser pipeline is ready (or its load failed/cancelled). */ function browserModelReady(): Promise { return loadGate.current?.promise ?? Promise.resolve(); } function takeLiveRequest(requestId: string | undefined) { if (requestId === undefined) return undefined; const pending = pendingLive.current.get(requestId); if (pending) pendingLive.current.delete(requestId); return pending; } function ensureWorker(): Worker { if (workerRef.current) return workerRef.current; // Inference always runs in a dedicated Web Worker (D-003), never on the // main thread (Architecture.md §5). The worker script is bundled by the // server at startup and served at a stable URL (see src/index.ts) because // Bun's bundler does not follow `new Worker(new URL(...))` graphs yet. const worker = new Worker("inference.worker.js", { type: "module" }); worker.addEventListener("message", (event: MessageEvent) => { const message = event.data; if (!isWorkerToUiMessage(message)) return; switch (message.type) { case "load-progress": setLoadProgress((state) => applyLoadProgress(state, message)); if (message.status === "ready") { // Surface the backend the worker actually selected (D-004) and // refresh the downloaded badges now that weights are cached. setActiveDevice(message.device); setActiveThreads(message.threads ?? null); refreshDownloaded(); // Release any transcribe posts waiting on this load (D-025). loadGate.current?.resolve(); } break; case "partial": // Live segment passes (requestId set) resolve wholesale via // "result"; their token stream is not rendered incrementally. if (message.requestId === undefined) { setPartialText((text) => text + message.text); } break; case "transcribe-progress": // File flow only — live passes never carry progress (D-016). if (message.requestId === undefined) { setTranscribeProgress({ processedSeconds: message.processedSeconds, totalSeconds: message.totalSeconds, }); } break; case "result": { const pending = takeLiveRequest(message.requestId); if (pending) { pending.resolve({ text: message.text, chunks: message.chunks }); } else if (message.requestId === undefined) { setResult({ text: message.text, chunks: message.chunks }); setTranscribeProgress(null); setBusy(false); setRunEngine(null); } break; } case "error": { const pending = takeLiveRequest(message.requestId); if (pending) { pending.reject(new Error(message.message)); } else if (message.requestId === undefined) { // Load-stage failures and legacy id-less passes. Engine-routed // file passes carry an id and surface through the engine promise. if (message.stage === "load") { loadGate.current?.reject(new Error(message.message)); } const classified = classifyError(message.message, { stage: message.stage, offline: isOffline(), }); setError({ classified, retryKind: classified.retryable ? (message.stage === "load" ? "load" : "file") : undefined, }); setTranscribeProgress(null); setBusy(false); setRunEngine(null); } break; } case "load-restart": { // The accelerator attempt failed after it had already initialized // onnxruntime's WASM runtime single-threaded (D-025). This worker // can never run multithreaded WASM again, so: remember the outcome // for this session, replace the worker, and reload with the // explicit fallback device. The load gate stays pending — the fresh // worker's "ready" releases the waiting transcribe posts, which // then flow to the new worker. deviceOverrides.current.set( overrideKey(message.modelId, message.preference), message.device, ); // Defensive: the gate means nothing should be in flight, but never // leave a promise awaiting a worker we are about to terminate. const restart = new Error( "Inference restarted to use all CPU cores — this pass was skipped.", ); browserEngineRef.current?.failAll(restart); for (const pendingPass of pendingLive.current.values()) { pendingPass.reject(restart); } pendingLive.current.clear(); workerRef.current?.terminate(); workerRef.current = null; browserEngineRef.current = null; setLoadProgress(initialLoadProgress); setActiveDevice(null); setActiveThreads(null); const fresh = ensureWorker(); postToWorker(fresh, { type: "load", modelId: message.modelId, device: message.device, sequentialSessionLoad: isMobileLoadRisk(message.modelId, message.device), }); break; } } }); worker.addEventListener("error", (event) => { // Classify on the raw (possibly empty) message: an empty message is // itself the signal for a silent OOM tab-kill (D-027 — no JS error // event carries details when that happens), which a fallback string // would otherwise hide from the classifier. const classified = classifyError(event.message, { stage: "worker-crash", offline: isOffline(), }); const failure = new Error(classified.message); // Unblock any live passes so a live session can drain and stop. for (const pending of pendingLive.current.values()) pending.reject(failure); pendingLive.current.clear(); loadGate.current?.reject(failure); setError({ classified, retryKind: classified.retryable ? "load" : undefined }); setBusy(false); }); workerRef.current = worker; // The BrowserEngine shares this worker and only sees its own request ids // (engine-*), so it coexists with the live loop's live-* ids above. browserEngineRef.current = new BrowserEngine(worker); return worker; } function postToWorker(worker: Worker, message: UiToWorkerMessage, transfer?: Transferable[]) { worker.postMessage(message, transfer ?? []); } /** * Post a load request and reset per-load UI state so the progress panel * restarts cleanly on model/device switches. Idempotent in the worker for * an unchanged model+device; a change disposes the old pipeline first. */ function startLoad(nextModelId: string, preference: DevicePreference): Worker { const worker = ensureWorker(); setLoadProgress(initialLoadProgress); setLoadStarted(true); // D-033: `downloaded` (refreshed via getDownloadedModels, D-005) already // tells us whether this model's weights are cached before we ask the // worker to load it — capture that now, since the worker/Transformers.js // progress callback itself cannot distinguish a cached read from a // network download (see the ADR for why we don't patch that upstream). setLoadedFromCache(downloaded.has(nextModelId)); setError(null); newLoadGate(); const device = effectiveDevice(nextModelId, preference); postToWorker(worker, { type: "load", modelId: nextModelId, device, // D-027/D-029: sub-model sessions load sequentially (memory-safe, but // slower — halves the CDN concurrency a fast connection could otherwise // use) only where the peak-memory crash risk is real; everywhere else // (desktop, low-risk models) the worker loads them in parallel. sequentialSessionLoad: isMobileLoadRisk(nextModelId, guessDevice(device)), }); return worker; } function handleModelSelect(nextModelId: string) { if (busy || liveStatus !== "idle" || nextModelId === modelId) return; setModelId(nextModelId); // A "Translate to English" chosen on a model that can translate must not // silently carry over to one that cannot (D-WS4a). setTask((current) => taskForModel(nextModelId, current)); setActiveDevice(null); setActiveThreads(null); setPartialText(""); setTranscribeProgress(null); setResult(null); // Selection never downloads (D-020): the load starts when a transcription // needs it (file drop / live start on the browser engine) or when the // user explicitly clicks Download. Reset the load state so the next load // reports fresh progress for this model. setLoadStarted(false); setLoadProgress(initialLoadProgress); setLoadedFromCache(false); loadGate.current = null; } function handleDevicePreference(preference: DevicePreference) { if (busy || liveStatus !== "idle" || preference === devicePreference) return; setDevicePreference(preference); setActiveDevice(null); setActiveThreads(null); // No eager reload (D-020) — the next load picks up the new preference. setLoadStarted(false); setLoadProgress(initialLoadProgress); setLoadedFromCache(false); loadGate.current = null; } /** Explicit model download (D-020) — pre-caches weights for offline use. */ function handleDownloadModel() { if (busy || liveStatus !== "idle" || loadStarted) return; if (!confirmRiskyMobileLoad(modelId, devicePreference)) return; startLoad(modelId, devicePreference); } /** * Cancel the in-flight file transcription (D-018). Browser runs: no abort * API exists for a running `asr()` call — and on the WASM backend the pass * blocks the worker's event loop, so even a "cancel" message would sit * unread until the pass finished. Terminating the worker is the only * reliable preemption; it also aborts an in-flight model download. The next * run spawns a fresh worker and reloads the pipeline — cheap, because the * weights stay in the browser Cache API (D-005). Server runs: abort the * `/api/transcribe` fetch (the server cancels the sidecar pass, D-011) and * leave the worker — and its loaded pipeline — untouched. */ function cancelTranscription() { if (!busy) return; fileRun.current++; // any decode still running is now stale const cancelled = new Error("Transcription cancelled."); if (runEngine === "server") { abortRef.current?.abort(); abortRef.current = null; } else { browserEngineRef.current?.failAll(cancelled); browserEngineRef.current = null; workerRef.current?.terminate(); workerRef.current = null; // The worker (and any pipeline / partial download) is gone — reset the // load state so the next transcription reloads from a clean slate. // Rejecting the gate unblocks a transcribe post still waiting on it. loadGate.current?.reject(cancelled); loadGate.current = null; setLoadStarted(false); setLoadProgress(initialLoadProgress); setLoadedFromCache(false); setActiveDevice(null); setActiveThreads(null); } // The file flow never has live passes pending, but if anything is in the // map (defensive), unblock it so nothing awaits a dead worker forever. for (const pending of pendingLive.current.values()) pending.reject(cancelled); pendingLive.current.clear(); setBusy(false); setRunEngine(null); setPartialText(""); setTranscribeProgress(null); setResult(null); setFileName(null); setError(null); } async function handleFile(file: File) { if (busy || liveStatus !== "idle") return; lastFileRef.current = file; // The toggle's choice only takes effect with a session and an available // server engine (D-010); everything else runs on-device as always. const useServer = engineChoice === "server" && auth.user !== null && serverStatus?.available === true; // Browser-engine runs may need to load the model right away (D-020); bail // out before any state changes if the user declines a mobile crash risk // warning (D-027). if (!useServer && !confirmRiskyMobileLoad(modelId, devicePreference)) return; const run = ++fileRun.current; setBusy(true); setRunEngine(useServer ? "server" : "browser"); setError(null); setPartialText(""); setTranscribeProgress(null); setResult(null); setFileName(file.name); let engine: Engine | null = null; if (useServer) { engine = serverEngineRef.current ??= new ServerEngine(); } else { // Kick off the (idempotent) model load immediately so the download runs // while we decode the audio locally. Server runs skip this — they don't // need the browser pipeline. The engine itself is resolved after the // load gate below: a load-restart (D-025) may replace the worker (and // its BrowserEngine) while the audio decodes. ensureWorker(); if (!loadStarted) { startLoad(modelId, devicePreference); } else { const device = effectiveDevice(modelId, devicePreference); postToWorker(workerRef.current!, { type: "load", modelId, device, sequentialSessionLoad: isMobileLoadRisk(modelId, guessDevice(device)), }); } } const multilingual = getModel(modelId)?.multilingual ?? false; // Decoded entirely on-device via Web Audio (D-012); for browser runs the // audio never touches the network (Architecture.md §5) — server runs // upload the decoded PCM, which is the point of the toggle. setDecoding(true); let audio: Float32Array; try { audio = await decodeAudioFile(file); } catch (decodeError) { if (fileRun.current === run) { setError({ classified: classifyError(decodeError, { stage: "decode" }) }); setBusy(false); setRunEngine(null); } return; } finally { setDecoding(false); } if (fileRun.current !== run) return; // cancelled while decoding (D-018) if (engine === null) { // Browser run: wait for the pipeline before posting the pass, so the // audio never sits queued in a worker a load-restart may terminate // (D-025). The worker serialized the pass on the load anyway, so this // changes nothing about when inference actually starts. try { await browserModelReady(); } catch (loadError) { if (fileRun.current !== run) return; // cancelled while loading const classified = classifyError(loadError, { stage: "load", offline: isOffline() }); setError({ classified, retryKind: classified.retryable ? "load" : undefined }); setBusy(false); setRunEngine(null); return; } if (fileRun.current !== run) return; engine = browserEngineRef.current!; } // Server runs cancel by aborting the fetch (D-018 amendment); browser // runs cancel by terminating the worker, which rejects the promise via // BrowserEngine.failAll. const controller = useServer ? new AbortController() : null; abortRef.current = controller; try { const outcome = await engine.transcribe(audio, { // Language/task only apply to multilingual models; `.en` variants // take neither. The browser engine ignores modelId (worker holds it). modelId, ...(multilingual && { language, task: taskForModel(modelId, task) }), onPartial: (fragment) => { if (fileRun.current === run) setPartialText((text) => text + fragment); }, ...(!useServer && { onProgress: (progress: { processedSeconds: number; totalSeconds: number }) => { if (fileRun.current === run) setTranscribeProgress(progress); }, }), ...(controller && { signal: controller.signal }), }); if (fileRun.current !== run) return; // cancelled mid-pass setResult(outcome); setTranscribeProgress(null); setBusy(false); setRunEngine(null); } catch (transcribeError) { if (fileRun.current !== run) return; // cancellation rejections are expected const classified = classifyError(transcribeError, { stage: "transcribe", offline: isOffline(), }); setError({ classified, retryKind: classified.retryable ? "file" : undefined }); setTranscribeProgress(null); setBusy(false); setRunEngine(null); } finally { if (abortRef.current === controller) abortRef.current = null; } } // ------------------------------------------------------------------------- // Live microphone transcription (M7, D-016) // ------------------------------------------------------------------------- /** One inference pass over a live segment, correlated via requestId. */ async function transcribeLiveSegment( audio: Float32Array, options: { language?: string; task?: WhisperTask }, ): Promise { // Wait for the pipeline before posting: a load-restart (D-025) may swap // the worker while the model loads, and a pass queued in the old worker // would die with it (its audio buffer is transferred, so it cannot be // re-posted). The worker is re-read after the wait for the same reason. await browserModelReady(); const worker = workerRef.current; if (!worker) throw new Error("The inference worker is not running."); const requestId = `live-${++nextLiveRequest.current}`; return new Promise((resolve, reject) => { pendingLive.current.set(requestId, { resolve, reject }); // The controller hands over ownership, so the buffer can transfer. postToWorker(worker, { type: "transcribe", audio, requestId, ...options }, [ audio.buffer as ArrayBuffer, ]); }); } async function startLive() { if (busy || liveStatus !== "idle") return; const useServer = engineChoice === "server" && auth.user !== null && serverStatus?.available === true; // Browser-engine live sessions may need to load the model right away // (D-020); bail out before any state changes if the user declines a // mobile crash risk warning (D-027). if (!useServer && !confirmRiskyMobileLoad(modelId, devicePreference)) return; setError(null); setResult(null); setPartialText(""); setFileName(null); setLiveCommitted(""); setLiveTentative(""); setLivePending(0); setLiveStatus("starting"); const options = (getModel(modelId)?.multilingual ?? false) ? { language, task: taskForModel(modelId, task) } : {}; if (useServer) { const wsUrl = new URL(`/api/transcribe/live?model=${modelId}`, window.location.href); wsUrl.protocol = wsUrl.protocol === "https:" ? "wss:" : "ws:"; if (options.language) wsUrl.searchParams.set("language", options.language); if (options.task) wsUrl.searchParams.set("task", options.task); const ws = new WebSocket(wsUrl); // Ends the session locally: stops the mic (releasing the hardware and // the browser's recording indicator — nothing else holds this // MediaStream) and clears the refs so a stale onclose/onerror firing // later is a no-op (guarded by `liveWsRef.current === ws` below). const endSession = (message: string | null) => { liveWsRef.current = null; const session = liveRef.current; liveRef.current = null; void session?.mic.stop().catch(() => {}); setLiveCommitted(""); setLiveTentative(""); setLiveStatus("idle"); if (message) setError({ classified: classifyError(message) }); }; ws.onmessage = (event) => { let data: { type: string; [key: string]: unknown }; try { data = JSON.parse(event.data); } catch { // Not a frame we understand — ignore rather than crash the handler // and strand the session in "listening"/"stopping" forever. return; } if (data.type === "update") { const update = data.update as { committedText: string; tentativeText: string; pendingFinalPasses?: number; error?: string; }; setLiveCommitted(update.committedText); setLiveTentative(update.tentativeText); setLivePending(update.pendingFinalPasses ?? 0); if (update.error) setError({ classified: classifyError(update.error) }); } else if (data.type === "result") { setResult(data.result as TranscriptResult); endSession(null); } else if (data.type === "error") { endSession(typeof data.error === "string" ? data.error : "Live transcription failed."); } }; ws.onerror = () => { // A genuine connection failure also fires "close" right after, which // does the actual cleanup — this only needs to supply the message // (onclose can't distinguish a clean vs. abrupt closure on its own). if (liveWsRef.current === ws) { setError({ classified: classifyError( "Lost connection to the server during live transcription. The microphone has been stopped.", { offline: isOffline() }, ), }); } }; ws.onclose = () => { // Fires after a normal finish too (onmessage already nulled the ref // there), so this only acts on an *unexpected* disconnect — otherwise // stop() would hang forever waiting for a "result" that never comes, // and the mic would stay open with no code left holding a reference // to it. if (liveWsRef.current === ws) endSession(null); }; ws.onopen = async () => { try { const mic = await startMicCapture((chunk) => { if (ws.readyState === WebSocket.OPEN) { // The byte view narrows the buffer typing to what send() // accepts (same pattern as ServerEngine's request body). ws.send( new Uint8Array( chunk.buffer as ArrayBuffer, chunk.byteOffset, chunk.byteLength, ), ); } }); liveWsRef.current = ws; liveRef.current = { mic, transcriber: null }; setLiveStatus("listening"); } catch (micError) { ws.close(); setLiveStatus("idle"); const message = micError instanceof DOMException && micError.name === "NotAllowedError" ? "Microphone access was denied. Allow it in your browser's site settings and try again." : micError instanceof Error ? micError.message : String(micError); setError({ classified: classifyError(message) }); } }; return; } // Idempotent model load, like the file flow — a pending download runs // while the mic opens; segment passes wait on the load gate until ready. const worker = ensureWorker(); if (!loadStarted) { startLoad(modelId, devicePreference); } else { const device = effectiveDevice(modelId, devicePreference); postToWorker(worker, { type: "load", modelId, device, sequentialSessionLoad: isMobileLoadRisk(modelId, guessDevice(device)), }); } const transcriber = new LiveTranscriber({ ...LIVE_TUNING, transcribe: (audio) => transcribeLiveSegment(audio, options), onUpdate: (update) => { setLiveCommitted(update.committedText); setLiveTentative(update.tentativeText); setLivePending(update.pendingFinalPasses); if (update.error) setError({ classified: classifyError(update.error) }); }, }); try { const mic = await startMicCapture((chunk) => transcriber.push(chunk)); const session = { mic, transcriber }; liveRef.current = session; if (loadProgress.ready) { // Pipeline already loaded for the current model/device — nothing to // wait on, go straight to "listening" as before. setLiveStatus("listening"); } else { // Model not ready yet (first use, or a model/device switch): don't // claim "Listening" until the pipeline actually is one, otherwise the // live-dot/hint imply an active session while nothing is being // transcribed (D-034). `transcribeLiveSegment` already awaits the // same gate before posting, so no segments are lost either way. setLiveStatus("loading-model"); try { await browserModelReady(); // The session could have been torn down (Stop, or another start) // while we were waiting — only advance if it's still this one. if (liveRef.current === session) setLiveStatus("listening"); } catch { // Load failed; the worker's own "error" handler already calls // setError() for stage:"load" failures, so don't duplicate it here. if (liveRef.current === session) { liveRef.current = null; await mic.stop().catch(() => {}); setLiveStatus("idle"); } } } } catch (micError) { setLiveStatus("idle"); const message = micError instanceof DOMException && micError.name === "NotAllowedError" ? "Microphone access was denied. Allow it in your browser's site settings and try again." : micError instanceof Error ? micError.message : String(micError); setError({ classified: classifyError(message) }); } } async function stopLive() { const session = liveRef.current; if (!session || liveStatus !== "listening") return; setLiveStatus("stopping"); // Stop feeding audio first, then drain the remaining segment passes. await session.mic.stop(); if (liveWsRef.current) { if (liveWsRef.current.readyState === WebSocket.OPEN) { liveWsRef.current.send(JSON.stringify({ type: "stop" })); } else { // The connection is already closing/closed — send() would silently // drop the "stop" frame (WebSocket.send() does not throw for a // non-OPEN socket), and nothing would ever answer with a "result", // leaving the UI stuck on "Finishing up…" forever. Its close/error // handler already stops the mic; just clear the ref and settle. liveWsRef.current = null; liveRef.current = null; setLiveStatus("idle"); setError({ classified: classifyError( "Live transcription lost its connection to the server before it could finish.", { offline: isOffline() }, ), }); } } else { try { const final = await session.transcriber!.stop(); setResult(final); setLiveCommitted(""); setLiveTentative(""); } finally { liveRef.current = null; setLiveStatus("idle"); } } } const model = getModel(modelId); const modelReady = loadProgress.ready; const showModelProgress = loadStarted && !modelReady && !error; // Server runs never load the browser model, so they are "transcribing" the // moment decoding ends. const transcribing = busy && !decoding && (runEngine === "server" || modelReady); const liveActive = liveStatus !== "idle"; // What the next file run will use (mirrors handleFile's decision). const serverSelected = engineChoice === "server" && auth.user !== null && serverStatus?.available === true; const availableModels = serverSelected ? MODELS.filter((m) => m.fasterWhisperModel === "large-v3-turbo" || m.fasterWhisperModel === "medium.en") : MODELS; useEffect(() => { if (serverSelected) { const allowed = ["onnx-community/whisper-large-v3-turbo", "onnx-community/whisper-medium.en_timestamped"]; if (!allowed.includes(modelId) && !busy && liveStatus === "idle") { setModelId("onnx-community/whisper-large-v3-turbo"); setTask((current) => taskForModel("onnx-community/whisper-large-v3-turbo", current)); setActiveDevice(null); setActiveThreads(null); setPartialText(""); setTranscribeProgress(null); setResult(null); setLoadStarted(false); setLoadProgress(initialLoadProgress); setLoadedFromCache(false); } } }, [serverSelected, modelId, busy, liveStatus]); return (

Whisper Web

Speech-to-text that runs entirely in your browser. Your audio never leaves this device.

{/* Session UI (M4). Anonymous-first: logged out, this is only an unobtrusive "Sign in" button — everything else is unchanged. Renders nothing until the /me probe confirms the server has auth enabled (runtime flag, D-015). */}
{model?.multilingual && ( )} {/* Engine toggle (M5, D-010): signed-in users choose where uploaded files are transcribed. Live mode always stays on-device (D-016). */} {auth.user && ( { setEngineChoice(choice); saveEngineChoice(choice); }} status={serverStatus} disabled={busy || liveActive} /> )} void startLive()} onStop={() => void stopLive()} disabled={busy} /> {/* Live on the WASM backend cannot keep up with real time for the big models — every pass pads to a full 30 s Whisper window (D-024), so tentative text lags and the stop drain grows. Say so instead of looking frozen. */} {liveActive && activeDevice === "wasm" && (model?.encoderParamsM ?? 0) >= 300 && (

{model?.displayName} runs slower than real time on the WASM backend, so live text will lag behind speech. A smaller model (Tiny, Base or Small) keeps up better on this device.

)} {fileName && (

File: {fileName} {decoding && " — decoding audio…"} {busy && ( // Cancel the whole run (D-018): decode, pending model download, // or the transcription pass itself. )}

)} {error && (

{error.classified.message}

{error.classified.suggestion && (

{error.classified.suggestion}

)} {error.retryKind === "load" && ( )} {error.retryKind === "file" && lastFileRef.current && ( )}
)} {showModelProgress && ( )} 0 ? `Finishing up… transcribing ${livePending} remaining segment${ livePending === 1 ? "" : "s" }` : "Finishing up…" : liveActive ? "Listening…" : undefined } /> {/* Text translation of the finished transcript (D-WS4b) — any engine, file or live; secondary to the transcript itself. */} {result && !liveActive && ( )} {applyUpdate && (
A new version of Whisper Web is available.
)} {/* Storage management (M6): cached models with per-model delete. */}
); } export default App;