import { useEffect, useRef, useState } from "react"; import "./index.css"; import { AuthForms } from "./components/AuthForms"; import { EngineToggle } from "./components/EngineToggle"; import { FileDropzone } from "./components/FileDropzone"; import { LiveControls, type LiveStatus } from "./components/LiveControls"; import { ModelLoadProgress } from "./components/ModelLoadProgress"; import { ModelPicker } from "./components/ModelPicker"; import { StorageManager } from "./components/StorageManager"; import { TranscriptView } from "./components/TranscriptView"; import { TranscriptionOptions } from "./components/TranscriptionOptions"; import { decodeAudioFile } from "./lib/audio"; import { BrowserEngine, ServerEngine, fetchServerEngineStatus, loadEngineChoice, saveEngineChoice, type Engine, type EngineKind, type ServerEngineStatus, } from "./lib/engine"; import { useAuth } from "./lib/useAuth"; import { LIVE_TUNING, LiveTranscriber, type LiveTranscribeResult } from "./lib/live"; import { startMicCapture, type MicCapture } from "./lib/mic"; import { getDownloadedModels } from "./lib/modelCache"; import { registerServiceWorker } from "./pwa/register"; import { AUTO_LANGUAGE, DEFAULT_DEVICE_PREFERENCE, DEFAULT_MODEL_ID, DEFAULT_TASK, MODELS, getModel, } from "./lib/models"; import { applyLoadProgress, initialLoadProgress, isWorkerToUiMessage, type DevicePreference, type InferenceDevice, type TranscriptChunk, type UiToWorkerMessage, type WhisperTask, } from "./worker/protocol"; interface TranscriptResult { text: string; chunks: TranscriptChunk[]; } const MODEL_IDS = MODELS.map((model) => model.id); export function App() { const workerRef = useRef(null); // Live-mode session (M7, D-016). The mic and transcriber live outside React // state; only display text goes through setState. Segment passes are // correlated with worker responses via requestId. const liveRef = useRef<{ mic: MicCapture; transcriber: LiveTranscriber | null } | null>( null, ); const liveWsRef = useRef(null); const pendingLive = useRef( new Map< string, { resolve: (result: LiveTranscribeResult) => void; reject: (error: Error) => void; } >(), ); const nextLiveRequest = useRef(0); // File-flow run counter (D-018). Cancel bumps it, so a decode that was in // flight when the user cancelled finds itself stale and never posts its // audio to the (fresh) worker. Decoding itself cannot be aborted. const fileRun = useRef(0); // Engine plumbing (M5, D-010). The BrowserEngine is bound to the worker's // lifetime (recreated together on cancel); the ServerEngine is stateless // and lives for the app. A server run's fetch is cancelled via abortRef. const browserEngineRef = useRef(null); const serverEngineRef = useRef(null); const abortRef = useRef(null); const [busy, setBusy] = useState(false); // Which engine the current file run uses; null while idle. Server runs // don't need the browser model, so parts of the UI key off this. const [runEngine, setRunEngine] = useState(null); const [decoding, setDecoding] = useState(false); const [loadStarted, setLoadStarted] = useState(false); const [loadProgress, setLoadProgress] = useState(initialLoadProgress); const [partialText, setPartialText] = useState(""); const [transcribeProgress, setTranscribeProgress] = useState<{ processedSeconds: number; totalSeconds: number; } | null>(null); const [result, setResult] = useState(null); const [error, setError] = useState(null); const [fileName, setFileName] = useState(null); const [liveStatus, setLiveStatus] = useState("idle"); const [liveCommitted, setLiveCommitted] = useState(""); const [liveTentative, setLiveTentative] = useState(""); const [modelId, setModelId] = useState(DEFAULT_MODEL_ID); const [devicePreference, setDevicePreference] = useState( DEFAULT_DEVICE_PREFERENCE, ); const [activeDevice, setActiveDevice] = useState(null); const [language, setLanguage] = useState(AUTO_LANGUAGE); const [task, setTask] = useState(DEFAULT_TASK); const [downloaded, setDownloaded] = useState>(new Set()); // PWA state (M3, D-007): is a service worker controlling this page (app // shell available offline), and is an updated version waiting? const [swControlled, setSwControlled] = useState(false); const [applyUpdate, setApplyUpdate] = useState<(() => void) | null>(null); // Session + engine choice (M4/M5). One useAuth for the whole app — the // header forms and the EngineToggle must agree on the session. The choice // persists across reloads; it only takes effect while signed in AND the // deployment reports the server engine available (anonymous-first: logged // out, everything runs on-device exactly as before). const auth = useAuth(); const [engineChoice, setEngineChoice] = useState(() => loadEngineChoice()); const [serverStatus, setServerStatus] = useState(null); function refreshDownloaded() { // Downloaded badges (D-005): query Cache Storage for cached weights. getDownloadedModels(MODEL_IDS).then(setDownloaded, () => {}); } // Terminate the worker when the app unmounts (also keeps HMR tidy). useEffect(() => { refreshDownloaded(); // Registration is idempotent, so the StrictMode double-invoke is fine. registerServiceWorker({ onControlled: () => setSwControlled(true), onUpdateReady: (apply) => setApplyUpdate(() => apply), }); return () => { void liveRef.current?.mic.stop().catch(() => {}); liveRef.current = null; liveWsRef.current?.close(); liveWsRef.current = null; workerRef.current?.terminate(); workerRef.current = null; browserEngineRef.current = null; abortRef.current?.abort(); abortRef.current = null; }; }, []); // Probe server-engine availability once a session exists (the endpoint is // auth-guarded, so probing while anonymous is pointless). Never rejects. useEffect(() => { if (!auth.user) { setServerStatus(null); return; } let cancelled = false; fetchServerEngineStatus().then((status) => { if (!cancelled) setServerStatus(status); }); return () => { cancelled = true; }; }, [auth.user]); function takeLiveRequest(requestId: string | undefined) { if (requestId === undefined) return undefined; const pending = pendingLive.current.get(requestId); if (pending) pendingLive.current.delete(requestId); return pending; } function ensureWorker(): Worker { if (workerRef.current) return workerRef.current; // Inference always runs in a dedicated Web Worker (D-003), never on the // main thread (Architecture.md §5). The worker script is bundled by the // server at startup and served at a stable URL (see src/index.ts) because // Bun's bundler does not follow `new Worker(new URL(...))` graphs yet. const worker = new Worker("inference.worker.js", { type: "module" }); worker.addEventListener("message", (event: MessageEvent) => { const message = event.data; if (!isWorkerToUiMessage(message)) return; switch (message.type) { case "load-progress": setLoadProgress((state) => applyLoadProgress(state, message)); if (message.status === "ready") { // Surface the backend the worker actually selected (D-004) and // refresh the downloaded badges now that weights are cached. setActiveDevice(message.device); refreshDownloaded(); } break; case "partial": // Live segment passes (requestId set) resolve wholesale via // "result"; their token stream is not rendered incrementally. if (message.requestId === undefined) { setPartialText((text) => text + message.text); } break; case "transcribe-progress": // File flow only — live passes never carry progress (D-016). if (message.requestId === undefined) { setTranscribeProgress({ processedSeconds: message.processedSeconds, totalSeconds: message.totalSeconds, }); } break; case "result": { const pending = takeLiveRequest(message.requestId); if (pending) { pending.resolve({ text: message.text, chunks: message.chunks }); } else if (message.requestId === undefined) { setResult({ text: message.text, chunks: message.chunks }); setTranscribeProgress(null); setBusy(false); setRunEngine(null); } break; } case "error": { const pending = takeLiveRequest(message.requestId); if (pending) { pending.reject(new Error(message.message)); } else if (message.requestId === undefined) { // Load-stage failures and legacy id-less passes. Engine-routed // file passes carry an id and surface through the engine promise. setError(message.message); setTranscribeProgress(null); setBusy(false); setRunEngine(null); } break; } } }); worker.addEventListener("error", (event) => { const failure = new Error( event.message || "The inference worker crashed unexpectedly.", ); // Unblock any live passes so a live session can drain and stop. for (const pending of pendingLive.current.values()) pending.reject(failure); pendingLive.current.clear(); setError(failure.message); setBusy(false); }); workerRef.current = worker; // The BrowserEngine shares this worker and only sees its own request ids // (engine-*), so it coexists with the live loop's live-* ids above. browserEngineRef.current = new BrowserEngine(worker); return worker; } function postToWorker(worker: Worker, message: UiToWorkerMessage, transfer?: Transferable[]) { worker.postMessage(message, transfer ?? []); } /** * Post a load request and reset per-load UI state so the progress panel * restarts cleanly on model/device switches. Idempotent in the worker for * an unchanged model+device; a change disposes the old pipeline first. */ function startLoad(nextModelId: string, preference: DevicePreference): Worker { const worker = ensureWorker(); setLoadProgress(initialLoadProgress); setLoadStarted(true); setError(null); postToWorker(worker, { type: "load", modelId: nextModelId, device: preference }); return worker; } function handleModelSelect(nextModelId: string) { if (busy || liveStatus !== "idle" || nextModelId === modelId) return; setModelId(nextModelId); setActiveDevice(null); setPartialText(""); setTranscribeProgress(null); setResult(null); // Selection never downloads (D-020): the load starts when a transcription // needs it (file drop / live start on the browser engine) or when the // user explicitly clicks Download. Reset the load state so the next load // reports fresh progress for this model. setLoadStarted(false); setLoadProgress(initialLoadProgress); } function handleDevicePreference(preference: DevicePreference) { if (busy || liveStatus !== "idle" || preference === devicePreference) return; setDevicePreference(preference); setActiveDevice(null); // No eager reload (D-020) — the next load picks up the new preference. setLoadStarted(false); setLoadProgress(initialLoadProgress); } /** Explicit model download (D-020) — pre-caches weights for offline use. */ function handleDownloadModel() { if (busy || liveStatus !== "idle" || loadStarted) return; startLoad(modelId, devicePreference); } /** * Cancel the in-flight file transcription (D-018). Browser runs: no abort * API exists for a running `asr()` call — and on the WASM backend the pass * blocks the worker's event loop, so even a "cancel" message would sit * unread until the pass finished. Terminating the worker is the only * reliable preemption; it also aborts an in-flight model download. The next * run spawns a fresh worker and reloads the pipeline — cheap, because the * weights stay in the browser Cache API (D-005). Server runs: abort the * `/api/transcribe` fetch (the server cancels the sidecar pass, D-011) and * leave the worker — and its loaded pipeline — untouched. */ function cancelTranscription() { if (!busy) return; fileRun.current++; // any decode still running is now stale const cancelled = new Error("Transcription cancelled."); if (runEngine === "server") { abortRef.current?.abort(); abortRef.current = null; } else { browserEngineRef.current?.failAll(cancelled); browserEngineRef.current = null; workerRef.current?.terminate(); workerRef.current = null; // The worker (and any pipeline / partial download) is gone — reset the // load state so the next transcription reloads from a clean slate. setLoadStarted(false); setLoadProgress(initialLoadProgress); setActiveDevice(null); } // The file flow never has live passes pending, but if anything is in the // map (defensive), unblock it so nothing awaits a dead worker forever. for (const pending of pendingLive.current.values()) pending.reject(cancelled); pendingLive.current.clear(); setBusy(false); setRunEngine(null); setPartialText(""); setTranscribeProgress(null); setResult(null); setFileName(null); setError(null); } async function handleFile(file: File) { if (busy || liveStatus !== "idle") return; const run = ++fileRun.current; // The toggle's choice only takes effect with a session and an available // server engine (D-010); everything else runs on-device as always. const useServer = engineChoice === "server" && auth.user !== null && serverStatus?.available === true; setBusy(true); setRunEngine(useServer ? "server" : "browser"); setError(null); setPartialText(""); setTranscribeProgress(null); setResult(null); setFileName(file.name); let engine: Engine; if (useServer) { engine = serverEngineRef.current ??= new ServerEngine(); } else { // Kick off the (idempotent) model load immediately so the download runs // while we decode the audio locally. Server runs skip this — they don't // need the browser pipeline. ensureWorker(); if (!loadStarted) { startLoad(modelId, devicePreference); } else { postToWorker(workerRef.current!, { type: "load", modelId, device: devicePreference, }); } engine = browserEngineRef.current!; } const multilingual = getModel(modelId)?.multilingual ?? false; // Decoded entirely on-device via Web Audio (D-012); for browser runs the // audio never touches the network (Architecture.md §5) — server runs // upload the decoded PCM, which is the point of the toggle. setDecoding(true); let audio: Float32Array; try { audio = await decodeAudioFile(file); } catch (decodeError) { if (fileRun.current === run) { setError( decodeError instanceof Error ? decodeError.message : String(decodeError), ); setBusy(false); setRunEngine(null); } return; } finally { setDecoding(false); } if (fileRun.current !== run) return; // cancelled while decoding (D-018) // Server runs cancel by aborting the fetch (D-018 amendment); browser // runs cancel by terminating the worker, which rejects the promise via // BrowserEngine.failAll. const controller = useServer ? new AbortController() : null; abortRef.current = controller; try { const outcome = await engine.transcribe(audio, { // Language/task only apply to multilingual models; `.en` variants // take neither. The browser engine ignores modelId (worker holds it). modelId, ...(multilingual && { language, task }), onPartial: (fragment) => { if (fileRun.current === run) setPartialText((text) => text + fragment); }, ...(!useServer && { onProgress: (progress: { processedSeconds: number; totalSeconds: number }) => { if (fileRun.current === run) setTranscribeProgress(progress); }, }), ...(controller && { signal: controller.signal }), }); if (fileRun.current !== run) return; // cancelled mid-pass setResult(outcome); setTranscribeProgress(null); setBusy(false); setRunEngine(null); } catch (transcribeError) { if (fileRun.current !== run) return; // cancellation rejections are expected setError( transcribeError instanceof Error ? transcribeError.message : String(transcribeError), ); setTranscribeProgress(null); setBusy(false); setRunEngine(null); } finally { if (abortRef.current === controller) abortRef.current = null; } } // ------------------------------------------------------------------------- // Live microphone transcription (M7, D-016) // ------------------------------------------------------------------------- /** One inference pass over a live segment, correlated via requestId. */ function transcribeLiveSegment( worker: Worker, audio: Float32Array, options: { language?: string; task?: WhisperTask }, ): Promise { const requestId = `live-${++nextLiveRequest.current}`; return new Promise((resolve, reject) => { pendingLive.current.set(requestId, { resolve, reject }); // The controller hands over ownership, so the buffer can transfer. postToWorker(worker, { type: "transcribe", audio, requestId, ...options }, [ audio.buffer as ArrayBuffer, ]); }); } async function startLive() { if (busy || liveStatus !== "idle") return; setError(null); setResult(null); setPartialText(""); setFileName(null); setLiveCommitted(""); setLiveTentative(""); setLiveStatus("starting"); const useServer = engineChoice === "server" && auth.user !== null && serverStatus?.available === true; const options = (getModel(modelId)?.multilingual ?? false) ? { language, task } : {}; if (useServer) { const wsUrl = new URL(`/api/transcribe/live?model=${modelId}`, window.location.href); wsUrl.protocol = wsUrl.protocol === "https:" ? "wss:" : "ws:"; if (options.language) wsUrl.searchParams.set("language", options.language); if (options.task) wsUrl.searchParams.set("task", options.task); const ws = new WebSocket(wsUrl); ws.onmessage = (event) => { const data = JSON.parse(event.data); if (data.type === "update") { setLiveCommitted(data.update.committedText); setLiveTentative(data.update.tentativeText); if (data.update.error) setError(data.update.error); } else if (data.type === "result") { setResult(data.result); setLiveCommitted(""); setLiveTentative(""); setLiveStatus("idle"); liveWsRef.current = null; liveRef.current = null; } else if (data.type === "error") { setError(data.error); setLiveStatus("idle"); liveWsRef.current = null; liveRef.current = null; } }; ws.onerror = () => { setError("WebSocket connection error."); setLiveStatus("idle"); liveWsRef.current = null; liveRef.current = null; }; ws.onopen = async () => { try { const mic = await startMicCapture((chunk) => { if (ws.readyState === WebSocket.OPEN) { ws.send(chunk.buffer); } }); liveWsRef.current = ws; liveRef.current = { mic, transcriber: null }; setLiveStatus("listening"); } catch (micError) { ws.close(); setLiveStatus("idle"); setError( micError instanceof DOMException && micError.name === "NotAllowedError" ? "Microphone access was denied. Allow it in your browser's site settings and try again." : micError instanceof Error ? micError.message : String(micError), ); } }; return; } // Idempotent model load, like the file flow — a pending download runs // while the mic opens; segment passes queue in the worker until ready. const worker = ensureWorker(); if (!loadStarted) { startLoad(modelId, devicePreference); } else { postToWorker(worker, { type: "load", modelId, device: devicePreference }); } const transcriber = new LiveTranscriber({ ...LIVE_TUNING, transcribe: (audio) => transcribeLiveSegment(worker, audio, options), onUpdate: (update) => { setLiveCommitted(update.committedText); setLiveTentative(update.tentativeText); if (update.error) setError(update.error); }, }); try { const mic = await startMicCapture((chunk) => transcriber.push(chunk)); liveRef.current = { mic, transcriber }; setLiveStatus("listening"); } catch (micError) { setLiveStatus("idle"); setError( micError instanceof DOMException && micError.name === "NotAllowedError" ? "Microphone access was denied. Allow it in your browser's site settings and try again." : micError instanceof Error ? micError.message : String(micError), ); } } async function stopLive() { const session = liveRef.current; if (!session || liveStatus !== "listening") return; setLiveStatus("stopping"); // Stop feeding audio first, then drain the remaining segment passes. await session.mic.stop(); if (liveWsRef.current) { liveWsRef.current.send(JSON.stringify({ type: "stop" })); } else { try { const final = await session.transcriber!.stop(); setResult(final); setLiveCommitted(""); setLiveTentative(""); } finally { liveRef.current = null; setLiveStatus("idle"); } } } const model = getModel(modelId); const modelReady = loadProgress.ready; const showModelProgress = loadStarted && !modelReady && !error; // Server runs never load the browser model, so they are "transcribing" the // moment decoding ends. const transcribing = busy && !decoding && (runEngine === "server" || modelReady); const liveActive = liveStatus !== "idle"; // What the next file run will use (mirrors handleFile's decision). const serverSelected = engineChoice === "server" && auth.user !== null && serverStatus?.available === true; return (

AU Whisper Web

Speech-to-text that runs entirely in your browser. Your audio never leaves this device.

{/* Session UI (M4). Anonymous-first: logged out, this is only an unobtrusive "Sign in" button — everything else is unchanged. Renders nothing until the /me probe confirms the server has auth enabled (runtime flag, D-015). */}
{model?.multilingual && ( )} {/* Engine toggle (M5, D-010): signed-in users choose where uploaded files are transcribed. Live mode always stays on-device (D-016). */} {auth.user && ( { setEngineChoice(choice); saveEngineChoice(choice); }} status={serverStatus} disabled={busy || liveActive} /> )} void startLive()} onStop={() => void stopLive()} disabled={busy} /> {fileName && (

File: {fileName} {decoding && " — decoding audio…"} {busy && ( // Cancel the whole run (D-018): decode, pending model download, // or the transcription pass itself. )}

)} {error && (

{error}

)} {showModelProgress && ( )} {applyUpdate && (
A new version of Whisper Web is available.
)} {/* Storage management (M6): cached models with per-model delete. */}
); } export default App;