Start on live transcription and python sidecar (faster-whisper)

src/App.tsx

@@ -3,11 +3,14 @@ import "./index.css";
 
 import { AuthForms } from "./components/AuthForms";
 import { FileDropzone } from "./components/FileDropzone";
+import { LiveControls, type LiveStatus } from "./components/LiveControls";
 import { ModelLoadProgress } from "./components/ModelLoadProgress";
 import { ModelPicker } from "./components/ModelPicker";
 import { TranscriptView } from "./components/TranscriptView";
 import { TranscriptionOptions } from "./components/TranscriptionOptions";
 import { decodeAudioFile } from "./lib/audio";
+import { LiveTranscriber, type LiveTranscribeResult } from "./lib/live";
+import { startMicCapture, type MicCapture } from "./lib/mic";
 import { getDownloadedModels } from "./lib/modelCache";
 import { registerServiceWorker } from "./pwa/register";
 import {
@@ -39,6 +42,23 @@ const MODEL_IDS = MODELS.map((model) => model.id);
 export function App() {
   const workerRef = useRef<Worker | null>(null);
 
+  // Live-mode session (M7, D-016). The mic and transcriber live outside React
+  // state; only display text goes through setState. Segment passes are
+  // correlated with worker responses via requestId.
+  const liveRef = useRef<{ mic: MicCapture; transcriber: LiveTranscriber } | null>(
+    null,
+  );
+  const pendingLive = useRef(
+    new Map<
+      string,
+      {
+        resolve: (result: LiveTranscribeResult) => void;
+        reject: (error: Error) => void;
+      }
+    >(),
+  );
+  const nextLiveRequest = useRef(0);
+
   const [busy, setBusy] = useState(false);
   const [decoding, setDecoding] = useState(false);
   const [loadStarted, setLoadStarted] = useState(false);
@@ -48,6 +68,10 @@ export function App() {
   const [error, setError] = useState<string | null>(null);
   const [fileName, setFileName] = useState<string | null>(null);
 
+  const [liveStatus, setLiveStatus] = useState<LiveStatus>("idle");
+  const [liveCommitted, setLiveCommitted] = useState("");
+  const [liveTentative, setLiveTentative] = useState("");
+
   const [modelId, setModelId] = useState(DEFAULT_MODEL_ID);
   const [devicePreference, setDevicePreference] = useState<DevicePreference>(
     DEFAULT_DEVICE_PREFERENCE,
@@ -76,11 +100,20 @@ export function App() {
       onUpdateReady: (apply) => setApplyUpdate(() => apply),
     });
     return () => {
+      void liveRef.current?.mic.stop().catch(() => {});
+      liveRef.current = null;
       workerRef.current?.terminate();
       workerRef.current = null;
     };
   }, []);
 
+  function takeLiveRequest(requestId: string | undefined) {
+    if (requestId === undefined) return undefined;
+    const pending = pendingLive.current.get(requestId);
+    if (pending) pendingLive.current.delete(requestId);
+    return pending;
+  }
+
   function ensureWorker(): Worker {
     if (workerRef.current) return workerRef.current;
     // Inference always runs in a dedicated Web Worker (D-003), never on the
@@ -102,20 +135,42 @@ export function App() {
           }
           break;
         case "partial":
-          setPartialText((text) => text + message.text);
+          // Live segment passes (requestId set) resolve wholesale via
+          // "result"; their token stream is not rendered incrementally.
+          if (message.requestId === undefined) {
+            setPartialText((text) => text + message.text);
+          }
           break;
-        case "result":
-          setResult({ text: message.text, chunks: message.chunks });
-          setBusy(false);
+        case "result": {
+          const pending = takeLiveRequest(message.requestId);
+          if (pending) {
+            pending.resolve({ text: message.text, chunks: message.chunks });
+          } else if (message.requestId === undefined) {
+            setResult({ text: message.text, chunks: message.chunks });
+            setBusy(false);
+          }
           break;
-        case "error":
-          setError(message.message);
-          setBusy(false);
+        }
+        case "error": {
+          const pending = takeLiveRequest(message.requestId);
+          if (pending) {
+            pending.reject(new Error(message.message));
+          } else {
+            setError(message.message);
+            setBusy(false);
+          }
           break;
+        }
       }
     });
     worker.addEventListener("error", (event) => {
-      setError(event.message || "The inference worker crashed unexpectedly.");
+      const failure = new Error(
+        event.message || "The inference worker crashed unexpectedly.",
+      );
+      // Unblock any live passes so a live session can drain and stop.
+      for (const pending of pendingLive.current.values()) pending.reject(failure);
+      pendingLive.current.clear();
+      setError(failure.message);
       setBusy(false);
     });
     workerRef.current = worker;
@@ -141,7 +196,7 @@ export function App() {
   }
 
   function handleModelSelect(nextModelId: string) {
-    if (busy || nextModelId === modelId) return;
+    if (busy || liveStatus !== "idle" || nextModelId === modelId) return;
     setModelId(nextModelId);
     setActiveDevice(null);
     setPartialText("");
@@ -150,7 +205,7 @@ export function App() {
   }
 
   function handleDevicePreference(preference: DevicePreference) {
-    if (busy || preference === devicePreference) return;
+    if (busy || liveStatus !== "idle" || preference === devicePreference) return;
     setDevicePreference(preference);
     setActiveDevice(null);
     // Only reload eagerly if a load was already in motion; otherwise the next
@@ -159,7 +214,7 @@ export function App() {
   }
 
   async function handleFile(file: File) {
-    if (busy) return;
+    if (busy || liveStatus !== "idle") return;
     setBusy(true);
     setError(null);
     setPartialText("");
@@ -199,10 +254,95 @@ export function App() {
     }
   }
 
+  // -------------------------------------------------------------------------
+  // Live microphone transcription (M7, D-016)
+  // -------------------------------------------------------------------------
+
+  /** One inference pass over a live segment, correlated via requestId. */
+  function transcribeLiveSegment(
+    worker: Worker,
+    audio: Float32Array,
+    options: { language?: string; task?: WhisperTask },
+  ): Promise<LiveTranscribeResult> {
+    const requestId = `live-${++nextLiveRequest.current}`;
+    return new Promise((resolve, reject) => {
+      pendingLive.current.set(requestId, { resolve, reject });
+      // The controller hands over ownership, so the buffer can transfer.
+      postToWorker(worker, { type: "transcribe", audio, requestId, ...options }, [
+        audio.buffer as ArrayBuffer,
+      ]);
+    });
+  }
+
+  async function startLive() {
+    if (busy || liveStatus !== "idle") return;
+    setError(null);
+    setResult(null);
+    setPartialText("");
+    setFileName(null);
+    setLiveCommitted("");
+    setLiveTentative("");
+    setLiveStatus("starting");
+
+    // Idempotent model load, like the file flow — a pending download runs
+    // while the mic opens; segment passes queue in the worker until ready.
+    const worker = ensureWorker();
+    if (!loadStarted) {
+      startLoad(modelId, devicePreference);
+    } else {
+      postToWorker(worker, { type: "load", modelId, device: devicePreference });
+    }
+
+    // Language/task are fixed for the session; the options are disabled while
+    // live. `.en` models take neither (same rule as the file flow).
+    const options = (getModel(modelId)?.multilingual ?? false) ? { language, task } : {};
+    const transcriber = new LiveTranscriber({
+      transcribe: (audio) => transcribeLiveSegment(worker, audio, options),
+      onUpdate: (update) => {
+        setLiveCommitted(update.committedText);
+        setLiveTentative(update.tentativeText);
+        if (update.error) setError(update.error);
+      },
+    });
+
+    try {
+      const mic = await startMicCapture((chunk) => transcriber.push(chunk));
+      liveRef.current = { mic, transcriber };
+      setLiveStatus("listening");
+    } catch (micError) {
+      setLiveStatus("idle");
+      setError(
+        micError instanceof DOMException && micError.name === "NotAllowedError"
+          ? "Microphone access was denied. Allow it in your browser's site settings and try again."
+          : micError instanceof Error
+            ? micError.message
+            : String(micError),
+      );
+    }
+  }
+
+  async function stopLive() {
+    const session = liveRef.current;
+    if (!session || liveStatus !== "listening") return;
+    setLiveStatus("stopping");
+    // Stop feeding audio first, then drain the remaining segment passes.
+    await session.mic.stop();
+    try {
+      const final = await session.transcriber.stop();
+      setResult(final);
+      setLiveCommitted("");
+      setLiveTentative("");
+    } finally {
+      liveRef.current = null;
+      setLiveStatus("idle");
+    }
+  }
+
   const model = getModel(modelId);
   const modelReady = loadProgress.ready;
   const showModelProgress = loadStarted && !modelReady && !error;
   const transcribing = busy && !decoding && modelReady;
+  const liveActive = liveStatus !== "idle";
 
   return (
     <main className="app">
@@ -230,7 +370,7 @@ export function App() {
         activeDevice={activeDevice}
         downloaded={downloaded}
         offlineReady={swControlled}
-        disabled={busy}
+        disabled={busy || liveActive}
       />
 
       {model?.multilingual && (
@@ -239,11 +379,18 @@ export function App() {
           onLanguageChange={setLanguage}
           task={task}
           onTaskChange={setTask}
-          disabled={busy}
+          disabled={busy || liveActive}
         />
       )}
 
-      <FileDropzone onFile={handleFile} disabled={busy} />
+      <FileDropzone onFile={handleFile} disabled={busy || liveActive} />
+
+      <LiveControls
+        status={liveStatus}
+        onStart={() => void startLive()}
+        onStop={() => void stopLive()}
+        disabled={busy}
+      />
 
       {fileName && (
         <p className="file-name">
@@ -263,9 +410,17 @@ export function App() {
       )}
 
       <TranscriptView
-        partialText={partialText}
+        partialText={liveActive ? liveCommitted : partialText}
+        tentativeText={liveActive ? liveTentative : ""}
         result={result}
-        transcribing={transcribing}
+        transcribing={liveActive || transcribing}
+        activeLabel={
+          liveStatus === "stopping"
+            ? "Finishing up…"
+            : liveActive
+              ? "Listening…"
+              : undefined
+        }
       />
 
       {applyUpdate && (