Add live transcription via remote server.

This commit is contained in:
Lucas Winther committed 2026-07-14 20:21:48 +02:00
1 parent 695b97be76
commit 42fe27d2dd
9 files changed
+227 -21

No files matched your search

+86 -13
View File
@@ -58,9 +58,10 @@ export function App() {
// Live-mode session (M7, D-016). The mic and transcriber live outside React
// state; only display text goes through setState. Segment passes are
// correlated with worker responses via requestId.
const liveRef = useRef<{ mic: MicCapture; transcriber: LiveTranscriber } | null>(
const liveRef = useRef<{ mic: MicCapture; transcriber: LiveTranscriber | null } | null>(
null,
);
const liveWsRef = useRef<WebSocket | null>(null);
const pendingLive = useRef(
new Map<
string,
@@ -143,6 +144,8 @@ export function App() {
return () => {
void liveRef.current?.mic.stop().catch(() => {});
liveRef.current = null;
liveWsRef.current?.close();
liveWsRef.current = null;
workerRef.current?.terminate();
workerRef.current = null;
browserEngineRef.current = null;
@@ -465,6 +468,74 @@ export function App() {
setLiveTentative("");
setLiveStatus("starting");
const useServer =
engineChoice === "server" &&
auth.user !== null &&
serverStatus?.available === true;
const options = (getModel(modelId)?.multilingual ?? false) ? { language, task } : {};
if (useServer) {
const wsUrl = new URL(`/api/transcribe/live?model=${modelId}`, window.location.href);
wsUrl.protocol = wsUrl.protocol === "https:" ? "wss:" : "ws:";
if (options.language) wsUrl.searchParams.set("language", options.language);
if (options.task) wsUrl.searchParams.set("task", options.task);
const ws = new WebSocket(wsUrl);
ws.onmessage = (event) => {
const data = JSON.parse(event.data);
if (data.type === "update") {
setLiveCommitted(data.update.committedText);
setLiveTentative(data.update.tentativeText);
if (data.update.error) setError(data.update.error);
} else if (data.type === "result") {
setResult(data.result);
setLiveCommitted("");
setLiveTentative("");
setLiveStatus("idle");
liveWsRef.current = null;
liveRef.current = null;
} else if (data.type === "error") {
setError(data.error);
setLiveStatus("idle");
liveWsRef.current = null;
liveRef.current = null;
}
};
ws.onerror = () => {
setError("WebSocket connection error.");
setLiveStatus("idle");
liveWsRef.current = null;
liveRef.current = null;
};
ws.onopen = async () => {
try {
const mic = await startMicCapture((chunk) => {
if (ws.readyState === WebSocket.OPEN) {
ws.send(chunk.buffer);
}
});
liveWsRef.current = ws;
liveRef.current = { mic, transcriber: null };
setLiveStatus("listening");
} catch (micError) {
ws.close();
setLiveStatus("idle");
setError(
micError instanceof DOMException && micError.name === "NotAllowedError"
? "Microphone access was denied. Allow it in your browser's site settings and try again."
: micError instanceof Error
? micError.message
: String(micError),
);
}
};
return;
}
// Idempotent model load, like the file flow — a pending download runs
// while the mic opens; segment passes queue in the worker until ready.
const worker = ensureWorker();
@@ -474,9 +545,6 @@ export function App() {
postToWorker(worker, { type: "load", modelId, device: devicePreference });
}
// Language/task are fixed for the session; the options are disabled while
// live. `.en` models take neither (same rule as the file flow).
const options = (getModel(modelId)?.multilingual ?? false) ? { language, task } : {};
const transcriber = new LiveTranscriber({
...LIVE_TUNING,
transcribe: (audio) => transcribeLiveSegment(worker, audio, options),
@@ -509,14 +577,19 @@ export function App() {
setLiveStatus("stopping");
// Stop feeding audio first, then drain the remaining segment passes.
await session.mic.stop();
try {
const final = await session.transcriber.stop();
setResult(final);
setLiveCommitted("");
setLiveTentative("");
} finally {
liveRef.current = null;
setLiveStatus("idle");
if (liveWsRef.current) {
liveWsRef.current.send(JSON.stringify({ type: "stop" }));
} else {
try {
const final = await session.transcriber!.stop();
setResult(final);
setLiveCommitted("");
setLiveTentative("");
} finally {
liveRef.current = null;
setLiveStatus("idle");
}
}
}
@@ -667,7 +740,7 @@ export function App() {
<p>
Model: <code>{modelId}</code> · Engine:{" "}
{serverSelected
? "server (faster-whisper) — files upload to this deployment"
? "server (faster-whisper) — audio streams to this deployment"
: activeDevice === "webgpu"
? "WebGPU (on-device)"
: activeDevice === "webnn"