Add live transcription via remote server.
This commit is contained in:
1 parent
695b97be76
commit
42fe27d2dd
9 files changed
+227
-21
No files matched your search
+86
-13
@@ -58,9 +58,10 @@ export function App() {
|
||||
// Live-mode session (M7, D-016). The mic and transcriber live outside React
|
||||
// state; only display text goes through setState. Segment passes are
|
||||
// correlated with worker responses via requestId.
|
||||
const liveRef = useRef<{ mic: MicCapture; transcriber: LiveTranscriber } | null>(
|
||||
const liveRef = useRef<{ mic: MicCapture; transcriber: LiveTranscriber | null } | null>(
|
||||
null,
|
||||
);
|
||||
const liveWsRef = useRef<WebSocket | null>(null);
|
||||
const pendingLive = useRef(
|
||||
new Map<
|
||||
string,
|
||||
@@ -143,6 +144,8 @@ export function App() {
|
||||
return () => {
|
||||
void liveRef.current?.mic.stop().catch(() => {});
|
||||
liveRef.current = null;
|
||||
liveWsRef.current?.close();
|
||||
liveWsRef.current = null;
|
||||
workerRef.current?.terminate();
|
||||
workerRef.current = null;
|
||||
browserEngineRef.current = null;
|
||||
@@ -465,6 +468,74 @@ export function App() {
|
||||
setLiveTentative("");
|
||||
setLiveStatus("starting");
|
||||
|
||||
const useServer =
|
||||
engineChoice === "server" &&
|
||||
auth.user !== null &&
|
||||
serverStatus?.available === true;
|
||||
|
||||
const options = (getModel(modelId)?.multilingual ?? false) ? { language, task } : {};
|
||||
|
||||
if (useServer) {
|
||||
const wsUrl = new URL(`/api/transcribe/live?model=${modelId}`, window.location.href);
|
||||
wsUrl.protocol = wsUrl.protocol === "https:" ? "wss:" : "ws:";
|
||||
if (options.language) wsUrl.searchParams.set("language", options.language);
|
||||
if (options.task) wsUrl.searchParams.set("task", options.task);
|
||||
|
||||
const ws = new WebSocket(wsUrl);
|
||||
|
||||
ws.onmessage = (event) => {
|
||||
const data = JSON.parse(event.data);
|
||||
if (data.type === "update") {
|
||||
setLiveCommitted(data.update.committedText);
|
||||
setLiveTentative(data.update.tentativeText);
|
||||
if (data.update.error) setError(data.update.error);
|
||||
} else if (data.type === "result") {
|
||||
setResult(data.result);
|
||||
setLiveCommitted("");
|
||||
setLiveTentative("");
|
||||
setLiveStatus("idle");
|
||||
liveWsRef.current = null;
|
||||
liveRef.current = null;
|
||||
} else if (data.type === "error") {
|
||||
setError(data.error);
|
||||
setLiveStatus("idle");
|
||||
liveWsRef.current = null;
|
||||
liveRef.current = null;
|
||||
}
|
||||
};
|
||||
|
||||
ws.onerror = () => {
|
||||
setError("WebSocket connection error.");
|
||||
setLiveStatus("idle");
|
||||
liveWsRef.current = null;
|
||||
liveRef.current = null;
|
||||
};
|
||||
|
||||
ws.onopen = async () => {
|
||||
try {
|
||||
const mic = await startMicCapture((chunk) => {
|
||||
if (ws.readyState === WebSocket.OPEN) {
|
||||
ws.send(chunk.buffer);
|
||||
}
|
||||
});
|
||||
liveWsRef.current = ws;
|
||||
liveRef.current = { mic, transcriber: null };
|
||||
setLiveStatus("listening");
|
||||
} catch (micError) {
|
||||
ws.close();
|
||||
setLiveStatus("idle");
|
||||
setError(
|
||||
micError instanceof DOMException && micError.name === "NotAllowedError"
|
||||
? "Microphone access was denied. Allow it in your browser's site settings and try again."
|
||||
: micError instanceof Error
|
||||
? micError.message
|
||||
: String(micError),
|
||||
);
|
||||
}
|
||||
};
|
||||
return;
|
||||
}
|
||||
|
||||
// Idempotent model load, like the file flow — a pending download runs
|
||||
// while the mic opens; segment passes queue in the worker until ready.
|
||||
const worker = ensureWorker();
|
||||
@@ -474,9 +545,6 @@ export function App() {
|
||||
postToWorker(worker, { type: "load", modelId, device: devicePreference });
|
||||
}
|
||||
|
||||
// Language/task are fixed for the session; the options are disabled while
|
||||
// live. `.en` models take neither (same rule as the file flow).
|
||||
const options = (getModel(modelId)?.multilingual ?? false) ? { language, task } : {};
|
||||
const transcriber = new LiveTranscriber({
|
||||
...LIVE_TUNING,
|
||||
transcribe: (audio) => transcribeLiveSegment(worker, audio, options),
|
||||
@@ -509,14 +577,19 @@ export function App() {
|
||||
setLiveStatus("stopping");
|
||||
// Stop feeding audio first, then drain the remaining segment passes.
|
||||
await session.mic.stop();
|
||||
try {
|
||||
const final = await session.transcriber.stop();
|
||||
setResult(final);
|
||||
setLiveCommitted("");
|
||||
setLiveTentative("");
|
||||
} finally {
|
||||
liveRef.current = null;
|
||||
setLiveStatus("idle");
|
||||
|
||||
if (liveWsRef.current) {
|
||||
liveWsRef.current.send(JSON.stringify({ type: "stop" }));
|
||||
} else {
|
||||
try {
|
||||
const final = await session.transcriber!.stop();
|
||||
setResult(final);
|
||||
setLiveCommitted("");
|
||||
setLiveTentative("");
|
||||
} finally {
|
||||
liveRef.current = null;
|
||||
setLiveStatus("idle");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -667,7 +740,7 @@ export function App() {
|
||||
<p>
|
||||
Model: <code>{modelId}</code> · Engine:{" "}
|
||||
{serverSelected
|
||||
? "server (faster-whisper) — files upload to this deployment"
|
||||
? "server (faster-whisper) — audio streams to this deployment"
|
||||
: activeDevice === "webgpu"
|
||||
? "WebGPU (on-device)"
|
||||
: activeDevice === "webnn"
|
||||
|
||||
Reference in new issue
Block a user