Start on live transcription and python sidecar (faster-whisper)

This commit is contained in:
Lucas Winther committed 2026-07-10 09:28:42 +02:00
1 parent 15bf14728a
commit 033ea32968
14 files changed
+1376 -61

No files matched your search

+171 -16
View File
@@ -3,11 +3,14 @@ import "./index.css";
import { AuthForms } from "./components/AuthForms";
import { FileDropzone } from "./components/FileDropzone";
import { LiveControls, type LiveStatus } from "./components/LiveControls";
import { ModelLoadProgress } from "./components/ModelLoadProgress";
import { ModelPicker } from "./components/ModelPicker";
import { TranscriptView } from "./components/TranscriptView";
import { TranscriptionOptions } from "./components/TranscriptionOptions";
import { decodeAudioFile } from "./lib/audio";
import { LiveTranscriber, type LiveTranscribeResult } from "./lib/live";
import { startMicCapture, type MicCapture } from "./lib/mic";
import { getDownloadedModels } from "./lib/modelCache";
import { registerServiceWorker } from "./pwa/register";
import {
@@ -39,6 +42,23 @@ const MODEL_IDS = MODELS.map((model) => model.id);
export function App() {
const workerRef = useRef<Worker | null>(null);
// Live-mode session (M7, D-016). The mic and transcriber live outside React
// state; only display text goes through setState. Segment passes are
// correlated with worker responses via requestId.
const liveRef = useRef<{ mic: MicCapture; transcriber: LiveTranscriber } | null>(
null,
);
const pendingLive = useRef(
new Map<
string,
{
resolve: (result: LiveTranscribeResult) => void;
reject: (error: Error) => void;
}
>(),
);
const nextLiveRequest = useRef(0);
const [busy, setBusy] = useState(false);
const [decoding, setDecoding] = useState(false);
const [loadStarted, setLoadStarted] = useState(false);
@@ -48,6 +68,10 @@ export function App() {
const [error, setError] = useState<string | null>(null);
const [fileName, setFileName] = useState<string | null>(null);
const [liveStatus, setLiveStatus] = useState<LiveStatus>("idle");
const [liveCommitted, setLiveCommitted] = useState("");
const [liveTentative, setLiveTentative] = useState("");
const [modelId, setModelId] = useState(DEFAULT_MODEL_ID);
const [devicePreference, setDevicePreference] = useState<DevicePreference>(
DEFAULT_DEVICE_PREFERENCE,
@@ -76,11 +100,20 @@ export function App() {
onUpdateReady: (apply) => setApplyUpdate(() => apply),
});
return () => {
void liveRef.current?.mic.stop().catch(() => {});
liveRef.current = null;
workerRef.current?.terminate();
workerRef.current = null;
};
}, []);
function takeLiveRequest(requestId: string | undefined) {
if (requestId === undefined) return undefined;
const pending = pendingLive.current.get(requestId);
if (pending) pendingLive.current.delete(requestId);
return pending;
}
function ensureWorker(): Worker {
if (workerRef.current) return workerRef.current;
// Inference always runs in a dedicated Web Worker (D-003), never on the
@@ -102,20 +135,42 @@ export function App() {
}
break;
case "partial":
setPartialText((text) => text + message.text);
// Live segment passes (requestId set) resolve wholesale via
// "result"; their token stream is not rendered incrementally.
if (message.requestId === undefined) {
setPartialText((text) => text + message.text);
}
break;
case "result":
setResult({ text: message.text, chunks: message.chunks });
setBusy(false);
case "result": {
const pending = takeLiveRequest(message.requestId);
if (pending) {
pending.resolve({ text: message.text, chunks: message.chunks });
} else if (message.requestId === undefined) {
setResult({ text: message.text, chunks: message.chunks });
setBusy(false);
}
break;
case "error":
setError(message.message);
setBusy(false);
}
case "error": {
const pending = takeLiveRequest(message.requestId);
if (pending) {
pending.reject(new Error(message.message));
} else {
setError(message.message);
setBusy(false);
}
break;
}
}
});
worker.addEventListener("error", (event) => {
setError(event.message || "The inference worker crashed unexpectedly.");
const failure = new Error(
event.message || "The inference worker crashed unexpectedly.",
);
// Unblock any live passes so a live session can drain and stop.
for (const pending of pendingLive.current.values()) pending.reject(failure);
pendingLive.current.clear();
setError(failure.message);
setBusy(false);
});
workerRef.current = worker;
@@ -141,7 +196,7 @@ export function App() {
}
function handleModelSelect(nextModelId: string) {
if (busy || nextModelId === modelId) return;
if (busy || liveStatus !== "idle" || nextModelId === modelId) return;
setModelId(nextModelId);
setActiveDevice(null);
setPartialText("");
@@ -150,7 +205,7 @@ export function App() {
}
function handleDevicePreference(preference: DevicePreference) {
if (busy || preference === devicePreference) return;
if (busy || liveStatus !== "idle" || preference === devicePreference) return;
setDevicePreference(preference);
setActiveDevice(null);
// Only reload eagerly if a load was already in motion; otherwise the next
@@ -159,7 +214,7 @@ export function App() {
}
async function handleFile(file: File) {
if (busy) return;
if (busy || liveStatus !== "idle") return;
setBusy(true);
setError(null);
setPartialText("");
@@ -199,10 +254,95 @@ export function App() {
}
}
// -------------------------------------------------------------------------
// Live microphone transcription (M7, D-016)
// -------------------------------------------------------------------------
/** One inference pass over a live segment, correlated via requestId. */
function transcribeLiveSegment(
worker: Worker,
audio: Float32Array,
options: { language?: string; task?: WhisperTask },
): Promise<LiveTranscribeResult> {
const requestId = `live-${++nextLiveRequest.current}`;
return new Promise((resolve, reject) => {
pendingLive.current.set(requestId, { resolve, reject });
// The controller hands over ownership, so the buffer can transfer.
postToWorker(worker, { type: "transcribe", audio, requestId, ...options }, [
audio.buffer as ArrayBuffer,
]);
});
}
async function startLive() {
if (busy || liveStatus !== "idle") return;
setError(null);
setResult(null);
setPartialText("");
setFileName(null);
setLiveCommitted("");
setLiveTentative("");
setLiveStatus("starting");
// Idempotent model load, like the file flow — a pending download runs
// while the mic opens; segment passes queue in the worker until ready.
const worker = ensureWorker();
if (!loadStarted) {
startLoad(modelId, devicePreference);
} else {
postToWorker(worker, { type: "load", modelId, device: devicePreference });
}
// Language/task are fixed for the session; the options are disabled while
// live. `.en` models take neither (same rule as the file flow).
const options = (getModel(modelId)?.multilingual ?? false) ? { language, task } : {};
const transcriber = new LiveTranscriber({
transcribe: (audio) => transcribeLiveSegment(worker, audio, options),
onUpdate: (update) => {
setLiveCommitted(update.committedText);
setLiveTentative(update.tentativeText);
if (update.error) setError(update.error);
},
});
try {
const mic = await startMicCapture((chunk) => transcriber.push(chunk));
liveRef.current = { mic, transcriber };
setLiveStatus("listening");
} catch (micError) {
setLiveStatus("idle");
setError(
micError instanceof DOMException && micError.name === "NotAllowedError"
? "Microphone access was denied. Allow it in your browser's site settings and try again."
: micError instanceof Error
? micError.message
: String(micError),
);
}
}
async function stopLive() {
const session = liveRef.current;
if (!session || liveStatus !== "listening") return;
setLiveStatus("stopping");
// Stop feeding audio first, then drain the remaining segment passes.
await session.mic.stop();
try {
const final = await session.transcriber.stop();
setResult(final);
setLiveCommitted("");
setLiveTentative("");
} finally {
liveRef.current = null;
setLiveStatus("idle");
}
}
const model = getModel(modelId);
const modelReady = loadProgress.ready;
const showModelProgress = loadStarted && !modelReady && !error;
const transcribing = busy && !decoding && modelReady;
const liveActive = liveStatus !== "idle";
return (
<main className="app">
@@ -230,7 +370,7 @@ export function App() {
activeDevice={activeDevice}
downloaded={downloaded}
offlineReady={swControlled}
disabled={busy}
disabled={busy || liveActive}
/>
{model?.multilingual && (
@@ -239,11 +379,18 @@ export function App() {
onLanguageChange={setLanguage}
task={task}
onTaskChange={setTask}
disabled={busy}
disabled={busy || liveActive}
/>
)}
<FileDropzone onFile={handleFile} disabled={busy} />
<FileDropzone onFile={handleFile} disabled={busy || liveActive} />
<LiveControls
status={liveStatus}
onStart={() => void startLive()}
onStop={() => void stopLive()}
disabled={busy}
/>
{fileName && (
<p className="file-name">
@@ -263,9 +410,17 @@ export function App() {
)}
<TranscriptView
partialText={partialText}
partialText={liveActive ? liveCommitted : partialText}
tentativeText={liveActive ? liveTentative : ""}
result={result}
transcribing={transcribing}
transcribing={liveActive || transcribing}
activeLabel={
liveStatus === "stopping"
? "Finishing up…"
: liveActive
? "Listening…"
: undefined
}
/>
{applyUpdate && (
+25 -8
View File
@@ -2,12 +2,19 @@ import { useEffect, useState } from "react";
import type { TranscriptChunk } from "../worker/protocol";
interface TranscriptViewProps {
/** Text streamed so far (accumulated partial messages). */
/** Text streamed so far (accumulated partials / committed live segments). */
partialText: string;
/** Final transcript, once available. */
result: { text: string; chunks: TranscriptChunk[] } | null;
/** True while a transcription is in flight. */
transcribing: boolean;
/**
* Rolling live-mode text that may still change (D-016) — rendered visually
* distinct from the committed text it follows.
*/
tentativeText?: string;
/** Heading while running; defaults to "Transcribing…". */
activeLabel?: string;
}
function formatTimestamp(seconds: number): string {
@@ -17,9 +24,16 @@ function formatTimestamp(seconds: number): string {
}
/** Streaming transcript: live partials while running, final text + copy button after. */
export function TranscriptView({ partialText, result, transcribing }: TranscriptViewProps) {
export function TranscriptView({
partialText,
result,
transcribing,
tentativeText = "",
activeLabel = "Transcribing…",
}: TranscriptViewProps) {
const [copied, setCopied] = useState(false);
const text = result?.text ?? partialText;
const copyText = `${text}${tentativeText}`.trim();
useEffect(() => {
if (!copied) return;
@@ -27,29 +41,32 @@ export function TranscriptView({ partialText, result, transcribing }: Transcript
return () => clearTimeout(timer);
}, [copied]);
if (!transcribing && !text) return null;
if (!transcribing && !text && !tentativeText) return null;
async function copy() {
if (!text) return;
await navigator.clipboard.writeText(text);
if (!copyText) return;
await navigator.clipboard.writeText(copyText);
setCopied(true);
}
return (
<section className="transcript">
<div className="transcript-header">
<h2>{result ? "Transcript" : "Transcribing…"}</h2>
<h2>{result ? "Transcript" : activeLabel}</h2>
<button
type="button"
className="copy-button"
onClick={copy}
disabled={!text}
disabled={!copyText}
>
{copied ? "Copied!" : "Copy"}
</button>
</div>
<p className="transcript-text" aria-live="polite">
{text || "Listening for the first words…"}
{text || (tentativeText ? "" : "Listening for the first words…")}
{tentativeText && (
<span className="transcript-tentative">{tentativeText}</span>
)}
{transcribing && <span className="cursor" aria-hidden="true" />}
</p>
{result && result.chunks.length > 0 && (
+62
View File
@@ -397,6 +397,61 @@ body {
font-size: 0.9rem;
}
/* --- LiveControls (M7, D-016) --- */
.live-controls {
display: flex;
align-items: center;
gap: 0.75rem;
}
.live-button {
border: 1px solid var(--border);
background: transparent;
color: inherit;
border-radius: 8px;
padding: 0.3rem 0.9rem;
font-size: 0.85rem;
cursor: pointer;
transition: border-color 0.15s;
white-space: nowrap;
}
.live-button:hover:enabled {
border-color: var(--accent);
}
.live-button:disabled {
opacity: 0.5;
cursor: default;
}
.live-button-active,
.live-button-active:hover:enabled {
border-color: #e0605e;
}
.live-dot {
flex-shrink: 0;
width: 0.6rem;
height: 0.6rem;
border-radius: 999px;
background: #e0605e;
animation: pulse 1.2s ease-in-out infinite;
}
@keyframes pulse {
50% {
opacity: 0.25;
}
}
.live-hint {
margin: 0;
color: var(--muted);
font-size: 0.9rem;
}
.file-name {
margin: 0;
color: var(--muted);
@@ -512,6 +567,13 @@ body {
overflow-wrap: anywhere;
}
/* Live-mode text that may still change (D-016) — visually distinct from the
committed text it follows. */
.transcript-tentative {
color: var(--muted);
font-style: italic;
}
.cursor {
display: inline-block;
width: 0.55em;
+14 -2
View File
@@ -4,6 +4,7 @@ import { ensureIcons, ICONS_DIR } from "../scripts/generate-icons.ts";
import { injectPrecacheManifest } from "./pwa/precache.ts";
import { auth } from "./server/auth.ts";
import { AUTH_ENABLED } from "./server/flags.ts";
import { transcribe } from "./server/transcribe.ts";
const ROOT = join(import.meta.dir, "..");
const IS_PRODUCTION = process.env.NODE_ENV === "production";
@@ -58,8 +59,14 @@ async function serveOrtFile(dir: string, name: string): Promise<Response> {
async function handleApiRequest(request: Request): Promise<Response> {
const { pathname } = new URL(request.url);
// Runtime feature flag (D-015): with auth disabled, the endpoints don't
// exist — the client's /me probe reads the 404 as "feature off".
if (!AUTH_ENABLED && pathname.startsWith("/api/auth/")) return NOT_FOUND();
// exist — the client's /me probe reads the 404 as "feature off". The
// server engine (M5) is session-gated, so it disappears along with auth.
if (
!AUTH_ENABLED &&
(pathname.startsWith("/api/auth/") || pathname.startsWith("/api/transcribe"))
) {
return NOT_FOUND();
}
switch (`${request.method} ${pathname}`) {
case "POST /api/auth/register":
return auth.register(request);
@@ -69,6 +76,11 @@ async function handleApiRequest(request: Request): Promise<Response> {
return auth.logout(request);
case "GET /api/auth/me":
return auth.me(request);
// Server engine (M5, D-011/D-013): availability probe + streamed run.
case "GET /api/transcribe/status":
return transcribe.status(request);
case "POST /api/transcribe":
return transcribe.post(request);
default:
return NOT_FOUND();
}
+385 -7
View File
@@ -1,11 +1,389 @@
/**
* Engine abstraction (Decisions.md D-010).
* Engine abstraction (Decisions.md D-010): the UI transcribes through one
* `Engine` interface and never branches on where inference runs beyond the
* EngineToggle. `BrowserEngine` wraps the inference Web Worker (D-003);
* `ServerEngine` wraps the streamed `POST /api/transcribe` fetch (D-011,
* faster-whisper sidecar per D-013).
*
* TODO(M5): define the `Engine` interface — `{ load(model), transcribe(audio),
* onProgress }` — plus `BrowserEngine` (wraps the inference worker) and
* `ServerEngine` (wraps the streamed `POST /api/transcribe` fetch, D-011).
*
* Until then the UI talks to the worker directly via src/worker/protocol.ts.
* Deliberately framework-free like authClient.ts: the worker and fetch are
* injectable, so `bun test` exercises everything without a DOM or a server
* (tests/engine.test.ts). This module is also where the HTTP stream's wire
* format lives — src/server/transcribe.ts imports it so client and server
* can never drift apart.
*/
export {};
import {
isWorkerToUiMessage,
type TranscribeMessage,
type TranscriptChunk,
type WhisperTask,
} from "../worker/protocol";
import type { FetchLike } from "./authClient";
// ---------------------------------------------------------------------------
// The Engine interface (D-010)
// ---------------------------------------------------------------------------
export type EngineKind = "browser" | "server";
export interface EngineTranscribeOptions {
/**
* Catalog model id (src/lib/models.ts). Required by the server engine —
* each request names its model since the server holds no per-client state.
* The browser engine ignores it: its model is whatever the worker loaded.
*/
modelId?: string;
/** Whisper language code; "auto"/undefined lets Whisper detect. */
language?: string;
/** transcribe | translate (multilingual models only). */
task?: WhisperTask;
/**
* Streaming transcript fragments — append each to the text shown so far.
* Browser engine: token fragments; server engine: whole segments.
*/
onPartial?: (fragment: string) => void;
/**
* Abort the run. Honored by the server engine (the request is cancelled
* server-side too); the browser worker cannot abort a pass yet (M6).
*/
signal?: AbortSignal;
}
export interface EngineTranscribeResult {
text: string;
chunks: TranscriptChunk[];
}
export interface Engine {
readonly kind: EngineKind;
/**
* Transcribe 16 kHz mono PCM (src/lib/audio.ts). Takes ownership of
* `audio` — the buffer may be transferred or consumed.
*/
transcribe(
audio: Float32Array,
options?: EngineTranscribeOptions,
): Promise<EngineTranscribeResult>;
}
// ---------------------------------------------------------------------------
// Browser engine — wraps the inference worker (D-003)
// ---------------------------------------------------------------------------
/** The slice of `Worker` the engine touches, injectable for tests. */
export interface WorkerLike {
postMessage(message: unknown, transfer?: Transferable[]): void;
addEventListener(
type: "message" | "error",
listener: (event: { data?: unknown; message?: string }) => void,
): void;
}
interface PendingRequest {
resolve: (result: EngineTranscribeResult) => void;
reject: (error: Error) => void;
onPartial?: (fragment: string) => void;
}
/**
* Correlates transcribe requests with worker responses via the protocol's
* `requestId` (D-016). Model loading stays outside: the UI drives `load`
* messages and the progress panel directly against the worker, and both can
* share the worker because the engine ignores messages without one of its
* request ids.
*/
let nextEngineInstance = 0;
export class BrowserEngine implements Engine {
readonly kind = "browser";
private readonly pending = new Map<string, PendingRequest>();
/** Ids carry an instance discriminator so engines can share one worker. */
private readonly instance = ++nextEngineInstance;
private nextRequestId = 0;
constructor(private readonly worker: WorkerLike) {
worker.addEventListener("message", (event) => this.handleMessage(event.data));
worker.addEventListener("error", (event) => {
this.failAll(
new Error(event.message || "The inference worker crashed unexpectedly."),
);
});
}
transcribe(
audio: Float32Array,
options: EngineTranscribeOptions = {},
): Promise<EngineTranscribeResult> {
const requestId = `engine-${this.instance}-${++this.nextRequestId}`;
return new Promise((resolve, reject) => {
this.pending.set(requestId, { resolve, reject, onPartial: options.onPartial });
const message: TranscribeMessage = {
type: "transcribe",
audio,
requestId,
...(options.language !== undefined &&
options.language !== "auto" && { language: options.language }),
...(options.task !== undefined && { task: options.task }),
};
// Ownership of the audio moves to the worker — zero-copy transfer.
this.worker.postMessage(message, [audio.buffer as ArrayBuffer]);
});
}
/** Reject every in-flight request (worker crash / teardown). */
failAll(error: Error): void {
const requests = [...this.pending.values()];
this.pending.clear();
for (const request of requests) request.reject(error);
}
private handleMessage(data: unknown): void {
if (!isWorkerToUiMessage(data)) return;
if (!("requestId" in data) || data.requestId === undefined) return;
const request = this.pending.get(data.requestId);
if (!request) return;
switch (data.type) {
case "partial":
request.onPartial?.(data.text);
break;
case "result":
this.pending.delete(data.requestId);
request.resolve({ text: data.text, chunks: data.chunks });
break;
case "error":
this.pending.delete(data.requestId);
request.reject(new Error(data.message));
break;
}
}
}
// ---------------------------------------------------------------------------
// Server transcription wire format (D-011: streamed JSON lines)
// ---------------------------------------------------------------------------
/**
* One line of POST /api/transcribe's `application/x-ndjson` response body.
* Produced by src/server/transcribe.ts, consumed by ServerEngine. Errors
* after the 200 status line travel in-stream as an "error" line.
*/
export type TranscribeStreamLine =
| { type: "segment"; text: string; start: number; end: number }
| { type: "result"; text: string; chunks: TranscriptChunk[] }
| { type: "error"; message: string };
export function isTranscribeStreamLine(
value: unknown,
): value is TranscribeStreamLine {
if (typeof value !== "object" || value === null) return false;
const line = value as Record<string, unknown>;
switch (line.type) {
case "segment":
return (
typeof line.text === "string" &&
typeof line.start === "number" &&
typeof line.end === "number"
);
case "result":
return typeof line.text === "string" && Array.isArray(line.chunks);
case "error":
return typeof line.message === "string";
default:
return false;
}
}
/** Split a byte stream into complete lines (trailing partial line included). */
async function* streamLines(
stream: ReadableStream<Uint8Array>,
): AsyncGenerator<string> {
const decoder = new TextDecoder();
let buffer = "";
for await (const chunk of stream) {
buffer += decoder.decode(chunk, { stream: true });
let newline: number;
while ((newline = buffer.indexOf("\n")) !== -1) {
yield buffer.slice(0, newline);
buffer = buffer.slice(newline + 1);
}
}
buffer += decoder.decode();
if (buffer.trim().length > 0) yield buffer;
}
// ---------------------------------------------------------------------------
// Server engine — wraps POST /api/transcribe (D-011, D-013)
// ---------------------------------------------------------------------------
const defaultFetch: FetchLike = (input, init) => globalThis.fetch(input, init);
/** UI-safe message for a non-OK /api/transcribe response. */
async function serverErrorMessage(response: Response): Promise<string> {
let detail: string | undefined;
try {
const body: unknown = await response.json();
const error = (body as { error?: unknown } | null)?.error;
if (typeof error === "string") detail = error;
} catch {
// Non-JSON body — fall through to the status-based message.
}
switch (response.status) {
case 401:
return "Sign in to use the server engine.";
case 404:
return "The server engine is not available on this deployment.";
case 413:
return detail ?? "The audio is too large for the server engine.";
case 503:
return detail
? `The server engine is unavailable: ${detail}`
: "The server engine is unavailable right now.";
default:
return detail ?? `Server transcription failed (${response.status}).`;
}
}
export class ServerEngine implements Engine {
readonly kind = "server";
constructor(private readonly fetchFn: FetchLike = defaultFetch) {}
async transcribe(
audio: Float32Array,
options: EngineTranscribeOptions = {},
): Promise<EngineTranscribeResult> {
if (!options.modelId) {
throw new Error("The server engine needs a model id per request.");
}
const params = new URLSearchParams({ model: options.modelId });
if (options.language !== undefined && options.language !== "auto") {
params.set("language", options.language);
}
if (options.task !== undefined) params.set("task", options.task);
let response: Response;
try {
response = await this.fetchFn(`/api/transcribe?${params}`, {
method: "POST",
credentials: "same-origin",
headers: { "Content-Type": "application/octet-stream" },
// Raw float32 PCM — exactly what the browser engine consumes, so the
// server never has to decode containers (server-py/sidecar.py). The
// byte view narrows the buffer typing to what BodyInit accepts.
body: new Uint8Array(audio.buffer as ArrayBuffer, audio.byteOffset, audio.byteLength),
signal: options.signal,
});
} catch (error) {
if (error instanceof DOMException && error.name === "AbortError") throw error;
throw new Error("Could not reach the server. Check your connection.");
}
if (!response.ok) throw new Error(await serverErrorMessage(response));
if (!response.body) {
throw new Error("The server returned an empty transcription stream.");
}
for await (const rawLine of streamLines(response.body)) {
if (rawLine.trim().length === 0) continue;
let parsed: unknown;
try {
parsed = JSON.parse(rawLine);
} catch {
throw new Error("The server returned a malformed transcription stream.");
}
if (!isTranscribeStreamLine(parsed)) {
throw new Error("The server returned a malformed transcription stream.");
}
switch (parsed.type) {
case "segment":
options.onPartial?.(parsed.text);
break;
case "result":
return { text: parsed.text, chunks: parsed.chunks };
case "error":
throw new Error(parsed.message);
}
}
// The stream closed without a result line — the connection was cut.
throw new Error("The connection was lost before the transcript finished.");
}
}
// ---------------------------------------------------------------------------
// Server engine availability (EngineToggle probes this after sign-in)
// ---------------------------------------------------------------------------
export interface ServerEngineStatus {
available: boolean;
/** Human-readable explanation when unavailable (e.g. sidecar missing). */
reason?: string;
}
/**
* GET /api/transcribe/status → availability. Never rejects: any failure
* (401, 404 when the deployment has no server engine, offline) resolves to
* unavailable, mirroring fetchAuthStatus's offline-first behavior.
*/
export async function fetchServerEngineStatus(
fetchFn: FetchLike = defaultFetch,
): Promise<ServerEngineStatus> {
try {
const response = await fetchFn("/api/transcribe/status", {
method: "GET",
credentials: "same-origin",
});
if (!response.ok) {
return { available: false, reason: await serverErrorMessage(response) };
}
const body: unknown = await response.json().catch(() => null);
const status = body as { available?: unknown; reason?: unknown } | null;
if (typeof status?.available !== "boolean") return { available: false };
return {
available: status.available,
...(typeof status.reason === "string" && { reason: status.reason }),
};
} catch {
return { available: false, reason: "The server cannot be reached." };
}
}
// ---------------------------------------------------------------------------
// Engine choice persistence (EngineToggle, survives reloads)
// ---------------------------------------------------------------------------
export const ENGINE_CHOICE_KEY = "whisper-engine-choice";
export interface StorageLike {
getItem(key: string): string | null;
setItem(key: string, value: string): void;
}
function defaultStorage(): StorageLike | null {
try {
return (globalThis as { localStorage?: StorageLike }).localStorage ?? null;
} catch {
return null; // storage access can throw (privacy modes)
}
}
/** Persisted toggle choice; anything unreadable falls back to "browser". */
export function loadEngineChoice(
storage: StorageLike | null = defaultStorage(),
): EngineKind {
try {
const value = storage?.getItem(ENGINE_CHOICE_KEY);
return value === "server" ? "server" : "browser";
} catch {
return "browser";
}
}
export function saveEngineChoice(
choice: EngineKind,
storage: StorageLike | null = defaultStorage(),
): void {
try {
storage?.setItem(ENGINE_CHOICE_KEY, choice);
} catch {
// Not persistable (private mode / quota) — the choice just won't stick.
}
}
+26
View File
@@ -48,6 +48,14 @@ export interface WhisperModel {
decoderParamsM: number;
/** Per-device dtype defaults (D-004 / D-006). */
defaults: Record<InferenceDevice, DtypeDefault>;
/**
* Model name the server engine passes to faster-whisper (D-013). These are
* faster-whisper's own aliases, which it resolves to the official
* CTranslate2 conversions on the Hub (`Systran/faster-whisper-*`;
* large-v3-turbo → `mobiuslabsgmbh/faster-whisper-large-v3-turbo`), so the
* pinned library version stays the single authority on the exact repo.
*/
fasterWhisperModel: string;
}
// ---------------------------------------------------------------------------
@@ -121,6 +129,7 @@ const WEBGPU_FP16_Q4: DtypeDefault = {
export const MODELS: readonly WhisperModel[] = [
{
id: "onnx-community/whisper-tiny.en",
fasterWhisperModel: "tiny.en",
displayName: "Whisper Tiny (English)",
multilingual: false,
encoderParamsM: 8,
@@ -133,6 +142,7 @@ export const MODELS: readonly WhisperModel[] = [
},
{
id: "onnx-community/whisper-tiny",
fasterWhisperModel: "tiny",
displayName: "Whisper Tiny",
multilingual: true,
encoderParamsM: 8,
@@ -141,6 +151,7 @@ export const MODELS: readonly WhisperModel[] = [
},
{
id: "onnx-community/whisper-base.en",
fasterWhisperModel: "base.en",
displayName: "Whisper Base (English)",
multilingual: false,
encoderParamsM: 20,
@@ -149,6 +160,7 @@ export const MODELS: readonly WhisperModel[] = [
},
{
id: "onnx-community/whisper-base",
fasterWhisperModel: "base",
displayName: "Whisper Base",
multilingual: true,
encoderParamsM: 20,
@@ -157,6 +169,7 @@ export const MODELS: readonly WhisperModel[] = [
},
{
id: "onnx-community/whisper-small.en",
fasterWhisperModel: "small.en",
displayName: "Whisper Small (English)",
multilingual: false,
encoderParamsM: 88,
@@ -165,6 +178,7 @@ export const MODELS: readonly WhisperModel[] = [
},
{
id: "onnx-community/whisper-small",
fasterWhisperModel: "small",
displayName: "Whisper Small",
multilingual: true,
encoderParamsM: 88,
@@ -173,6 +187,7 @@ export const MODELS: readonly WhisperModel[] = [
},
{
id: "onnx-community/whisper-medium.en",
fasterWhisperModel: "medium.en",
displayName: "Whisper Medium (English)",
multilingual: false,
encoderParamsM: 306,
@@ -181,6 +196,7 @@ export const MODELS: readonly WhisperModel[] = [
},
{
id: "onnx-community/whisper-medium",
fasterWhisperModel: "medium",
displayName: "Whisper Medium",
multilingual: true,
encoderParamsM: 306,
@@ -189,6 +205,7 @@ export const MODELS: readonly WhisperModel[] = [
},
{
id: "onnx-community/whisper-large-v3-turbo",
fasterWhisperModel: "large-v3-turbo",
displayName: "Whisper Large v3 Turbo",
multilingual: true,
encoderParamsM: 635,
@@ -201,6 +218,15 @@ export function getModel(id: string): WhisperModel | undefined {
return MODELS.find((model) => model.id === id);
}
/**
* faster-whisper model name for a catalog id (server engine, D-013), or
* undefined for ids outside the catalog — the transcribe endpoint answers 400
* rather than forwarding arbitrary strings to the sidecar.
*/
export function getFasterWhisperModel(id: string): string | undefined {
return getModel(id)?.fasterWhisperModel;
}
/** Default dtype for a model on a given device. Unknown ids fall back to q8. */
export function resolveDefaultDtype(
modelId: string,
+473 -6
View File
@@ -1,10 +1,477 @@
/**
* Server-side transcription engine — TODO(M5).
* Server-side transcription engine (M5, Decisions.md D-013): bridges the
* faster-whisper Python sidecar (server-py/sidecar.py) to the streamed
* JSON-lines HTTP contract (D-011, wire format in src/lib/engine.ts).
*
* POST /api/transcribe: auth-guarded, runs the same Transformers.js pipeline
* under Bun (ONNX Runtime, CPU), streams progress + partial transcripts as a
* chunked JSON-lines response (Decisions.md D-011). Reuses the shared model
* catalog in src/lib/models.ts; keep pipelines warm across requests.
* Routes (wired in src/index.ts, auth-guarded — the server engine is the one
* logged-in-only feature, Architecture.md §5):
* GET /api/transcribe/status → 200 {available, reason?} | 401
* POST /api/transcribe?model=<catalog id>[&language=..][&task=..]
* body: raw float32 mono 16 kHz PCM (application/octet-stream)
* → 200 application/x-ndjson (segment* then result | error lines)
* | 400 | 401 | 413 | 503 {error}
*
* The sidecar is spawned lazily on first use and supervised: a crash rejects
* the requests it was carrying and the next request respawns it, while a
* missing runtime (no python3, faster-whisper not installed) is remembered as
* a permanent "unavailable" reason — fixing that requires a deployment
* change, so we never busy-respawn. Either way the browser/anonymous path is
* untouched: this module holds the only Python dependency in the app.
*
* Warm-up strategy (WorkBreakdown M5): the sidecar keeps the last-used model
* loaded between requests, so only the first request for a model (or a model
* switch) pays the load; weights download once into the sidecar's cache.
*/
export {};
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { TranscribeStreamLine } from "../lib/engine";
import { getFasterWhisperModel } from "../lib/models";
import type { TranscriptChunk, WhisperTask } from "../worker/protocol";
import { getSessionUser as appGetSessionUser, type SessionUser } from "./auth.ts";
// ---------------------------------------------------------------------------
// Sidecar interface (injectable — tests use a fake, see tests/transcribe.test.ts)
// ---------------------------------------------------------------------------
export interface SidecarSegment {
start: number;
end: number;
text: string;
}
export interface SidecarTranscribeRequest {
/** faster-whisper model name (catalog mapping, src/lib/models.ts). */
model: string;
/** Path to a file of raw float32 mono 16 kHz PCM. */
audioPath: string;
language?: string;
task?: WhisperTask;
}
export interface SidecarStatus {
available: boolean;
reason?: string;
}
export interface Sidecar {
/** Spawns the process if needed; reports whether inference can run. */
status(): Promise<SidecarStatus>;
/**
* Run one transcription, streaming segments as they decode. Resolves after
* the sidecar's "done" (cancelled: true when `signal` fired first);
* rejects on sidecar errors or crashes.
*/
transcribe(
request: SidecarTranscribeRequest,
onSegment: (segment: SidecarSegment) => void,
signal?: AbortSignal,
): Promise<{ cancelled: boolean }>;
}
// ---------------------------------------------------------------------------
// The real sidecar: Bun.spawn + JSON-lines over stdio (D-013)
// ---------------------------------------------------------------------------
const SIDECAR_SCRIPT = join(import.meta.dir, "../../server-py/sidecar.py");
/** How long the spawned process gets to import faster-whisper and report. */
const READY_TIMEOUT_MS = 60_000;
interface SidecarProcess {
proc: ReturnType<typeof Bun.spawn>;
ready: Promise<void>;
}
interface PendingTranscription {
onSegment: (segment: SidecarSegment) => void;
resolve: (outcome: { cancelled: boolean }) => void;
reject: (error: Error) => void;
}
export function createSidecar(command?: readonly string[]): Sidecar {
// "python3" resolved per spawn — SIDECAR_PYTHON overrides for exotic setups.
const cmd = command ?? [process.env.SIDECAR_PYTHON || "python3", SIDECAR_SCRIPT];
let current: SidecarProcess | null = null;
/** Deployment-level failure (no python, deps missing): never respawn. */
let permanentFailure: string | null = null;
const pending = new Map<string, PendingTranscription>();
let nextRequestId = 0;
function failAllPending(error: Error): void {
const requests = [...pending.values()];
pending.clear();
for (const request of requests) request.reject(error);
}
function routeLine(rawLine: string): void {
let message: Record<string, unknown>;
try {
message = JSON.parse(rawLine) as Record<string, unknown>;
} catch {
return; // stray non-JSON output must not kill the bridge
}
const request = pending.get(String(message.id));
if (!request) return;
switch (message.type) {
case "segment":
if (
typeof message.start === "number" &&
typeof message.end === "number" &&
typeof message.text === "string"
) {
request.onSegment({
start: message.start,
end: message.end,
text: message.text,
});
}
break;
case "done":
pending.delete(String(message.id));
request.resolve({ cancelled: message.cancelled === true });
break;
case "error":
pending.delete(String(message.id));
request.reject(new Error(String(message.message ?? "sidecar error")));
break;
}
}
function spawn(): SidecarProcess {
const proc = Bun.spawn({
cmd: [...cmd],
stdin: "pipe",
stdout: "pipe",
stderr: "inherit", // model download progress / tracebacks → server log
});
const ready = new Promise<void>((resolve, reject) => {
let settled = false;
const timeout = setTimeout(() => {
if (settled) return;
settled = true;
proc.kill();
reject(new Error("the transcription sidecar did not start in time"));
}, READY_TIMEOUT_MS);
void (async () => {
// First line decides readiness; everything after is request traffic.
const decoder = new TextDecoder();
let buffer = "";
for await (const chunk of proc.stdout as ReadableStream<Uint8Array>) {
buffer += decoder.decode(chunk, { stream: true });
let newline: number;
while ((newline = buffer.indexOf("\n")) !== -1) {
const line = buffer.slice(0, newline);
buffer = buffer.slice(newline + 1);
if (!settled) {
settled = true;
clearTimeout(timeout);
let handshake: Record<string, unknown> = {};
try {
handshake = JSON.parse(line) as Record<string, unknown>;
} catch {
// fall through — treated as a bad handshake below
}
if (handshake.type === "ready") {
resolve();
} else {
// "fatal": faster-whisper missing — a deployment problem.
permanentFailure = String(
handshake.message ?? "the transcription sidecar failed to start",
);
reject(new Error(permanentFailure));
}
} else {
routeLine(line);
}
}
}
// stdout closed: the process exited (crash or shutdown).
if (!settled) {
settled = true;
clearTimeout(timeout);
reject(new Error("the transcription sidecar exited during startup"));
}
failAllPending(new Error("the transcription sidecar exited unexpectedly"));
if (current?.proc === proc) current = null; // next request respawns
})();
});
// A failed startup must not surface as an unhandled rejection when nobody
// is awaiting yet (e.g. crash between requests).
ready.catch(() => {});
return { proc, ready };
}
async function ensure(): Promise<SidecarProcess> {
if (permanentFailure) throw new Error(permanentFailure);
if (current) return current;
try {
current = spawn();
} catch (error) {
// Bun.spawn throws synchronously when the binary is missing.
permanentFailure = `python3 is not available on this server (${
error instanceof Error ? error.message : String(error)
})`;
throw new Error(permanentFailure);
}
return current;
}
function send(proc: ReturnType<typeof Bun.spawn>, message: unknown): void {
const stdin = proc.stdin as { write(data: string): unknown; flush(): unknown };
stdin.write(`${JSON.stringify(message)}\n`);
stdin.flush();
}
return {
async status() {
try {
const sidecar = await ensure();
await sidecar.ready;
return { available: true };
} catch (error) {
return {
available: false,
reason: error instanceof Error ? error.message : String(error),
};
}
},
async transcribe(request, onSegment, signal) {
const sidecar = await ensure();
await sidecar.ready;
const id = `req-${++nextRequestId}`;
return new Promise<{ cancelled: boolean }>((resolve, reject) => {
pending.set(id, { onSegment, resolve, reject });
signal?.addEventListener(
"abort",
() => {
// The sidecar checks between segments and still answers "done"
// (cancelled: true), which resolves this promise normally.
if (pending.has(id) && current?.proc === sidecar.proc) {
send(sidecar.proc, { type: "cancel", id });
}
},
{ once: true },
);
try {
send(sidecar.proc, {
type: "transcribe",
id,
model: request.model,
audio_path: request.audioPath,
language: request.language ?? null,
task: request.task ?? "transcribe",
});
} catch (error) {
pending.delete(id);
reject(error instanceof Error ? error : new Error(String(error)));
}
});
},
};
}
// ---------------------------------------------------------------------------
// HTTP handlers
// ---------------------------------------------------------------------------
/** Default cap ≈ 2¼ hours of raw float32 16 kHz mono (64 kB/s). */
const DEFAULT_MAX_UPLOAD_BYTES = 512 * 1024 * 1024;
const LANGUAGE_RE = /^[a-z]{2,3}$/;
export interface TranscribeHandlers {
status(request: Request): Promise<Response>;
post(request: Request): Promise<Response>;
}
export interface TranscribeDeps {
sidecar: Sidecar;
getSessionUser(request: Request): SessionUser | null;
maxUploadBytes?: number;
/** Directory for uploaded-PCM temp files. Default: the OS temp dir. */
tempDir?: string;
}
function json(body: unknown, status: number): Response {
return new Response(JSON.stringify(body), {
status,
headers: { "Content-Type": "application/json" },
});
}
interface TranscribeParams {
modelId: string;
fasterWhisperModel: string;
language?: string;
task: WhisperTask;
}
/** Parse + validate query params; a string is the 400 error text. */
function parseParams(url: URL): TranscribeParams | string {
const modelId = url.searchParams.get("model");
if (!modelId) return "missing model parameter";
const fasterWhisperModel = getFasterWhisperModel(modelId);
// Catalog-only (D-006/D-013): never forward arbitrary ids to the sidecar.
if (!fasterWhisperModel) return `unknown model: ${modelId}`;
const rawLanguage = url.searchParams.get("language");
let language: string | undefined;
if (rawLanguage !== null && rawLanguage !== "" && rawLanguage !== "auto") {
if (!LANGUAGE_RE.test(rawLanguage)) return `invalid language: ${rawLanguage}`;
language = rawLanguage;
}
const rawTask = url.searchParams.get("task") ?? "transcribe";
if (rawTask !== "transcribe" && rawTask !== "translate") {
return `invalid task: ${rawTask}`;
}
return { modelId, fasterWhisperModel, language, task: rawTask };
}
/**
* Stream the request body to `path`, enforcing the size limit even for
* chunked bodies. Returns bytes written, or a Response to answer with.
*/
async function writeBodyToFile(
request: Request,
path: string,
maxBytes: number,
): Promise<number | Response> {
const declared = Number(request.headers.get("content-length"));
if (Number.isFinite(declared) && declared > maxBytes) {
return json({ error: `audio exceeds the ${maxBytes}-byte upload limit` }, 413);
}
if (!request.body) return json({ error: "missing request body" }, 400);
const sink = Bun.file(path).writer();
let written = 0;
try {
for await (const chunk of request.body) {
written += chunk.byteLength;
if (written > maxBytes) {
return json({ error: `audio exceeds the ${maxBytes}-byte upload limit` }, 413);
}
sink.write(chunk);
}
} finally {
await sink.end();
}
if (written === 0) return json({ error: "missing request body" }, 400);
if (written % 4 !== 0) {
return json({ error: "body must be raw float32 PCM (length % 4 == 0)" }, 400);
}
return written;
}
export function createTranscribeHandlers(deps: TranscribeDeps): TranscribeHandlers {
const maxUploadBytes = deps.maxUploadBytes ?? DEFAULT_MAX_UPLOAD_BYTES;
const tempDir = deps.tempDir ?? tmpdir();
return {
async status(request) {
if (!deps.getSessionUser(request)) {
return json({ error: "not authenticated" }, 401);
}
const status = await deps.sidecar.status();
return json(status, 200);
},
async post(request) {
if (!deps.getSessionUser(request)) {
return json({ error: "not authenticated" }, 401);
}
const params = parseParams(new URL(request.url));
if (typeof params === "string") return json({ error: params }, 400);
// Refuse before consuming a potentially huge body when inference can't
// run anyway (degrade gracefully — D-013).
const status = await deps.sidecar.status();
if (!status.available) {
return json({ error: status.reason ?? "server engine unavailable" }, 503);
}
const audioPath = join(tempDir, `whisper-upload-${crypto.randomUUID()}.f32`);
const written = await writeBodyToFile(request, audioPath, maxUploadBytes);
if (written instanceof Response) {
await Bun.file(audioPath).unlink().catch(() => {});
return written;
}
// 200 with an ndjson stream (D-011); failures beyond this point travel
// in-stream as an "error" line since the status is already committed.
const encoder = new TextEncoder();
const stream = new ReadableStream<Uint8Array>({
async start(controller) {
const enqueue = (line: TranscribeStreamLine) =>
controller.enqueue(encoder.encode(`${JSON.stringify(line)}\n`));
const chunks: TranscriptChunk[] = [];
try {
await deps.sidecar.transcribe(
{
model: params.fasterWhisperModel,
audioPath,
language: params.language,
task: params.task,
},
(segment) => {
chunks.push({
text: segment.text,
timestamp: [segment.start, segment.end],
});
enqueue({
type: "segment",
text: segment.text,
start: segment.start,
end: segment.end,
});
},
// Client gone (abort/refresh) → cancel inference between segments.
request.signal,
);
enqueue({
type: "result",
text: chunks.map((chunk) => chunk.text).join("").trim(),
chunks,
});
} catch (error) {
enqueue({
type: "error",
message: error instanceof Error ? error.message : String(error),
});
} finally {
await Bun.file(audioPath).unlink().catch(() => {});
try {
controller.close();
} catch {
// Already errored/cancelled — the temp file cleanup above is
// what actually matters here.
}
}
},
});
return new Response(stream, {
headers: {
"Content-Type": "application/x-ndjson; charset=utf-8",
"Cache-Control": "no-store",
},
});
},
};
}
// ---------------------------------------------------------------------------
// Default instance (real sidecar + app sessions), wired in src/index.ts
// ---------------------------------------------------------------------------
export const transcribe: TranscribeHandlers = createTranscribeHandlers({
sidecar: createSidecar(),
getSessionUser: appGetSessionUser,
maxUploadBytes: Number(process.env.TRANSCRIBE_MAX_UPLOAD_BYTES) || undefined,
});
+10 -2
View File
@@ -194,6 +194,7 @@ async function transcribe(message: TranscribeMessage): Promise<void> {
type: "error",
stage: "transcribe",
message: "No model loaded — send a \"load\" message first.",
requestId: message.requestId,
});
return;
}
@@ -204,7 +205,8 @@ async function transcribe(message: TranscribeMessage): Promise<void> {
asr.tokenizer as ConstructorParameters<typeof WhisperTextStreamer>[0],
{
skip_prompt: true,
callback_function: (text: string) => post({ type: "partial", text }),
callback_function: (text: string) =>
post({ type: "partial", text, requestId: message.requestId }),
},
);
@@ -233,6 +235,7 @@ async function transcribe(message: TranscribeMessage): Promise<void> {
type: "result",
text: (single.text ?? "").trim(),
chunks: single.chunks ?? [],
requestId: message.requestId,
};
post(result);
}
@@ -255,7 +258,12 @@ scope.addEventListener("message", (event) => {
break;
case "transcribe":
transcribe(data).catch((error: unknown) => {
post({ type: "error", stage: "transcribe", message: errorMessage(error) });
post({
type: "error",
stage: "transcribe",
message: errorMessage(error),
requestId: data.requestId,
});
});
break;
}
+27 -5
View File
@@ -61,6 +61,12 @@ export interface TranscribeMessage {
language?: string;
/** Only meaningful for multilingual models. Defaults to "transcribe". */
task?: WhisperTask;
/**
* Optional correlation id, echoed on the partial/result/error messages this
* request produces (D-016). The live-transcription loop uses it to match
* responses to segment passes; the file flow omits it and is unchanged.
*/
requestId?: string;
}
export type UiToWorkerMessage = LoadMessage | TranscribeMessage;
@@ -96,6 +102,8 @@ export type LoadProgressMessage =
export interface PartialMessage {
type: "partial";
text: string;
/** Echo of the originating transcribe request's id, if it carried one. */
requestId?: string;
}
/** One timestamped segment of the final transcript. */
@@ -109,12 +117,16 @@ export interface ResultMessage {
type: "result";
text: string;
chunks: TranscriptChunk[];
/** Echo of the originating transcribe request's id, if it carried one. */
requestId?: string;
}
export interface ErrorMessage {
type: "error";
stage: "load" | "transcribe";
message: string;
/** Echo of the failing transcribe request's id, if it carried one. */
requestId?: string;
}
export type WorkerToUiMessage =
@@ -131,6 +143,10 @@ function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === "object" && value !== null;
}
function isOptionalString(value: unknown): value is string | undefined {
return value === undefined || typeof value === "string";
}
const DEVICES: readonly string[] = ["webgpu", "wasm"];
const DTYPES: readonly string[] = ["fp32", "fp16", "q8", "q4"];
@@ -165,10 +181,11 @@ export function isUiToWorkerMessage(value: unknown): value is UiToWorkerMessage
case "transcribe":
return (
value.audio instanceof Float32Array &&
(value.language === undefined || typeof value.language === "string") &&
isOptionalString(value.language) &&
(value.task === undefined ||
value.task === "transcribe" ||
value.task === "translate")
value.task === "translate") &&
isOptionalString(value.requestId)
);
default:
return false;
@@ -197,13 +214,18 @@ export function isWorkerToUiMessage(value: unknown): value is WorkerToUiMessage
return false;
}
case "partial":
return typeof value.text === "string";
return typeof value.text === "string" && isOptionalString(value.requestId);
case "result":
return typeof value.text === "string" && Array.isArray(value.chunks);
return (
typeof value.text === "string" &&
Array.isArray(value.chunks) &&
isOptionalString(value.requestId)
);
case "error":
return (
typeof value.message === "string" &&
(value.stage === "load" || value.stage === "transcribe")
(value.stage === "load" || value.stage === "transcribe") &&
isOptionalString(value.requestId)
);
default:
return false;