390 lines
14 KiB
TypeScript
390 lines
14 KiB
TypeScript
/**
|
|
* Engine abstraction (Decisions.md D-010): the UI transcribes through one
|
|
* `Engine` interface and never branches on where inference runs beyond the
|
|
* EngineToggle. `BrowserEngine` wraps the inference Web Worker (D-003);
|
|
* `ServerEngine` wraps the streamed `POST /api/transcribe` fetch (D-011,
|
|
* faster-whisper sidecar per D-013).
|
|
*
|
|
* Deliberately framework-free like authClient.ts: the worker and fetch are
|
|
* injectable, so `bun test` exercises everything without a DOM or a server
|
|
* (tests/engine.test.ts). This module is also where the HTTP stream's wire
|
|
* format lives — src/server/transcribe.ts imports it so client and server
|
|
* can never drift apart.
|
|
*/
|
|
|
|
import {
|
|
isWorkerToUiMessage,
|
|
type TranscribeMessage,
|
|
type TranscriptChunk,
|
|
type WhisperTask,
|
|
} from "../worker/protocol";
|
|
import type { FetchLike } from "./authClient";
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// The Engine interface (D-010)
|
|
// ---------------------------------------------------------------------------
|
|
|
|
export type EngineKind = "browser" | "server";
|
|
|
|
export interface EngineTranscribeOptions {
|
|
/**
|
|
* Catalog model id (src/lib/models.ts). Required by the server engine —
|
|
* each request names its model since the server holds no per-client state.
|
|
* The browser engine ignores it: its model is whatever the worker loaded.
|
|
*/
|
|
modelId?: string;
|
|
/** Whisper language code; "auto"/undefined lets Whisper detect. */
|
|
language?: string;
|
|
/** transcribe | translate (multilingual models only). */
|
|
task?: WhisperTask;
|
|
/**
|
|
* Streaming transcript fragments — append each to the text shown so far.
|
|
* Browser engine: token fragments; server engine: whole segments.
|
|
*/
|
|
onPartial?: (fragment: string) => void;
|
|
/**
|
|
* Abort the run. Honored by the server engine (the request is cancelled
|
|
* server-side too); the browser worker cannot abort a pass yet (M6).
|
|
*/
|
|
signal?: AbortSignal;
|
|
}
|
|
|
|
export interface EngineTranscribeResult {
|
|
text: string;
|
|
chunks: TranscriptChunk[];
|
|
}
|
|
|
|
export interface Engine {
|
|
readonly kind: EngineKind;
|
|
/**
|
|
* Transcribe 16 kHz mono PCM (src/lib/audio.ts). Takes ownership of
|
|
* `audio` — the buffer may be transferred or consumed.
|
|
*/
|
|
transcribe(
|
|
audio: Float32Array,
|
|
options?: EngineTranscribeOptions,
|
|
): Promise<EngineTranscribeResult>;
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Browser engine — wraps the inference worker (D-003)
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/** The slice of `Worker` the engine touches, injectable for tests. */
|
|
export interface WorkerLike {
|
|
postMessage(message: unknown, transfer?: Transferable[]): void;
|
|
addEventListener(
|
|
type: "message" | "error",
|
|
listener: (event: { data?: unknown; message?: string }) => void,
|
|
): void;
|
|
}
|
|
|
|
interface PendingRequest {
|
|
resolve: (result: EngineTranscribeResult) => void;
|
|
reject: (error: Error) => void;
|
|
onPartial?: (fragment: string) => void;
|
|
}
|
|
|
|
/**
|
|
* Correlates transcribe requests with worker responses via the protocol's
|
|
* `requestId` (D-016). Model loading stays outside: the UI drives `load`
|
|
* messages and the progress panel directly against the worker, and both can
|
|
* share the worker because the engine ignores messages without one of its
|
|
* request ids.
|
|
*/
|
|
let nextEngineInstance = 0;
|
|
|
|
export class BrowserEngine implements Engine {
|
|
readonly kind = "browser";
|
|
private readonly pending = new Map<string, PendingRequest>();
|
|
/** Ids carry an instance discriminator so engines can share one worker. */
|
|
private readonly instance = ++nextEngineInstance;
|
|
private nextRequestId = 0;
|
|
|
|
constructor(private readonly worker: WorkerLike) {
|
|
worker.addEventListener("message", (event) => this.handleMessage(event.data));
|
|
worker.addEventListener("error", (event) => {
|
|
this.failAll(
|
|
new Error(event.message || "The inference worker crashed unexpectedly."),
|
|
);
|
|
});
|
|
}
|
|
|
|
transcribe(
|
|
audio: Float32Array,
|
|
options: EngineTranscribeOptions = {},
|
|
): Promise<EngineTranscribeResult> {
|
|
const requestId = `engine-${this.instance}-${++this.nextRequestId}`;
|
|
return new Promise((resolve, reject) => {
|
|
this.pending.set(requestId, { resolve, reject, onPartial: options.onPartial });
|
|
const message: TranscribeMessage = {
|
|
type: "transcribe",
|
|
audio,
|
|
requestId,
|
|
...(options.language !== undefined &&
|
|
options.language !== "auto" && { language: options.language }),
|
|
...(options.task !== undefined && { task: options.task }),
|
|
};
|
|
// Ownership of the audio moves to the worker — zero-copy transfer.
|
|
this.worker.postMessage(message, [audio.buffer as ArrayBuffer]);
|
|
});
|
|
}
|
|
|
|
/** Reject every in-flight request (worker crash / teardown). */
|
|
failAll(error: Error): void {
|
|
const requests = [...this.pending.values()];
|
|
this.pending.clear();
|
|
for (const request of requests) request.reject(error);
|
|
}
|
|
|
|
private handleMessage(data: unknown): void {
|
|
if (!isWorkerToUiMessage(data)) return;
|
|
if (!("requestId" in data) || data.requestId === undefined) return;
|
|
const request = this.pending.get(data.requestId);
|
|
if (!request) return;
|
|
switch (data.type) {
|
|
case "partial":
|
|
request.onPartial?.(data.text);
|
|
break;
|
|
case "result":
|
|
this.pending.delete(data.requestId);
|
|
request.resolve({ text: data.text, chunks: data.chunks });
|
|
break;
|
|
case "error":
|
|
this.pending.delete(data.requestId);
|
|
request.reject(new Error(data.message));
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Server transcription wire format (D-011: streamed JSON lines)
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/**
|
|
* One line of POST /api/transcribe's `application/x-ndjson` response body.
|
|
* Produced by src/server/transcribe.ts, consumed by ServerEngine. Errors
|
|
* after the 200 status line travel in-stream as an "error" line.
|
|
*/
|
|
export type TranscribeStreamLine =
|
|
| { type: "segment"; text: string; start: number; end: number }
|
|
| { type: "result"; text: string; chunks: TranscriptChunk[] }
|
|
| { type: "error"; message: string };
|
|
|
|
export function isTranscribeStreamLine(
|
|
value: unknown,
|
|
): value is TranscribeStreamLine {
|
|
if (typeof value !== "object" || value === null) return false;
|
|
const line = value as Record<string, unknown>;
|
|
switch (line.type) {
|
|
case "segment":
|
|
return (
|
|
typeof line.text === "string" &&
|
|
typeof line.start === "number" &&
|
|
typeof line.end === "number"
|
|
);
|
|
case "result":
|
|
return typeof line.text === "string" && Array.isArray(line.chunks);
|
|
case "error":
|
|
return typeof line.message === "string";
|
|
default:
|
|
return false;
|
|
}
|
|
}
|
|
|
|
/** Split a byte stream into complete lines (trailing partial line included). */
|
|
async function* streamLines(
|
|
stream: ReadableStream<Uint8Array>,
|
|
): AsyncGenerator<string> {
|
|
const decoder = new TextDecoder();
|
|
let buffer = "";
|
|
for await (const chunk of stream) {
|
|
buffer += decoder.decode(chunk, { stream: true });
|
|
let newline: number;
|
|
while ((newline = buffer.indexOf("\n")) !== -1) {
|
|
yield buffer.slice(0, newline);
|
|
buffer = buffer.slice(newline + 1);
|
|
}
|
|
}
|
|
buffer += decoder.decode();
|
|
if (buffer.trim().length > 0) yield buffer;
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Server engine — wraps POST /api/transcribe (D-011, D-013)
|
|
// ---------------------------------------------------------------------------
|
|
|
|
const defaultFetch: FetchLike = (input, init) => globalThis.fetch(input, init);
|
|
|
|
/** UI-safe message for a non-OK /api/transcribe response. */
|
|
async function serverErrorMessage(response: Response): Promise<string> {
|
|
let detail: string | undefined;
|
|
try {
|
|
const body: unknown = await response.json();
|
|
const error = (body as { error?: unknown } | null)?.error;
|
|
if (typeof error === "string") detail = error;
|
|
} catch {
|
|
// Non-JSON body — fall through to the status-based message.
|
|
}
|
|
switch (response.status) {
|
|
case 401:
|
|
return "Sign in to use the server engine.";
|
|
case 404:
|
|
return "The server engine is not available on this deployment.";
|
|
case 413:
|
|
return detail ?? "The audio is too large for the server engine.";
|
|
case 503:
|
|
return detail
|
|
? `The server engine is unavailable: ${detail}`
|
|
: "The server engine is unavailable right now.";
|
|
default:
|
|
return detail ?? `Server transcription failed (${response.status}).`;
|
|
}
|
|
}
|
|
|
|
export class ServerEngine implements Engine {
|
|
readonly kind = "server";
|
|
|
|
constructor(private readonly fetchFn: FetchLike = defaultFetch) {}
|
|
|
|
async transcribe(
|
|
audio: Float32Array,
|
|
options: EngineTranscribeOptions = {},
|
|
): Promise<EngineTranscribeResult> {
|
|
if (!options.modelId) {
|
|
throw new Error("The server engine needs a model id per request.");
|
|
}
|
|
const params = new URLSearchParams({ model: options.modelId });
|
|
if (options.language !== undefined && options.language !== "auto") {
|
|
params.set("language", options.language);
|
|
}
|
|
if (options.task !== undefined) params.set("task", options.task);
|
|
|
|
let response: Response;
|
|
try {
|
|
response = await this.fetchFn(`/api/transcribe?${params}`, {
|
|
method: "POST",
|
|
credentials: "same-origin",
|
|
headers: { "Content-Type": "application/octet-stream" },
|
|
// Raw float32 PCM — exactly what the browser engine consumes, so the
|
|
// server never has to decode containers (server-py/sidecar.py). The
|
|
// byte view narrows the buffer typing to what BodyInit accepts.
|
|
body: new Uint8Array(audio.buffer as ArrayBuffer, audio.byteOffset, audio.byteLength),
|
|
signal: options.signal,
|
|
});
|
|
} catch (error) {
|
|
if (error instanceof DOMException && error.name === "AbortError") throw error;
|
|
throw new Error("Could not reach the server. Check your connection.");
|
|
}
|
|
if (!response.ok) throw new Error(await serverErrorMessage(response));
|
|
if (!response.body) {
|
|
throw new Error("The server returned an empty transcription stream.");
|
|
}
|
|
|
|
for await (const rawLine of streamLines(response.body)) {
|
|
if (rawLine.trim().length === 0) continue;
|
|
let parsed: unknown;
|
|
try {
|
|
parsed = JSON.parse(rawLine);
|
|
} catch {
|
|
throw new Error("The server returned a malformed transcription stream.");
|
|
}
|
|
if (!isTranscribeStreamLine(parsed)) {
|
|
throw new Error("The server returned a malformed transcription stream.");
|
|
}
|
|
switch (parsed.type) {
|
|
case "segment":
|
|
options.onPartial?.(parsed.text);
|
|
break;
|
|
case "result":
|
|
return { text: parsed.text, chunks: parsed.chunks };
|
|
case "error":
|
|
throw new Error(parsed.message);
|
|
}
|
|
}
|
|
// The stream closed without a result line — the connection was cut.
|
|
throw new Error("The connection was lost before the transcript finished.");
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Server engine availability (EngineToggle probes this after sign-in)
|
|
// ---------------------------------------------------------------------------
|
|
|
|
export interface ServerEngineStatus {
|
|
available: boolean;
|
|
/** Human-readable explanation when unavailable (e.g. sidecar missing). */
|
|
reason?: string;
|
|
}
|
|
|
|
/**
|
|
* GET /api/transcribe/status → availability. Never rejects: any failure
|
|
* (401, 404 when the deployment has no server engine, offline) resolves to
|
|
* unavailable, mirroring fetchAuthStatus's offline-first behavior.
|
|
*/
|
|
export async function fetchServerEngineStatus(
|
|
fetchFn: FetchLike = defaultFetch,
|
|
): Promise<ServerEngineStatus> {
|
|
try {
|
|
const response = await fetchFn("/api/transcribe/status", {
|
|
method: "GET",
|
|
credentials: "same-origin",
|
|
});
|
|
if (!response.ok) {
|
|
return { available: false, reason: await serverErrorMessage(response) };
|
|
}
|
|
const body: unknown = await response.json().catch(() => null);
|
|
const status = body as { available?: unknown; reason?: unknown } | null;
|
|
if (typeof status?.available !== "boolean") return { available: false };
|
|
return {
|
|
available: status.available,
|
|
...(typeof status.reason === "string" && { reason: status.reason }),
|
|
};
|
|
} catch {
|
|
return { available: false, reason: "The server cannot be reached." };
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Engine choice persistence (EngineToggle, survives reloads)
|
|
// ---------------------------------------------------------------------------
|
|
|
|
export const ENGINE_CHOICE_KEY = "whisper-engine-choice";
|
|
|
|
export interface StorageLike {
|
|
getItem(key: string): string | null;
|
|
setItem(key: string, value: string): void;
|
|
}
|
|
|
|
function defaultStorage(): StorageLike | null {
|
|
try {
|
|
return (globalThis as { localStorage?: StorageLike }).localStorage ?? null;
|
|
} catch {
|
|
return null; // storage access can throw (privacy modes)
|
|
}
|
|
}
|
|
|
|
/** Persisted toggle choice; anything unreadable falls back to "browser". */
|
|
export function loadEngineChoice(
|
|
storage: StorageLike | null = defaultStorage(),
|
|
): EngineKind {
|
|
try {
|
|
const value = storage?.getItem(ENGINE_CHOICE_KEY);
|
|
return value === "server" ? "server" : "browser";
|
|
} catch {
|
|
return "browser";
|
|
}
|
|
}
|
|
|
|
export function saveEngineChoice(
|
|
choice: EngineKind,
|
|
storage: StorageLike | null = defaultStorage(),
|
|
): void {
|
|
try {
|
|
storage?.setItem(ENGINE_CHOICE_KEY, choice);
|
|
} catch {
|
|
// Not persistable (private mode / quota) — the choice just won't stick.
|
|
}
|
|
}
|