Start on live transcription and python sidecar (faster-whisper)

src/lib/engine.ts

@@ -1,11 +1,389 @@
 /**
- * Engine abstraction (Decisions.md D-010).
+ * Engine abstraction (Decisions.md D-010): the UI transcribes through one
+ * `Engine` interface and never branches on where inference runs beyond the
+ * EngineToggle. `BrowserEngine` wraps the inference Web Worker (D-003);
+ * `ServerEngine` wraps the streamed `POST /api/transcribe` fetch (D-011,
+ * faster-whisper sidecar per D-013).
  *
- * TODO(M5): define the `Engine` interface — `{ load(model), transcribe(audio),
- * onProgress }` — plus `BrowserEngine` (wraps the inference worker) and
- * `ServerEngine` (wraps the streamed `POST /api/transcribe` fetch, D-011).
- *
- * Until then the UI talks to the worker directly via src/worker/protocol.ts.
+ * Deliberately framework-free like authClient.ts: the worker and fetch are
+ * injectable, so `bun test` exercises everything without a DOM or a server
+ * (tests/engine.test.ts). This module is also where the HTTP stream's wire
+ * format lives — src/server/transcribe.ts imports it so client and server
+ * can never drift apart.
  */
 
-export {};
+import {
+  isWorkerToUiMessage,
+  type TranscribeMessage,
+  type TranscriptChunk,
+  type WhisperTask,
+} from "../worker/protocol";
+import type { FetchLike } from "./authClient";
+
+// ---------------------------------------------------------------------------
+// The Engine interface (D-010)
+// ---------------------------------------------------------------------------
+
+export type EngineKind = "browser" | "server";
+
+export interface EngineTranscribeOptions {
+  /**
+   * Catalog model id (src/lib/models.ts). Required by the server engine —
+   * each request names its model since the server holds no per-client state.
+   * The browser engine ignores it: its model is whatever the worker loaded.
+   */
+  modelId?: string;
+  /** Whisper language code; "auto"/undefined lets Whisper detect. */
+  language?: string;
+  /** transcribe | translate (multilingual models only). */
+  task?: WhisperTask;
+  /**
+   * Streaming transcript fragments — append each to the text shown so far.
+   * Browser engine: token fragments; server engine: whole segments.
+   */
+  onPartial?: (fragment: string) => void;
+  /**
+   * Abort the run. Honored by the server engine (the request is cancelled
+   * server-side too); the browser worker cannot abort a pass yet (M6).
+   */
+  signal?: AbortSignal;
+}
+
+export interface EngineTranscribeResult {
+  text: string;
+  chunks: TranscriptChunk[];
+}
+
+export interface Engine {
+  readonly kind: EngineKind;
+  /**
+   * Transcribe 16 kHz mono PCM (src/lib/audio.ts). Takes ownership of
+   * `audio` — the buffer may be transferred or consumed.
+   */
+  transcribe(
+    audio: Float32Array,
+    options?: EngineTranscribeOptions,
+  ): Promise<EngineTranscribeResult>;
+}
+
+// ---------------------------------------------------------------------------
+// Browser engine — wraps the inference worker (D-003)
+// ---------------------------------------------------------------------------
+
+/** The slice of `Worker` the engine touches, injectable for tests. */
+export interface WorkerLike {
+  postMessage(message: unknown, transfer?: Transferable[]): void;
+  addEventListener(
+    type: "message" | "error",
+    listener: (event: { data?: unknown; message?: string }) => void,
+  ): void;
+}
+
+interface PendingRequest {
+  resolve: (result: EngineTranscribeResult) => void;
+  reject: (error: Error) => void;
+  onPartial?: (fragment: string) => void;
+}
+
+/**
+ * Correlates transcribe requests with worker responses via the protocol's
+ * `requestId` (D-016). Model loading stays outside: the UI drives `load`
+ * messages and the progress panel directly against the worker, and both can
+ * share the worker because the engine ignores messages without one of its
+ * request ids.
+ */
+let nextEngineInstance = 0;
+
+export class BrowserEngine implements Engine {
+  readonly kind = "browser";
+  private readonly pending = new Map<string, PendingRequest>();
+  /** Ids carry an instance discriminator so engines can share one worker. */
+  private readonly instance = ++nextEngineInstance;
+  private nextRequestId = 0;
+
+  constructor(private readonly worker: WorkerLike) {
+    worker.addEventListener("message", (event) => this.handleMessage(event.data));
+    worker.addEventListener("error", (event) => {
+      this.failAll(
+        new Error(event.message || "The inference worker crashed unexpectedly."),
+      );
+    });
+  }
+
+  transcribe(
+    audio: Float32Array,
+    options: EngineTranscribeOptions = {},
+  ): Promise<EngineTranscribeResult> {
+    const requestId = `engine-${this.instance}-${++this.nextRequestId}`;
+    return new Promise((resolve, reject) => {
+      this.pending.set(requestId, { resolve, reject, onPartial: options.onPartial });
+      const message: TranscribeMessage = {
+        type: "transcribe",
+        audio,
+        requestId,
+        ...(options.language !== undefined &&
+          options.language !== "auto" && { language: options.language }),
+        ...(options.task !== undefined && { task: options.task }),
+      };
+      // Ownership of the audio moves to the worker — zero-copy transfer.
+      this.worker.postMessage(message, [audio.buffer as ArrayBuffer]);
+    });
+  }
+
+  /** Reject every in-flight request (worker crash / teardown). */
+  failAll(error: Error): void {
+    const requests = [...this.pending.values()];
+    this.pending.clear();
+    for (const request of requests) request.reject(error);
+  }
+
+  private handleMessage(data: unknown): void {
+    if (!isWorkerToUiMessage(data)) return;
+    if (!("requestId" in data) || data.requestId === undefined) return;
+    const request = this.pending.get(data.requestId);
+    if (!request) return;
+    switch (data.type) {
+      case "partial":
+        request.onPartial?.(data.text);
+        break;
+      case "result":
+        this.pending.delete(data.requestId);
+        request.resolve({ text: data.text, chunks: data.chunks });
+        break;
+      case "error":
+        this.pending.delete(data.requestId);
+        request.reject(new Error(data.message));
+        break;
+    }
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Server transcription wire format (D-011: streamed JSON lines)
+// ---------------------------------------------------------------------------
+
+/**
+ * One line of POST /api/transcribe's `application/x-ndjson` response body.
+ * Produced by src/server/transcribe.ts, consumed by ServerEngine. Errors
+ * after the 200 status line travel in-stream as an "error" line.
+ */
+export type TranscribeStreamLine =
+  | { type: "segment"; text: string; start: number; end: number }
+  | { type: "result"; text: string; chunks: TranscriptChunk[] }
+  | { type: "error"; message: string };
+
+export function isTranscribeStreamLine(
+  value: unknown,
+): value is TranscribeStreamLine {
+  if (typeof value !== "object" || value === null) return false;
+  const line = value as Record<string, unknown>;
+  switch (line.type) {
+    case "segment":
+      return (
+        typeof line.text === "string" &&
+        typeof line.start === "number" &&
+        typeof line.end === "number"
+      );
+    case "result":
+      return typeof line.text === "string" && Array.isArray(line.chunks);
+    case "error":
+      return typeof line.message === "string";
+    default:
+      return false;
+  }
+}
+
+/** Split a byte stream into complete lines (trailing partial line included). */
+async function* streamLines(
+  stream: ReadableStream<Uint8Array>,
+): AsyncGenerator<string> {
+  const decoder = new TextDecoder();
+  let buffer = "";
+  for await (const chunk of stream) {
+    buffer += decoder.decode(chunk, { stream: true });
+    let newline: number;
+    while ((newline = buffer.indexOf("\n")) !== -1) {
+      yield buffer.slice(0, newline);
+      buffer = buffer.slice(newline + 1);
+    }
+  }
+  buffer += decoder.decode();
+  if (buffer.trim().length > 0) yield buffer;
+}
+
+// ---------------------------------------------------------------------------
+// Server engine — wraps POST /api/transcribe (D-011, D-013)
+// ---------------------------------------------------------------------------
+
+const defaultFetch: FetchLike = (input, init) => globalThis.fetch(input, init);
+
+/** UI-safe message for a non-OK /api/transcribe response. */
+async function serverErrorMessage(response: Response): Promise<string> {
+  let detail: string | undefined;
+  try {
+    const body: unknown = await response.json();
+    const error = (body as { error?: unknown } | null)?.error;
+    if (typeof error === "string") detail = error;
+  } catch {
+    // Non-JSON body — fall through to the status-based message.
+  }
+  switch (response.status) {
+    case 401:
+      return "Sign in to use the server engine.";
+    case 404:
+      return "The server engine is not available on this deployment.";
+    case 413:
+      return detail ?? "The audio is too large for the server engine.";
+    case 503:
+      return detail
+        ? `The server engine is unavailable: ${detail}`
+        : "The server engine is unavailable right now.";
+    default:
+      return detail ?? `Server transcription failed (${response.status}).`;
+  }
+}
+
+export class ServerEngine implements Engine {
+  readonly kind = "server";
+
+  constructor(private readonly fetchFn: FetchLike = defaultFetch) {}
+
+  async transcribe(
+    audio: Float32Array,
+    options: EngineTranscribeOptions = {},
+  ): Promise<EngineTranscribeResult> {
+    if (!options.modelId) {
+      throw new Error("The server engine needs a model id per request.");
+    }
+    const params = new URLSearchParams({ model: options.modelId });
+    if (options.language !== undefined && options.language !== "auto") {
+      params.set("language", options.language);
+    }
+    if (options.task !== undefined) params.set("task", options.task);
+
+    let response: Response;
+    try {
+      response = await this.fetchFn(`/api/transcribe?${params}`, {
+        method: "POST",
+        credentials: "same-origin",
+        headers: { "Content-Type": "application/octet-stream" },
+        // Raw float32 PCM — exactly what the browser engine consumes, so the
+        // server never has to decode containers (server-py/sidecar.py). The
+        // byte view narrows the buffer typing to what BodyInit accepts.
+        body: new Uint8Array(audio.buffer as ArrayBuffer, audio.byteOffset, audio.byteLength),
+        signal: options.signal,
+      });
+    } catch (error) {
+      if (error instanceof DOMException && error.name === "AbortError") throw error;
+      throw new Error("Could not reach the server. Check your connection.");
+    }
+    if (!response.ok) throw new Error(await serverErrorMessage(response));
+    if (!response.body) {
+      throw new Error("The server returned an empty transcription stream.");
+    }
+
+    for await (const rawLine of streamLines(response.body)) {
+      if (rawLine.trim().length === 0) continue;
+      let parsed: unknown;
+      try {
+        parsed = JSON.parse(rawLine);
+      } catch {
+        throw new Error("The server returned a malformed transcription stream.");
+      }
+      if (!isTranscribeStreamLine(parsed)) {
+        throw new Error("The server returned a malformed transcription stream.");
+      }
+      switch (parsed.type) {
+        case "segment":
+          options.onPartial?.(parsed.text);
+          break;
+        case "result":
+          return { text: parsed.text, chunks: parsed.chunks };
+        case "error":
+          throw new Error(parsed.message);
+      }
+    }
+    // The stream closed without a result line — the connection was cut.
+    throw new Error("The connection was lost before the transcript finished.");
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Server engine availability (EngineToggle probes this after sign-in)
+// ---------------------------------------------------------------------------
+
+export interface ServerEngineStatus {
+  available: boolean;
+  /** Human-readable explanation when unavailable (e.g. sidecar missing). */
+  reason?: string;
+}
+
+/**
+ * GET /api/transcribe/status → availability. Never rejects: any failure
+ * (401, 404 when the deployment has no server engine, offline) resolves to
+ * unavailable, mirroring fetchAuthStatus's offline-first behavior.
+ */
+export async function fetchServerEngineStatus(
+  fetchFn: FetchLike = defaultFetch,
+): Promise<ServerEngineStatus> {
+  try {
+    const response = await fetchFn("/api/transcribe/status", {
+      method: "GET",
+      credentials: "same-origin",
+    });
+    if (!response.ok) {
+      return { available: false, reason: await serverErrorMessage(response) };
+    }
+    const body: unknown = await response.json().catch(() => null);
+    const status = body as { available?: unknown; reason?: unknown } | null;
+    if (typeof status?.available !== "boolean") return { available: false };
+    return {
+      available: status.available,
+      ...(typeof status.reason === "string" && { reason: status.reason }),
+    };
+  } catch {
+    return { available: false, reason: "The server cannot be reached." };
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Engine choice persistence (EngineToggle, survives reloads)
+// ---------------------------------------------------------------------------
+
+export const ENGINE_CHOICE_KEY = "whisper-engine-choice";
+
+export interface StorageLike {
+  getItem(key: string): string | null;
+  setItem(key: string, value: string): void;
+}
+
+function defaultStorage(): StorageLike | null {
+  try {
+    return (globalThis as { localStorage?: StorageLike }).localStorage ?? null;
+  } catch {
+    return null; // storage access can throw (privacy modes)
+  }
+}
+
+/** Persisted toggle choice; anything unreadable falls back to "browser". */
+export function loadEngineChoice(
+  storage: StorageLike | null = defaultStorage(),
+): EngineKind {
+  try {
+    const value = storage?.getItem(ENGINE_CHOICE_KEY);
+    return value === "server" ? "server" : "browser";
+  } catch {
+    return "browser";
+  }
+}
+
+export function saveEngineChoice(
+  choice: EngineKind,
+  storage: StorageLike | null = defaultStorage(),
+): void {
+  try {
+    storage?.setItem(ENGINE_CHOICE_KEY, choice);
+  } catch {
+    // Not persistable (private mode / quota) — the choice just won't stick.
+  }
+}

src/lib/models.ts

@@ -48,6 +48,14 @@ export interface WhisperModel {
   decoderParamsM: number;
   /** Per-device dtype defaults (D-004 / D-006). */
   defaults: Record<InferenceDevice, DtypeDefault>;
+  /**
+   * Model name the server engine passes to faster-whisper (D-013). These are
+   * faster-whisper's own aliases, which it resolves to the official
+   * CTranslate2 conversions on the Hub (`Systran/faster-whisper-*`;
+   * large-v3-turbo → `mobiuslabsgmbh/faster-whisper-large-v3-turbo`), so the
+   * pinned library version stays the single authority on the exact repo.
+   */
+  fasterWhisperModel: string;
 }
 
 // ---------------------------------------------------------------------------
@@ -121,6 +129,7 @@ const WEBGPU_FP16_Q4: DtypeDefault = {
 export const MODELS: readonly WhisperModel[] = [
   {
     id: "onnx-community/whisper-tiny.en",
+    fasterWhisperModel: "tiny.en",
     displayName: "Whisper Tiny (English)",
     multilingual: false,
     encoderParamsM: 8,
@@ -133,6 +142,7 @@ export const MODELS: readonly WhisperModel[] = [
   },
   {
     id: "onnx-community/whisper-tiny",
+    fasterWhisperModel: "tiny",
     displayName: "Whisper Tiny",
     multilingual: true,
     encoderParamsM: 8,
@@ -141,6 +151,7 @@ export const MODELS: readonly WhisperModel[] = [
   },
   {
     id: "onnx-community/whisper-base.en",
+    fasterWhisperModel: "base.en",
     displayName: "Whisper Base (English)",
     multilingual: false,
     encoderParamsM: 20,
@@ -149,6 +160,7 @@ export const MODELS: readonly WhisperModel[] = [
   },
   {
     id: "onnx-community/whisper-base",
+    fasterWhisperModel: "base",
     displayName: "Whisper Base",
     multilingual: true,
     encoderParamsM: 20,
@@ -157,6 +169,7 @@ export const MODELS: readonly WhisperModel[] = [
   },
   {
     id: "onnx-community/whisper-small.en",
+    fasterWhisperModel: "small.en",
     displayName: "Whisper Small (English)",
     multilingual: false,
     encoderParamsM: 88,
@@ -165,6 +178,7 @@ export const MODELS: readonly WhisperModel[] = [
   },
   {
     id: "onnx-community/whisper-small",
+    fasterWhisperModel: "small",
     displayName: "Whisper Small",
     multilingual: true,
     encoderParamsM: 88,
@@ -173,6 +187,7 @@ export const MODELS: readonly WhisperModel[] = [
   },
   {
     id: "onnx-community/whisper-medium.en",
+    fasterWhisperModel: "medium.en",
     displayName: "Whisper Medium (English)",
     multilingual: false,
     encoderParamsM: 306,
@@ -181,6 +196,7 @@ export const MODELS: readonly WhisperModel[] = [
   },
   {
     id: "onnx-community/whisper-medium",
+    fasterWhisperModel: "medium",
     displayName: "Whisper Medium",
     multilingual: true,
     encoderParamsM: 306,
@@ -189,6 +205,7 @@ export const MODELS: readonly WhisperModel[] = [
   },
   {
     id: "onnx-community/whisper-large-v3-turbo",
+    fasterWhisperModel: "large-v3-turbo",
     displayName: "Whisper Large v3 Turbo",
     multilingual: true,
     encoderParamsM: 635,
@@ -201,6 +218,15 @@ export function getModel(id: string): WhisperModel | undefined {
   return MODELS.find((model) => model.id === id);
 }
 
+/**
+ * faster-whisper model name for a catalog id (server engine, D-013), or
+ * undefined for ids outside the catalog — the transcribe endpoint answers 400
+ * rather than forwarding arbitrary strings to the sidecar.
+ */
+export function getFasterWhisperModel(id: string): string | undefined {
+  return getModel(id)?.fasterWhisperModel;
+}
+
 /** Default dtype for a model on a given device. Unknown ids fall back to q8. */
 export function resolveDefaultDtype(
   modelId: string,