From 69602f046b0ebf3ee1b478f537bdec29e80bf556 Mon Sep 17 00:00:00 2001
From: Lucas Winther
Date: Wed, 29 Jul 2026 09:20:32 +0200
Subject: [PATCH] Usability and test infrastructure updates.
---
docs/Architecture.md | 42 ++++--
docs/Decisions.md | 69 +++++++++
docs/WorkBreakdown.md | 28 +++-
src/components/LiveControls.tsx | 8 ++
src/components/ModelLoadProgress.tsx | 6 +-
src/components/TranscriptView.tsx | 20 ++-
src/index.css | 12 ++
src/index.ts | 27 +++-
src/server/live.ts | 204 +++++++++++++++------------
src/server/transcribe.ts | 34 +++--
tests/transcribe.test.ts | 41 ++++++
11 files changed, 364 insertions(+), 127 deletions(-)
diff --git a/docs/Architecture.md b/docs/Architecture.md
index 71b1362..c539ff0 100644
--- a/docs/Architecture.md
+++ b/docs/Architecture.md
@@ -84,24 +84,26 @@ src/
auth.ts # Passport.js wiring + session middleware adapter
db.ts # bun:sqlite schema + queries
transcribe.ts # server-side ASR endpoint (bridges the sidecar, M5)
+ live.ts # server-engine live mic over WebSocket (D-019, M7)
server-py/ # faster-whisper sidecar (Python, D-013; M5)
tests/ # bun test
```
-(Current state: M0–M3 built — browser transcription with the curated model
+(Current state: M0–M6 built — browser transcription with the curated model
catalog, WebNN→WebGPU→WASM auto-selection (D-017), model switching,
-language/task options,
-and the installable offline PWA (`pwa/`, `scripts/build.ts` production build,
-`dist/` serving in `src/index.ts`; browser-level offline behavior pending the
-manual run in `tests/offline.md`). M4 accounts are built (`server/db.ts`,
-`server/auth.ts`, `/api/auth/*`, `AuthForms`). M7 live microphone
-transcription is built (`lib/live.ts`, `lib/mic.ts`, `LiveControls`, D-016;
-in-browser run pending per `tests/live.md`). M5's server side is built —
-`server-py/sidecar.py`, `server/transcribe.ts` (supervisor +
-`/api/transcribe` + `/status`), `lib/engine.ts` (both engines, D-010), and
-the catalog mapping in `lib/models.ts` — with graceful degradation when
-Python/faster-whisper is absent; the `EngineToggle` component and App wiring
-are the remaining M5 work; see `WorkBreakdown.md`.)
+language/task options, the installable offline PWA (`pwa/`,
+`scripts/build.ts` production build, `dist/` serving in `src/index.ts`;
+browser-level offline behavior pending the manual run in `tests/offline.md`),
+M4 accounts (`server/db.ts`, `server/auth.ts`, `/api/auth/*`, `AuthForms`),
+and the M5 server engine end to end — `server-py/sidecar.py`,
+`server/transcribe.ts` (supervisor + `/api/transcribe` + `/status`),
+`lib/engine.ts` (both engines, D-010), `EngineToggle`, and the catalog
+mapping in `lib/models.ts` — with graceful degradation when
+Python/faster-whisper is absent. M7 live microphone transcription is built
+for both engines: browser (`lib/live.ts`, `lib/mic.ts`, `LiveControls`,
+D-016) and server (`server/live.ts`, D-019); in-browser manual verification
+of both is still pending per `tests/live.md` (no real browser in the dev
+environment). See `WorkBreakdown.md` for the itemized checklist.)
## 4. Key flows
@@ -160,8 +162,18 @@ are the remaining M5 work; see `WorkBreakdown.md`.)
4. Stopping flushes the segmenter, drains pending final passes (merged, with
a "N remaining segments" countdown in the UI), and yields a normal
transcript (text + absolute-timestamped chunks) in `TranscriptView`.
- Everything runs on-device; works offline like the file flow. Server-engine
- live input is a recorded future want (WorkBreakdown M7, D-016).
+ Everything runs on-device; works offline like the file flow.
+5. **Server-engine live (logged in, D-019).** Same `LiveControls` UI, but
+ `App.tsx` opens a `GET /api/transcribe/live` WebSocket instead of running
+ the segmenter locally: the browser only captures/resamples mic audio
+ (`lib/mic.ts`) and streams raw PCM frames over the socket; the *same* pure
+ `LiveTranscriber`/segmenter (`lib/live.ts`) runs server-side
+ (`server/live.ts`), writing each segment to a temp file and bridging it to
+ the faster-whisper sidecar (D-013) the same way `POST /api/transcribe`
+ does. `{type:"update"}` frames carry committed/tentative text back;
+ `{type:"stop"}` drains and closes with a `{type:"result"}` frame. Selected
+ only when signed in with the server engine available (`EngineToggle`);
+ anonymous/offline users always get the browser-engine path above.
### 4.2 Offline (PWA)
- Service worker (`pwa/sw.ts`, hand-rolled — D-007) precaches the built app shell
diff --git a/docs/Decisions.md b/docs/Decisions.md
index a40df3e..047b8fb 100644
--- a/docs/Decisions.md
+++ b/docs/Decisions.md
@@ -836,3 +836,72 @@ baked-in models, either accept runtime downloads for the extra ones (point
`FASTER_WHISPER_DOWNLOAD_ROOT` at `/app/data` for those, per the README's
override note) or extend the Dockerfile's pre-download `RUN` line —
re-measure and re-document the image size either way.
+
+## D-032 — Server-live WebSocket brought in line with the HTTP engine's DI/validation/test conventions
+**Status:** Accepted (2026-07-29)
+D-019's server-engine live transcription (`GET /api/transcribe/live` →
+`src/server/live.ts`) had landed and was wired end-to-end in `App.tsx`, but
+three things about it lagged the standard the rest of M5 set with
+`src/server/transcribe.ts`:
+1. **Not injectable.** `live.ts` imported the real `sidecar` singleton
+ directly and called it from inside the `LiveTranscriber`'s `transcribe`
+ closure, unlike `createTranscribeHandlers(deps)`, whose whole point (per
+ its own comment) is that `Sidecar` is "injectable — tests use a fake".
+ There was no way to unit-test the WebSocket bridge without a real Python
+ sidecar.
+2. **Unvalidated query params.** `POST /api/transcribe`'s `parseParams`
+ validates `language` against `LANGUAGE_RE` and `task` against
+ `"transcribe" | "translate"` before it ever reaches the sidecar protocol.
+ The WS upgrade route in `src/index.ts` skipped both checks —
+ `url.searchParams.get("task") || undefined` forwarded an arbitrary string
+ straight into `LiveSessionData.task` and on to the Python sidecar's
+ JSON-lines protocol. Also skipped: the upfront `sidecar.status()` check the HTTP
+ route makes before doing any work (D-013's "degrade gracefully" —
+ opening a socket that can only report unavailability after the first
+ segment reaches the sidecar is a worse failure mode than a prompt 503).
+3. **Zero test coverage.** No `tests/server-live.*` file existed; every
+ other stateful server module in the app (`transcribe.ts`, `auth.ts`,
+ `db.ts`) has one.
+**Fix:**
+1. `src/server/transcribe.ts` exports `parseLanguageAndTask(url)` (extracted
+ from `parseParams`, same validation, same 400-text-on-string-return
+ contract); `src/index.ts`'s `GET /api/transcribe/live` case now calls it
+ and 400s on an invalid value, then calls `sidecar.status()` and 503s with
+ the reason before upgrading — matching `POST /api/transcribe` exactly.
+2. `src/server/live.ts` exports `createLiveWebSocket(deps: { sidecar:
+ Sidecar })`; `liveWebSocket` (the instance `src/index.ts` wires into
+ `Bun.serve`) is just `createLiveWebSocket({ sidecar })` against the real
+ singleton. `LiveSessionData.user`/`task` are now typed `SessionUser`/
+ `WhisperTask` instead of `any`/`string | undefined` (task always has a
+ value now that the WS route defaults it like the HTTP route does), which
+ let the pre-existing `options.websocket = liveWebSocket as any` cast in
+ `src/index.ts` drop to a plain assignment — it was papering over the
+ untyped `user`/`task` fields, not a real incompatibility with `Bun.serve`'s
+ generic `websocket` option.
+3. `tests/server-live.test.ts` exercises `createLiveWebSocket` against a fake
+ `Sidecar` (same shape as `tests/transcribe.test.ts`'s): a committed
+ segment writes exactly its own PCM bytes to the temp file and cleans up
+ after, model/language/task pass through, a sidecar error surfaces via the
+ `update` message's `error` field instead of crashing the session, a
+ `{"type":"stop"}` frame drains and closes with a `result` message, and a
+ malformed text frame is ignored. `tests/transcribe.test.ts` gained direct
+ cases for `parseLanguageAndTask`.
+**Incidental correctness fix found while rewriting the `transcribe` closure:**
+the original code wrote `new Uint8Array(audio.buffer as ArrayBuffer)` to the
+temp file — the whole underlying `ArrayBuffer`, ignoring `audio.byteOffset`/
+`audio.byteLength`. Every real caller today happens to pass a fresh
+`Float32Array` from `lib/live.ts`'s `concat()` helper (always offset 0, full
+length), so this never manifested, but `LiveTranscribeFn`'s type signature
+(`(audio: Float32Array) => ...`) makes no such promise to callers. Restored
+the original's `audio.byteOffset, audio.byteLength` slice (which the
+pre-refactor code already had — it would have been an actual regression to
+drop it during the DI rewrite).
+**Consequence:** the server-live path now has the same testability and
+input-validation posture as the HTTP path; no behavior change for a valid
+client, since `App.tsx` already only ever sends catalog model ids and
+`"transcribe" | "translate"` task values.
+**Alternatives rejected:** leaving `live.ts` untested and testing only
+through a real spawned sidecar (matches `createSidecar`'s own integration
+tests, but that suite already covers the sidecar process itself — the gap
+was specifically the WebSocket/`LiveTranscriber` glue, which needs no Python
+process to exercise).
diff --git a/docs/WorkBreakdown.md b/docs/WorkBreakdown.md
index cde0254..0be4a50 100644
--- a/docs/WorkBreakdown.md
+++ b/docs/WorkBreakdown.md
@@ -273,7 +273,19 @@ Notes (M5):
(`estimatePeakLoadBytes`) is risky on a phone/tablet, covering the
explicit Download button and the lazy auto-load on file-drop/live-start
alike.
-- [ ] Basic a11y pass (keyboard, focus, aria on progress/live transcript)
+- [x] Basic a11y pass (keyboard, focus, aria on progress/live transcript) —
+ the model-download and transcription `
+ {/* Announce mode transitions for screen-reader users not focused on the
+ button — the button's own label change is only reliably heard by
+ whoever triggered it. */}
+
+ {status === "starting" && "Requesting microphone access"}
+ {status === "listening" && "Live transcription started"}
+ {status === "stopping" && "Finishing live transcription"}
+
);
}
diff --git a/src/components/ModelLoadProgress.tsx b/src/components/ModelLoadProgress.tsx
index cb7c081..3748ed2 100644
--- a/src/components/ModelLoadProgress.tsx
+++ b/src/components/ModelLoadProgress.tsx
@@ -39,7 +39,11 @@ export function ModelLoadProgress({ modelId, progress }: ModelLoadProgressProps)
: "…"}
-
+
);
})}
diff --git a/src/components/TranscriptView.tsx b/src/components/TranscriptView.tsx
index d0cb010..188b832 100644
--- a/src/components/TranscriptView.tsx
+++ b/src/components/TranscriptView.tsx
@@ -1,4 +1,4 @@
-import { useEffect, useState } from "react";
+import { useEffect, useRef, useState } from "react";
import {
exportFilename,
transcriptToSrt,
@@ -64,6 +64,8 @@ export function TranscriptView({
exportName = null,
}: TranscriptViewProps) {
const [copied, setCopied] = useState(false);
+ const headingRef = useRef(null);
+ const prevResultRef = useRef(null);
const text = result?.text ?? partialText;
const copyText = `${text}${tentativeText}`.trim();
const showProgress =
@@ -78,6 +80,16 @@ export function TranscriptView({
return () => clearTimeout(timer);
}, [copied]);
+ // Move focus to the transcript heading the moment a final result lands, so
+ // keyboard/screen-reader users land on the new content instead of having to
+ // hunt for it after a file or live run finishes.
+ useEffect(() => {
+ if (result && !prevResultRef.current) {
+ headingRef.current?.focus();
+ }
+ prevResultRef.current = result;
+ }, [result]);
+
if (!transcribing && !text && !tentativeText) return null;
async function copy() {
@@ -89,7 +101,7 @@ export function TranscriptView({
return (
-
+
{result ? "Transcript" : activeLabel}
{percent !== null && (
{percent}%
@@ -150,6 +162,9 @@ export function TranscriptView({
>
{copied ? "Copied!" : "Copy"}
+
+ {copied ? "Copied to clipboard" : ""}
+
{showProgress && (
@@ -157,6 +172,7 @@ export function TranscriptView({
{formatTimestamp(progress.processedSeconds)} /{" "}
diff --git a/src/index.css b/src/index.css
index 51a9f0c..0a499bd 100644
--- a/src/index.css
+++ b/src/index.css
@@ -16,6 +16,18 @@ body {
min-height: 100vh;
}
+.visually-hidden {
+ position: absolute;
+ width: 1px;
+ height: 1px;
+ padding: 0;
+ margin: -1px;
+ overflow: hidden;
+ clip: rect(0, 0, 0, 0);
+ white-space: nowrap;
+ border: 0;
+}
+
.app {
max-width: 720px;
margin: 0 auto;
diff --git a/src/index.ts b/src/index.ts
index 3893dfa..c5b7837 100644
--- a/src/index.ts
+++ b/src/index.ts
@@ -1,10 +1,11 @@
import { serve } from "bun";
+import { tmpdir } from "node:os";
import { join, normalize, sep } from "node:path";
import { ensureIcons, ICONS_DIR } from "../scripts/generate-icons.ts";
import { injectPrecacheManifest } from "./pwa/precache.ts";
import { auth } from "./server/auth.ts";
import { AUTH_ENABLED } from "./server/flags.ts";
-import { MAX_UPLOAD_BYTES, transcribe } from "./server/transcribe.ts";
+import { MAX_UPLOAD_BYTES, parseLanguageAndTask, sidecar, transcribe } from "./server/transcribe.ts";
import { liveWebSocket } from "./server/live.ts";
import { getFasterWhisperModel } from "./lib/models.ts";
@@ -132,15 +133,27 @@ async function handleApiRequest(request: Request, server: any): Promise) {
- // Initialize the transcriber for this connection.
- const transcriber = new LiveTranscriber({
- ...LIVE_TUNING,
- transcribe: async (audio: Float32Array) => {
- const audioPath = join(
- ws.data.tempDir,
- `whisper-live-${crypto.randomUUID()}.f32`
- );
- const sink = Bun.file(audioPath).writer();
- // audio.buffer is an ArrayBuffer, we need a Uint8Array view
- sink.write(new Uint8Array(audio.buffer as ArrayBuffer, audio.byteOffset, audio.byteLength));
- await sink.end();
+export interface LiveWebSocketDeps {
+ sidecar: Sidecar;
+}
- return new Promise((resolve, reject) => {
- let text = "";
- let chunks: any[] = [];
-
- sidecar.transcribe(
- {
- model: ws.data.model,
- audioPath,
- language: ws.data.language,
- task: ws.data.task,
- },
- (segment) => {
- text += segment.text;
- chunks.push({
- text: segment.text,
- timestamp: [segment.start, segment.end],
- });
- }
- )
- .then(() => resolve({ text: text.trim(), chunks }))
- .catch(reject)
- .finally(() => Bun.file(audioPath).unlink().catch(() => {}));
- });
- },
- onUpdate: (update: LiveUpdate) => {
- ws.send(JSON.stringify({ type: "update", update }));
- },
- });
-
- // Store it on the websocket
- (ws as any).transcriber = transcriber;
- },
+/** Exported for tests (tests/server-live.test.ts), which fake this shape. */
+export type LiveSocket = ServerWebSocket & { transcriber?: LiveTranscriber };
- message(ws: ServerWebSocket, message: string | Buffer | Uint8Array) {
- if (typeof message === "string") {
- let parsed: any;
- try {
- parsed = JSON.parse(message);
- } catch {
- // Not a frame we understand — ignore rather than crash the handler
- // (which would otherwise take the whole connection down for every
- // client sharing this Bun.serve process, not just this session).
+export function createLiveWebSocket(deps: LiveWebSocketDeps) {
+ return {
+ open(ws: LiveSocket) {
+ const transcriber = new LiveTranscriber({
+ ...LIVE_TUNING,
+ transcribe: async (audio) => {
+ const audioPath = join(
+ ws.data.tempDir,
+ `whisper-live-${crypto.randomUUID()}.f32`,
+ );
+ // audio may be a view into a larger buffer — byteOffset/byteLength
+ // must be preserved, not just the underlying buffer.
+ await Bun.write(
+ audioPath,
+ new Uint8Array(audio.buffer as ArrayBuffer, audio.byteOffset, audio.byteLength),
+ );
+ try {
+ const text: string[] = [];
+ const chunks: { text: string; timestamp: [number, number | null] }[] = [];
+ await deps.sidecar.transcribe(
+ {
+ model: ws.data.model,
+ audioPath,
+ language: ws.data.language,
+ task: ws.data.task,
+ },
+ (segment) => {
+ text.push(segment.text);
+ chunks.push({ text: segment.text, timestamp: [segment.start, segment.end] });
+ },
+ );
+ return { text: text.join("").trim(), chunks };
+ } finally {
+ await Bun.file(audioPath).unlink().catch(() => {});
+ }
+ },
+ onUpdate: (update: LiveUpdate) => {
+ ws.send(JSON.stringify({ type: "update", update }));
+ },
+ });
+ ws.transcriber = transcriber;
+ },
+
+ message(ws: LiveSocket, message: string | Buffer) {
+ if (typeof message === "string") {
+ let parsed: { type?: string };
+ try {
+ parsed = JSON.parse(message);
+ } catch {
+ // Not a frame we understand — ignore rather than crash the handler
+ // (which would otherwise take the whole connection down for every
+ // client sharing this Bun.serve process, not just this session).
+ return;
+ }
+ if (parsed.type === "stop") {
+ ws.transcriber
+ ?.stop()
+ .then((result) => {
+ ws.send(JSON.stringify({ type: "result", result }));
+ ws.close();
+ })
+ .catch((err) => {
+ ws.send(
+ JSON.stringify({
+ type: "error",
+ error: err instanceof Error ? err.message : String(err),
+ }),
+ );
+ ws.close();
+ });
+ }
return;
}
- if (parsed.type === "stop") {
- const transcriber = (ws as any).transcriber as LiveTranscriber;
- transcriber.stop().then((result) => {
- ws.send(JSON.stringify({ type: "result", result }));
- ws.close();
- }).catch((err) => {
- ws.send(JSON.stringify({ type: "error", error: err instanceof Error ? err.message : String(err) }));
- ws.close();
- });
- }
- } else {
- // Audio chunk arrived
- const transcriber = (ws as any).transcriber as LiveTranscriber;
- // Convert raw bytes to Float32Array (16kHz mono)
- const buffer = message instanceof Buffer ? message.buffer : message.buffer;
- const byteOffset = message instanceof Buffer ? message.byteOffset : (message as Uint8Array).byteOffset;
- const byteLength = message.byteLength;
- const audio = new Float32Array(buffer, byteOffset, byteLength / 4);
- transcriber.push(audio);
- }
- },
+ // Binary frame: a raw float32 mono 16 kHz PCM chunk (lib/mic.ts already
+ // resampled it on the main thread, same contract as the browser engine).
+ const audio = new Float32Array(
+ message.buffer,
+ message.byteOffset,
+ message.byteLength / 4,
+ );
+ ws.transcriber?.push(audio);
+ },
- close(ws: ServerWebSocket) {
- const transcriber = (ws as any).transcriber as LiveTranscriber;
- transcriber?.stop().catch(() => {});
- },
-};
+ close(ws: LiveSocket) {
+ ws.transcriber?.stop().catch(() => {});
+ },
+ };
+}
+
+// ---------------------------------------------------------------------------
+// Default instance (real sidecar), wired in src/index.ts
+// ---------------------------------------------------------------------------
+
+export const liveWebSocket = createLiveWebSocket({ sidecar });
diff --git a/src/server/transcribe.ts b/src/server/transcribe.ts
index 1740a8d..eb95835 100644
--- a/src/server/transcribe.ts
+++ b/src/server/transcribe.ts
@@ -324,14 +324,16 @@ interface TranscribeParams {
task: WhisperTask;
}
-/** Parse + validate query params; a string is the 400 error text. */
-function parseParams(url: URL): TranscribeParams | string {
- const modelId = url.searchParams.get("model");
- if (!modelId) return "missing model parameter";
- const fasterWhisperModel = getFasterWhisperModel(modelId);
- // Catalog-only (D-006/D-013): never forward arbitrary ids to the sidecar.
- if (!fasterWhisperModel) return `unknown model: ${modelId}`;
-
+/**
+ * Parse + validate the `language`/`task` query params shared by both the
+ * HTTP (`POST /api/transcribe`) and WebSocket (`GET /api/transcribe/live`,
+ * src/server/live.ts) entry points, so a live session can't forward an
+ * unvalidated string into the sidecar protocol just because it skips
+ * `parseParams` below. A string return is the 400 error text.
+ */
+export function parseLanguageAndTask(
+ url: URL,
+): { language?: string; task: WhisperTask } | string {
const rawLanguage = url.searchParams.get("language");
let language: string | undefined;
if (rawLanguage !== null && rawLanguage !== "" && rawLanguage !== "auto") {
@@ -344,7 +346,21 @@ function parseParams(url: URL): TranscribeParams | string {
return `invalid task: ${rawTask}`;
}
- return { modelId, fasterWhisperModel, language, task: rawTask };
+ return { language, task: rawTask };
+}
+
+/** Parse + validate query params; a string is the 400 error text. */
+function parseParams(url: URL): TranscribeParams | string {
+ const modelId = url.searchParams.get("model");
+ if (!modelId) return "missing model parameter";
+ const fasterWhisperModel = getFasterWhisperModel(modelId);
+ // Catalog-only (D-006/D-013): never forward arbitrary ids to the sidecar.
+ if (!fasterWhisperModel) return `unknown model: ${modelId}`;
+
+ const options = parseLanguageAndTask(url);
+ if (typeof options === "string") return options;
+
+ return { modelId, fasterWhisperModel, ...options };
}
/**
diff --git a/tests/transcribe.test.ts b/tests/transcribe.test.ts
index 0189e39..2b54b66 100644
--- a/tests/transcribe.test.ts
+++ b/tests/transcribe.test.ts
@@ -5,6 +5,7 @@ import { join } from "node:path";
import {
createSidecar,
createTranscribeHandlers,
+ parseLanguageAndTask,
type Sidecar,
type SidecarSegment,
type SidecarTranscribeRequest,
@@ -124,6 +125,46 @@ async function readStreamLines(response: Response): Promise {
+ test("defaults to transcribe with no language", () => {
+ expect(parseLanguageAndTask(new URL("http://localhost/x"))).toEqual({
+ language: undefined,
+ task: "transcribe",
+ });
+ });
+
+ test("accepts a valid language and task", () => {
+ expect(parseLanguageAndTask(new URL("http://localhost/x?language=da&task=translate"))).toEqual(
+ { language: "da", task: "translate" },
+ );
+ });
+
+ test('"auto"/empty language means no language override', () => {
+ expect(parseLanguageAndTask(new URL("http://localhost/x?language=auto"))).toEqual({
+ language: undefined,
+ task: "transcribe",
+ });
+ expect(parseLanguageAndTask(new URL("http://localhost/x?language="))).toEqual({
+ language: undefined,
+ task: "transcribe",
+ });
+ });
+
+ test("rejects an invalid language or task", () => {
+ expect(parseLanguageAndTask(new URL("http://localhost/x?language=Danish!"))).toBe(
+ "invalid language: Danish!",
+ );
+ expect(parseLanguageAndTask(new URL("http://localhost/x?task=summarize"))).toBe(
+ "invalid task: summarize",
+ );
+ });
+});
+
describe("GET /api/transcribe/status", () => {
test("401 without a session", async () => {
const handlers = createTranscribeHandlers({