Start on live transcription and python sidecar (faster-whisper)
This commit is contained in:
Lucas Winther
@@ -0,0 +1,373 @@
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import {
|
||||
BrowserEngine,
|
||||
ENGINE_CHOICE_KEY,
|
||||
ServerEngine,
|
||||
fetchServerEngineStatus,
|
||||
isTranscribeStreamLine,
|
||||
loadEngineChoice,
|
||||
saveEngineChoice,
|
||||
type StorageLike,
|
||||
type WorkerLike,
|
||||
} from "../src/lib/engine";
|
||||
import type { FetchLike } from "../src/lib/authClient";
|
||||
import type { TranscriptChunk } from "../src/worker/protocol";
|
||||
|
||||
// D-010: both engines behind one interface, testable without a DOM/worker/server.
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// BrowserEngine — requestId correlation against a fake worker
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
class FakeWorker implements WorkerLike {
|
||||
posted: { message: Record<string, unknown>; transfer?: Transferable[] }[] = [];
|
||||
private listeners = { message: [] as ((e: { data: unknown }) => void)[], error: [] as ((e: { message?: string }) => void)[] };
|
||||
|
||||
postMessage(message: unknown, transfer?: Transferable[]): void {
|
||||
this.posted.push({ message: message as Record<string, unknown>, transfer });
|
||||
}
|
||||
|
||||
addEventListener(
|
||||
type: "message" | "error",
|
||||
listener: (event: { data?: unknown; message?: string }) => void,
|
||||
): void {
|
||||
this.listeners[type].push(listener);
|
||||
}
|
||||
|
||||
emit(data: unknown): void {
|
||||
for (const listener of this.listeners.message) listener({ data });
|
||||
}
|
||||
|
||||
emitError(message: string): void {
|
||||
for (const listener of this.listeners.error) listener({ message });
|
||||
}
|
||||
|
||||
lastRequestId(): string {
|
||||
return this.posted.at(-1)!.message.requestId as string;
|
||||
}
|
||||
}
|
||||
|
||||
const CHUNKS: TranscriptChunk[] = [{ text: " hi", timestamp: [0, 1] }];
|
||||
|
||||
describe("BrowserEngine", () => {
|
||||
test("posts a transcribe message with a requestId and transfers the buffer", () => {
|
||||
const worker = new FakeWorker();
|
||||
const engine = new BrowserEngine(worker);
|
||||
const audio = new Float32Array([0.1, 0.2]);
|
||||
void engine.transcribe(audio, { language: "da", task: "translate" });
|
||||
|
||||
expect(worker.posted).toHaveLength(1);
|
||||
const { message, transfer } = worker.posted[0]!;
|
||||
expect(message.type).toBe("transcribe");
|
||||
expect(message.language).toBe("da");
|
||||
expect(message.task).toBe("translate");
|
||||
expect(typeof message.requestId).toBe("string");
|
||||
expect(transfer).toEqual([audio.buffer as ArrayBuffer]);
|
||||
});
|
||||
|
||||
test('omits language "auto" and the modelId (worker holds the model)', () => {
|
||||
const worker = new FakeWorker();
|
||||
const engine = new BrowserEngine(worker);
|
||||
void engine.transcribe(new Float32Array(4), {
|
||||
language: "auto",
|
||||
modelId: "onnx-community/whisper-tiny.en",
|
||||
});
|
||||
const { message } = worker.posted[0]!;
|
||||
expect("language" in message).toBe(false);
|
||||
expect("modelId" in message).toBe(false);
|
||||
});
|
||||
|
||||
test("resolves on the matching result and routes partials to onPartial", async () => {
|
||||
const worker = new FakeWorker();
|
||||
const engine = new BrowserEngine(worker);
|
||||
const fragments: string[] = [];
|
||||
const promise = engine.transcribe(new Float32Array(4), {
|
||||
onPartial: (fragment) => fragments.push(fragment),
|
||||
});
|
||||
const requestId = worker.lastRequestId();
|
||||
|
||||
// Noise the engine must ignore: other ids, id-less file-flow messages,
|
||||
// load progress.
|
||||
worker.emit({ type: "partial", text: "IGNORED", requestId: "other" });
|
||||
worker.emit({ type: "partial", text: "IGNORED" });
|
||||
worker.emit({ type: "load-progress", status: "ready", device: "wasm" });
|
||||
|
||||
worker.emit({ type: "partial", text: "Hel", requestId });
|
||||
worker.emit({ type: "partial", text: "lo", requestId });
|
||||
worker.emit({ type: "result", text: "Hello", chunks: CHUNKS, requestId });
|
||||
worker.emit({ type: "result", text: "STALE", chunks: [], requestId: "other" });
|
||||
|
||||
expect(await promise).toEqual({ text: "Hello", chunks: CHUNKS });
|
||||
expect(fragments).toEqual(["Hel", "lo"]);
|
||||
});
|
||||
|
||||
test("rejects on a matching error message", async () => {
|
||||
const worker = new FakeWorker();
|
||||
const engine = new BrowserEngine(worker);
|
||||
const promise = engine.transcribe(new Float32Array(4));
|
||||
worker.emit({
|
||||
type: "error",
|
||||
stage: "transcribe",
|
||||
message: "out of memory",
|
||||
requestId: worker.lastRequestId(),
|
||||
});
|
||||
expect(promise).rejects.toThrow("out of memory");
|
||||
});
|
||||
|
||||
test("a worker crash rejects every in-flight request", async () => {
|
||||
const worker = new FakeWorker();
|
||||
const engine = new BrowserEngine(worker);
|
||||
const first = engine.transcribe(new Float32Array(4));
|
||||
const second = engine.transcribe(new Float32Array(4));
|
||||
worker.emitError("worker exploded");
|
||||
expect(first).rejects.toThrow("worker exploded");
|
||||
expect(second).rejects.toThrow("worker exploded");
|
||||
});
|
||||
|
||||
test("two engines can share one worker without stealing responses", async () => {
|
||||
const worker = new FakeWorker();
|
||||
const engineA = new BrowserEngine(worker);
|
||||
const engineB = new BrowserEngine(worker);
|
||||
const promiseA = engineA.transcribe(new Float32Array(4));
|
||||
const promiseB = engineB.transcribe(new Float32Array(4));
|
||||
const [idA, idB] = worker.posted.map(
|
||||
(post) => post.message.requestId as string,
|
||||
);
|
||||
expect(idA).not.toBe(idB);
|
||||
worker.emit({ type: "result", text: "for A", chunks: [], requestId: idA });
|
||||
worker.emit({ type: "result", text: "for B", chunks: [], requestId: idB });
|
||||
expect((await promiseA).text).toBe("for A");
|
||||
expect((await promiseB).text).toBe("for B");
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// ServerEngine — ndjson stream over an injected fetch
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function ndjsonResponse(lines: unknown[], status = 200): Response {
|
||||
const body = lines.map((line) => `${JSON.stringify(line)}\n`).join("");
|
||||
return new Response(body, {
|
||||
status,
|
||||
headers: { "Content-Type": "application/x-ndjson" },
|
||||
});
|
||||
}
|
||||
|
||||
describe("ServerEngine", () => {
|
||||
test("streams segments to onPartial and resolves with the result line", async () => {
|
||||
const calls: { url: string; init?: RequestInit }[] = [];
|
||||
const fetchFn: FetchLike = async (url, init) => {
|
||||
calls.push({ url, init });
|
||||
return ndjsonResponse([
|
||||
{ type: "segment", text: " Hello", start: 0, end: 1.5 },
|
||||
{ type: "segment", text: " world.", start: 1.5, end: 3 },
|
||||
{ type: "result", text: "Hello world.", chunks: CHUNKS },
|
||||
]);
|
||||
};
|
||||
const engine = new ServerEngine(fetchFn);
|
||||
const fragments: string[] = [];
|
||||
const audio = new Float32Array([0.5, -0.5, 0.25]);
|
||||
const result = await engine.transcribe(audio, {
|
||||
modelId: "onnx-community/whisper-tiny.en",
|
||||
language: "en",
|
||||
task: "transcribe",
|
||||
onPartial: (fragment) => fragments.push(fragment),
|
||||
});
|
||||
|
||||
expect(result).toEqual({ text: "Hello world.", chunks: CHUNKS });
|
||||
expect(fragments).toEqual([" Hello", " world."]);
|
||||
|
||||
const call = calls[0]!;
|
||||
const url = new URL(call.url, "http://localhost");
|
||||
expect(url.pathname).toBe("/api/transcribe");
|
||||
expect(url.searchParams.get("model")).toBe("onnx-community/whisper-tiny.en");
|
||||
expect(url.searchParams.get("language")).toBe("en");
|
||||
expect(url.searchParams.get("task")).toBe("transcribe");
|
||||
expect(call.init?.method).toBe("POST");
|
||||
expect(call.init?.credentials).toBe("same-origin");
|
||||
// The body must be the raw PCM bytes of the Float32Array.
|
||||
const sent = new Float32Array((call.init?.body as Uint8Array).buffer.slice(0));
|
||||
expect([...sent]).toEqual([...audio]);
|
||||
});
|
||||
|
||||
test('drops language "auto" from the query', async () => {
|
||||
let url = "";
|
||||
const engine = new ServerEngine(async (input) => {
|
||||
url = input;
|
||||
return ndjsonResponse([{ type: "result", text: "", chunks: [] }]);
|
||||
});
|
||||
await engine.transcribe(new Float32Array(4), {
|
||||
modelId: "onnx-community/whisper-tiny",
|
||||
language: "auto",
|
||||
});
|
||||
expect(url).not.toContain("language");
|
||||
});
|
||||
|
||||
test("requires a modelId", () => {
|
||||
const engine = new ServerEngine(async () => ndjsonResponse([]));
|
||||
expect(engine.transcribe(new Float32Array(4))).rejects.toThrow(/model id/);
|
||||
});
|
||||
|
||||
test("rejects with the in-stream error line's message", () => {
|
||||
const engine = new ServerEngine(async () =>
|
||||
ndjsonResponse([
|
||||
{ type: "segment", text: " partial", start: 0, end: 1 },
|
||||
{ type: "error", message: "sidecar exploded" },
|
||||
]),
|
||||
);
|
||||
expect(
|
||||
engine.transcribe(new Float32Array(4), { modelId: "onnx-community/whisper-tiny" }),
|
||||
).rejects.toThrow("sidecar exploded");
|
||||
});
|
||||
|
||||
test("maps HTTP statuses to UI-safe messages", async () => {
|
||||
const cases: [number, RegExp][] = [
|
||||
[401, /sign in/i],
|
||||
[404, /not available/i],
|
||||
[413, /too large|limit/i],
|
||||
[503, /unavailable/i],
|
||||
];
|
||||
for (const [status, pattern] of cases) {
|
||||
const engine = new ServerEngine(async () => new Response("{}", { status }));
|
||||
expect(
|
||||
engine.transcribe(new Float32Array(4), {
|
||||
modelId: "onnx-community/whisper-tiny",
|
||||
}),
|
||||
).rejects.toThrow(pattern);
|
||||
}
|
||||
});
|
||||
|
||||
test("surfaces the server's {error} detail on 503", () => {
|
||||
const engine = new ServerEngine(async () =>
|
||||
new Response(JSON.stringify({ error: "faster-whisper is not installed" }), {
|
||||
status: 503,
|
||||
}),
|
||||
);
|
||||
expect(
|
||||
engine.transcribe(new Float32Array(4), { modelId: "onnx-community/whisper-tiny" }),
|
||||
).rejects.toThrow(/faster-whisper is not installed/);
|
||||
});
|
||||
|
||||
test("a stream that ends without a result line is a lost connection", () => {
|
||||
const engine = new ServerEngine(async () =>
|
||||
ndjsonResponse([{ type: "segment", text: " cut", start: 0, end: 1 }]),
|
||||
);
|
||||
expect(
|
||||
engine.transcribe(new Float32Array(4), { modelId: "onnx-community/whisper-tiny" }),
|
||||
).rejects.toThrow(/connection was lost/);
|
||||
});
|
||||
|
||||
test("network failure gets a friendly message", () => {
|
||||
const engine = new ServerEngine(async () => {
|
||||
throw new TypeError("fetch failed");
|
||||
});
|
||||
expect(
|
||||
engine.transcribe(new Float32Array(4), { modelId: "onnx-community/whisper-tiny" }),
|
||||
).rejects.toThrow(/could not reach the server/i);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Wire-format guard
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe("isTranscribeStreamLine", () => {
|
||||
test("accepts the three line shapes", () => {
|
||||
expect(
|
||||
isTranscribeStreamLine({ type: "segment", text: " a", start: 0, end: 1 }),
|
||||
).toBe(true);
|
||||
expect(isTranscribeStreamLine({ type: "result", text: "a", chunks: [] })).toBe(true);
|
||||
expect(isTranscribeStreamLine({ type: "error", message: "x" })).toBe(true);
|
||||
});
|
||||
|
||||
test("rejects malformed lines", () => {
|
||||
expect(isTranscribeStreamLine(null)).toBe(false);
|
||||
expect(isTranscribeStreamLine({ type: "segment", text: " a" })).toBe(false);
|
||||
expect(isTranscribeStreamLine({ type: "result", text: "a" })).toBe(false);
|
||||
expect(isTranscribeStreamLine({ type: "unknown" })).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Availability probe
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe("fetchServerEngineStatus", () => {
|
||||
test("passes through an available status", async () => {
|
||||
const status = await fetchServerEngineStatus(async () =>
|
||||
new Response(JSON.stringify({ available: true }), { status: 200 }),
|
||||
);
|
||||
expect(status).toEqual({ available: true });
|
||||
});
|
||||
|
||||
test("passes through unavailable + reason", async () => {
|
||||
const status = await fetchServerEngineStatus(async () =>
|
||||
new Response(
|
||||
JSON.stringify({ available: false, reason: "sidecar missing" }),
|
||||
{ status: 200 },
|
||||
),
|
||||
);
|
||||
expect(status).toEqual({ available: false, reason: "sidecar missing" });
|
||||
});
|
||||
|
||||
test("404 (deployment without the feature) resolves unavailable", async () => {
|
||||
const status = await fetchServerEngineStatus(async () =>
|
||||
new Response("Not Found", { status: 404 }),
|
||||
);
|
||||
expect(status.available).toBe(false);
|
||||
});
|
||||
|
||||
test("never rejects on network failure", async () => {
|
||||
const status = await fetchServerEngineStatus(async () => {
|
||||
throw new TypeError("offline");
|
||||
});
|
||||
expect(status.available).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Engine choice persistence
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function fakeStorage(initial: Record<string, string> = {}): StorageLike & {
|
||||
data: Record<string, string>;
|
||||
} {
|
||||
const data = { ...initial };
|
||||
return {
|
||||
data,
|
||||
getItem: (key) => (key in data ? data[key]! : null),
|
||||
setItem: (key, value) => {
|
||||
data[key] = value;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
describe("engine choice persistence", () => {
|
||||
test("round-trips through storage", () => {
|
||||
const storage = fakeStorage();
|
||||
saveEngineChoice("server", storage);
|
||||
expect(storage.data[ENGINE_CHOICE_KEY]).toBe("server");
|
||||
expect(loadEngineChoice(storage)).toBe("server");
|
||||
});
|
||||
|
||||
test('defaults to "browser" for missing or garbage values', () => {
|
||||
expect(loadEngineChoice(fakeStorage())).toBe("browser");
|
||||
expect(
|
||||
loadEngineChoice(fakeStorage({ [ENGINE_CHOICE_KEY]: "cloud" })),
|
||||
).toBe("browser");
|
||||
expect(loadEngineChoice(null)).toBe("browser");
|
||||
});
|
||||
|
||||
test("throwing storage is survived", () => {
|
||||
const storage: StorageLike = {
|
||||
getItem: () => {
|
||||
throw new Error("denied");
|
||||
},
|
||||
setItem: () => {
|
||||
throw new Error("denied");
|
||||
},
|
||||
};
|
||||
expect(loadEngineChoice(storage)).toBe("browser");
|
||||
expect(() => saveEngineChoice("server", storage)).not.toThrow();
|
||||
});
|
||||
});
|
||||
Vendored
+58
@@ -0,0 +1,58 @@
|
||||
"""Protocol-faithful stand-in for server-py/sidecar.py (no faster-whisper).
|
||||
|
||||
Used by tests/transcribe.test.ts to exercise the real Bun-side supervisor
|
||||
(createSidecar) against an actual child process. Behavior is keyed off the
|
||||
requested model name:
|
||||
|
||||
"ok" two segments (the second reports the audio byte count,
|
||||
language and task for pass-through assertions), then done
|
||||
"boom" an error line
|
||||
"die" exit(1) mid-request (crash simulation)
|
||||
"wait-for-cancel" one segment, then block until the next stdin line
|
||||
(the cancel), then done {cancelled: true}
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
|
||||
def emit(message):
|
||||
sys.stdout.write(json.dumps(message) + "\n")
|
||||
sys.stdout.flush()
|
||||
|
||||
|
||||
emit({"type": "ready", "version": "fake"})
|
||||
|
||||
for line in sys.stdin:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
message = json.loads(line)
|
||||
kind = message.get("type")
|
||||
if kind == "shutdown":
|
||||
break
|
||||
if kind != "transcribe":
|
||||
continue
|
||||
|
||||
request_id = message.get("id")
|
||||
model = message.get("model")
|
||||
if model == "boom":
|
||||
emit({"type": "error", "id": request_id, "message": "kaboom"})
|
||||
elif model == "die":
|
||||
sys.exit(1)
|
||||
elif model == "wait-for-cancel":
|
||||
emit({"type": "segment", "id": request_id, "start": 0.0, "end": 1.0,
|
||||
"text": " before cancel"})
|
||||
next_line = json.loads(sys.stdin.readline())
|
||||
cancelled = (next_line.get("type") == "cancel"
|
||||
and next_line.get("id") == request_id)
|
||||
emit({"type": "done", "id": request_id, "cancelled": cancelled})
|
||||
else:
|
||||
size = os.path.getsize(message["audio_path"])
|
||||
emit({"type": "segment", "id": request_id, "start": 0.0, "end": 1.5,
|
||||
"text": " Hello"})
|
||||
emit({"type": "segment", "id": request_id, "start": 1.5, "end": 3.0,
|
||||
"text": " world. [model=%s bytes=%d lang=%s task=%s]"
|
||||
% (model, size, message.get("language"), message.get("task"))})
|
||||
emit({"type": "done", "id": request_id, "cancelled": False})
|
||||
@@ -0,0 +1,77 @@
|
||||
# Manual live-transcription verification (M7, D-016)
|
||||
|
||||
The segment loop (`src/lib/live.ts`) and the protocol changes are covered by
|
||||
unit tests; microphone capture, the AudioWorklet path, and end-to-end live
|
||||
quality can only be exercised in a real browser. This is that procedure.
|
||||
Re-run it after any change to `src/lib/live.ts`, `src/lib/mic.ts`,
|
||||
`src/components/LiveControls.tsx`, or the live wiring in `src/App.tsx`.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- A machine with a working microphone.
|
||||
- Chrome/Edge (WebGPU) and ideally also Firefox and Safari — the three differ
|
||||
exactly where mic capture is fragile (context sample rates, AudioWorklet
|
||||
Blob-URL modules; Safari exercises the ScriptProcessorNode fallback if the
|
||||
worklet path fails there).
|
||||
- `getUserMedia` requires a secure context; `http://localhost` qualifies:
|
||||
```sh
|
||||
bun dev # or: bun run build && bun run start
|
||||
```
|
||||
|
||||
## 1. Permission flow
|
||||
|
||||
1. Open the app, pick `whisper-tiny.en`, click **Start live transcription**.
|
||||
2. The browser prompts for the microphone. **Deny** it: the button must return
|
||||
to idle and the error "Microphone access was denied…" must appear; the app
|
||||
stays usable (file flow unaffected).
|
||||
3. Click again and **allow**: button becomes **Stop** with a pulsing red dot;
|
||||
the browser's mic indicator turns on. If the model wasn't downloaded yet,
|
||||
the download progress panel appears and live text simply starts once it is
|
||||
ready (segment passes queue in the worker).
|
||||
|
||||
## 2. Live transcription behavior
|
||||
|
||||
1. Speak a few sentences with natural pauses. Expect:
|
||||
- While speaking: dimmed italic *tentative* text appears within a couple of
|
||||
seconds (tiny.en; slower models lag proportionally) and refreshes.
|
||||
- After a ~1 s pause: the sentence commits — the text turns from italic to
|
||||
regular ("committed") and no longer changes.
|
||||
- Silence alone (say nothing for 30 s) must produce **no** text — the
|
||||
energy gate drops silent segments, so no hallucinated "thank you."-style
|
||||
output.
|
||||
2. DevTools → Network: **zero** requests while speaking (weights may download
|
||||
at the start; the audio itself never leaves the device).
|
||||
3. During a live session, ModelPicker / options / dropzone are disabled.
|
||||
4. Model choice sanity: repeat briefly with a multilingual model + a fixed
|
||||
language and confirm the language/task options are honored.
|
||||
|
||||
## 3. Stop & result
|
||||
|
||||
1. Click **Stop** mid-sentence. The button shows "Finishing…" while the last
|
||||
segment gets its final pass, then the normal **Transcript** panel appears:
|
||||
full text, Copy button, and a Timestamps section whose segment times are
|
||||
**absolute** (a sentence spoken ~10 s in must not be timestamped 0:00).
|
||||
2. The browser's mic indicator must turn off immediately on Stop.
|
||||
3. Start a new live session afterwards, then also transcribe a file — both
|
||||
must still work (session cleanup, request correlation).
|
||||
|
||||
## 4. Quality/latency knobs (only if needed)
|
||||
|
||||
If commits are too eager/lazy in real rooms, tune `LiveSegmenterOptions`
|
||||
(`speechRms` gate 0.015, `commitSilenceSeconds` 0.75, `maxSegmentSeconds` 25,
|
||||
`minSpeechSeconds` 0.25) and the `LiveTranscriber` throttle
|
||||
(`tentativeIntervalMs` 1500) — all injected from `src/App.tsx` defaults, all
|
||||
unit-tested in `tests/live.test.ts`. Record any tuned values in D-016.
|
||||
|
||||
## 5. Offline live transcription
|
||||
|
||||
With the PWA installed and a model cached (see `tests/offline.md`), go
|
||||
offline (airplane mode) and run a live session — it must work identically:
|
||||
capture, worklet (Blob URL, no network), inference, stop/transcript.
|
||||
|
||||
## Recording the result
|
||||
|
||||
When this passes in a real browser, flip the `[~]` manual-verification item
|
||||
in `WorkBreakdown.md` M7 to `[x]` and note browser + version here.
|
||||
|
||||
- [ ] Verified on: _browser/version, date_
|
||||
@@ -0,0 +1,313 @@
|
||||
import { describe, test, expect } from "bun:test";
|
||||
import {
|
||||
LiveSegmenter,
|
||||
LiveTranscriber,
|
||||
rms,
|
||||
type LiveTranscribeResult,
|
||||
type LiveUpdate,
|
||||
} from "../src/lib/live";
|
||||
|
||||
// Pure segment-loop logic only (D-016) — no mic, no worker, no model. Audio
|
||||
// is synthetic: "speech" is a ±0.1 square wave (RMS 0.1, far above the 0.015
|
||||
// gate), "silence" is zeros.
|
||||
|
||||
const RATE = 16_000;
|
||||
const FRAME = 512; // default frameSeconds 0.032 × 16 kHz
|
||||
|
||||
function speech(seconds: number): Float32Array {
|
||||
const out = new Float32Array(Math.round(seconds * RATE));
|
||||
for (let i = 0; i < out.length; i++) out[i] = i % 2 === 0 ? 0.1 : -0.1;
|
||||
return out;
|
||||
}
|
||||
|
||||
function silence(seconds: number): Float32Array {
|
||||
return new Float32Array(Math.round(seconds * RATE));
|
||||
}
|
||||
|
||||
/** Let pending promise chains (and a macrotask) settle. */
|
||||
function tick(): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, 0));
|
||||
}
|
||||
|
||||
describe("rms", () => {
|
||||
test("zero for empty and silent blocks, level for a square wave", () => {
|
||||
expect(rms(new Float32Array(0))).toBe(0);
|
||||
expect(rms(new Float32Array(100))).toBe(0);
|
||||
expect(rms(speech(0.1))).toBeCloseTo(0.1, 5);
|
||||
});
|
||||
});
|
||||
|
||||
describe("LiveSegmenter", () => {
|
||||
test("pure silence never produces a segment", () => {
|
||||
const segmenter = new LiveSegmenter();
|
||||
expect(segmenter.push(silence(3))).toEqual([]);
|
||||
expect(segmenter.currentSegmentId).toBeNull();
|
||||
expect(segmenter.currentSegmentSamples).toBe(0);
|
||||
expect(segmenter.currentSegment()).toBeNull();
|
||||
expect(segmenter.flush()).toBeNull();
|
||||
});
|
||||
|
||||
test("speech followed by trailing silence commits with pre-roll", () => {
|
||||
const segmenter = new LiveSegmenter();
|
||||
// 16 frames silence, 32 frames speech, then silence until commit.
|
||||
expect(segmenter.push(silence(0.512))).toEqual([]);
|
||||
expect(segmenter.push(speech(1.024))).toEqual([]);
|
||||
expect(segmenter.currentSegmentId).not.toBeNull();
|
||||
|
||||
const committed = segmenter.push(silence(1.024));
|
||||
expect(committed).toHaveLength(1);
|
||||
const segment = committed[0]!;
|
||||
expect(segment.reason).toBe("silence");
|
||||
// Pre-roll keeps 10 frames (0.3 s / 0.032 s) of the leading silence.
|
||||
expect(segment.startSample).toBe((16 - 10) * FRAME);
|
||||
// 10 pre-roll + 32 speech + 24 trailing-silence frames (ceil(0.75/0.032)).
|
||||
expect(segment.audio.length).toBe((10 + 32 + 24) * FRAME);
|
||||
// Back to idle after the commit.
|
||||
expect(segmenter.currentSegmentId).toBeNull();
|
||||
});
|
||||
|
||||
test("near-silent blips are dropped, not committed", () => {
|
||||
const segmenter = new LiveSegmenter();
|
||||
// 4 speech frames = 0.128 s < minSpeechSeconds 0.25 s.
|
||||
expect(segmenter.push(speech(0.128))).toEqual([]);
|
||||
expect(segmenter.currentSegmentId).not.toBeNull();
|
||||
expect(segmenter.push(silence(2))).toEqual([]);
|
||||
expect(segmenter.currentSegmentId).toBeNull();
|
||||
expect(segmenter.flush()).toBeNull();
|
||||
});
|
||||
|
||||
test("continuous speech commits at the max segment length and restarts", () => {
|
||||
const segmenter = new LiveSegmenter({ maxSegmentSeconds: 2 });
|
||||
const committed = segmenter.push(speech(3));
|
||||
expect(committed).toHaveLength(1);
|
||||
expect(committed[0]!.reason).toBe("max-length");
|
||||
expect(committed[0]!.startSample).toBe(0);
|
||||
// 2 s cap → first frame count at/over 32 000 samples.
|
||||
expect(committed[0]!.audio.length).toBe(Math.ceil((2 * RATE) / FRAME) * FRAME);
|
||||
// The rest of the speech opened a fresh segment.
|
||||
expect(segmenter.currentSegmentId).not.toBeNull();
|
||||
expect(segmenter.currentSegmentSamples).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
test("flush commits the in-progress segment, including the sub-frame tail", () => {
|
||||
const segmenter = new LiveSegmenter();
|
||||
// 1 s + 100 samples: the 100-sample tail is smaller than one frame.
|
||||
segmenter.push(speech(1));
|
||||
segmenter.push(speech(100 / RATE));
|
||||
const flushed = segmenter.flush();
|
||||
expect(flushed).not.toBeNull();
|
||||
expect(flushed!.reason).toBe("flush");
|
||||
expect(flushed!.audio.length).toBe(Math.round(1 * RATE) + 100);
|
||||
expect(segmenter.currentSegmentId).toBeNull();
|
||||
});
|
||||
|
||||
test("currentSegment snapshots the in-progress audio", () => {
|
||||
const segmenter = new LiveSegmenter();
|
||||
segmenter.push(speech(1.024));
|
||||
expect(segmenter.currentSegmentSamples).toBe(32 * FRAME);
|
||||
const snapshot = segmenter.currentSegment();
|
||||
expect(snapshot).not.toBeNull();
|
||||
expect(snapshot!.audio.length).toBe(32 * FRAME);
|
||||
expect(snapshot!.startSample).toBe(0);
|
||||
expect(rms(snapshot!.audio)).toBeCloseTo(0.1, 5);
|
||||
});
|
||||
|
||||
test("frame splitting is independent of push block sizes", () => {
|
||||
const bulk = new LiveSegmenter();
|
||||
const dribble = new LiveSegmenter();
|
||||
const audio = speech(1.024);
|
||||
bulk.push(audio);
|
||||
for (let i = 0; i < audio.length; i += 100) {
|
||||
dribble.push(audio.slice(i, Math.min(i + 100, audio.length)));
|
||||
}
|
||||
expect(dribble.currentSegmentSamples).toBe(bulk.currentSegmentSamples);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// LiveTranscriber — fake engine with manually resolved passes
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
interface FakeCall {
|
||||
audio: Float32Array;
|
||||
resolve: (result: LiveTranscribeResult) => void;
|
||||
reject: (error: Error) => void;
|
||||
}
|
||||
|
||||
function fakeEngine() {
|
||||
const calls: FakeCall[] = [];
|
||||
const updates: LiveUpdate[] = [];
|
||||
let now = 0;
|
||||
return {
|
||||
calls,
|
||||
updates,
|
||||
advance(ms: number) {
|
||||
now += ms;
|
||||
},
|
||||
make(options: { tentativeIntervalMs?: number; minTentativeSeconds?: number } = {}) {
|
||||
return new LiveTranscriber({
|
||||
transcribe: (audio) =>
|
||||
new Promise<LiveTranscribeResult>((resolve, reject) => {
|
||||
calls.push({ audio, resolve, reject });
|
||||
}),
|
||||
onUpdate: (update) => updates.push(update),
|
||||
now: () => now,
|
||||
...options,
|
||||
});
|
||||
},
|
||||
last() {
|
||||
return updates[updates.length - 1]!;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
describe("LiveTranscriber", () => {
|
||||
test("runs a throttled tentative pass over the growing segment", async () => {
|
||||
const engine = fakeEngine();
|
||||
const live = engine.make();
|
||||
|
||||
live.push(speech(1.5));
|
||||
expect(engine.calls).toHaveLength(1); // ≥ minTentativeSeconds → first pass
|
||||
expect(engine.calls[0]!.audio.length).toBeGreaterThanOrEqual(1.4 * RATE);
|
||||
|
||||
// Single-flight: more audio while a pass is running starts nothing new.
|
||||
live.push(speech(0.5));
|
||||
expect(engine.calls).toHaveLength(1);
|
||||
|
||||
engine.calls[0]!.resolve({ text: " hello", chunks: [] });
|
||||
await tick();
|
||||
expect(engine.last().tentativeText).toBe(" hello");
|
||||
expect(engine.last().committedText).toBe("");
|
||||
|
||||
// Same segment, interval not elapsed → still no second pass on push.
|
||||
live.push(speech(0.2));
|
||||
expect(engine.calls).toHaveLength(1);
|
||||
|
||||
// Interval elapsed + new samples → refresh.
|
||||
engine.advance(2000);
|
||||
live.push(speech(0.2));
|
||||
expect(engine.calls).toHaveLength(2);
|
||||
});
|
||||
|
||||
test("committed segments get a final pass that freezes their text", async () => {
|
||||
const engine = fakeEngine();
|
||||
const live = engine.make();
|
||||
|
||||
live.push(speech(1.5));
|
||||
engine.calls[0]!.resolve({ text: " helo", chunks: [] });
|
||||
await tick();
|
||||
|
||||
// Trailing silence commits the segment; tentative text turns provisional.
|
||||
live.push(silence(1.5));
|
||||
expect(engine.last().committedText).toBe("helo");
|
||||
expect(engine.last().pendingFinalPasses).toBe(1);
|
||||
expect(engine.calls).toHaveLength(2); // final pass started immediately
|
||||
|
||||
engine.calls[1]!.resolve({
|
||||
text: " hello world",
|
||||
chunks: [{ text: " hello world", timestamp: [0, 1.4] }],
|
||||
});
|
||||
await tick();
|
||||
expect(engine.last().committedText).toBe("hello world");
|
||||
expect(engine.last().tentativeText).toBe("");
|
||||
expect(engine.last().pendingFinalPasses).toBe(0);
|
||||
// Speech started at sample 0 → no offset on the chunk timestamps.
|
||||
expect(engine.last().chunks[0]!.timestamp[0]).toBeCloseTo(0, 5);
|
||||
expect(engine.last().chunks[0]!.timestamp[1]!).toBeCloseTo(1.4, 5);
|
||||
});
|
||||
|
||||
test("chunk timestamps are offset by the segment's session position", async () => {
|
||||
const engine = fakeEngine();
|
||||
const live = engine.make({ minTentativeSeconds: 9999 }); // finals only
|
||||
|
||||
live.push(silence(0.512)); // 16 frames; pre-roll keeps the last 10
|
||||
live.push(speech(1));
|
||||
const stopping = live.stop();
|
||||
await tick();
|
||||
expect(engine.calls).toHaveLength(1);
|
||||
engine.calls[0]!.resolve({
|
||||
text: " offset",
|
||||
chunks: [{ text: " offset", timestamp: [0, 1] }],
|
||||
});
|
||||
const result = await stopping;
|
||||
|
||||
const offset = ((16 - 10) * FRAME) / RATE;
|
||||
expect(result.text).toBe("offset");
|
||||
expect(result.chunks[0]!.timestamp[0]).toBeCloseTo(offset, 5);
|
||||
expect(result.chunks[0]!.timestamp[1]!).toBeCloseTo(1 + offset, 5);
|
||||
expect(result.chunks[0]!.timestamp[1]).not.toBeNull();
|
||||
});
|
||||
|
||||
test("stop flushes, drains pending passes, and joins the transcript", async () => {
|
||||
const engine = fakeEngine();
|
||||
const live = engine.make({ minTentativeSeconds: 9999 });
|
||||
|
||||
live.push(speech(1));
|
||||
live.push(silence(1.5)); // segment 1 commits → final pass 1
|
||||
live.push(speech(1)); // segment 2 in progress
|
||||
const stopping = live.stop(); // flush commits segment 2
|
||||
|
||||
let resolved = false;
|
||||
void stopping.then(() => {
|
||||
resolved = true;
|
||||
});
|
||||
await tick();
|
||||
expect(resolved).toBe(false); // still draining
|
||||
|
||||
engine.calls[0]!.resolve({ text: " one.", chunks: [] });
|
||||
await tick();
|
||||
engine.calls[1]!.resolve({ text: " two.", chunks: [] });
|
||||
const result = await stopping;
|
||||
expect(result.text).toBe("one. two.");
|
||||
|
||||
// The session is over: further audio is ignored.
|
||||
live.push(speech(2));
|
||||
expect(engine.calls).toHaveLength(2);
|
||||
});
|
||||
|
||||
test("a failing pass keeps the provisional text and the session alive", async () => {
|
||||
const engine = fakeEngine();
|
||||
const live = engine.make();
|
||||
|
||||
live.push(speech(1.5));
|
||||
engine.calls[0]!.resolve({ text: " provisional", chunks: [] });
|
||||
await tick();
|
||||
live.push(silence(1.5)); // commit → final pass
|
||||
engine.calls[1]!.reject(new Error("inference exploded"));
|
||||
await tick();
|
||||
|
||||
expect(engine.last().error).toBe("inference exploded");
|
||||
expect(engine.last().committedText).toBe("provisional");
|
||||
expect(engine.last().pendingFinalPasses).toBe(0);
|
||||
|
||||
// Later segments still work.
|
||||
engine.advance(5000);
|
||||
live.push(speech(1.5));
|
||||
expect(engine.calls).toHaveLength(3);
|
||||
const stopping = live.stop();
|
||||
await tick();
|
||||
engine.calls[2]!.resolve({ text: " ignored (stale tentative)", chunks: [] });
|
||||
await tick();
|
||||
engine.calls[3]!.resolve({ text: " recovered", chunks: [] });
|
||||
const result = await stopping;
|
||||
expect(result.text).toBe("provisional recovered");
|
||||
});
|
||||
|
||||
test("a tentative result for an already-committed segment is discarded", async () => {
|
||||
const engine = fakeEngine();
|
||||
const live = engine.make();
|
||||
|
||||
live.push(speech(1.5)); // tentative pass in flight
|
||||
live.push(silence(1.5)); // segment commits while the pass runs
|
||||
expect(engine.last().committedText).toBe(""); // no text known yet
|
||||
|
||||
engine.calls[0]!.resolve({ text: " stale", chunks: [] });
|
||||
await tick();
|
||||
// Stale text was not applied as tentative; the final pass owns the text.
|
||||
expect(engine.last().tentativeText).toBe("");
|
||||
expect(engine.calls).toHaveLength(2);
|
||||
engine.calls[1]!.resolve({ text: " final", chunks: [] });
|
||||
await tick();
|
||||
expect(engine.last().committedText).toBe("final");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,51 @@
|
||||
# Manual verification: server engine (M5, D-013)
|
||||
|
||||
The dev environment has no Python package manager and no browser, so — like
|
||||
`tests/offline.md` and `tests/live.md` — the real-inference path needs a
|
||||
manual run on a machine with Python ≥ 3.9. Everything protocol- and
|
||||
HTTP-level is already covered by `bun test` (`tests/transcribe.test.ts`
|
||||
exercises the real `Bun.spawn` supervisor against
|
||||
`tests/fixtures/fake_sidecar.py`, plus the degradation paths against the real
|
||||
`server-py/sidecar.py`).
|
||||
|
||||
## 1. Install the sidecar dependencies
|
||||
|
||||
```sh
|
||||
python3 -m pip install -r server-py/requirements.txt
|
||||
```
|
||||
|
||||
## 2. Opt-in automated pass (no browser needed)
|
||||
|
||||
```sh
|
||||
FW_INTEGRATION=1 bun test tests/transcribe.test.ts
|
||||
```
|
||||
|
||||
Runs the real sidecar end to end (spawn → ready handshake → `tiny.en` load →
|
||||
transcribe → done). First run downloads ~75 MB of weights from the Hub.
|
||||
|
||||
- [ ] The `real faster-whisper sidecar` describe block passes.
|
||||
|
||||
## 3. In-browser run (both engines, same audio)
|
||||
|
||||
1. `bun run dev`, open the app, register/sign in.
|
||||
2. `GET /api/transcribe/status` (with the session cookie) returns
|
||||
`{"available": true}` — e.g. check the Network tab.
|
||||
3. Pick `Whisper Tiny (English)`; transcribe a short clean English clip
|
||||
(e.g. the JFK "ask not…" sample) **on device**. Note the transcript.
|
||||
4. Switch the engine toggle to **Server** (visible only while signed in;
|
||||
requires the M5 UI wiring — until that lands, drive it manually:
|
||||
`curl -X POST --data-binary @audio.f32 -b "sid=…" \
|
||||
"http://localhost:3000/api/transcribe?model=onnx-community/whisper-tiny.en"`
|
||||
where `audio.f32` is raw float32 mono 16 kHz PCM).
|
||||
5. Transcribe the same clip through the server engine.
|
||||
|
||||
- [ ] Segments stream in as JSON lines (not one blob at the end).
|
||||
- [ ] Both transcripts agree in content (minor punctuation/casing drift
|
||||
between Transformers.js and faster-whisper is expected).
|
||||
- [ ] Signed out, the toggle is gone and `POST /api/transcribe` answers 401.
|
||||
- [ ] Second request on the same model is visibly faster (no reload —
|
||||
warm-up strategy).
|
||||
- [ ] `AUTH_ENABLED=false bun run dev`: `/api/transcribe*` answers 404 and
|
||||
the anonymous browser flow is unaffected.
|
||||
|
||||
When this passes, check off the M5 integration item in `WorkBreakdown.md`.
|
||||
@@ -0,0 +1,448 @@
|
||||
import { afterAll, describe, expect, test } from "bun:test";
|
||||
import { mkdtemp, readdir, rm } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import {
|
||||
createSidecar,
|
||||
createTranscribeHandlers,
|
||||
type Sidecar,
|
||||
type SidecarSegment,
|
||||
type SidecarTranscribeRequest,
|
||||
} from "../src/server/transcribe";
|
||||
import { isTranscribeStreamLine, type TranscribeStreamLine } from "../src/lib/engine";
|
||||
import type { SessionUser } from "../src/server/auth";
|
||||
|
||||
// M5 server engine: HTTP handlers against a fake sidecar, and the real
|
||||
// Bun.spawn supervisor against a protocol-faithful Python stand-in
|
||||
// (tests/fixtures/fake_sidecar.py) — no faster-whisper needed. The end-to-end
|
||||
// run with real inference is manual/opt-in: see tests/server-engine.md.
|
||||
|
||||
const FIXTURE_SIDECAR = join(import.meta.dir, "fixtures/fake_sidecar.py");
|
||||
const REAL_SIDECAR = join(import.meta.dir, "../server-py/sidecar.py");
|
||||
|
||||
const tempDirs: string[] = [];
|
||||
async function makeTempDir(): Promise<string> {
|
||||
const dir = await mkdtemp(join(tmpdir(), "transcribe-test-"));
|
||||
tempDirs.push(dir);
|
||||
return dir;
|
||||
}
|
||||
afterAll(async () => {
|
||||
await Promise.all(tempDirs.map((dir) => rm(dir, { recursive: true, force: true })));
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const USER: SessionUser = { id: 1, username: "tester" };
|
||||
const sessionFromCookie = (request: Request): SessionUser | null =>
|
||||
request.headers.get("cookie") === "sid=ok" ? USER : null;
|
||||
|
||||
interface FakeSidecar extends Sidecar {
|
||||
requests: SidecarTranscribeRequest[];
|
||||
/** Bytes of each request's audio file, captured before cleanup. */
|
||||
audioPayloads: Uint8Array[];
|
||||
}
|
||||
|
||||
function fakeSidecar(
|
||||
behavior: {
|
||||
available?: boolean;
|
||||
reason?: string;
|
||||
segments?: SidecarSegment[];
|
||||
error?: string;
|
||||
waitForAbort?: boolean;
|
||||
} = {},
|
||||
): FakeSidecar {
|
||||
const segments = behavior.segments ?? [
|
||||
{ start: 0, end: 1.5, text: " Hello" },
|
||||
{ start: 1.5, end: 3, text: " world." },
|
||||
];
|
||||
const sidecar: FakeSidecar = {
|
||||
requests: [],
|
||||
audioPayloads: [],
|
||||
async status() {
|
||||
return behavior.available === false
|
||||
? { available: false, reason: behavior.reason ?? "nope" }
|
||||
: { available: true };
|
||||
},
|
||||
async transcribe(request, onSegment, signal) {
|
||||
sidecar.requests.push(request);
|
||||
sidecar.audioPayloads.push(
|
||||
new Uint8Array(await Bun.file(request.audioPath).arrayBuffer()),
|
||||
);
|
||||
if (behavior.error) throw new Error(behavior.error);
|
||||
onSegment(segments[0]!);
|
||||
if (behavior.waitForAbort) {
|
||||
await new Promise<void>((resolve) => {
|
||||
if (signal?.aborted) return resolve();
|
||||
signal?.addEventListener("abort", () => resolve(), { once: true });
|
||||
});
|
||||
return { cancelled: true };
|
||||
}
|
||||
for (const segment of segments.slice(1)) onSegment(segment);
|
||||
return { cancelled: false };
|
||||
},
|
||||
};
|
||||
return sidecar;
|
||||
}
|
||||
|
||||
function pcmBody(samples: number[]): Uint8Array<ArrayBuffer> {
|
||||
const audio = new Float32Array(samples);
|
||||
return new Uint8Array(audio.buffer as ArrayBuffer);
|
||||
}
|
||||
|
||||
function postRequest(
|
||||
query: string,
|
||||
body: BodyInit | null,
|
||||
init: { cookie?: string; signal?: AbortSignal; contentLength?: boolean } = {},
|
||||
): Request {
|
||||
const headers: Record<string, string> = {
|
||||
"Content-Type": "application/octet-stream",
|
||||
};
|
||||
if (init.cookie !== undefined) headers.cookie = init.cookie;
|
||||
return new Request(`http://localhost/api/transcribe${query}`, {
|
||||
method: "POST",
|
||||
headers,
|
||||
body,
|
||||
signal: init.signal,
|
||||
});
|
||||
}
|
||||
|
||||
async function readStreamLines(response: Response): Promise<TranscribeStreamLine[]> {
|
||||
const text = await response.text();
|
||||
return text
|
||||
.split("\n")
|
||||
.filter((line) => line.trim().length > 0)
|
||||
.map((line) => {
|
||||
const parsed: unknown = JSON.parse(line);
|
||||
if (!isTranscribeStreamLine(parsed)) throw new Error(`bad line: ${line}`);
|
||||
return parsed;
|
||||
});
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// HTTP handlers (fake sidecar)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe("GET /api/transcribe/status", () => {
|
||||
test("401 without a session", async () => {
|
||||
const handlers = createTranscribeHandlers({
|
||||
sidecar: fakeSidecar(),
|
||||
getSessionUser: sessionFromCookie,
|
||||
});
|
||||
const response = await handlers.status(
|
||||
new Request("http://localhost/api/transcribe/status"),
|
||||
);
|
||||
expect(response.status).toBe(401);
|
||||
});
|
||||
|
||||
test("passes the sidecar's availability through", async () => {
|
||||
const handlers = createTranscribeHandlers({
|
||||
sidecar: fakeSidecar({ available: false, reason: "python3 missing" }),
|
||||
getSessionUser: sessionFromCookie,
|
||||
});
|
||||
const response = await handlers.status(
|
||||
new Request("http://localhost/api/transcribe/status", {
|
||||
headers: { cookie: "sid=ok" },
|
||||
}),
|
||||
);
|
||||
expect(response.status).toBe(200);
|
||||
expect(await response.json()).toEqual({
|
||||
available: false,
|
||||
reason: "python3 missing",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("POST /api/transcribe", () => {
|
||||
test("401 without a session — the server engine is logged-in only", async () => {
|
||||
const sidecar = fakeSidecar();
|
||||
const handlers = createTranscribeHandlers({
|
||||
sidecar,
|
||||
getSessionUser: sessionFromCookie,
|
||||
});
|
||||
const response = await handlers.post(
|
||||
postRequest("?model=onnx-community/whisper-tiny.en", pcmBody([0.1])),
|
||||
);
|
||||
expect(response.status).toBe(401);
|
||||
expect(sidecar.requests).toHaveLength(0);
|
||||
});
|
||||
|
||||
test("400 for missing/unknown model, bad language, bad task", async () => {
|
||||
const handlers = createTranscribeHandlers({
|
||||
sidecar: fakeSidecar(),
|
||||
getSessionUser: sessionFromCookie,
|
||||
tempDir: await makeTempDir(),
|
||||
});
|
||||
const cases = [
|
||||
"",
|
||||
"?model=someone/other-model",
|
||||
"?model=onnx-community/whisper-tiny&language=Danish!",
|
||||
"?model=onnx-community/whisper-tiny&task=summarize",
|
||||
];
|
||||
for (const query of cases) {
|
||||
const response = await handlers.post(
|
||||
postRequest(query, pcmBody([0.1]), { cookie: "sid=ok" }),
|
||||
);
|
||||
expect(response.status).toBe(400);
|
||||
}
|
||||
});
|
||||
|
||||
test("503 with the reason when the sidecar is unavailable", async () => {
|
||||
const sidecar = fakeSidecar({ available: false, reason: "faster-whisper is not installed" });
|
||||
const handlers = createTranscribeHandlers({
|
||||
sidecar,
|
||||
getSessionUser: sessionFromCookie,
|
||||
});
|
||||
const response = await handlers.post(
|
||||
postRequest("?model=onnx-community/whisper-tiny.en", pcmBody([0.1]), {
|
||||
cookie: "sid=ok",
|
||||
}),
|
||||
);
|
||||
expect(response.status).toBe(503);
|
||||
expect(await response.json()).toEqual({ error: "faster-whisper is not installed" });
|
||||
expect(sidecar.requests).toHaveLength(0);
|
||||
});
|
||||
|
||||
test("413 over the upload limit; 400 for empty or non-float32 bodies", async () => {
|
||||
const tempDir = await makeTempDir();
|
||||
const handlers = createTranscribeHandlers({
|
||||
sidecar: fakeSidecar(),
|
||||
getSessionUser: sessionFromCookie,
|
||||
maxUploadBytes: 16,
|
||||
tempDir,
|
||||
});
|
||||
const over = await handlers.post(
|
||||
postRequest("?model=onnx-community/whisper-tiny.en", pcmBody([1, 2, 3, 4, 5]), {
|
||||
cookie: "sid=ok",
|
||||
}),
|
||||
);
|
||||
expect(over.status).toBe(413);
|
||||
|
||||
const empty = await handlers.post(
|
||||
postRequest("?model=onnx-community/whisper-tiny.en", new Uint8Array(0), {
|
||||
cookie: "sid=ok",
|
||||
}),
|
||||
);
|
||||
expect(empty.status).toBe(400);
|
||||
|
||||
const misaligned = await handlers.post(
|
||||
postRequest("?model=onnx-community/whisper-tiny.en", new Uint8Array(6), {
|
||||
cookie: "sid=ok",
|
||||
}),
|
||||
);
|
||||
expect(misaligned.status).toBe(400);
|
||||
|
||||
// Rejected uploads must not leave temp files behind.
|
||||
expect(await readdir(tempDir)).toEqual([]);
|
||||
});
|
||||
|
||||
test("streams segments then the result, maps the model, cleans up", async () => {
|
||||
const tempDir = await makeTempDir();
|
||||
const sidecar = fakeSidecar();
|
||||
const handlers = createTranscribeHandlers({
|
||||
sidecar,
|
||||
getSessionUser: sessionFromCookie,
|
||||
tempDir,
|
||||
});
|
||||
const audio = [0.25, -0.25, 0.5, -0.5];
|
||||
const response = await handlers.post(
|
||||
postRequest(
|
||||
"?model=onnx-community/whisper-base&language=da&task=translate",
|
||||
pcmBody(audio),
|
||||
{ cookie: "sid=ok" },
|
||||
),
|
||||
);
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.headers.get("content-type")).toContain("application/x-ndjson");
|
||||
|
||||
const lines = await readStreamLines(response);
|
||||
expect(lines).toEqual([
|
||||
{ type: "segment", text: " Hello", start: 0, end: 1.5 },
|
||||
{ type: "segment", text: " world.", start: 1.5, end: 3 },
|
||||
{
|
||||
type: "result",
|
||||
text: "Hello world.",
|
||||
chunks: [
|
||||
{ text: " Hello", timestamp: [0, 1.5] },
|
||||
{ text: " world.", timestamp: [1.5, 3] },
|
||||
],
|
||||
},
|
||||
]);
|
||||
|
||||
// Catalog id → faster-whisper name (D-013), options passed through.
|
||||
expect(sidecar.requests).toEqual([
|
||||
expect.objectContaining({ model: "base", language: "da", task: "translate" }),
|
||||
]);
|
||||
// The sidecar saw exactly the uploaded PCM bytes…
|
||||
expect([...new Float32Array(sidecar.audioPayloads[0]!.buffer.slice(0))]).toEqual(
|
||||
audio,
|
||||
);
|
||||
// …and the temp file is gone once the stream finished.
|
||||
expect(await readdir(tempDir)).toEqual([]);
|
||||
});
|
||||
|
||||
test("sidecar errors travel in-stream after the 200 header", async () => {
|
||||
const tempDir = await makeTempDir();
|
||||
const handlers = createTranscribeHandlers({
|
||||
sidecar: fakeSidecar({ error: "model load failed" }),
|
||||
getSessionUser: sessionFromCookie,
|
||||
tempDir,
|
||||
});
|
||||
const response = await handlers.post(
|
||||
postRequest("?model=onnx-community/whisper-tiny.en", pcmBody([0.1]), {
|
||||
cookie: "sid=ok",
|
||||
}),
|
||||
);
|
||||
expect(response.status).toBe(200);
|
||||
const lines = await readStreamLines(response);
|
||||
expect(lines).toEqual([{ type: "error", message: "model load failed" }]);
|
||||
expect(await readdir(tempDir)).toEqual([]);
|
||||
});
|
||||
|
||||
test("an aborted request cancels the sidecar run", async () => {
|
||||
const tempDir = await makeTempDir();
|
||||
const sidecar = fakeSidecar({ waitForAbort: true });
|
||||
const handlers = createTranscribeHandlers({
|
||||
sidecar,
|
||||
getSessionUser: sessionFromCookie,
|
||||
tempDir,
|
||||
});
|
||||
const controller = new AbortController();
|
||||
const response = await handlers.post(
|
||||
postRequest("?model=onnx-community/whisper-tiny.en", pcmBody([0.1]), {
|
||||
cookie: "sid=ok",
|
||||
signal: controller.signal,
|
||||
}),
|
||||
);
|
||||
expect(response.status).toBe(200);
|
||||
controller.abort();
|
||||
// Cancelled runs still terminate the stream with the partial result.
|
||||
const lines = await readStreamLines(response);
|
||||
expect(lines.at(-1)?.type).toBe("result");
|
||||
expect(await readdir(tempDir)).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// createSidecar — the real Bun.spawn supervisor against a Python child
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe("createSidecar (fake python sidecar)", () => {
|
||||
test("handshake, segment streaming, and request/option pass-through", async () => {
|
||||
const sidecar = createSidecar(["python3", FIXTURE_SIDECAR]);
|
||||
expect(await sidecar.status()).toEqual({ available: true });
|
||||
|
||||
const audioPath = join(await makeTempDir(), "audio.f32");
|
||||
await Bun.write(audioPath, pcmBody([0.1, 0.2, 0.3]));
|
||||
|
||||
const segments: SidecarSegment[] = [];
|
||||
const outcome = await sidecar.transcribe(
|
||||
{ model: "ok", audioPath, language: "da", task: "translate" },
|
||||
(segment) => segments.push(segment),
|
||||
);
|
||||
expect(outcome).toEqual({ cancelled: false });
|
||||
expect(segments).toHaveLength(2);
|
||||
expect(segments[0]).toEqual({ start: 0, end: 1.5, text: " Hello" });
|
||||
// The fixture echoes what it received (audio bytes, language, task).
|
||||
expect(segments[1]!.text).toContain("model=ok bytes=12 lang=da task=translate");
|
||||
});
|
||||
|
||||
test("sidecar 'error' lines reject that request only", async () => {
|
||||
const sidecar = createSidecar(["python3", FIXTURE_SIDECAR]);
|
||||
const audioPath = join(await makeTempDir(), "audio.f32");
|
||||
await Bun.write(audioPath, pcmBody([0.1]));
|
||||
|
||||
expect(
|
||||
sidecar.transcribe({ model: "boom", audioPath }, () => {}),
|
||||
).rejects.toThrow("kaboom");
|
||||
// The process survived; the next request works.
|
||||
const outcome = await sidecar.transcribe({ model: "ok", audioPath }, () => {});
|
||||
expect(outcome).toEqual({ cancelled: false });
|
||||
});
|
||||
|
||||
test("a crash rejects in-flight requests and the next request respawns", async () => {
|
||||
const sidecar = createSidecar(["python3", FIXTURE_SIDECAR]);
|
||||
const audioPath = join(await makeTempDir(), "audio.f32");
|
||||
await Bun.write(audioPath, pcmBody([0.1]));
|
||||
|
||||
expect(
|
||||
sidecar.transcribe({ model: "die", audioPath }, () => {}),
|
||||
).rejects.toThrow(/exited unexpectedly/);
|
||||
// Supervision: a fresh process serves the next request.
|
||||
const segments: SidecarSegment[] = [];
|
||||
const outcome = await sidecar.transcribe(
|
||||
{ model: "ok", audioPath },
|
||||
(segment) => segments.push(segment),
|
||||
);
|
||||
expect(outcome).toEqual({ cancelled: false });
|
||||
expect(segments).toHaveLength(2);
|
||||
});
|
||||
|
||||
test("abort sends a cancel and resolves {cancelled: true}", async () => {
|
||||
const sidecar = createSidecar(["python3", FIXTURE_SIDECAR]);
|
||||
const audioPath = join(await makeTempDir(), "audio.f32");
|
||||
await Bun.write(audioPath, pcmBody([0.1]));
|
||||
|
||||
const controller = new AbortController();
|
||||
const outcome = await sidecar.transcribe(
|
||||
{ model: "wait-for-cancel", audioPath },
|
||||
() => controller.abort(), // abort as soon as the first segment arrives
|
||||
controller.signal,
|
||||
);
|
||||
expect(outcome).toEqual({ cancelled: true });
|
||||
});
|
||||
});
|
||||
|
||||
describe("createSidecar (degradation, D-013)", () => {
|
||||
test("the real sidecar without faster-whisper reports a permanent reason", async () => {
|
||||
// -S/-I keep site-packages off sys.path, so this behaves like a
|
||||
// Bun-only deployment even on machines that have faster-whisper.
|
||||
const sidecar = createSidecar(["python3", "-S", "-I", REAL_SIDECAR]);
|
||||
const first = await sidecar.status();
|
||||
expect(first.available).toBe(false);
|
||||
expect(first.reason).toContain("faster-whisper is not installed");
|
||||
// Remembered as permanent — no respawn storm, same answer immediately.
|
||||
const second = await sidecar.status();
|
||||
expect(second).toEqual(first);
|
||||
});
|
||||
|
||||
test("a missing python binary reports unavailable", async () => {
|
||||
const sidecar = createSidecar(["definitely-not-a-python-binary"]);
|
||||
const status = await sidecar.status();
|
||||
expect(status.available).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Opt-in: real faster-whisper end to end (needs python3 + requirements.txt
|
||||
// installed and network/cached weights). See tests/server-engine.md.
|
||||
// FW_INTEGRATION=1 bun test tests/transcribe.test.ts
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe.skipIf(!process.env.FW_INTEGRATION)("real faster-whisper sidecar", () => {
|
||||
test(
|
||||
"transcribes PCM through the real sidecar",
|
||||
async () => {
|
||||
const sidecar = createSidecar();
|
||||
const status = await sidecar.status();
|
||||
expect(status).toEqual({ available: true });
|
||||
|
||||
// One second of a 440 Hz tone — not speech, but exercises model load,
|
||||
// numpy decode, and the full protocol round-trip.
|
||||
const samples = new Float32Array(16_000);
|
||||
for (let i = 0; i < samples.length; i++) {
|
||||
samples[i] = 0.1 * Math.sin((2 * Math.PI * 440 * i) / 16_000);
|
||||
}
|
||||
const audioPath = join(await makeTempDir(), "tone.f32");
|
||||
await Bun.write(audioPath, new Uint8Array(samples.buffer));
|
||||
|
||||
const outcome = await sidecar.transcribe(
|
||||
{ model: "tiny.en", audioPath },
|
||||
() => {},
|
||||
);
|
||||
expect(outcome.cancelled).toBe(false);
|
||||
},
|
||||
{ timeout: 300_000 },
|
||||
);
|
||||
});
|
||||
Reference in New Issue
Block a user