From 2d668b3fb8647ed44041e9d009cf92d9815f5b70 Mon Sep 17 00:00:00 2001 From: Lucas Winther Date: Fri, 10 Jul 2026 09:41:02 +0200 Subject: [PATCH] Start on live transcription and python sidecar (faster-whisper) --- server-py/requirements.txt | 4 + server-py/sidecar.py | 159 +++++++++++ src/components/LiveControls.tsx | 46 +++ src/lib/live.ts | 476 ++++++++++++++++++++++++++++++++ src/lib/mic.ts | 169 ++++++++++++ tests/engine.test.ts | 373 +++++++++++++++++++++++++ tests/fixtures/fake_sidecar.py | 58 ++++ tests/live.md | 77 ++++++ tests/live.test.ts | 313 +++++++++++++++++++++ tests/server-engine.md | 51 ++++ tests/transcribe.test.ts | 448 ++++++++++++++++++++++++++++++ 11 files changed, 2174 insertions(+) create mode 100644 server-py/requirements.txt create mode 100644 server-py/sidecar.py create mode 100644 src/components/LiveControls.tsx create mode 100644 src/lib/live.ts create mode 100644 src/lib/mic.ts create mode 100644 tests/engine.test.ts create mode 100644 tests/fixtures/fake_sidecar.py create mode 100644 tests/live.md create mode 100644 tests/live.test.ts create mode 100644 tests/server-engine.md create mode 100644 tests/transcribe.test.ts diff --git a/server-py/requirements.txt b/server-py/requirements.txt new file mode 100644 index 0000000..f4f712c --- /dev/null +++ b/server-py/requirements.txt @@ -0,0 +1,4 @@ +# Server-engine sidecar dependencies (Decisions.md D-013). Only needed on +# deployments that enable the server engine — the browser/anonymous path +# deploys with Bun alone. Install with: python3 -m pip install -r requirements.txt +faster-whisper>=1.1,<2 diff --git a/server-py/sidecar.py b/server-py/sidecar.py new file mode 100644 index 0000000..95f8d3b --- /dev/null +++ b/server-py/sidecar.py @@ -0,0 +1,159 @@ +"""faster-whisper sidecar (Decisions.md D-013). + +Bun spawns and supervises this process (src/server/transcribe.ts); it is the +only place server-side inference happens. The interface is JSON-lines over +stdio — one JSON object per line, no framing beyond that: + + stdout <- {"type": "ready", "version": "1.1.1"} # startup handshake + stdout <- {"type": "fatal", "message": "..."} # then exit 1 + + stdin -> {"type": "transcribe", "id": "1", "model": "tiny.en", + "audio_path": "/tmp/...", "language": "en" | null, + "task": "transcribe" | "translate"} + stdout <- {"type": "segment", "id": "1", "start": 0.0, "end": 4.2, + "text": " Hello world."} # 0..n of these + stdout <- {"type": "done", "id": "1", "cancelled": false} + stdout <- {"type": "error", "id": "1", "message": "..."} # instead of done + + stdin -> {"type": "cancel", "id": "1"} # stop between segments; still "done" + stdin -> {"type": "shutdown"} # clean exit (EOF does the same) + +Audio arrives as a file of raw little-endian float32 mono PCM at 16 kHz — the +exact representation the browser already produces for its own engine — so no +container decoding happens here. Requests are processed strictly one at a +time; a reader thread keeps consuming stdin meanwhile so `cancel` can +interrupt the segment loop of the request in flight. + +The last-used model stays loaded between requests (WorkBreakdown M5 warm-up: +never reload weights per request) and is replaced — not accumulated — when a +request names a different model, bounding memory to one model. + +Environment: + FASTER_WHISPER_DEVICE ctranslate2 device (default "auto") + FASTER_WHISPER_COMPUTE compute type (default "int8" — the fast, + low-memory CPU choice; CT2 converts on load) + FASTER_WHISPER_DOWNLOAD_ROOT optional model download/cache directory + +Requires Python >= 3.9 and the packages in requirements.txt. This file must +keep working without them up to the import: a missing dependency is reported +as a "fatal" line, which the Bun side surfaces as "server engine unavailable" +(the anonymous/browser path never needs any of this). +""" + +import json +import os +import queue +import sys +import threading + +SAMPLE_RATE = 16000 + + +def emit(message): + """Write one JSON line to stdout. Only the main thread emits.""" + sys.stdout.write(json.dumps(message) + "\n") + sys.stdout.flush() + + +def main(): + try: + import faster_whisper + import numpy as np + except ImportError as error: + emit( + { + "type": "fatal", + "message": "faster-whisper is not installed (%s) — " + "see server-py/requirements.txt" % error, + } + ) + return 1 + + device = os.environ.get("FASTER_WHISPER_DEVICE", "auto") + compute_type = os.environ.get("FASTER_WHISPER_COMPUTE", "int8") + download_root = os.environ.get("FASTER_WHISPER_DOWNLOAD_ROOT") or None + + requests = queue.Queue() + cancelled = set() # request ids; GIL-atomic add/contains is enough here + SHUTDOWN = object() + + def read_stdin(): + """Reader thread: queue work, flag cancels, shut down on EOF.""" + for line in sys.stdin: + line = line.strip() + if not line: + continue + try: + message = json.loads(line) + except ValueError: + continue # a malformed line must not kill the sidecar + kind = message.get("type") + if kind == "transcribe": + requests.put(message) + elif kind == "cancel": + cancelled.add(message.get("id")) + elif kind == "shutdown": + requests.put(SHUTDOWN) + return + requests.put(SHUTDOWN) # supervisor closed the pipe / died + + threading.Thread(target=read_stdin, daemon=True).start() + + cached_name = None + cached_model = None + + emit({"type": "ready", "version": getattr(faster_whisper, "__version__", "?")}) + + while True: + request = requests.get() + if request is SHUTDOWN: + return 0 + + request_id = request.get("id") + try: + model_name = request["model"] + audio_path = request["audio_path"] + language = request.get("language") or None + task = request.get("task") or "transcribe" + + if model_name != cached_name: + cached_model = None # release before loading the replacement + cached_model = faster_whisper.WhisperModel( + model_name, + device=device, + compute_type=compute_type, + download_root=download_root, + ) + cached_name = model_name + + audio = np.fromfile(audio_path, dtype=np.float32) + segments, _info = cached_model.transcribe( + audio, language=language, task=task + ) + + was_cancelled = False + # `segments` is a generator: inference happens during iteration, + # so each segment can stream out (and cancel can cut in) while the + # next one is still being decoded. + for segment in segments: + if request_id in cancelled: + was_cancelled = True + break + emit( + { + "type": "segment", + "id": request_id, + "start": round(float(segment.start), 3), + "end": round(float(segment.end), 3), + "text": segment.text, + } + ) + emit({"type": "done", "id": request_id, "cancelled": was_cancelled}) + except Exception as error: # noqa: BLE001 — one bad request must not exit + emit({"type": "error", "id": request_id, "message": str(error)}) + finally: + cancelled.discard(request_id) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/components/LiveControls.tsx b/src/components/LiveControls.tsx new file mode 100644 index 0000000..902d2e4 --- /dev/null +++ b/src/components/LiveControls.tsx @@ -0,0 +1,46 @@ +/** Live microphone transcription start/stop (M7, Decisions.md D-016). */ + +export type LiveStatus = "idle" | "starting" | "listening" | "stopping"; + +interface LiveControlsProps { + status: LiveStatus; + onStart: () => void; + onStop: () => void; + /** True while the file flow is busy — the two modes are mutually exclusive. */ + disabled?: boolean; +} + +const LABELS: Record = { + idle: "Start live transcription", + starting: "Requesting microphone…", + listening: "Stop", + stopping: "Finishing…", +}; + +export function LiveControls({ + status, + onStart, + onStop, + disabled = false, +}: LiveControlsProps) { + const listening = status === "listening"; + return ( +
+ {listening &&
+ ); +} diff --git a/src/lib/live.ts b/src/lib/live.ts new file mode 100644 index 0000000..0ff5f17 --- /dev/null +++ b/src/lib/live.ts @@ -0,0 +1,476 @@ +/** + * Live microphone transcription — the segment loop (Decisions.md D-016). + * + * Whisper is not a streaming model, so "live" is approximated by splitting + * the mic signal into speech segments with a simple RMS energy gate and + * re-transcribing the current segment on a rolling basis (tentative text) + * until it commits, at which point it gets one final pass and freezes. + * + * Everything here is pure and free of browser globals so it runs under + * `bun test`: audio arrives via `push()` (mic capture lives in lib/mic.ts) + * and inference is an injected async function (the UI wraps the worker). + */ + +import type { TranscriptChunk } from "../worker/protocol"; +import { TARGET_SAMPLE_RATE } from "./audio"; + +/** Root-mean-square level of a sample block. Pure. */ +export function rms(samples: Float32Array): number { + if (samples.length === 0) return 0; + let sum = 0; + for (let i = 0; i < samples.length; i++) { + sum += samples[i]! * samples[i]!; + } + return Math.sqrt(sum / samples.length); +} + +function concat(parts: readonly Float32Array[], length: number): Float32Array { + const out = new Float32Array(length); + let offset = 0; + for (const part of parts) { + out.set(part, offset); + offset += part.length; + } + return out; +} + +// --------------------------------------------------------------------------- +// Segmenter: RMS energy gate → speech segments +// --------------------------------------------------------------------------- + +export interface LiveSegmenterOptions { + /** Sample rate of the incoming (and outgoing) PCM. Default 16 kHz. */ + sampleRate?: number; + /** Analysis frame length in seconds. Default 0.032 (~32 ms). */ + frameSeconds?: number; + /** A frame at/above this RMS counts as speech. Default 0.015. */ + speechRms?: number; + /** Trailing silence that commits the current segment. Default 0.75 s. */ + commitSilenceSeconds?: number; + /** + * Hard cap per segment — must stay under Whisper's 30 s window so a live + * segment is always a single-chunk transcription. Default 25 s. + */ + maxSegmentSeconds?: number; + /** Audio kept from just before speech onset. Default 0.3 s. */ + prerollSeconds?: number; + /** + * Segments with less accumulated speech than this are dropped instead of + * committed — Whisper hallucinates on (near-)silence. Default 0.25 s. + */ + minSpeechSeconds?: number; +} + +export type CommitReason = "silence" | "max-length" | "flush"; + +/** One committed speech segment, ready for a final transcription pass. */ +export interface CommittedSegment { + audio: Float32Array; + /** Absolute offset of `audio[0]` in samples since the session started. */ + startSample: number; + reason: CommitReason; +} + +interface Frame { + samples: Float32Array; + /** Absolute offset of this frame's first sample. */ + start: number; +} + +/** + * Splits a continuous PCM stream into speech segments. Push mic audio in any + * block sizes; committed segments come back from `push()`/`flush()`. + */ +export class LiveSegmenter { + private readonly sampleRate: number; + private readonly frameSamples: number; + private readonly speechRms: number; + private readonly commitSilenceFrames: number; + private readonly maxSegmentSamples: number; + private readonly maxPrerollFrames: number; + private readonly minSpeechSamples: number; + + /** Leftover samples smaller than one analysis frame. */ + private carry = new Float32Array(0); + /** Absolute offset of the next unprocessed frame's first sample. */ + private absSample = 0; + /** Recent silence kept while idle, prepended at speech onset. */ + private preroll: Frame[] = []; + private segment: Frame[] | null = null; + private segmentStart = 0; + private segmentSamples = 0; + private speechSamples = 0; + private silenceFrames = 0; + private nextSegmentId = 1; + private segmentId: number | null = null; + + constructor(options: LiveSegmenterOptions = {}) { + this.sampleRate = options.sampleRate ?? TARGET_SAMPLE_RATE; + const frameSeconds = options.frameSeconds ?? 0.032; + this.frameSamples = Math.max(1, Math.round(frameSeconds * this.sampleRate)); + this.speechRms = options.speechRms ?? 0.015; + this.commitSilenceFrames = Math.max( + 1, + Math.ceil((options.commitSilenceSeconds ?? 0.75) / frameSeconds), + ); + this.maxSegmentSamples = Math.round( + (options.maxSegmentSeconds ?? 25) * this.sampleRate, + ); + this.maxPrerollFrames = Math.ceil((options.prerollSeconds ?? 0.3) / frameSeconds); + this.minSpeechSamples = Math.round( + (options.minSpeechSeconds ?? 0.25) * this.sampleRate, + ); + } + + /** Identity of the in-progress segment; null while idle (no speech yet). */ + get currentSegmentId(): number | null { + return this.segmentId; + } + + /** Samples accumulated in the in-progress segment (0 while idle). */ + get currentSegmentSamples(): number { + return this.segment ? this.segmentSamples : 0; + } + + /** Copy of the in-progress segment's audio, for a tentative pass. */ + currentSegment(): { audio: Float32Array; startSample: number } | null { + if (!this.segment) return null; + return { + audio: concat( + this.segment.map((frame) => frame.samples), + this.segmentSamples, + ), + startSample: this.segmentStart, + }; + } + + /** Feed mic samples; returns segments that committed during this block. */ + push(samples: Float32Array): CommittedSegment[] { + const committed: CommittedSegment[] = []; + let buffer: Float32Array; + if (this.carry.length > 0) { + buffer = concat([this.carry, samples], this.carry.length + samples.length); + } else { + buffer = samples; + } + + let offset = 0; + while (buffer.length - offset >= this.frameSamples) { + // Copy the frame — the caller may reuse/transfer its buffer. + const frame: Frame = { + samples: buffer.slice(offset, offset + this.frameSamples), + start: this.absSample, + }; + offset += this.frameSamples; + this.absSample += this.frameSamples; + const segment = this.processFrame(frame, rms(frame.samples) >= this.speechRms); + if (segment) committed.push(segment); + } + this.carry = buffer.slice(offset); + return committed; + } + + /** Commit whatever is in progress (session stop). */ + flush(): CommittedSegment | null { + if (this.segment && this.carry.length > 0) { + // Sub-frame tail — too short to classify, but part of the speech. + this.segment.push({ samples: this.carry, start: this.absSample }); + this.segmentSamples += this.carry.length; + this.absSample += this.carry.length; + this.carry = new Float32Array(0); + } + return this.commit("flush"); + } + + private processFrame(frame: Frame, speech: boolean): CommittedSegment | null { + if (!this.segment) { + if (!speech) { + this.preroll.push(frame); + if (this.preroll.length > this.maxPrerollFrames) this.preroll.shift(); + return null; + } + this.segment = [...this.preroll, frame]; + this.segmentStart = this.segment[0]!.start; + this.segmentSamples = this.segment.length * this.frameSamples; + this.speechSamples = this.frameSamples; + this.silenceFrames = 0; + this.preroll = []; + this.segmentId = this.nextSegmentId++; + return null; + } + + this.segment.push(frame); + this.segmentSamples += this.frameSamples; + if (speech) { + this.speechSamples += this.frameSamples; + this.silenceFrames = 0; + } else { + this.silenceFrames += 1; + if (this.silenceFrames >= this.commitSilenceFrames) return this.commit("silence"); + } + if (this.segmentSamples >= this.maxSegmentSamples) return this.commit("max-length"); + return null; + } + + private commit(reason: CommitReason): CommittedSegment | null { + const segment = this.segment; + if (!segment) return null; + const result: CommittedSegment | null = + this.speechSamples >= this.minSpeechSamples + ? { + audio: concat( + segment.map((frame) => frame.samples), + this.segmentSamples, + ), + startSample: this.segmentStart, + reason, + } + : null; // near-silence: drop, never transcribe (D-016) + this.segment = null; + this.segmentSamples = 0; + this.speechSamples = 0; + this.silenceFrames = 0; + this.segmentId = null; + return result; + } +} + +// --------------------------------------------------------------------------- +// Transcriber: schedules inference passes over the segment stream +// --------------------------------------------------------------------------- + +export interface LiveTranscribeResult { + text: string; + /** Timestamps relative to the transcribed audio's start, Whisper-style. */ + chunks: TranscriptChunk[]; +} + +/** + * One inference pass. Takes ownership of `audio` (it may be transferred to + * the worker). The controller never runs two passes concurrently. + */ +export type LiveTranscribeFn = (audio: Float32Array) => Promise; + +export interface LiveUpdate { + /** Frozen (or provisionally frozen) text of committed segments, joined. */ + committedText: string; + /** Rolling text of the in-progress segment; replaced every pass. */ + tentativeText: string; + /** Finalized chunks with absolute session timestamps. */ + chunks: TranscriptChunk[]; + /** Committed segments still waiting for their final pass. */ + pendingFinalPasses: number; + /** Last inference error, if any (the session keeps going). */ + error: string | null; +} + +export interface LiveTranscriberOptions { + transcribe: LiveTranscribeFn; + onUpdate: (update: LiveUpdate) => void; + segmenter?: LiveSegmenterOptions; + /** Minimum ms between tentative passes. Default 1500. */ + tentativeIntervalMs?: number; + /** Minimum in-progress segment length before tentative passes. Default 1 s. */ + minTentativeSeconds?: number; + /** Injectable clock (tests). Default Date.now. */ + now?: () => number; +} + +interface SegmentState { + startSample: number; + /** Handed to the final pass, then released. */ + audio: Float32Array | null; + /** Tentative text until the final pass overwrites it. */ + text: string; + chunks: TranscriptChunk[]; + final: boolean; +} + +/** + * Drives the live session: feeds mic audio to the segmenter, keeps exactly + * one inference pass in flight (final passes for committed segments first, + * then throttled tentative refreshes of the current segment), and reports + * every text change through `onUpdate`. Scheduling is driven entirely by + * `push()` calls and pass completions — no timers, so it is fully testable. + */ +export class LiveTranscriber { + private readonly segmenter: LiveSegmenter; + private readonly transcribe: LiveTranscribeFn; + private readonly onUpdate: (update: LiveUpdate) => void; + private readonly tentativeIntervalMs: number; + private readonly minTentativeSamples: number; + private readonly sampleRate: number; + private readonly now: () => number; + + private segments: SegmentState[] = []; + private tentativeText = ""; + private lastTentative: { segmentId: number; samples: number; at: number } | null = + null; + private inFlight = false; + private stopped = false; + private error: string | null = null; + private drainResolvers: (() => void)[] = []; + + constructor(options: LiveTranscriberOptions) { + this.segmenter = new LiveSegmenter(options.segmenter); + this.transcribe = options.transcribe; + this.onUpdate = options.onUpdate; + this.tentativeIntervalMs = options.tentativeIntervalMs ?? 1500; + this.sampleRate = options.segmenter?.sampleRate ?? TARGET_SAMPLE_RATE; + this.minTentativeSamples = Math.round( + (options.minTentativeSeconds ?? 1) * this.sampleRate, + ); + this.now = options.now ?? Date.now; + } + + /** Feed a block of 16 kHz mono mic samples. Ignored after stop(). */ + push(samples: Float32Array): void { + if (this.stopped) return; + for (const committed of this.segmenter.push(samples)) { + this.acceptSegment(committed); + } + this.pump(); + } + + /** + * End the session: commit the in-progress segment, wait for the remaining + * final passes, and return the full transcript. + */ + async stop(): Promise { + if (!this.stopped) { + this.stopped = true; + const flushed = this.segmenter.flush(); + if (flushed) this.acceptSegment(flushed); + // Anything tentative either moved into the flushed segment or belonged + // to audio the segmenter dropped as near-silence. + this.tentativeText = ""; + this.pump(); + } + await this.whenDrained(); + this.emit(); + return { text: this.committedText(), chunks: this.allChunks() }; + } + + private acceptSegment(committed: CommittedSegment): void { + // The committing segment is the one the tentative text described; carry + // it as the provisional text until the final pass replaces it. + this.segments.push({ + startSample: committed.startSample, + audio: committed.audio, + text: this.tentativeText, + chunks: [], + final: false, + }); + this.tentativeText = ""; + this.lastTentative = null; + this.emit(); + } + + /** Single-flight scheduler: pick and start at most one inference pass. */ + private pump(): void { + if (this.inFlight) return; + const pending = this.segments.find((segment) => !segment.final); + if (pending) { + void this.runFinalPass(pending); + return; + } + if (!this.stopped) { + this.maybeRunTentativePass(); + return; + } + // Stopped and drained — release stop() waiters. + const resolvers = this.drainResolvers; + this.drainResolvers = []; + for (const resolve of resolvers) resolve(); + } + + private async runFinalPass(segment: SegmentState): Promise { + this.inFlight = true; + const audio = segment.audio!; + segment.audio = null; // ownership moves to the transcribe fn + const offsetSeconds = segment.startSample / this.sampleRate; + try { + const result = await this.transcribe(audio); + segment.text = result.text; + segment.chunks = result.chunks.map( + (chunk): TranscriptChunk => ({ + text: chunk.text, + timestamp: [ + chunk.timestamp[0] + offsetSeconds, + chunk.timestamp[1] === null ? null : chunk.timestamp[1] + offsetSeconds, + ], + }), + ); + } catch (error) { + // Keep the provisional (tentative) text rather than losing the segment. + this.error = error instanceof Error ? error.message : String(error); + } + segment.final = true; + this.inFlight = false; + this.emit(); + this.pump(); + } + + private maybeRunTentativePass(): void { + const segmentId = this.segmenter.currentSegmentId; + if (segmentId === null) return; + const samples = this.segmenter.currentSegmentSamples; + if (samples < this.minTentativeSamples) return; + const last = this.lastTentative; + if (last && this.now() - last.at < this.tentativeIntervalMs) return; + if (last && last.segmentId === segmentId && samples <= last.samples) return; + + const snapshot = this.segmenter.currentSegment(); + if (!snapshot) return; + this.lastTentative = { segmentId, samples, at: this.now() }; + this.inFlight = true; + void this.transcribe(snapshot.audio) + .then( + (result) => { + // Discard stale passes: the segment may have committed (its final + // pass supersedes this) or been dropped while we were busy. + if (this.segmenter.currentSegmentId === segmentId) { + this.tentativeText = result.text; + } + }, + (error: unknown) => { + this.error = error instanceof Error ? error.message : String(error); + }, + ) + .then(() => { + this.inFlight = false; + this.emit(); + this.pump(); + }); + } + + private committedText(): string { + return this.segments + .map((segment) => segment.text.trim()) + .filter(Boolean) + .join(" "); + } + + private allChunks(): TranscriptChunk[] { + return this.segments.flatMap((segment) => segment.chunks); + } + + private emit(): void { + this.onUpdate({ + committedText: this.committedText(), + tentativeText: this.tentativeText, + chunks: this.allChunks(), + pendingFinalPasses: this.segments.filter((segment) => !segment.final).length, + error: this.error, + }); + } + + private whenDrained(): Promise { + const drained = + !this.inFlight && this.segments.every((segment) => segment.final); + if (drained) return Promise.resolve(); + return new Promise((resolve) => { + this.drainResolvers.push(resolve); + }); + } +} diff --git a/src/lib/mic.ts b/src/lib/mic.ts new file mode 100644 index 0000000..8a14b41 --- /dev/null +++ b/src/lib/mic.ts @@ -0,0 +1,169 @@ +/** + * Microphone capture for live transcription (Decisions.md D-016). Browser-only. + * + * `getUserMedia` → `AudioContext` at the device's **native** rate (Firefox + * cannot connect a MediaStreamAudioSourceNode across sample rates, so the + * context is never forced to 16 kHz) → an AudioWorklet chunker posts ~100 ms + * Float32 blocks to the main thread, which resamples them to 16 kHz with the + * pure `resampleLinear` before handing them to the live segment loop + * (src/lib/live.ts). + * + * The worklet module is created from an inline source string via a Blob URL: + * no extra served file, no build or service-worker changes, and capture keeps + * working offline. Browsers without AudioWorklet (or that reject Blob-URL + * modules) fall back to a ScriptProcessorNode. + * + * Mic audio is processed entirely on-device, like every browser-engine path + * (Architecture.md §5). + */ + +import { TARGET_SAMPLE_RATE, resampleLinear } from "./audio"; + +export interface MicCapture { + /** Native rate of the capture context (chunks are resampled to 16 kHz). */ + sampleRate: number; + /** True when the AudioWorklet path failed and ScriptProcessorNode is used. */ + usingFallback: boolean; + /** Release the mic (stops the permission indicator) and the audio graph. */ + stop(): Promise; +} + +const WORKLET_NAME = "live-mic-chunker"; + +// Runs in the AudioWorklet global scope — plain JS, kept dependency-free. +// Accumulates the 128-sample render quanta into ~100 ms chunks so the main +// thread handles ~10 messages/s instead of ~375. +const WORKLET_SOURCE = ` +registerProcessor("${WORKLET_NAME}", class extends AudioWorkletProcessor { + constructor() { + super(); + this.blocks = []; + this.length = 0; + this.target = Math.max(128, Math.round(sampleRate / 10)); + } + process(inputs) { + const channel = inputs[0] && inputs[0][0]; + if (channel && channel.length > 0) { + this.blocks.push(new Float32Array(channel)); + this.length += channel.length; + if (this.length >= this.target) { + const out = new Float32Array(this.length); + let offset = 0; + for (const block of this.blocks) { + out.set(block, offset); + offset += block.length; + } + this.blocks = []; + this.length = 0; + this.port.postMessage(out, [out.buffer]); + } + } + return true; + } +}); +`; + +async function attachWorklet( + context: AudioContext, + source: MediaStreamAudioSourceNode, + deliver: (samples: Float32Array) => void, +): Promise<() => void> { + const url = URL.createObjectURL( + new Blob([WORKLET_SOURCE], { type: "text/javascript" }), + ); + try { + await context.audioWorklet.addModule(url); + } finally { + URL.revokeObjectURL(url); + } + const node = new AudioWorkletNode(context, WORKLET_NAME, { + numberOfInputs: 1, + numberOfOutputs: 0, + channelCount: 1, + }); + node.port.onmessage = (event: MessageEvent) => deliver(event.data as Float32Array); + source.connect(node); + return () => { + node.port.onmessage = null; + source.disconnect(node); + }; +} + +function attachScriptProcessor( + context: AudioContext, + source: MediaStreamAudioSourceNode, + deliver: (samples: Float32Array) => void, +): () => void { + const node = context.createScriptProcessor(4096, 1, 1); + node.onaudioprocess = (event) => { + // The input buffer is reused between callbacks — copy before handing off. + deliver(new Float32Array(event.inputBuffer.getChannelData(0))); + }; + // ScriptProcessorNode only fires when connected to the destination; route + // it through a muted gain so the mic is never played back. + const mute = context.createGain(); + mute.gain.value = 0; + source.connect(node); + node.connect(mute); + mute.connect(context.destination); + return () => { + node.onaudioprocess = null; + source.disconnect(node); + node.disconnect(); + mute.disconnect(); + }; +} + +/** + * Open the microphone and stream 16 kHz mono Float32 chunks to `onChunk` + * until `stop()`. Rejects if the user denies mic permission (surface the + * error; do not retry without a user gesture). + */ +export async function startMicCapture( + onChunk: (samples: Float32Array) => void, +): Promise { + if (!navigator.mediaDevices?.getUserMedia) { + throw new Error( + "Microphone capture is not supported in this browser (requires a secure context).", + ); + } + const stream = await navigator.mediaDevices.getUserMedia({ + audio: { echoCancellation: true, noiseSuppression: true, autoGainControl: true }, + }); + + const context = new AudioContext(); + // Contexts created outside a user gesture start suspended; the start button + // click satisfies the gesture requirement, so this resolves immediately. + await context.resume().catch(() => {}); + const source = context.createMediaStreamSource(stream); + + const deliver = (samples: Float32Array) => { + onChunk( + context.sampleRate === TARGET_SAMPLE_RATE + ? samples + : resampleLinear(samples, context.sampleRate, TARGET_SAMPLE_RATE), + ); + }; + + let detach: () => void; + let usingFallback = false; + try { + detach = await attachWorklet(context, source, deliver); + } catch { + detach = attachScriptProcessor(context, source, deliver); + usingFallback = true; + } + + let stopped = false; + return { + sampleRate: context.sampleRate, + usingFallback, + async stop() { + if (stopped) return; + stopped = true; + detach(); + for (const track of stream.getTracks()) track.stop(); + await context.close().catch(() => {}); + }, + }; +} diff --git a/tests/engine.test.ts b/tests/engine.test.ts new file mode 100644 index 0000000..2eb4f23 --- /dev/null +++ b/tests/engine.test.ts @@ -0,0 +1,373 @@ +import { describe, expect, test } from "bun:test"; +import { + BrowserEngine, + ENGINE_CHOICE_KEY, + ServerEngine, + fetchServerEngineStatus, + isTranscribeStreamLine, + loadEngineChoice, + saveEngineChoice, + type StorageLike, + type WorkerLike, +} from "../src/lib/engine"; +import type { FetchLike } from "../src/lib/authClient"; +import type { TranscriptChunk } from "../src/worker/protocol"; + +// D-010: both engines behind one interface, testable without a DOM/worker/server. + +// --------------------------------------------------------------------------- +// BrowserEngine — requestId correlation against a fake worker +// --------------------------------------------------------------------------- + +class FakeWorker implements WorkerLike { + posted: { message: Record; transfer?: Transferable[] }[] = []; + private listeners = { message: [] as ((e: { data: unknown }) => void)[], error: [] as ((e: { message?: string }) => void)[] }; + + postMessage(message: unknown, transfer?: Transferable[]): void { + this.posted.push({ message: message as Record, transfer }); + } + + addEventListener( + type: "message" | "error", + listener: (event: { data?: unknown; message?: string }) => void, + ): void { + this.listeners[type].push(listener); + } + + emit(data: unknown): void { + for (const listener of this.listeners.message) listener({ data }); + } + + emitError(message: string): void { + for (const listener of this.listeners.error) listener({ message }); + } + + lastRequestId(): string { + return this.posted.at(-1)!.message.requestId as string; + } +} + +const CHUNKS: TranscriptChunk[] = [{ text: " hi", timestamp: [0, 1] }]; + +describe("BrowserEngine", () => { + test("posts a transcribe message with a requestId and transfers the buffer", () => { + const worker = new FakeWorker(); + const engine = new BrowserEngine(worker); + const audio = new Float32Array([0.1, 0.2]); + void engine.transcribe(audio, { language: "da", task: "translate" }); + + expect(worker.posted).toHaveLength(1); + const { message, transfer } = worker.posted[0]!; + expect(message.type).toBe("transcribe"); + expect(message.language).toBe("da"); + expect(message.task).toBe("translate"); + expect(typeof message.requestId).toBe("string"); + expect(transfer).toEqual([audio.buffer as ArrayBuffer]); + }); + + test('omits language "auto" and the modelId (worker holds the model)', () => { + const worker = new FakeWorker(); + const engine = new BrowserEngine(worker); + void engine.transcribe(new Float32Array(4), { + language: "auto", + modelId: "onnx-community/whisper-tiny.en", + }); + const { message } = worker.posted[0]!; + expect("language" in message).toBe(false); + expect("modelId" in message).toBe(false); + }); + + test("resolves on the matching result and routes partials to onPartial", async () => { + const worker = new FakeWorker(); + const engine = new BrowserEngine(worker); + const fragments: string[] = []; + const promise = engine.transcribe(new Float32Array(4), { + onPartial: (fragment) => fragments.push(fragment), + }); + const requestId = worker.lastRequestId(); + + // Noise the engine must ignore: other ids, id-less file-flow messages, + // load progress. + worker.emit({ type: "partial", text: "IGNORED", requestId: "other" }); + worker.emit({ type: "partial", text: "IGNORED" }); + worker.emit({ type: "load-progress", status: "ready", device: "wasm" }); + + worker.emit({ type: "partial", text: "Hel", requestId }); + worker.emit({ type: "partial", text: "lo", requestId }); + worker.emit({ type: "result", text: "Hello", chunks: CHUNKS, requestId }); + worker.emit({ type: "result", text: "STALE", chunks: [], requestId: "other" }); + + expect(await promise).toEqual({ text: "Hello", chunks: CHUNKS }); + expect(fragments).toEqual(["Hel", "lo"]); + }); + + test("rejects on a matching error message", async () => { + const worker = new FakeWorker(); + const engine = new BrowserEngine(worker); + const promise = engine.transcribe(new Float32Array(4)); + worker.emit({ + type: "error", + stage: "transcribe", + message: "out of memory", + requestId: worker.lastRequestId(), + }); + expect(promise).rejects.toThrow("out of memory"); + }); + + test("a worker crash rejects every in-flight request", async () => { + const worker = new FakeWorker(); + const engine = new BrowserEngine(worker); + const first = engine.transcribe(new Float32Array(4)); + const second = engine.transcribe(new Float32Array(4)); + worker.emitError("worker exploded"); + expect(first).rejects.toThrow("worker exploded"); + expect(second).rejects.toThrow("worker exploded"); + }); + + test("two engines can share one worker without stealing responses", async () => { + const worker = new FakeWorker(); + const engineA = new BrowserEngine(worker); + const engineB = new BrowserEngine(worker); + const promiseA = engineA.transcribe(new Float32Array(4)); + const promiseB = engineB.transcribe(new Float32Array(4)); + const [idA, idB] = worker.posted.map( + (post) => post.message.requestId as string, + ); + expect(idA).not.toBe(idB); + worker.emit({ type: "result", text: "for A", chunks: [], requestId: idA }); + worker.emit({ type: "result", text: "for B", chunks: [], requestId: idB }); + expect((await promiseA).text).toBe("for A"); + expect((await promiseB).text).toBe("for B"); + }); +}); + +// --------------------------------------------------------------------------- +// ServerEngine — ndjson stream over an injected fetch +// --------------------------------------------------------------------------- + +function ndjsonResponse(lines: unknown[], status = 200): Response { + const body = lines.map((line) => `${JSON.stringify(line)}\n`).join(""); + return new Response(body, { + status, + headers: { "Content-Type": "application/x-ndjson" }, + }); +} + +describe("ServerEngine", () => { + test("streams segments to onPartial and resolves with the result line", async () => { + const calls: { url: string; init?: RequestInit }[] = []; + const fetchFn: FetchLike = async (url, init) => { + calls.push({ url, init }); + return ndjsonResponse([ + { type: "segment", text: " Hello", start: 0, end: 1.5 }, + { type: "segment", text: " world.", start: 1.5, end: 3 }, + { type: "result", text: "Hello world.", chunks: CHUNKS }, + ]); + }; + const engine = new ServerEngine(fetchFn); + const fragments: string[] = []; + const audio = new Float32Array([0.5, -0.5, 0.25]); + const result = await engine.transcribe(audio, { + modelId: "onnx-community/whisper-tiny.en", + language: "en", + task: "transcribe", + onPartial: (fragment) => fragments.push(fragment), + }); + + expect(result).toEqual({ text: "Hello world.", chunks: CHUNKS }); + expect(fragments).toEqual([" Hello", " world."]); + + const call = calls[0]!; + const url = new URL(call.url, "http://localhost"); + expect(url.pathname).toBe("/api/transcribe"); + expect(url.searchParams.get("model")).toBe("onnx-community/whisper-tiny.en"); + expect(url.searchParams.get("language")).toBe("en"); + expect(url.searchParams.get("task")).toBe("transcribe"); + expect(call.init?.method).toBe("POST"); + expect(call.init?.credentials).toBe("same-origin"); + // The body must be the raw PCM bytes of the Float32Array. + const sent = new Float32Array((call.init?.body as Uint8Array).buffer.slice(0)); + expect([...sent]).toEqual([...audio]); + }); + + test('drops language "auto" from the query', async () => { + let url = ""; + const engine = new ServerEngine(async (input) => { + url = input; + return ndjsonResponse([{ type: "result", text: "", chunks: [] }]); + }); + await engine.transcribe(new Float32Array(4), { + modelId: "onnx-community/whisper-tiny", + language: "auto", + }); + expect(url).not.toContain("language"); + }); + + test("requires a modelId", () => { + const engine = new ServerEngine(async () => ndjsonResponse([])); + expect(engine.transcribe(new Float32Array(4))).rejects.toThrow(/model id/); + }); + + test("rejects with the in-stream error line's message", () => { + const engine = new ServerEngine(async () => + ndjsonResponse([ + { type: "segment", text: " partial", start: 0, end: 1 }, + { type: "error", message: "sidecar exploded" }, + ]), + ); + expect( + engine.transcribe(new Float32Array(4), { modelId: "onnx-community/whisper-tiny" }), + ).rejects.toThrow("sidecar exploded"); + }); + + test("maps HTTP statuses to UI-safe messages", async () => { + const cases: [number, RegExp][] = [ + [401, /sign in/i], + [404, /not available/i], + [413, /too large|limit/i], + [503, /unavailable/i], + ]; + for (const [status, pattern] of cases) { + const engine = new ServerEngine(async () => new Response("{}", { status })); + expect( + engine.transcribe(new Float32Array(4), { + modelId: "onnx-community/whisper-tiny", + }), + ).rejects.toThrow(pattern); + } + }); + + test("surfaces the server's {error} detail on 503", () => { + const engine = new ServerEngine(async () => + new Response(JSON.stringify({ error: "faster-whisper is not installed" }), { + status: 503, + }), + ); + expect( + engine.transcribe(new Float32Array(4), { modelId: "onnx-community/whisper-tiny" }), + ).rejects.toThrow(/faster-whisper is not installed/); + }); + + test("a stream that ends without a result line is a lost connection", () => { + const engine = new ServerEngine(async () => + ndjsonResponse([{ type: "segment", text: " cut", start: 0, end: 1 }]), + ); + expect( + engine.transcribe(new Float32Array(4), { modelId: "onnx-community/whisper-tiny" }), + ).rejects.toThrow(/connection was lost/); + }); + + test("network failure gets a friendly message", () => { + const engine = new ServerEngine(async () => { + throw new TypeError("fetch failed"); + }); + expect( + engine.transcribe(new Float32Array(4), { modelId: "onnx-community/whisper-tiny" }), + ).rejects.toThrow(/could not reach the server/i); + }); +}); + +// --------------------------------------------------------------------------- +// Wire-format guard +// --------------------------------------------------------------------------- + +describe("isTranscribeStreamLine", () => { + test("accepts the three line shapes", () => { + expect( + isTranscribeStreamLine({ type: "segment", text: " a", start: 0, end: 1 }), + ).toBe(true); + expect(isTranscribeStreamLine({ type: "result", text: "a", chunks: [] })).toBe(true); + expect(isTranscribeStreamLine({ type: "error", message: "x" })).toBe(true); + }); + + test("rejects malformed lines", () => { + expect(isTranscribeStreamLine(null)).toBe(false); + expect(isTranscribeStreamLine({ type: "segment", text: " a" })).toBe(false); + expect(isTranscribeStreamLine({ type: "result", text: "a" })).toBe(false); + expect(isTranscribeStreamLine({ type: "unknown" })).toBe(false); + }); +}); + +// --------------------------------------------------------------------------- +// Availability probe +// --------------------------------------------------------------------------- + +describe("fetchServerEngineStatus", () => { + test("passes through an available status", async () => { + const status = await fetchServerEngineStatus(async () => + new Response(JSON.stringify({ available: true }), { status: 200 }), + ); + expect(status).toEqual({ available: true }); + }); + + test("passes through unavailable + reason", async () => { + const status = await fetchServerEngineStatus(async () => + new Response( + JSON.stringify({ available: false, reason: "sidecar missing" }), + { status: 200 }, + ), + ); + expect(status).toEqual({ available: false, reason: "sidecar missing" }); + }); + + test("404 (deployment without the feature) resolves unavailable", async () => { + const status = await fetchServerEngineStatus(async () => + new Response("Not Found", { status: 404 }), + ); + expect(status.available).toBe(false); + }); + + test("never rejects on network failure", async () => { + const status = await fetchServerEngineStatus(async () => { + throw new TypeError("offline"); + }); + expect(status.available).toBe(false); + }); +}); + +// --------------------------------------------------------------------------- +// Engine choice persistence +// --------------------------------------------------------------------------- + +function fakeStorage(initial: Record = {}): StorageLike & { + data: Record; +} { + const data = { ...initial }; + return { + data, + getItem: (key) => (key in data ? data[key]! : null), + setItem: (key, value) => { + data[key] = value; + }, + }; +} + +describe("engine choice persistence", () => { + test("round-trips through storage", () => { + const storage = fakeStorage(); + saveEngineChoice("server", storage); + expect(storage.data[ENGINE_CHOICE_KEY]).toBe("server"); + expect(loadEngineChoice(storage)).toBe("server"); + }); + + test('defaults to "browser" for missing or garbage values', () => { + expect(loadEngineChoice(fakeStorage())).toBe("browser"); + expect( + loadEngineChoice(fakeStorage({ [ENGINE_CHOICE_KEY]: "cloud" })), + ).toBe("browser"); + expect(loadEngineChoice(null)).toBe("browser"); + }); + + test("throwing storage is survived", () => { + const storage: StorageLike = { + getItem: () => { + throw new Error("denied"); + }, + setItem: () => { + throw new Error("denied"); + }, + }; + expect(loadEngineChoice(storage)).toBe("browser"); + expect(() => saveEngineChoice("server", storage)).not.toThrow(); + }); +}); diff --git a/tests/fixtures/fake_sidecar.py b/tests/fixtures/fake_sidecar.py new file mode 100644 index 0000000..ac4f3e6 --- /dev/null +++ b/tests/fixtures/fake_sidecar.py @@ -0,0 +1,58 @@ +"""Protocol-faithful stand-in for server-py/sidecar.py (no faster-whisper). + +Used by tests/transcribe.test.ts to exercise the real Bun-side supervisor +(createSidecar) against an actual child process. Behavior is keyed off the +requested model name: + + "ok" two segments (the second reports the audio byte count, + language and task for pass-through assertions), then done + "boom" an error line + "die" exit(1) mid-request (crash simulation) + "wait-for-cancel" one segment, then block until the next stdin line + (the cancel), then done {cancelled: true} +""" + +import json +import os +import sys + + +def emit(message): + sys.stdout.write(json.dumps(message) + "\n") + sys.stdout.flush() + + +emit({"type": "ready", "version": "fake"}) + +for line in sys.stdin: + line = line.strip() + if not line: + continue + message = json.loads(line) + kind = message.get("type") + if kind == "shutdown": + break + if kind != "transcribe": + continue + + request_id = message.get("id") + model = message.get("model") + if model == "boom": + emit({"type": "error", "id": request_id, "message": "kaboom"}) + elif model == "die": + sys.exit(1) + elif model == "wait-for-cancel": + emit({"type": "segment", "id": request_id, "start": 0.0, "end": 1.0, + "text": " before cancel"}) + next_line = json.loads(sys.stdin.readline()) + cancelled = (next_line.get("type") == "cancel" + and next_line.get("id") == request_id) + emit({"type": "done", "id": request_id, "cancelled": cancelled}) + else: + size = os.path.getsize(message["audio_path"]) + emit({"type": "segment", "id": request_id, "start": 0.0, "end": 1.5, + "text": " Hello"}) + emit({"type": "segment", "id": request_id, "start": 1.5, "end": 3.0, + "text": " world. [model=%s bytes=%d lang=%s task=%s]" + % (model, size, message.get("language"), message.get("task"))}) + emit({"type": "done", "id": request_id, "cancelled": False}) diff --git a/tests/live.md b/tests/live.md new file mode 100644 index 0000000..348dc39 --- /dev/null +++ b/tests/live.md @@ -0,0 +1,77 @@ +# Manual live-transcription verification (M7, D-016) + +The segment loop (`src/lib/live.ts`) and the protocol changes are covered by +unit tests; microphone capture, the AudioWorklet path, and end-to-end live +quality can only be exercised in a real browser. This is that procedure. +Re-run it after any change to `src/lib/live.ts`, `src/lib/mic.ts`, +`src/components/LiveControls.tsx`, or the live wiring in `src/App.tsx`. + +## Prerequisites + +- A machine with a working microphone. +- Chrome/Edge (WebGPU) and ideally also Firefox and Safari — the three differ + exactly where mic capture is fragile (context sample rates, AudioWorklet + Blob-URL modules; Safari exercises the ScriptProcessorNode fallback if the + worklet path fails there). +- `getUserMedia` requires a secure context; `http://localhost` qualifies: + ```sh + bun dev # or: bun run build && bun run start + ``` + +## 1. Permission flow + +1. Open the app, pick `whisper-tiny.en`, click **Start live transcription**. +2. The browser prompts for the microphone. **Deny** it: the button must return + to idle and the error "Microphone access was denied…" must appear; the app + stays usable (file flow unaffected). +3. Click again and **allow**: button becomes **Stop** with a pulsing red dot; + the browser's mic indicator turns on. If the model wasn't downloaded yet, + the download progress panel appears and live text simply starts once it is + ready (segment passes queue in the worker). + +## 2. Live transcription behavior + +1. Speak a few sentences with natural pauses. Expect: + - While speaking: dimmed italic *tentative* text appears within a couple of + seconds (tiny.en; slower models lag proportionally) and refreshes. + - After a ~1 s pause: the sentence commits — the text turns from italic to + regular ("committed") and no longer changes. + - Silence alone (say nothing for 30 s) must produce **no** text — the + energy gate drops silent segments, so no hallucinated "thank you."-style + output. +2. DevTools → Network: **zero** requests while speaking (weights may download + at the start; the audio itself never leaves the device). +3. During a live session, ModelPicker / options / dropzone are disabled. +4. Model choice sanity: repeat briefly with a multilingual model + a fixed + language and confirm the language/task options are honored. + +## 3. Stop & result + +1. Click **Stop** mid-sentence. The button shows "Finishing…" while the last + segment gets its final pass, then the normal **Transcript** panel appears: + full text, Copy button, and a Timestamps section whose segment times are + **absolute** (a sentence spoken ~10 s in must not be timestamped 0:00). +2. The browser's mic indicator must turn off immediately on Stop. +3. Start a new live session afterwards, then also transcribe a file — both + must still work (session cleanup, request correlation). + +## 4. Quality/latency knobs (only if needed) + +If commits are too eager/lazy in real rooms, tune `LiveSegmenterOptions` +(`speechRms` gate 0.015, `commitSilenceSeconds` 0.75, `maxSegmentSeconds` 25, +`minSpeechSeconds` 0.25) and the `LiveTranscriber` throttle +(`tentativeIntervalMs` 1500) — all injected from `src/App.tsx` defaults, all +unit-tested in `tests/live.test.ts`. Record any tuned values in D-016. + +## 5. Offline live transcription + +With the PWA installed and a model cached (see `tests/offline.md`), go +offline (airplane mode) and run a live session — it must work identically: +capture, worklet (Blob URL, no network), inference, stop/transcript. + +## Recording the result + +When this passes in a real browser, flip the `[~]` manual-verification item +in `WorkBreakdown.md` M7 to `[x]` and note browser + version here. + +- [ ] Verified on: _browser/version, date_ diff --git a/tests/live.test.ts b/tests/live.test.ts new file mode 100644 index 0000000..c877e39 --- /dev/null +++ b/tests/live.test.ts @@ -0,0 +1,313 @@ +import { describe, test, expect } from "bun:test"; +import { + LiveSegmenter, + LiveTranscriber, + rms, + type LiveTranscribeResult, + type LiveUpdate, +} from "../src/lib/live"; + +// Pure segment-loop logic only (D-016) — no mic, no worker, no model. Audio +// is synthetic: "speech" is a ±0.1 square wave (RMS 0.1, far above the 0.015 +// gate), "silence" is zeros. + +const RATE = 16_000; +const FRAME = 512; // default frameSeconds 0.032 × 16 kHz + +function speech(seconds: number): Float32Array { + const out = new Float32Array(Math.round(seconds * RATE)); + for (let i = 0; i < out.length; i++) out[i] = i % 2 === 0 ? 0.1 : -0.1; + return out; +} + +function silence(seconds: number): Float32Array { + return new Float32Array(Math.round(seconds * RATE)); +} + +/** Let pending promise chains (and a macrotask) settle. */ +function tick(): Promise { + return new Promise((resolve) => setTimeout(resolve, 0)); +} + +describe("rms", () => { + test("zero for empty and silent blocks, level for a square wave", () => { + expect(rms(new Float32Array(0))).toBe(0); + expect(rms(new Float32Array(100))).toBe(0); + expect(rms(speech(0.1))).toBeCloseTo(0.1, 5); + }); +}); + +describe("LiveSegmenter", () => { + test("pure silence never produces a segment", () => { + const segmenter = new LiveSegmenter(); + expect(segmenter.push(silence(3))).toEqual([]); + expect(segmenter.currentSegmentId).toBeNull(); + expect(segmenter.currentSegmentSamples).toBe(0); + expect(segmenter.currentSegment()).toBeNull(); + expect(segmenter.flush()).toBeNull(); + }); + + test("speech followed by trailing silence commits with pre-roll", () => { + const segmenter = new LiveSegmenter(); + // 16 frames silence, 32 frames speech, then silence until commit. + expect(segmenter.push(silence(0.512))).toEqual([]); + expect(segmenter.push(speech(1.024))).toEqual([]); + expect(segmenter.currentSegmentId).not.toBeNull(); + + const committed = segmenter.push(silence(1.024)); + expect(committed).toHaveLength(1); + const segment = committed[0]!; + expect(segment.reason).toBe("silence"); + // Pre-roll keeps 10 frames (0.3 s / 0.032 s) of the leading silence. + expect(segment.startSample).toBe((16 - 10) * FRAME); + // 10 pre-roll + 32 speech + 24 trailing-silence frames (ceil(0.75/0.032)). + expect(segment.audio.length).toBe((10 + 32 + 24) * FRAME); + // Back to idle after the commit. + expect(segmenter.currentSegmentId).toBeNull(); + }); + + test("near-silent blips are dropped, not committed", () => { + const segmenter = new LiveSegmenter(); + // 4 speech frames = 0.128 s < minSpeechSeconds 0.25 s. + expect(segmenter.push(speech(0.128))).toEqual([]); + expect(segmenter.currentSegmentId).not.toBeNull(); + expect(segmenter.push(silence(2))).toEqual([]); + expect(segmenter.currentSegmentId).toBeNull(); + expect(segmenter.flush()).toBeNull(); + }); + + test("continuous speech commits at the max segment length and restarts", () => { + const segmenter = new LiveSegmenter({ maxSegmentSeconds: 2 }); + const committed = segmenter.push(speech(3)); + expect(committed).toHaveLength(1); + expect(committed[0]!.reason).toBe("max-length"); + expect(committed[0]!.startSample).toBe(0); + // 2 s cap → first frame count at/over 32 000 samples. + expect(committed[0]!.audio.length).toBe(Math.ceil((2 * RATE) / FRAME) * FRAME); + // The rest of the speech opened a fresh segment. + expect(segmenter.currentSegmentId).not.toBeNull(); + expect(segmenter.currentSegmentSamples).toBeGreaterThan(0); + }); + + test("flush commits the in-progress segment, including the sub-frame tail", () => { + const segmenter = new LiveSegmenter(); + // 1 s + 100 samples: the 100-sample tail is smaller than one frame. + segmenter.push(speech(1)); + segmenter.push(speech(100 / RATE)); + const flushed = segmenter.flush(); + expect(flushed).not.toBeNull(); + expect(flushed!.reason).toBe("flush"); + expect(flushed!.audio.length).toBe(Math.round(1 * RATE) + 100); + expect(segmenter.currentSegmentId).toBeNull(); + }); + + test("currentSegment snapshots the in-progress audio", () => { + const segmenter = new LiveSegmenter(); + segmenter.push(speech(1.024)); + expect(segmenter.currentSegmentSamples).toBe(32 * FRAME); + const snapshot = segmenter.currentSegment(); + expect(snapshot).not.toBeNull(); + expect(snapshot!.audio.length).toBe(32 * FRAME); + expect(snapshot!.startSample).toBe(0); + expect(rms(snapshot!.audio)).toBeCloseTo(0.1, 5); + }); + + test("frame splitting is independent of push block sizes", () => { + const bulk = new LiveSegmenter(); + const dribble = new LiveSegmenter(); + const audio = speech(1.024); + bulk.push(audio); + for (let i = 0; i < audio.length; i += 100) { + dribble.push(audio.slice(i, Math.min(i + 100, audio.length))); + } + expect(dribble.currentSegmentSamples).toBe(bulk.currentSegmentSamples); + }); +}); + +// --------------------------------------------------------------------------- +// LiveTranscriber — fake engine with manually resolved passes +// --------------------------------------------------------------------------- + +interface FakeCall { + audio: Float32Array; + resolve: (result: LiveTranscribeResult) => void; + reject: (error: Error) => void; +} + +function fakeEngine() { + const calls: FakeCall[] = []; + const updates: LiveUpdate[] = []; + let now = 0; + return { + calls, + updates, + advance(ms: number) { + now += ms; + }, + make(options: { tentativeIntervalMs?: number; minTentativeSeconds?: number } = {}) { + return new LiveTranscriber({ + transcribe: (audio) => + new Promise((resolve, reject) => { + calls.push({ audio, resolve, reject }); + }), + onUpdate: (update) => updates.push(update), + now: () => now, + ...options, + }); + }, + last() { + return updates[updates.length - 1]!; + }, + }; +} + +describe("LiveTranscriber", () => { + test("runs a throttled tentative pass over the growing segment", async () => { + const engine = fakeEngine(); + const live = engine.make(); + + live.push(speech(1.5)); + expect(engine.calls).toHaveLength(1); // ≥ minTentativeSeconds → first pass + expect(engine.calls[0]!.audio.length).toBeGreaterThanOrEqual(1.4 * RATE); + + // Single-flight: more audio while a pass is running starts nothing new. + live.push(speech(0.5)); + expect(engine.calls).toHaveLength(1); + + engine.calls[0]!.resolve({ text: " hello", chunks: [] }); + await tick(); + expect(engine.last().tentativeText).toBe(" hello"); + expect(engine.last().committedText).toBe(""); + + // Same segment, interval not elapsed → still no second pass on push. + live.push(speech(0.2)); + expect(engine.calls).toHaveLength(1); + + // Interval elapsed + new samples → refresh. + engine.advance(2000); + live.push(speech(0.2)); + expect(engine.calls).toHaveLength(2); + }); + + test("committed segments get a final pass that freezes their text", async () => { + const engine = fakeEngine(); + const live = engine.make(); + + live.push(speech(1.5)); + engine.calls[0]!.resolve({ text: " helo", chunks: [] }); + await tick(); + + // Trailing silence commits the segment; tentative text turns provisional. + live.push(silence(1.5)); + expect(engine.last().committedText).toBe("helo"); + expect(engine.last().pendingFinalPasses).toBe(1); + expect(engine.calls).toHaveLength(2); // final pass started immediately + + engine.calls[1]!.resolve({ + text: " hello world", + chunks: [{ text: " hello world", timestamp: [0, 1.4] }], + }); + await tick(); + expect(engine.last().committedText).toBe("hello world"); + expect(engine.last().tentativeText).toBe(""); + expect(engine.last().pendingFinalPasses).toBe(0); + // Speech started at sample 0 → no offset on the chunk timestamps. + expect(engine.last().chunks[0]!.timestamp[0]).toBeCloseTo(0, 5); + expect(engine.last().chunks[0]!.timestamp[1]!).toBeCloseTo(1.4, 5); + }); + + test("chunk timestamps are offset by the segment's session position", async () => { + const engine = fakeEngine(); + const live = engine.make({ minTentativeSeconds: 9999 }); // finals only + + live.push(silence(0.512)); // 16 frames; pre-roll keeps the last 10 + live.push(speech(1)); + const stopping = live.stop(); + await tick(); + expect(engine.calls).toHaveLength(1); + engine.calls[0]!.resolve({ + text: " offset", + chunks: [{ text: " offset", timestamp: [0, 1] }], + }); + const result = await stopping; + + const offset = ((16 - 10) * FRAME) / RATE; + expect(result.text).toBe("offset"); + expect(result.chunks[0]!.timestamp[0]).toBeCloseTo(offset, 5); + expect(result.chunks[0]!.timestamp[1]!).toBeCloseTo(1 + offset, 5); + expect(result.chunks[0]!.timestamp[1]).not.toBeNull(); + }); + + test("stop flushes, drains pending passes, and joins the transcript", async () => { + const engine = fakeEngine(); + const live = engine.make({ minTentativeSeconds: 9999 }); + + live.push(speech(1)); + live.push(silence(1.5)); // segment 1 commits → final pass 1 + live.push(speech(1)); // segment 2 in progress + const stopping = live.stop(); // flush commits segment 2 + + let resolved = false; + void stopping.then(() => { + resolved = true; + }); + await tick(); + expect(resolved).toBe(false); // still draining + + engine.calls[0]!.resolve({ text: " one.", chunks: [] }); + await tick(); + engine.calls[1]!.resolve({ text: " two.", chunks: [] }); + const result = await stopping; + expect(result.text).toBe("one. two."); + + // The session is over: further audio is ignored. + live.push(speech(2)); + expect(engine.calls).toHaveLength(2); + }); + + test("a failing pass keeps the provisional text and the session alive", async () => { + const engine = fakeEngine(); + const live = engine.make(); + + live.push(speech(1.5)); + engine.calls[0]!.resolve({ text: " provisional", chunks: [] }); + await tick(); + live.push(silence(1.5)); // commit → final pass + engine.calls[1]!.reject(new Error("inference exploded")); + await tick(); + + expect(engine.last().error).toBe("inference exploded"); + expect(engine.last().committedText).toBe("provisional"); + expect(engine.last().pendingFinalPasses).toBe(0); + + // Later segments still work. + engine.advance(5000); + live.push(speech(1.5)); + expect(engine.calls).toHaveLength(3); + const stopping = live.stop(); + await tick(); + engine.calls[2]!.resolve({ text: " ignored (stale tentative)", chunks: [] }); + await tick(); + engine.calls[3]!.resolve({ text: " recovered", chunks: [] }); + const result = await stopping; + expect(result.text).toBe("provisional recovered"); + }); + + test("a tentative result for an already-committed segment is discarded", async () => { + const engine = fakeEngine(); + const live = engine.make(); + + live.push(speech(1.5)); // tentative pass in flight + live.push(silence(1.5)); // segment commits while the pass runs + expect(engine.last().committedText).toBe(""); // no text known yet + + engine.calls[0]!.resolve({ text: " stale", chunks: [] }); + await tick(); + // Stale text was not applied as tentative; the final pass owns the text. + expect(engine.last().tentativeText).toBe(""); + expect(engine.calls).toHaveLength(2); + engine.calls[1]!.resolve({ text: " final", chunks: [] }); + await tick(); + expect(engine.last().committedText).toBe("final"); + }); +}); diff --git a/tests/server-engine.md b/tests/server-engine.md new file mode 100644 index 0000000..1b238f0 --- /dev/null +++ b/tests/server-engine.md @@ -0,0 +1,51 @@ +# Manual verification: server engine (M5, D-013) + +The dev environment has no Python package manager and no browser, so — like +`tests/offline.md` and `tests/live.md` — the real-inference path needs a +manual run on a machine with Python ≥ 3.9. Everything protocol- and +HTTP-level is already covered by `bun test` (`tests/transcribe.test.ts` +exercises the real `Bun.spawn` supervisor against +`tests/fixtures/fake_sidecar.py`, plus the degradation paths against the real +`server-py/sidecar.py`). + +## 1. Install the sidecar dependencies + +```sh +python3 -m pip install -r server-py/requirements.txt +``` + +## 2. Opt-in automated pass (no browser needed) + +```sh +FW_INTEGRATION=1 bun test tests/transcribe.test.ts +``` + +Runs the real sidecar end to end (spawn → ready handshake → `tiny.en` load → +transcribe → done). First run downloads ~75 MB of weights from the Hub. + +- [ ] The `real faster-whisper sidecar` describe block passes. + +## 3. In-browser run (both engines, same audio) + +1. `bun run dev`, open the app, register/sign in. +2. `GET /api/transcribe/status` (with the session cookie) returns + `{"available": true}` — e.g. check the Network tab. +3. Pick `Whisper Tiny (English)`; transcribe a short clean English clip + (e.g. the JFK "ask not…" sample) **on device**. Note the transcript. +4. Switch the engine toggle to **Server** (visible only while signed in; + requires the M5 UI wiring — until that lands, drive it manually: + `curl -X POST --data-binary @audio.f32 -b "sid=…" \ + "http://localhost:3000/api/transcribe?model=onnx-community/whisper-tiny.en"` + where `audio.f32` is raw float32 mono 16 kHz PCM). +5. Transcribe the same clip through the server engine. + +- [ ] Segments stream in as JSON lines (not one blob at the end). +- [ ] Both transcripts agree in content (minor punctuation/casing drift + between Transformers.js and faster-whisper is expected). +- [ ] Signed out, the toggle is gone and `POST /api/transcribe` answers 401. +- [ ] Second request on the same model is visibly faster (no reload — + warm-up strategy). +- [ ] `AUTH_ENABLED=false bun run dev`: `/api/transcribe*` answers 404 and + the anonymous browser flow is unaffected. + +When this passes, check off the M5 integration item in `WorkBreakdown.md`. diff --git a/tests/transcribe.test.ts b/tests/transcribe.test.ts new file mode 100644 index 0000000..0189e39 --- /dev/null +++ b/tests/transcribe.test.ts @@ -0,0 +1,448 @@ +import { afterAll, describe, expect, test } from "bun:test"; +import { mkdtemp, readdir, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + createSidecar, + createTranscribeHandlers, + type Sidecar, + type SidecarSegment, + type SidecarTranscribeRequest, +} from "../src/server/transcribe"; +import { isTranscribeStreamLine, type TranscribeStreamLine } from "../src/lib/engine"; +import type { SessionUser } from "../src/server/auth"; + +// M5 server engine: HTTP handlers against a fake sidecar, and the real +// Bun.spawn supervisor against a protocol-faithful Python stand-in +// (tests/fixtures/fake_sidecar.py) — no faster-whisper needed. The end-to-end +// run with real inference is manual/opt-in: see tests/server-engine.md. + +const FIXTURE_SIDECAR = join(import.meta.dir, "fixtures/fake_sidecar.py"); +const REAL_SIDECAR = join(import.meta.dir, "../server-py/sidecar.py"); + +const tempDirs: string[] = []; +async function makeTempDir(): Promise { + const dir = await mkdtemp(join(tmpdir(), "transcribe-test-")); + tempDirs.push(dir); + return dir; +} +afterAll(async () => { + await Promise.all(tempDirs.map((dir) => rm(dir, { recursive: true, force: true }))); +}); + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +const USER: SessionUser = { id: 1, username: "tester" }; +const sessionFromCookie = (request: Request): SessionUser | null => + request.headers.get("cookie") === "sid=ok" ? USER : null; + +interface FakeSidecar extends Sidecar { + requests: SidecarTranscribeRequest[]; + /** Bytes of each request's audio file, captured before cleanup. */ + audioPayloads: Uint8Array[]; +} + +function fakeSidecar( + behavior: { + available?: boolean; + reason?: string; + segments?: SidecarSegment[]; + error?: string; + waitForAbort?: boolean; + } = {}, +): FakeSidecar { + const segments = behavior.segments ?? [ + { start: 0, end: 1.5, text: " Hello" }, + { start: 1.5, end: 3, text: " world." }, + ]; + const sidecar: FakeSidecar = { + requests: [], + audioPayloads: [], + async status() { + return behavior.available === false + ? { available: false, reason: behavior.reason ?? "nope" } + : { available: true }; + }, + async transcribe(request, onSegment, signal) { + sidecar.requests.push(request); + sidecar.audioPayloads.push( + new Uint8Array(await Bun.file(request.audioPath).arrayBuffer()), + ); + if (behavior.error) throw new Error(behavior.error); + onSegment(segments[0]!); + if (behavior.waitForAbort) { + await new Promise((resolve) => { + if (signal?.aborted) return resolve(); + signal?.addEventListener("abort", () => resolve(), { once: true }); + }); + return { cancelled: true }; + } + for (const segment of segments.slice(1)) onSegment(segment); + return { cancelled: false }; + }, + }; + return sidecar; +} + +function pcmBody(samples: number[]): Uint8Array { + const audio = new Float32Array(samples); + return new Uint8Array(audio.buffer as ArrayBuffer); +} + +function postRequest( + query: string, + body: BodyInit | null, + init: { cookie?: string; signal?: AbortSignal; contentLength?: boolean } = {}, +): Request { + const headers: Record = { + "Content-Type": "application/octet-stream", + }; + if (init.cookie !== undefined) headers.cookie = init.cookie; + return new Request(`http://localhost/api/transcribe${query}`, { + method: "POST", + headers, + body, + signal: init.signal, + }); +} + +async function readStreamLines(response: Response): Promise { + const text = await response.text(); + return text + .split("\n") + .filter((line) => line.trim().length > 0) + .map((line) => { + const parsed: unknown = JSON.parse(line); + if (!isTranscribeStreamLine(parsed)) throw new Error(`bad line: ${line}`); + return parsed; + }); +} + +// --------------------------------------------------------------------------- +// HTTP handlers (fake sidecar) +// --------------------------------------------------------------------------- + +describe("GET /api/transcribe/status", () => { + test("401 without a session", async () => { + const handlers = createTranscribeHandlers({ + sidecar: fakeSidecar(), + getSessionUser: sessionFromCookie, + }); + const response = await handlers.status( + new Request("http://localhost/api/transcribe/status"), + ); + expect(response.status).toBe(401); + }); + + test("passes the sidecar's availability through", async () => { + const handlers = createTranscribeHandlers({ + sidecar: fakeSidecar({ available: false, reason: "python3 missing" }), + getSessionUser: sessionFromCookie, + }); + const response = await handlers.status( + new Request("http://localhost/api/transcribe/status", { + headers: { cookie: "sid=ok" }, + }), + ); + expect(response.status).toBe(200); + expect(await response.json()).toEqual({ + available: false, + reason: "python3 missing", + }); + }); +}); + +describe("POST /api/transcribe", () => { + test("401 without a session — the server engine is logged-in only", async () => { + const sidecar = fakeSidecar(); + const handlers = createTranscribeHandlers({ + sidecar, + getSessionUser: sessionFromCookie, + }); + const response = await handlers.post( + postRequest("?model=onnx-community/whisper-tiny.en", pcmBody([0.1])), + ); + expect(response.status).toBe(401); + expect(sidecar.requests).toHaveLength(0); + }); + + test("400 for missing/unknown model, bad language, bad task", async () => { + const handlers = createTranscribeHandlers({ + sidecar: fakeSidecar(), + getSessionUser: sessionFromCookie, + tempDir: await makeTempDir(), + }); + const cases = [ + "", + "?model=someone/other-model", + "?model=onnx-community/whisper-tiny&language=Danish!", + "?model=onnx-community/whisper-tiny&task=summarize", + ]; + for (const query of cases) { + const response = await handlers.post( + postRequest(query, pcmBody([0.1]), { cookie: "sid=ok" }), + ); + expect(response.status).toBe(400); + } + }); + + test("503 with the reason when the sidecar is unavailable", async () => { + const sidecar = fakeSidecar({ available: false, reason: "faster-whisper is not installed" }); + const handlers = createTranscribeHandlers({ + sidecar, + getSessionUser: sessionFromCookie, + }); + const response = await handlers.post( + postRequest("?model=onnx-community/whisper-tiny.en", pcmBody([0.1]), { + cookie: "sid=ok", + }), + ); + expect(response.status).toBe(503); + expect(await response.json()).toEqual({ error: "faster-whisper is not installed" }); + expect(sidecar.requests).toHaveLength(0); + }); + + test("413 over the upload limit; 400 for empty or non-float32 bodies", async () => { + const tempDir = await makeTempDir(); + const handlers = createTranscribeHandlers({ + sidecar: fakeSidecar(), + getSessionUser: sessionFromCookie, + maxUploadBytes: 16, + tempDir, + }); + const over = await handlers.post( + postRequest("?model=onnx-community/whisper-tiny.en", pcmBody([1, 2, 3, 4, 5]), { + cookie: "sid=ok", + }), + ); + expect(over.status).toBe(413); + + const empty = await handlers.post( + postRequest("?model=onnx-community/whisper-tiny.en", new Uint8Array(0), { + cookie: "sid=ok", + }), + ); + expect(empty.status).toBe(400); + + const misaligned = await handlers.post( + postRequest("?model=onnx-community/whisper-tiny.en", new Uint8Array(6), { + cookie: "sid=ok", + }), + ); + expect(misaligned.status).toBe(400); + + // Rejected uploads must not leave temp files behind. + expect(await readdir(tempDir)).toEqual([]); + }); + + test("streams segments then the result, maps the model, cleans up", async () => { + const tempDir = await makeTempDir(); + const sidecar = fakeSidecar(); + const handlers = createTranscribeHandlers({ + sidecar, + getSessionUser: sessionFromCookie, + tempDir, + }); + const audio = [0.25, -0.25, 0.5, -0.5]; + const response = await handlers.post( + postRequest( + "?model=onnx-community/whisper-base&language=da&task=translate", + pcmBody(audio), + { cookie: "sid=ok" }, + ), + ); + expect(response.status).toBe(200); + expect(response.headers.get("content-type")).toContain("application/x-ndjson"); + + const lines = await readStreamLines(response); + expect(lines).toEqual([ + { type: "segment", text: " Hello", start: 0, end: 1.5 }, + { type: "segment", text: " world.", start: 1.5, end: 3 }, + { + type: "result", + text: "Hello world.", + chunks: [ + { text: " Hello", timestamp: [0, 1.5] }, + { text: " world.", timestamp: [1.5, 3] }, + ], + }, + ]); + + // Catalog id → faster-whisper name (D-013), options passed through. + expect(sidecar.requests).toEqual([ + expect.objectContaining({ model: "base", language: "da", task: "translate" }), + ]); + // The sidecar saw exactly the uploaded PCM bytes… + expect([...new Float32Array(sidecar.audioPayloads[0]!.buffer.slice(0))]).toEqual( + audio, + ); + // …and the temp file is gone once the stream finished. + expect(await readdir(tempDir)).toEqual([]); + }); + + test("sidecar errors travel in-stream after the 200 header", async () => { + const tempDir = await makeTempDir(); + const handlers = createTranscribeHandlers({ + sidecar: fakeSidecar({ error: "model load failed" }), + getSessionUser: sessionFromCookie, + tempDir, + }); + const response = await handlers.post( + postRequest("?model=onnx-community/whisper-tiny.en", pcmBody([0.1]), { + cookie: "sid=ok", + }), + ); + expect(response.status).toBe(200); + const lines = await readStreamLines(response); + expect(lines).toEqual([{ type: "error", message: "model load failed" }]); + expect(await readdir(tempDir)).toEqual([]); + }); + + test("an aborted request cancels the sidecar run", async () => { + const tempDir = await makeTempDir(); + const sidecar = fakeSidecar({ waitForAbort: true }); + const handlers = createTranscribeHandlers({ + sidecar, + getSessionUser: sessionFromCookie, + tempDir, + }); + const controller = new AbortController(); + const response = await handlers.post( + postRequest("?model=onnx-community/whisper-tiny.en", pcmBody([0.1]), { + cookie: "sid=ok", + signal: controller.signal, + }), + ); + expect(response.status).toBe(200); + controller.abort(); + // Cancelled runs still terminate the stream with the partial result. + const lines = await readStreamLines(response); + expect(lines.at(-1)?.type).toBe("result"); + expect(await readdir(tempDir)).toEqual([]); + }); +}); + +// --------------------------------------------------------------------------- +// createSidecar — the real Bun.spawn supervisor against a Python child +// --------------------------------------------------------------------------- + +describe("createSidecar (fake python sidecar)", () => { + test("handshake, segment streaming, and request/option pass-through", async () => { + const sidecar = createSidecar(["python3", FIXTURE_SIDECAR]); + expect(await sidecar.status()).toEqual({ available: true }); + + const audioPath = join(await makeTempDir(), "audio.f32"); + await Bun.write(audioPath, pcmBody([0.1, 0.2, 0.3])); + + const segments: SidecarSegment[] = []; + const outcome = await sidecar.transcribe( + { model: "ok", audioPath, language: "da", task: "translate" }, + (segment) => segments.push(segment), + ); + expect(outcome).toEqual({ cancelled: false }); + expect(segments).toHaveLength(2); + expect(segments[0]).toEqual({ start: 0, end: 1.5, text: " Hello" }); + // The fixture echoes what it received (audio bytes, language, task). + expect(segments[1]!.text).toContain("model=ok bytes=12 lang=da task=translate"); + }); + + test("sidecar 'error' lines reject that request only", async () => { + const sidecar = createSidecar(["python3", FIXTURE_SIDECAR]); + const audioPath = join(await makeTempDir(), "audio.f32"); + await Bun.write(audioPath, pcmBody([0.1])); + + expect( + sidecar.transcribe({ model: "boom", audioPath }, () => {}), + ).rejects.toThrow("kaboom"); + // The process survived; the next request works. + const outcome = await sidecar.transcribe({ model: "ok", audioPath }, () => {}); + expect(outcome).toEqual({ cancelled: false }); + }); + + test("a crash rejects in-flight requests and the next request respawns", async () => { + const sidecar = createSidecar(["python3", FIXTURE_SIDECAR]); + const audioPath = join(await makeTempDir(), "audio.f32"); + await Bun.write(audioPath, pcmBody([0.1])); + + expect( + sidecar.transcribe({ model: "die", audioPath }, () => {}), + ).rejects.toThrow(/exited unexpectedly/); + // Supervision: a fresh process serves the next request. + const segments: SidecarSegment[] = []; + const outcome = await sidecar.transcribe( + { model: "ok", audioPath }, + (segment) => segments.push(segment), + ); + expect(outcome).toEqual({ cancelled: false }); + expect(segments).toHaveLength(2); + }); + + test("abort sends a cancel and resolves {cancelled: true}", async () => { + const sidecar = createSidecar(["python3", FIXTURE_SIDECAR]); + const audioPath = join(await makeTempDir(), "audio.f32"); + await Bun.write(audioPath, pcmBody([0.1])); + + const controller = new AbortController(); + const outcome = await sidecar.transcribe( + { model: "wait-for-cancel", audioPath }, + () => controller.abort(), // abort as soon as the first segment arrives + controller.signal, + ); + expect(outcome).toEqual({ cancelled: true }); + }); +}); + +describe("createSidecar (degradation, D-013)", () => { + test("the real sidecar without faster-whisper reports a permanent reason", async () => { + // -S/-I keep site-packages off sys.path, so this behaves like a + // Bun-only deployment even on machines that have faster-whisper. + const sidecar = createSidecar(["python3", "-S", "-I", REAL_SIDECAR]); + const first = await sidecar.status(); + expect(first.available).toBe(false); + expect(first.reason).toContain("faster-whisper is not installed"); + // Remembered as permanent — no respawn storm, same answer immediately. + const second = await sidecar.status(); + expect(second).toEqual(first); + }); + + test("a missing python binary reports unavailable", async () => { + const sidecar = createSidecar(["definitely-not-a-python-binary"]); + const status = await sidecar.status(); + expect(status.available).toBe(false); + }); +}); + +// --------------------------------------------------------------------------- +// Opt-in: real faster-whisper end to end (needs python3 + requirements.txt +// installed and network/cached weights). See tests/server-engine.md. +// FW_INTEGRATION=1 bun test tests/transcribe.test.ts +// --------------------------------------------------------------------------- + +describe.skipIf(!process.env.FW_INTEGRATION)("real faster-whisper sidecar", () => { + test( + "transcribes PCM through the real sidecar", + async () => { + const sidecar = createSidecar(); + const status = await sidecar.status(); + expect(status).toEqual({ available: true }); + + // One second of a 440 Hz tone — not speech, but exercises model load, + // numpy decode, and the full protocol round-trip. + const samples = new Float32Array(16_000); + for (let i = 0; i < samples.length; i++) { + samples[i] = 0.1 * Math.sin((2 * Math.PI * 440 * i) / 16_000); + } + const audioPath = join(await makeTempDir(), "tone.f32"); + await Bun.write(audioPath, new Uint8Array(samples.buffer)); + + const outcome = await sidecar.transcribe( + { model: "tiny.en", audioPath }, + () => {}, + ); + expect(outcome.cancelled).toBe(false); + }, + { timeout: 300_000 }, + ); +});