Start on live transcription and python sidecar (faster-whisper)
This commit is contained in:
Lucas Winther
@@ -0,0 +1,4 @@
|
||||
# Server-engine sidecar dependencies (Decisions.md D-013). Only needed on
|
||||
# deployments that enable the server engine — the browser/anonymous path
|
||||
# deploys with Bun alone. Install with: python3 -m pip install -r requirements.txt
|
||||
faster-whisper>=1.1,<2
|
||||
@@ -0,0 +1,159 @@
|
||||
"""faster-whisper sidecar (Decisions.md D-013).
|
||||
|
||||
Bun spawns and supervises this process (src/server/transcribe.ts); it is the
|
||||
only place server-side inference happens. The interface is JSON-lines over
|
||||
stdio — one JSON object per line, no framing beyond that:
|
||||
|
||||
stdout <- {"type": "ready", "version": "1.1.1"} # startup handshake
|
||||
stdout <- {"type": "fatal", "message": "..."} # then exit 1
|
||||
|
||||
stdin -> {"type": "transcribe", "id": "1", "model": "tiny.en",
|
||||
"audio_path": "/tmp/...", "language": "en" | null,
|
||||
"task": "transcribe" | "translate"}
|
||||
stdout <- {"type": "segment", "id": "1", "start": 0.0, "end": 4.2,
|
||||
"text": " Hello world."} # 0..n of these
|
||||
stdout <- {"type": "done", "id": "1", "cancelled": false}
|
||||
stdout <- {"type": "error", "id": "1", "message": "..."} # instead of done
|
||||
|
||||
stdin -> {"type": "cancel", "id": "1"} # stop between segments; still "done"
|
||||
stdin -> {"type": "shutdown"} # clean exit (EOF does the same)
|
||||
|
||||
Audio arrives as a file of raw little-endian float32 mono PCM at 16 kHz — the
|
||||
exact representation the browser already produces for its own engine — so no
|
||||
container decoding happens here. Requests are processed strictly one at a
|
||||
time; a reader thread keeps consuming stdin meanwhile so `cancel` can
|
||||
interrupt the segment loop of the request in flight.
|
||||
|
||||
The last-used model stays loaded between requests (WorkBreakdown M5 warm-up:
|
||||
never reload weights per request) and is replaced — not accumulated — when a
|
||||
request names a different model, bounding memory to one model.
|
||||
|
||||
Environment:
|
||||
FASTER_WHISPER_DEVICE ctranslate2 device (default "auto")
|
||||
FASTER_WHISPER_COMPUTE compute type (default "int8" — the fast,
|
||||
low-memory CPU choice; CT2 converts on load)
|
||||
FASTER_WHISPER_DOWNLOAD_ROOT optional model download/cache directory
|
||||
|
||||
Requires Python >= 3.9 and the packages in requirements.txt. This file must
|
||||
keep working without them up to the import: a missing dependency is reported
|
||||
as a "fatal" line, which the Bun side surfaces as "server engine unavailable"
|
||||
(the anonymous/browser path never needs any of this).
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import queue
|
||||
import sys
|
||||
import threading
|
||||
|
||||
SAMPLE_RATE = 16000
|
||||
|
||||
|
||||
def emit(message):
|
||||
"""Write one JSON line to stdout. Only the main thread emits."""
|
||||
sys.stdout.write(json.dumps(message) + "\n")
|
||||
sys.stdout.flush()
|
||||
|
||||
|
||||
def main():
|
||||
try:
|
||||
import faster_whisper
|
||||
import numpy as np
|
||||
except ImportError as error:
|
||||
emit(
|
||||
{
|
||||
"type": "fatal",
|
||||
"message": "faster-whisper is not installed (%s) — "
|
||||
"see server-py/requirements.txt" % error,
|
||||
}
|
||||
)
|
||||
return 1
|
||||
|
||||
device = os.environ.get("FASTER_WHISPER_DEVICE", "auto")
|
||||
compute_type = os.environ.get("FASTER_WHISPER_COMPUTE", "int8")
|
||||
download_root = os.environ.get("FASTER_WHISPER_DOWNLOAD_ROOT") or None
|
||||
|
||||
requests = queue.Queue()
|
||||
cancelled = set() # request ids; GIL-atomic add/contains is enough here
|
||||
SHUTDOWN = object()
|
||||
|
||||
def read_stdin():
|
||||
"""Reader thread: queue work, flag cancels, shut down on EOF."""
|
||||
for line in sys.stdin:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
message = json.loads(line)
|
||||
except ValueError:
|
||||
continue # a malformed line must not kill the sidecar
|
||||
kind = message.get("type")
|
||||
if kind == "transcribe":
|
||||
requests.put(message)
|
||||
elif kind == "cancel":
|
||||
cancelled.add(message.get("id"))
|
||||
elif kind == "shutdown":
|
||||
requests.put(SHUTDOWN)
|
||||
return
|
||||
requests.put(SHUTDOWN) # supervisor closed the pipe / died
|
||||
|
||||
threading.Thread(target=read_stdin, daemon=True).start()
|
||||
|
||||
cached_name = None
|
||||
cached_model = None
|
||||
|
||||
emit({"type": "ready", "version": getattr(faster_whisper, "__version__", "?")})
|
||||
|
||||
while True:
|
||||
request = requests.get()
|
||||
if request is SHUTDOWN:
|
||||
return 0
|
||||
|
||||
request_id = request.get("id")
|
||||
try:
|
||||
model_name = request["model"]
|
||||
audio_path = request["audio_path"]
|
||||
language = request.get("language") or None
|
||||
task = request.get("task") or "transcribe"
|
||||
|
||||
if model_name != cached_name:
|
||||
cached_model = None # release before loading the replacement
|
||||
cached_model = faster_whisper.WhisperModel(
|
||||
model_name,
|
||||
device=device,
|
||||
compute_type=compute_type,
|
||||
download_root=download_root,
|
||||
)
|
||||
cached_name = model_name
|
||||
|
||||
audio = np.fromfile(audio_path, dtype=np.float32)
|
||||
segments, _info = cached_model.transcribe(
|
||||
audio, language=language, task=task
|
||||
)
|
||||
|
||||
was_cancelled = False
|
||||
# `segments` is a generator: inference happens during iteration,
|
||||
# so each segment can stream out (and cancel can cut in) while the
|
||||
# next one is still being decoded.
|
||||
for segment in segments:
|
||||
if request_id in cancelled:
|
||||
was_cancelled = True
|
||||
break
|
||||
emit(
|
||||
{
|
||||
"type": "segment",
|
||||
"id": request_id,
|
||||
"start": round(float(segment.start), 3),
|
||||
"end": round(float(segment.end), 3),
|
||||
"text": segment.text,
|
||||
}
|
||||
)
|
||||
emit({"type": "done", "id": request_id, "cancelled": was_cancelled})
|
||||
except Exception as error: # noqa: BLE001 — one bad request must not exit
|
||||
emit({"type": "error", "id": request_id, "message": str(error)})
|
||||
finally:
|
||||
cancelled.discard(request_id)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,46 @@
|
||||
/** Live microphone transcription start/stop (M7, Decisions.md D-016). */
|
||||
|
||||
export type LiveStatus = "idle" | "starting" | "listening" | "stopping";
|
||||
|
||||
interface LiveControlsProps {
|
||||
status: LiveStatus;
|
||||
onStart: () => void;
|
||||
onStop: () => void;
|
||||
/** True while the file flow is busy — the two modes are mutually exclusive. */
|
||||
disabled?: boolean;
|
||||
}
|
||||
|
||||
const LABELS: Record<LiveStatus, string> = {
|
||||
idle: "Start live transcription",
|
||||
starting: "Requesting microphone…",
|
||||
listening: "Stop",
|
||||
stopping: "Finishing…",
|
||||
};
|
||||
|
||||
export function LiveControls({
|
||||
status,
|
||||
onStart,
|
||||
onStop,
|
||||
disabled = false,
|
||||
}: LiveControlsProps) {
|
||||
const listening = status === "listening";
|
||||
return (
|
||||
<div className="live-controls">
|
||||
{listening && <span className="live-dot" aria-hidden="true" />}
|
||||
<button
|
||||
type="button"
|
||||
className={`live-button${listening ? " live-button-active" : ""}`}
|
||||
onClick={status === "idle" ? onStart : onStop}
|
||||
disabled={disabled || status === "starting" || status === "stopping"}
|
||||
aria-pressed={listening}
|
||||
>
|
||||
{LABELS[status]}
|
||||
</button>
|
||||
<p className="live-hint">
|
||||
{listening
|
||||
? "Listening — like everything else, your audio stays on this device."
|
||||
: "…or transcribe live from your microphone."}
|
||||
</p>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
+476
@@ -0,0 +1,476 @@
|
||||
/**
|
||||
* Live microphone transcription — the segment loop (Decisions.md D-016).
|
||||
*
|
||||
* Whisper is not a streaming model, so "live" is approximated by splitting
|
||||
* the mic signal into speech segments with a simple RMS energy gate and
|
||||
* re-transcribing the current segment on a rolling basis (tentative text)
|
||||
* until it commits, at which point it gets one final pass and freezes.
|
||||
*
|
||||
* Everything here is pure and free of browser globals so it runs under
|
||||
* `bun test`: audio arrives via `push()` (mic capture lives in lib/mic.ts)
|
||||
* and inference is an injected async function (the UI wraps the worker).
|
||||
*/
|
||||
|
||||
import type { TranscriptChunk } from "../worker/protocol";
|
||||
import { TARGET_SAMPLE_RATE } from "./audio";
|
||||
|
||||
/** Root-mean-square level of a sample block. Pure. */
|
||||
export function rms(samples: Float32Array): number {
|
||||
if (samples.length === 0) return 0;
|
||||
let sum = 0;
|
||||
for (let i = 0; i < samples.length; i++) {
|
||||
sum += samples[i]! * samples[i]!;
|
||||
}
|
||||
return Math.sqrt(sum / samples.length);
|
||||
}
|
||||
|
||||
function concat(parts: readonly Float32Array[], length: number): Float32Array {
|
||||
const out = new Float32Array(length);
|
||||
let offset = 0;
|
||||
for (const part of parts) {
|
||||
out.set(part, offset);
|
||||
offset += part.length;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Segmenter: RMS energy gate → speech segments
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export interface LiveSegmenterOptions {
|
||||
/** Sample rate of the incoming (and outgoing) PCM. Default 16 kHz. */
|
||||
sampleRate?: number;
|
||||
/** Analysis frame length in seconds. Default 0.032 (~32 ms). */
|
||||
frameSeconds?: number;
|
||||
/** A frame at/above this RMS counts as speech. Default 0.015. */
|
||||
speechRms?: number;
|
||||
/** Trailing silence that commits the current segment. Default 0.75 s. */
|
||||
commitSilenceSeconds?: number;
|
||||
/**
|
||||
* Hard cap per segment — must stay under Whisper's 30 s window so a live
|
||||
* segment is always a single-chunk transcription. Default 25 s.
|
||||
*/
|
||||
maxSegmentSeconds?: number;
|
||||
/** Audio kept from just before speech onset. Default 0.3 s. */
|
||||
prerollSeconds?: number;
|
||||
/**
|
||||
* Segments with less accumulated speech than this are dropped instead of
|
||||
* committed — Whisper hallucinates on (near-)silence. Default 0.25 s.
|
||||
*/
|
||||
minSpeechSeconds?: number;
|
||||
}
|
||||
|
||||
export type CommitReason = "silence" | "max-length" | "flush";
|
||||
|
||||
/** One committed speech segment, ready for a final transcription pass. */
|
||||
export interface CommittedSegment {
|
||||
audio: Float32Array;
|
||||
/** Absolute offset of `audio[0]` in samples since the session started. */
|
||||
startSample: number;
|
||||
reason: CommitReason;
|
||||
}
|
||||
|
||||
interface Frame {
|
||||
samples: Float32Array;
|
||||
/** Absolute offset of this frame's first sample. */
|
||||
start: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Splits a continuous PCM stream into speech segments. Push mic audio in any
|
||||
* block sizes; committed segments come back from `push()`/`flush()`.
|
||||
*/
|
||||
export class LiveSegmenter {
|
||||
private readonly sampleRate: number;
|
||||
private readonly frameSamples: number;
|
||||
private readonly speechRms: number;
|
||||
private readonly commitSilenceFrames: number;
|
||||
private readonly maxSegmentSamples: number;
|
||||
private readonly maxPrerollFrames: number;
|
||||
private readonly minSpeechSamples: number;
|
||||
|
||||
/** Leftover samples smaller than one analysis frame. */
|
||||
private carry = new Float32Array(0);
|
||||
/** Absolute offset of the next unprocessed frame's first sample. */
|
||||
private absSample = 0;
|
||||
/** Recent silence kept while idle, prepended at speech onset. */
|
||||
private preroll: Frame[] = [];
|
||||
private segment: Frame[] | null = null;
|
||||
private segmentStart = 0;
|
||||
private segmentSamples = 0;
|
||||
private speechSamples = 0;
|
||||
private silenceFrames = 0;
|
||||
private nextSegmentId = 1;
|
||||
private segmentId: number | null = null;
|
||||
|
||||
constructor(options: LiveSegmenterOptions = {}) {
|
||||
this.sampleRate = options.sampleRate ?? TARGET_SAMPLE_RATE;
|
||||
const frameSeconds = options.frameSeconds ?? 0.032;
|
||||
this.frameSamples = Math.max(1, Math.round(frameSeconds * this.sampleRate));
|
||||
this.speechRms = options.speechRms ?? 0.015;
|
||||
this.commitSilenceFrames = Math.max(
|
||||
1,
|
||||
Math.ceil((options.commitSilenceSeconds ?? 0.75) / frameSeconds),
|
||||
);
|
||||
this.maxSegmentSamples = Math.round(
|
||||
(options.maxSegmentSeconds ?? 25) * this.sampleRate,
|
||||
);
|
||||
this.maxPrerollFrames = Math.ceil((options.prerollSeconds ?? 0.3) / frameSeconds);
|
||||
this.minSpeechSamples = Math.round(
|
||||
(options.minSpeechSeconds ?? 0.25) * this.sampleRate,
|
||||
);
|
||||
}
|
||||
|
||||
/** Identity of the in-progress segment; null while idle (no speech yet). */
|
||||
get currentSegmentId(): number | null {
|
||||
return this.segmentId;
|
||||
}
|
||||
|
||||
/** Samples accumulated in the in-progress segment (0 while idle). */
|
||||
get currentSegmentSamples(): number {
|
||||
return this.segment ? this.segmentSamples : 0;
|
||||
}
|
||||
|
||||
/** Copy of the in-progress segment's audio, for a tentative pass. */
|
||||
currentSegment(): { audio: Float32Array; startSample: number } | null {
|
||||
if (!this.segment) return null;
|
||||
return {
|
||||
audio: concat(
|
||||
this.segment.map((frame) => frame.samples),
|
||||
this.segmentSamples,
|
||||
),
|
||||
startSample: this.segmentStart,
|
||||
};
|
||||
}
|
||||
|
||||
/** Feed mic samples; returns segments that committed during this block. */
|
||||
push(samples: Float32Array): CommittedSegment[] {
|
||||
const committed: CommittedSegment[] = [];
|
||||
let buffer: Float32Array;
|
||||
if (this.carry.length > 0) {
|
||||
buffer = concat([this.carry, samples], this.carry.length + samples.length);
|
||||
} else {
|
||||
buffer = samples;
|
||||
}
|
||||
|
||||
let offset = 0;
|
||||
while (buffer.length - offset >= this.frameSamples) {
|
||||
// Copy the frame — the caller may reuse/transfer its buffer.
|
||||
const frame: Frame = {
|
||||
samples: buffer.slice(offset, offset + this.frameSamples),
|
||||
start: this.absSample,
|
||||
};
|
||||
offset += this.frameSamples;
|
||||
this.absSample += this.frameSamples;
|
||||
const segment = this.processFrame(frame, rms(frame.samples) >= this.speechRms);
|
||||
if (segment) committed.push(segment);
|
||||
}
|
||||
this.carry = buffer.slice(offset);
|
||||
return committed;
|
||||
}
|
||||
|
||||
/** Commit whatever is in progress (session stop). */
|
||||
flush(): CommittedSegment | null {
|
||||
if (this.segment && this.carry.length > 0) {
|
||||
// Sub-frame tail — too short to classify, but part of the speech.
|
||||
this.segment.push({ samples: this.carry, start: this.absSample });
|
||||
this.segmentSamples += this.carry.length;
|
||||
this.absSample += this.carry.length;
|
||||
this.carry = new Float32Array(0);
|
||||
}
|
||||
return this.commit("flush");
|
||||
}
|
||||
|
||||
private processFrame(frame: Frame, speech: boolean): CommittedSegment | null {
|
||||
if (!this.segment) {
|
||||
if (!speech) {
|
||||
this.preroll.push(frame);
|
||||
if (this.preroll.length > this.maxPrerollFrames) this.preroll.shift();
|
||||
return null;
|
||||
}
|
||||
this.segment = [...this.preroll, frame];
|
||||
this.segmentStart = this.segment[0]!.start;
|
||||
this.segmentSamples = this.segment.length * this.frameSamples;
|
||||
this.speechSamples = this.frameSamples;
|
||||
this.silenceFrames = 0;
|
||||
this.preroll = [];
|
||||
this.segmentId = this.nextSegmentId++;
|
||||
return null;
|
||||
}
|
||||
|
||||
this.segment.push(frame);
|
||||
this.segmentSamples += this.frameSamples;
|
||||
if (speech) {
|
||||
this.speechSamples += this.frameSamples;
|
||||
this.silenceFrames = 0;
|
||||
} else {
|
||||
this.silenceFrames += 1;
|
||||
if (this.silenceFrames >= this.commitSilenceFrames) return this.commit("silence");
|
||||
}
|
||||
if (this.segmentSamples >= this.maxSegmentSamples) return this.commit("max-length");
|
||||
return null;
|
||||
}
|
||||
|
||||
private commit(reason: CommitReason): CommittedSegment | null {
|
||||
const segment = this.segment;
|
||||
if (!segment) return null;
|
||||
const result: CommittedSegment | null =
|
||||
this.speechSamples >= this.minSpeechSamples
|
||||
? {
|
||||
audio: concat(
|
||||
segment.map((frame) => frame.samples),
|
||||
this.segmentSamples,
|
||||
),
|
||||
startSample: this.segmentStart,
|
||||
reason,
|
||||
}
|
||||
: null; // near-silence: drop, never transcribe (D-016)
|
||||
this.segment = null;
|
||||
this.segmentSamples = 0;
|
||||
this.speechSamples = 0;
|
||||
this.silenceFrames = 0;
|
||||
this.segmentId = null;
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Transcriber: schedules inference passes over the segment stream
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export interface LiveTranscribeResult {
|
||||
text: string;
|
||||
/** Timestamps relative to the transcribed audio's start, Whisper-style. */
|
||||
chunks: TranscriptChunk[];
|
||||
}
|
||||
|
||||
/**
|
||||
* One inference pass. Takes ownership of `audio` (it may be transferred to
|
||||
* the worker). The controller never runs two passes concurrently.
|
||||
*/
|
||||
export type LiveTranscribeFn = (audio: Float32Array) => Promise<LiveTranscribeResult>;
|
||||
|
||||
export interface LiveUpdate {
|
||||
/** Frozen (or provisionally frozen) text of committed segments, joined. */
|
||||
committedText: string;
|
||||
/** Rolling text of the in-progress segment; replaced every pass. */
|
||||
tentativeText: string;
|
||||
/** Finalized chunks with absolute session timestamps. */
|
||||
chunks: TranscriptChunk[];
|
||||
/** Committed segments still waiting for their final pass. */
|
||||
pendingFinalPasses: number;
|
||||
/** Last inference error, if any (the session keeps going). */
|
||||
error: string | null;
|
||||
}
|
||||
|
||||
export interface LiveTranscriberOptions {
|
||||
transcribe: LiveTranscribeFn;
|
||||
onUpdate: (update: LiveUpdate) => void;
|
||||
segmenter?: LiveSegmenterOptions;
|
||||
/** Minimum ms between tentative passes. Default 1500. */
|
||||
tentativeIntervalMs?: number;
|
||||
/** Minimum in-progress segment length before tentative passes. Default 1 s. */
|
||||
minTentativeSeconds?: number;
|
||||
/** Injectable clock (tests). Default Date.now. */
|
||||
now?: () => number;
|
||||
}
|
||||
|
||||
interface SegmentState {
|
||||
startSample: number;
|
||||
/** Handed to the final pass, then released. */
|
||||
audio: Float32Array | null;
|
||||
/** Tentative text until the final pass overwrites it. */
|
||||
text: string;
|
||||
chunks: TranscriptChunk[];
|
||||
final: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Drives the live session: feeds mic audio to the segmenter, keeps exactly
|
||||
* one inference pass in flight (final passes for committed segments first,
|
||||
* then throttled tentative refreshes of the current segment), and reports
|
||||
* every text change through `onUpdate`. Scheduling is driven entirely by
|
||||
* `push()` calls and pass completions — no timers, so it is fully testable.
|
||||
*/
|
||||
export class LiveTranscriber {
|
||||
private readonly segmenter: LiveSegmenter;
|
||||
private readonly transcribe: LiveTranscribeFn;
|
||||
private readonly onUpdate: (update: LiveUpdate) => void;
|
||||
private readonly tentativeIntervalMs: number;
|
||||
private readonly minTentativeSamples: number;
|
||||
private readonly sampleRate: number;
|
||||
private readonly now: () => number;
|
||||
|
||||
private segments: SegmentState[] = [];
|
||||
private tentativeText = "";
|
||||
private lastTentative: { segmentId: number; samples: number; at: number } | null =
|
||||
null;
|
||||
private inFlight = false;
|
||||
private stopped = false;
|
||||
private error: string | null = null;
|
||||
private drainResolvers: (() => void)[] = [];
|
||||
|
||||
constructor(options: LiveTranscriberOptions) {
|
||||
this.segmenter = new LiveSegmenter(options.segmenter);
|
||||
this.transcribe = options.transcribe;
|
||||
this.onUpdate = options.onUpdate;
|
||||
this.tentativeIntervalMs = options.tentativeIntervalMs ?? 1500;
|
||||
this.sampleRate = options.segmenter?.sampleRate ?? TARGET_SAMPLE_RATE;
|
||||
this.minTentativeSamples = Math.round(
|
||||
(options.minTentativeSeconds ?? 1) * this.sampleRate,
|
||||
);
|
||||
this.now = options.now ?? Date.now;
|
||||
}
|
||||
|
||||
/** Feed a block of 16 kHz mono mic samples. Ignored after stop(). */
|
||||
push(samples: Float32Array): void {
|
||||
if (this.stopped) return;
|
||||
for (const committed of this.segmenter.push(samples)) {
|
||||
this.acceptSegment(committed);
|
||||
}
|
||||
this.pump();
|
||||
}
|
||||
|
||||
/**
|
||||
* End the session: commit the in-progress segment, wait for the remaining
|
||||
* final passes, and return the full transcript.
|
||||
*/
|
||||
async stop(): Promise<LiveTranscribeResult> {
|
||||
if (!this.stopped) {
|
||||
this.stopped = true;
|
||||
const flushed = this.segmenter.flush();
|
||||
if (flushed) this.acceptSegment(flushed);
|
||||
// Anything tentative either moved into the flushed segment or belonged
|
||||
// to audio the segmenter dropped as near-silence.
|
||||
this.tentativeText = "";
|
||||
this.pump();
|
||||
}
|
||||
await this.whenDrained();
|
||||
this.emit();
|
||||
return { text: this.committedText(), chunks: this.allChunks() };
|
||||
}
|
||||
|
||||
private acceptSegment(committed: CommittedSegment): void {
|
||||
// The committing segment is the one the tentative text described; carry
|
||||
// it as the provisional text until the final pass replaces it.
|
||||
this.segments.push({
|
||||
startSample: committed.startSample,
|
||||
audio: committed.audio,
|
||||
text: this.tentativeText,
|
||||
chunks: [],
|
||||
final: false,
|
||||
});
|
||||
this.tentativeText = "";
|
||||
this.lastTentative = null;
|
||||
this.emit();
|
||||
}
|
||||
|
||||
/** Single-flight scheduler: pick and start at most one inference pass. */
|
||||
private pump(): void {
|
||||
if (this.inFlight) return;
|
||||
const pending = this.segments.find((segment) => !segment.final);
|
||||
if (pending) {
|
||||
void this.runFinalPass(pending);
|
||||
return;
|
||||
}
|
||||
if (!this.stopped) {
|
||||
this.maybeRunTentativePass();
|
||||
return;
|
||||
}
|
||||
// Stopped and drained — release stop() waiters.
|
||||
const resolvers = this.drainResolvers;
|
||||
this.drainResolvers = [];
|
||||
for (const resolve of resolvers) resolve();
|
||||
}
|
||||
|
||||
private async runFinalPass(segment: SegmentState): Promise<void> {
|
||||
this.inFlight = true;
|
||||
const audio = segment.audio!;
|
||||
segment.audio = null; // ownership moves to the transcribe fn
|
||||
const offsetSeconds = segment.startSample / this.sampleRate;
|
||||
try {
|
||||
const result = await this.transcribe(audio);
|
||||
segment.text = result.text;
|
||||
segment.chunks = result.chunks.map(
|
||||
(chunk): TranscriptChunk => ({
|
||||
text: chunk.text,
|
||||
timestamp: [
|
||||
chunk.timestamp[0] + offsetSeconds,
|
||||
chunk.timestamp[1] === null ? null : chunk.timestamp[1] + offsetSeconds,
|
||||
],
|
||||
}),
|
||||
);
|
||||
} catch (error) {
|
||||
// Keep the provisional (tentative) text rather than losing the segment.
|
||||
this.error = error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
segment.final = true;
|
||||
this.inFlight = false;
|
||||
this.emit();
|
||||
this.pump();
|
||||
}
|
||||
|
||||
private maybeRunTentativePass(): void {
|
||||
const segmentId = this.segmenter.currentSegmentId;
|
||||
if (segmentId === null) return;
|
||||
const samples = this.segmenter.currentSegmentSamples;
|
||||
if (samples < this.minTentativeSamples) return;
|
||||
const last = this.lastTentative;
|
||||
if (last && this.now() - last.at < this.tentativeIntervalMs) return;
|
||||
if (last && last.segmentId === segmentId && samples <= last.samples) return;
|
||||
|
||||
const snapshot = this.segmenter.currentSegment();
|
||||
if (!snapshot) return;
|
||||
this.lastTentative = { segmentId, samples, at: this.now() };
|
||||
this.inFlight = true;
|
||||
void this.transcribe(snapshot.audio)
|
||||
.then(
|
||||
(result) => {
|
||||
// Discard stale passes: the segment may have committed (its final
|
||||
// pass supersedes this) or been dropped while we were busy.
|
||||
if (this.segmenter.currentSegmentId === segmentId) {
|
||||
this.tentativeText = result.text;
|
||||
}
|
||||
},
|
||||
(error: unknown) => {
|
||||
this.error = error instanceof Error ? error.message : String(error);
|
||||
},
|
||||
)
|
||||
.then(() => {
|
||||
this.inFlight = false;
|
||||
this.emit();
|
||||
this.pump();
|
||||
});
|
||||
}
|
||||
|
||||
private committedText(): string {
|
||||
return this.segments
|
||||
.map((segment) => segment.text.trim())
|
||||
.filter(Boolean)
|
||||
.join(" ");
|
||||
}
|
||||
|
||||
private allChunks(): TranscriptChunk[] {
|
||||
return this.segments.flatMap((segment) => segment.chunks);
|
||||
}
|
||||
|
||||
private emit(): void {
|
||||
this.onUpdate({
|
||||
committedText: this.committedText(),
|
||||
tentativeText: this.tentativeText,
|
||||
chunks: this.allChunks(),
|
||||
pendingFinalPasses: this.segments.filter((segment) => !segment.final).length,
|
||||
error: this.error,
|
||||
});
|
||||
}
|
||||
|
||||
private whenDrained(): Promise<void> {
|
||||
const drained =
|
||||
!this.inFlight && this.segments.every((segment) => segment.final);
|
||||
if (drained) return Promise.resolve();
|
||||
return new Promise((resolve) => {
|
||||
this.drainResolvers.push(resolve);
|
||||
});
|
||||
}
|
||||
}
|
||||
+169
@@ -0,0 +1,169 @@
|
||||
/**
|
||||
* Microphone capture for live transcription (Decisions.md D-016). Browser-only.
|
||||
*
|
||||
* `getUserMedia` → `AudioContext` at the device's **native** rate (Firefox
|
||||
* cannot connect a MediaStreamAudioSourceNode across sample rates, so the
|
||||
* context is never forced to 16 kHz) → an AudioWorklet chunker posts ~100 ms
|
||||
* Float32 blocks to the main thread, which resamples them to 16 kHz with the
|
||||
* pure `resampleLinear` before handing them to the live segment loop
|
||||
* (src/lib/live.ts).
|
||||
*
|
||||
* The worklet module is created from an inline source string via a Blob URL:
|
||||
* no extra served file, no build or service-worker changes, and capture keeps
|
||||
* working offline. Browsers without AudioWorklet (or that reject Blob-URL
|
||||
* modules) fall back to a ScriptProcessorNode.
|
||||
*
|
||||
* Mic audio is processed entirely on-device, like every browser-engine path
|
||||
* (Architecture.md §5).
|
||||
*/
|
||||
|
||||
import { TARGET_SAMPLE_RATE, resampleLinear } from "./audio";
|
||||
|
||||
export interface MicCapture {
|
||||
/** Native rate of the capture context (chunks are resampled to 16 kHz). */
|
||||
sampleRate: number;
|
||||
/** True when the AudioWorklet path failed and ScriptProcessorNode is used. */
|
||||
usingFallback: boolean;
|
||||
/** Release the mic (stops the permission indicator) and the audio graph. */
|
||||
stop(): Promise<void>;
|
||||
}
|
||||
|
||||
const WORKLET_NAME = "live-mic-chunker";
|
||||
|
||||
// Runs in the AudioWorklet global scope — plain JS, kept dependency-free.
|
||||
// Accumulates the 128-sample render quanta into ~100 ms chunks so the main
|
||||
// thread handles ~10 messages/s instead of ~375.
|
||||
const WORKLET_SOURCE = `
|
||||
registerProcessor("${WORKLET_NAME}", class extends AudioWorkletProcessor {
|
||||
constructor() {
|
||||
super();
|
||||
this.blocks = [];
|
||||
this.length = 0;
|
||||
this.target = Math.max(128, Math.round(sampleRate / 10));
|
||||
}
|
||||
process(inputs) {
|
||||
const channel = inputs[0] && inputs[0][0];
|
||||
if (channel && channel.length > 0) {
|
||||
this.blocks.push(new Float32Array(channel));
|
||||
this.length += channel.length;
|
||||
if (this.length >= this.target) {
|
||||
const out = new Float32Array(this.length);
|
||||
let offset = 0;
|
||||
for (const block of this.blocks) {
|
||||
out.set(block, offset);
|
||||
offset += block.length;
|
||||
}
|
||||
this.blocks = [];
|
||||
this.length = 0;
|
||||
this.port.postMessage(out, [out.buffer]);
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
});
|
||||
`;
|
||||
|
||||
async function attachWorklet(
|
||||
context: AudioContext,
|
||||
source: MediaStreamAudioSourceNode,
|
||||
deliver: (samples: Float32Array) => void,
|
||||
): Promise<() => void> {
|
||||
const url = URL.createObjectURL(
|
||||
new Blob([WORKLET_SOURCE], { type: "text/javascript" }),
|
||||
);
|
||||
try {
|
||||
await context.audioWorklet.addModule(url);
|
||||
} finally {
|
||||
URL.revokeObjectURL(url);
|
||||
}
|
||||
const node = new AudioWorkletNode(context, WORKLET_NAME, {
|
||||
numberOfInputs: 1,
|
||||
numberOfOutputs: 0,
|
||||
channelCount: 1,
|
||||
});
|
||||
node.port.onmessage = (event: MessageEvent) => deliver(event.data as Float32Array);
|
||||
source.connect(node);
|
||||
return () => {
|
||||
node.port.onmessage = null;
|
||||
source.disconnect(node);
|
||||
};
|
||||
}
|
||||
|
||||
function attachScriptProcessor(
|
||||
context: AudioContext,
|
||||
source: MediaStreamAudioSourceNode,
|
||||
deliver: (samples: Float32Array) => void,
|
||||
): () => void {
|
||||
const node = context.createScriptProcessor(4096, 1, 1);
|
||||
node.onaudioprocess = (event) => {
|
||||
// The input buffer is reused between callbacks — copy before handing off.
|
||||
deliver(new Float32Array(event.inputBuffer.getChannelData(0)));
|
||||
};
|
||||
// ScriptProcessorNode only fires when connected to the destination; route
|
||||
// it through a muted gain so the mic is never played back.
|
||||
const mute = context.createGain();
|
||||
mute.gain.value = 0;
|
||||
source.connect(node);
|
||||
node.connect(mute);
|
||||
mute.connect(context.destination);
|
||||
return () => {
|
||||
node.onaudioprocess = null;
|
||||
source.disconnect(node);
|
||||
node.disconnect();
|
||||
mute.disconnect();
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Open the microphone and stream 16 kHz mono Float32 chunks to `onChunk`
|
||||
* until `stop()`. Rejects if the user denies mic permission (surface the
|
||||
* error; do not retry without a user gesture).
|
||||
*/
|
||||
export async function startMicCapture(
|
||||
onChunk: (samples: Float32Array) => void,
|
||||
): Promise<MicCapture> {
|
||||
if (!navigator.mediaDevices?.getUserMedia) {
|
||||
throw new Error(
|
||||
"Microphone capture is not supported in this browser (requires a secure context).",
|
||||
);
|
||||
}
|
||||
const stream = await navigator.mediaDevices.getUserMedia({
|
||||
audio: { echoCancellation: true, noiseSuppression: true, autoGainControl: true },
|
||||
});
|
||||
|
||||
const context = new AudioContext();
|
||||
// Contexts created outside a user gesture start suspended; the start button
|
||||
// click satisfies the gesture requirement, so this resolves immediately.
|
||||
await context.resume().catch(() => {});
|
||||
const source = context.createMediaStreamSource(stream);
|
||||
|
||||
const deliver = (samples: Float32Array) => {
|
||||
onChunk(
|
||||
context.sampleRate === TARGET_SAMPLE_RATE
|
||||
? samples
|
||||
: resampleLinear(samples, context.sampleRate, TARGET_SAMPLE_RATE),
|
||||
);
|
||||
};
|
||||
|
||||
let detach: () => void;
|
||||
let usingFallback = false;
|
||||
try {
|
||||
detach = await attachWorklet(context, source, deliver);
|
||||
} catch {
|
||||
detach = attachScriptProcessor(context, source, deliver);
|
||||
usingFallback = true;
|
||||
}
|
||||
|
||||
let stopped = false;
|
||||
return {
|
||||
sampleRate: context.sampleRate,
|
||||
usingFallback,
|
||||
async stop() {
|
||||
if (stopped) return;
|
||||
stopped = true;
|
||||
detach();
|
||||
for (const track of stream.getTracks()) track.stop();
|
||||
await context.close().catch(() => {});
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,373 @@
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import {
|
||||
BrowserEngine,
|
||||
ENGINE_CHOICE_KEY,
|
||||
ServerEngine,
|
||||
fetchServerEngineStatus,
|
||||
isTranscribeStreamLine,
|
||||
loadEngineChoice,
|
||||
saveEngineChoice,
|
||||
type StorageLike,
|
||||
type WorkerLike,
|
||||
} from "../src/lib/engine";
|
||||
import type { FetchLike } from "../src/lib/authClient";
|
||||
import type { TranscriptChunk } from "../src/worker/protocol";
|
||||
|
||||
// D-010: both engines behind one interface, testable without a DOM/worker/server.
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// BrowserEngine — requestId correlation against a fake worker
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
class FakeWorker implements WorkerLike {
|
||||
posted: { message: Record<string, unknown>; transfer?: Transferable[] }[] = [];
|
||||
private listeners = { message: [] as ((e: { data: unknown }) => void)[], error: [] as ((e: { message?: string }) => void)[] };
|
||||
|
||||
postMessage(message: unknown, transfer?: Transferable[]): void {
|
||||
this.posted.push({ message: message as Record<string, unknown>, transfer });
|
||||
}
|
||||
|
||||
addEventListener(
|
||||
type: "message" | "error",
|
||||
listener: (event: { data?: unknown; message?: string }) => void,
|
||||
): void {
|
||||
this.listeners[type].push(listener);
|
||||
}
|
||||
|
||||
emit(data: unknown): void {
|
||||
for (const listener of this.listeners.message) listener({ data });
|
||||
}
|
||||
|
||||
emitError(message: string): void {
|
||||
for (const listener of this.listeners.error) listener({ message });
|
||||
}
|
||||
|
||||
lastRequestId(): string {
|
||||
return this.posted.at(-1)!.message.requestId as string;
|
||||
}
|
||||
}
|
||||
|
||||
const CHUNKS: TranscriptChunk[] = [{ text: " hi", timestamp: [0, 1] }];
|
||||
|
||||
describe("BrowserEngine", () => {
|
||||
test("posts a transcribe message with a requestId and transfers the buffer", () => {
|
||||
const worker = new FakeWorker();
|
||||
const engine = new BrowserEngine(worker);
|
||||
const audio = new Float32Array([0.1, 0.2]);
|
||||
void engine.transcribe(audio, { language: "da", task: "translate" });
|
||||
|
||||
expect(worker.posted).toHaveLength(1);
|
||||
const { message, transfer } = worker.posted[0]!;
|
||||
expect(message.type).toBe("transcribe");
|
||||
expect(message.language).toBe("da");
|
||||
expect(message.task).toBe("translate");
|
||||
expect(typeof message.requestId).toBe("string");
|
||||
expect(transfer).toEqual([audio.buffer as ArrayBuffer]);
|
||||
});
|
||||
|
||||
test('omits language "auto" and the modelId (worker holds the model)', () => {
|
||||
const worker = new FakeWorker();
|
||||
const engine = new BrowserEngine(worker);
|
||||
void engine.transcribe(new Float32Array(4), {
|
||||
language: "auto",
|
||||
modelId: "onnx-community/whisper-tiny.en",
|
||||
});
|
||||
const { message } = worker.posted[0]!;
|
||||
expect("language" in message).toBe(false);
|
||||
expect("modelId" in message).toBe(false);
|
||||
});
|
||||
|
||||
test("resolves on the matching result and routes partials to onPartial", async () => {
|
||||
const worker = new FakeWorker();
|
||||
const engine = new BrowserEngine(worker);
|
||||
const fragments: string[] = [];
|
||||
const promise = engine.transcribe(new Float32Array(4), {
|
||||
onPartial: (fragment) => fragments.push(fragment),
|
||||
});
|
||||
const requestId = worker.lastRequestId();
|
||||
|
||||
// Noise the engine must ignore: other ids, id-less file-flow messages,
|
||||
// load progress.
|
||||
worker.emit({ type: "partial", text: "IGNORED", requestId: "other" });
|
||||
worker.emit({ type: "partial", text: "IGNORED" });
|
||||
worker.emit({ type: "load-progress", status: "ready", device: "wasm" });
|
||||
|
||||
worker.emit({ type: "partial", text: "Hel", requestId });
|
||||
worker.emit({ type: "partial", text: "lo", requestId });
|
||||
worker.emit({ type: "result", text: "Hello", chunks: CHUNKS, requestId });
|
||||
worker.emit({ type: "result", text: "STALE", chunks: [], requestId: "other" });
|
||||
|
||||
expect(await promise).toEqual({ text: "Hello", chunks: CHUNKS });
|
||||
expect(fragments).toEqual(["Hel", "lo"]);
|
||||
});
|
||||
|
||||
test("rejects on a matching error message", async () => {
|
||||
const worker = new FakeWorker();
|
||||
const engine = new BrowserEngine(worker);
|
||||
const promise = engine.transcribe(new Float32Array(4));
|
||||
worker.emit({
|
||||
type: "error",
|
||||
stage: "transcribe",
|
||||
message: "out of memory",
|
||||
requestId: worker.lastRequestId(),
|
||||
});
|
||||
expect(promise).rejects.toThrow("out of memory");
|
||||
});
|
||||
|
||||
test("a worker crash rejects every in-flight request", async () => {
|
||||
const worker = new FakeWorker();
|
||||
const engine = new BrowserEngine(worker);
|
||||
const first = engine.transcribe(new Float32Array(4));
|
||||
const second = engine.transcribe(new Float32Array(4));
|
||||
worker.emitError("worker exploded");
|
||||
expect(first).rejects.toThrow("worker exploded");
|
||||
expect(second).rejects.toThrow("worker exploded");
|
||||
});
|
||||
|
||||
test("two engines can share one worker without stealing responses", async () => {
|
||||
const worker = new FakeWorker();
|
||||
const engineA = new BrowserEngine(worker);
|
||||
const engineB = new BrowserEngine(worker);
|
||||
const promiseA = engineA.transcribe(new Float32Array(4));
|
||||
const promiseB = engineB.transcribe(new Float32Array(4));
|
||||
const [idA, idB] = worker.posted.map(
|
||||
(post) => post.message.requestId as string,
|
||||
);
|
||||
expect(idA).not.toBe(idB);
|
||||
worker.emit({ type: "result", text: "for A", chunks: [], requestId: idA });
|
||||
worker.emit({ type: "result", text: "for B", chunks: [], requestId: idB });
|
||||
expect((await promiseA).text).toBe("for A");
|
||||
expect((await promiseB).text).toBe("for B");
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// ServerEngine — ndjson stream over an injected fetch
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function ndjsonResponse(lines: unknown[], status = 200): Response {
|
||||
const body = lines.map((line) => `${JSON.stringify(line)}\n`).join("");
|
||||
return new Response(body, {
|
||||
status,
|
||||
headers: { "Content-Type": "application/x-ndjson" },
|
||||
});
|
||||
}
|
||||
|
||||
describe("ServerEngine", () => {
|
||||
test("streams segments to onPartial and resolves with the result line", async () => {
|
||||
const calls: { url: string; init?: RequestInit }[] = [];
|
||||
const fetchFn: FetchLike = async (url, init) => {
|
||||
calls.push({ url, init });
|
||||
return ndjsonResponse([
|
||||
{ type: "segment", text: " Hello", start: 0, end: 1.5 },
|
||||
{ type: "segment", text: " world.", start: 1.5, end: 3 },
|
||||
{ type: "result", text: "Hello world.", chunks: CHUNKS },
|
||||
]);
|
||||
};
|
||||
const engine = new ServerEngine(fetchFn);
|
||||
const fragments: string[] = [];
|
||||
const audio = new Float32Array([0.5, -0.5, 0.25]);
|
||||
const result = await engine.transcribe(audio, {
|
||||
modelId: "onnx-community/whisper-tiny.en",
|
||||
language: "en",
|
||||
task: "transcribe",
|
||||
onPartial: (fragment) => fragments.push(fragment),
|
||||
});
|
||||
|
||||
expect(result).toEqual({ text: "Hello world.", chunks: CHUNKS });
|
||||
expect(fragments).toEqual([" Hello", " world."]);
|
||||
|
||||
const call = calls[0]!;
|
||||
const url = new URL(call.url, "http://localhost");
|
||||
expect(url.pathname).toBe("/api/transcribe");
|
||||
expect(url.searchParams.get("model")).toBe("onnx-community/whisper-tiny.en");
|
||||
expect(url.searchParams.get("language")).toBe("en");
|
||||
expect(url.searchParams.get("task")).toBe("transcribe");
|
||||
expect(call.init?.method).toBe("POST");
|
||||
expect(call.init?.credentials).toBe("same-origin");
|
||||
// The body must be the raw PCM bytes of the Float32Array.
|
||||
const sent = new Float32Array((call.init?.body as Uint8Array).buffer.slice(0));
|
||||
expect([...sent]).toEqual([...audio]);
|
||||
});
|
||||
|
||||
test('drops language "auto" from the query', async () => {
|
||||
let url = "";
|
||||
const engine = new ServerEngine(async (input) => {
|
||||
url = input;
|
||||
return ndjsonResponse([{ type: "result", text: "", chunks: [] }]);
|
||||
});
|
||||
await engine.transcribe(new Float32Array(4), {
|
||||
modelId: "onnx-community/whisper-tiny",
|
||||
language: "auto",
|
||||
});
|
||||
expect(url).not.toContain("language");
|
||||
});
|
||||
|
||||
test("requires a modelId", () => {
|
||||
const engine = new ServerEngine(async () => ndjsonResponse([]));
|
||||
expect(engine.transcribe(new Float32Array(4))).rejects.toThrow(/model id/);
|
||||
});
|
||||
|
||||
test("rejects with the in-stream error line's message", () => {
|
||||
const engine = new ServerEngine(async () =>
|
||||
ndjsonResponse([
|
||||
{ type: "segment", text: " partial", start: 0, end: 1 },
|
||||
{ type: "error", message: "sidecar exploded" },
|
||||
]),
|
||||
);
|
||||
expect(
|
||||
engine.transcribe(new Float32Array(4), { modelId: "onnx-community/whisper-tiny" }),
|
||||
).rejects.toThrow("sidecar exploded");
|
||||
});
|
||||
|
||||
test("maps HTTP statuses to UI-safe messages", async () => {
|
||||
const cases: [number, RegExp][] = [
|
||||
[401, /sign in/i],
|
||||
[404, /not available/i],
|
||||
[413, /too large|limit/i],
|
||||
[503, /unavailable/i],
|
||||
];
|
||||
for (const [status, pattern] of cases) {
|
||||
const engine = new ServerEngine(async () => new Response("{}", { status }));
|
||||
expect(
|
||||
engine.transcribe(new Float32Array(4), {
|
||||
modelId: "onnx-community/whisper-tiny",
|
||||
}),
|
||||
).rejects.toThrow(pattern);
|
||||
}
|
||||
});
|
||||
|
||||
test("surfaces the server's {error} detail on 503", () => {
|
||||
const engine = new ServerEngine(async () =>
|
||||
new Response(JSON.stringify({ error: "faster-whisper is not installed" }), {
|
||||
status: 503,
|
||||
}),
|
||||
);
|
||||
expect(
|
||||
engine.transcribe(new Float32Array(4), { modelId: "onnx-community/whisper-tiny" }),
|
||||
).rejects.toThrow(/faster-whisper is not installed/);
|
||||
});
|
||||
|
||||
test("a stream that ends without a result line is a lost connection", () => {
|
||||
const engine = new ServerEngine(async () =>
|
||||
ndjsonResponse([{ type: "segment", text: " cut", start: 0, end: 1 }]),
|
||||
);
|
||||
expect(
|
||||
engine.transcribe(new Float32Array(4), { modelId: "onnx-community/whisper-tiny" }),
|
||||
).rejects.toThrow(/connection was lost/);
|
||||
});
|
||||
|
||||
test("network failure gets a friendly message", () => {
|
||||
const engine = new ServerEngine(async () => {
|
||||
throw new TypeError("fetch failed");
|
||||
});
|
||||
expect(
|
||||
engine.transcribe(new Float32Array(4), { modelId: "onnx-community/whisper-tiny" }),
|
||||
).rejects.toThrow(/could not reach the server/i);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Wire-format guard
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe("isTranscribeStreamLine", () => {
|
||||
test("accepts the three line shapes", () => {
|
||||
expect(
|
||||
isTranscribeStreamLine({ type: "segment", text: " a", start: 0, end: 1 }),
|
||||
).toBe(true);
|
||||
expect(isTranscribeStreamLine({ type: "result", text: "a", chunks: [] })).toBe(true);
|
||||
expect(isTranscribeStreamLine({ type: "error", message: "x" })).toBe(true);
|
||||
});
|
||||
|
||||
test("rejects malformed lines", () => {
|
||||
expect(isTranscribeStreamLine(null)).toBe(false);
|
||||
expect(isTranscribeStreamLine({ type: "segment", text: " a" })).toBe(false);
|
||||
expect(isTranscribeStreamLine({ type: "result", text: "a" })).toBe(false);
|
||||
expect(isTranscribeStreamLine({ type: "unknown" })).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Availability probe
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe("fetchServerEngineStatus", () => {
|
||||
test("passes through an available status", async () => {
|
||||
const status = await fetchServerEngineStatus(async () =>
|
||||
new Response(JSON.stringify({ available: true }), { status: 200 }),
|
||||
);
|
||||
expect(status).toEqual({ available: true });
|
||||
});
|
||||
|
||||
test("passes through unavailable + reason", async () => {
|
||||
const status = await fetchServerEngineStatus(async () =>
|
||||
new Response(
|
||||
JSON.stringify({ available: false, reason: "sidecar missing" }),
|
||||
{ status: 200 },
|
||||
),
|
||||
);
|
||||
expect(status).toEqual({ available: false, reason: "sidecar missing" });
|
||||
});
|
||||
|
||||
test("404 (deployment without the feature) resolves unavailable", async () => {
|
||||
const status = await fetchServerEngineStatus(async () =>
|
||||
new Response("Not Found", { status: 404 }),
|
||||
);
|
||||
expect(status.available).toBe(false);
|
||||
});
|
||||
|
||||
test("never rejects on network failure", async () => {
|
||||
const status = await fetchServerEngineStatus(async () => {
|
||||
throw new TypeError("offline");
|
||||
});
|
||||
expect(status.available).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Engine choice persistence
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function fakeStorage(initial: Record<string, string> = {}): StorageLike & {
|
||||
data: Record<string, string>;
|
||||
} {
|
||||
const data = { ...initial };
|
||||
return {
|
||||
data,
|
||||
getItem: (key) => (key in data ? data[key]! : null),
|
||||
setItem: (key, value) => {
|
||||
data[key] = value;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
describe("engine choice persistence", () => {
|
||||
test("round-trips through storage", () => {
|
||||
const storage = fakeStorage();
|
||||
saveEngineChoice("server", storage);
|
||||
expect(storage.data[ENGINE_CHOICE_KEY]).toBe("server");
|
||||
expect(loadEngineChoice(storage)).toBe("server");
|
||||
});
|
||||
|
||||
test('defaults to "browser" for missing or garbage values', () => {
|
||||
expect(loadEngineChoice(fakeStorage())).toBe("browser");
|
||||
expect(
|
||||
loadEngineChoice(fakeStorage({ [ENGINE_CHOICE_KEY]: "cloud" })),
|
||||
).toBe("browser");
|
||||
expect(loadEngineChoice(null)).toBe("browser");
|
||||
});
|
||||
|
||||
test("throwing storage is survived", () => {
|
||||
const storage: StorageLike = {
|
||||
getItem: () => {
|
||||
throw new Error("denied");
|
||||
},
|
||||
setItem: () => {
|
||||
throw new Error("denied");
|
||||
},
|
||||
};
|
||||
expect(loadEngineChoice(storage)).toBe("browser");
|
||||
expect(() => saveEngineChoice("server", storage)).not.toThrow();
|
||||
});
|
||||
});
|
||||
Vendored
+58
@@ -0,0 +1,58 @@
|
||||
"""Protocol-faithful stand-in for server-py/sidecar.py (no faster-whisper).
|
||||
|
||||
Used by tests/transcribe.test.ts to exercise the real Bun-side supervisor
|
||||
(createSidecar) against an actual child process. Behavior is keyed off the
|
||||
requested model name:
|
||||
|
||||
"ok" two segments (the second reports the audio byte count,
|
||||
language and task for pass-through assertions), then done
|
||||
"boom" an error line
|
||||
"die" exit(1) mid-request (crash simulation)
|
||||
"wait-for-cancel" one segment, then block until the next stdin line
|
||||
(the cancel), then done {cancelled: true}
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
|
||||
def emit(message):
|
||||
sys.stdout.write(json.dumps(message) + "\n")
|
||||
sys.stdout.flush()
|
||||
|
||||
|
||||
emit({"type": "ready", "version": "fake"})
|
||||
|
||||
for line in sys.stdin:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
message = json.loads(line)
|
||||
kind = message.get("type")
|
||||
if kind == "shutdown":
|
||||
break
|
||||
if kind != "transcribe":
|
||||
continue
|
||||
|
||||
request_id = message.get("id")
|
||||
model = message.get("model")
|
||||
if model == "boom":
|
||||
emit({"type": "error", "id": request_id, "message": "kaboom"})
|
||||
elif model == "die":
|
||||
sys.exit(1)
|
||||
elif model == "wait-for-cancel":
|
||||
emit({"type": "segment", "id": request_id, "start": 0.0, "end": 1.0,
|
||||
"text": " before cancel"})
|
||||
next_line = json.loads(sys.stdin.readline())
|
||||
cancelled = (next_line.get("type") == "cancel"
|
||||
and next_line.get("id") == request_id)
|
||||
emit({"type": "done", "id": request_id, "cancelled": cancelled})
|
||||
else:
|
||||
size = os.path.getsize(message["audio_path"])
|
||||
emit({"type": "segment", "id": request_id, "start": 0.0, "end": 1.5,
|
||||
"text": " Hello"})
|
||||
emit({"type": "segment", "id": request_id, "start": 1.5, "end": 3.0,
|
||||
"text": " world. [model=%s bytes=%d lang=%s task=%s]"
|
||||
% (model, size, message.get("language"), message.get("task"))})
|
||||
emit({"type": "done", "id": request_id, "cancelled": False})
|
||||
@@ -0,0 +1,77 @@
|
||||
# Manual live-transcription verification (M7, D-016)
|
||||
|
||||
The segment loop (`src/lib/live.ts`) and the protocol changes are covered by
|
||||
unit tests; microphone capture, the AudioWorklet path, and end-to-end live
|
||||
quality can only be exercised in a real browser. This is that procedure.
|
||||
Re-run it after any change to `src/lib/live.ts`, `src/lib/mic.ts`,
|
||||
`src/components/LiveControls.tsx`, or the live wiring in `src/App.tsx`.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- A machine with a working microphone.
|
||||
- Chrome/Edge (WebGPU) and ideally also Firefox and Safari — the three differ
|
||||
exactly where mic capture is fragile (context sample rates, AudioWorklet
|
||||
Blob-URL modules; Safari exercises the ScriptProcessorNode fallback if the
|
||||
worklet path fails there).
|
||||
- `getUserMedia` requires a secure context; `http://localhost` qualifies:
|
||||
```sh
|
||||
bun dev # or: bun run build && bun run start
|
||||
```
|
||||
|
||||
## 1. Permission flow
|
||||
|
||||
1. Open the app, pick `whisper-tiny.en`, click **Start live transcription**.
|
||||
2. The browser prompts for the microphone. **Deny** it: the button must return
|
||||
to idle and the error "Microphone access was denied…" must appear; the app
|
||||
stays usable (file flow unaffected).
|
||||
3. Click again and **allow**: button becomes **Stop** with a pulsing red dot;
|
||||
the browser's mic indicator turns on. If the model wasn't downloaded yet,
|
||||
the download progress panel appears and live text simply starts once it is
|
||||
ready (segment passes queue in the worker).
|
||||
|
||||
## 2. Live transcription behavior
|
||||
|
||||
1. Speak a few sentences with natural pauses. Expect:
|
||||
- While speaking: dimmed italic *tentative* text appears within a couple of
|
||||
seconds (tiny.en; slower models lag proportionally) and refreshes.
|
||||
- After a ~1 s pause: the sentence commits — the text turns from italic to
|
||||
regular ("committed") and no longer changes.
|
||||
- Silence alone (say nothing for 30 s) must produce **no** text — the
|
||||
energy gate drops silent segments, so no hallucinated "thank you."-style
|
||||
output.
|
||||
2. DevTools → Network: **zero** requests while speaking (weights may download
|
||||
at the start; the audio itself never leaves the device).
|
||||
3. During a live session, ModelPicker / options / dropzone are disabled.
|
||||
4. Model choice sanity: repeat briefly with a multilingual model + a fixed
|
||||
language and confirm the language/task options are honored.
|
||||
|
||||
## 3. Stop & result
|
||||
|
||||
1. Click **Stop** mid-sentence. The button shows "Finishing…" while the last
|
||||
segment gets its final pass, then the normal **Transcript** panel appears:
|
||||
full text, Copy button, and a Timestamps section whose segment times are
|
||||
**absolute** (a sentence spoken ~10 s in must not be timestamped 0:00).
|
||||
2. The browser's mic indicator must turn off immediately on Stop.
|
||||
3. Start a new live session afterwards, then also transcribe a file — both
|
||||
must still work (session cleanup, request correlation).
|
||||
|
||||
## 4. Quality/latency knobs (only if needed)
|
||||
|
||||
If commits are too eager/lazy in real rooms, tune `LiveSegmenterOptions`
|
||||
(`speechRms` gate 0.015, `commitSilenceSeconds` 0.75, `maxSegmentSeconds` 25,
|
||||
`minSpeechSeconds` 0.25) and the `LiveTranscriber` throttle
|
||||
(`tentativeIntervalMs` 1500) — all injected from `src/App.tsx` defaults, all
|
||||
unit-tested in `tests/live.test.ts`. Record any tuned values in D-016.
|
||||
|
||||
## 5. Offline live transcription
|
||||
|
||||
With the PWA installed and a model cached (see `tests/offline.md`), go
|
||||
offline (airplane mode) and run a live session — it must work identically:
|
||||
capture, worklet (Blob URL, no network), inference, stop/transcript.
|
||||
|
||||
## Recording the result
|
||||
|
||||
When this passes in a real browser, flip the `[~]` manual-verification item
|
||||
in `WorkBreakdown.md` M7 to `[x]` and note browser + version here.
|
||||
|
||||
- [ ] Verified on: _browser/version, date_
|
||||
@@ -0,0 +1,313 @@
|
||||
import { describe, test, expect } from "bun:test";
|
||||
import {
|
||||
LiveSegmenter,
|
||||
LiveTranscriber,
|
||||
rms,
|
||||
type LiveTranscribeResult,
|
||||
type LiveUpdate,
|
||||
} from "../src/lib/live";
|
||||
|
||||
// Pure segment-loop logic only (D-016) — no mic, no worker, no model. Audio
|
||||
// is synthetic: "speech" is a ±0.1 square wave (RMS 0.1, far above the 0.015
|
||||
// gate), "silence" is zeros.
|
||||
|
||||
const RATE = 16_000;
|
||||
const FRAME = 512; // default frameSeconds 0.032 × 16 kHz
|
||||
|
||||
function speech(seconds: number): Float32Array {
|
||||
const out = new Float32Array(Math.round(seconds * RATE));
|
||||
for (let i = 0; i < out.length; i++) out[i] = i % 2 === 0 ? 0.1 : -0.1;
|
||||
return out;
|
||||
}
|
||||
|
||||
function silence(seconds: number): Float32Array {
|
||||
return new Float32Array(Math.round(seconds * RATE));
|
||||
}
|
||||
|
||||
/** Let pending promise chains (and a macrotask) settle. */
|
||||
function tick(): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, 0));
|
||||
}
|
||||
|
||||
describe("rms", () => {
|
||||
test("zero for empty and silent blocks, level for a square wave", () => {
|
||||
expect(rms(new Float32Array(0))).toBe(0);
|
||||
expect(rms(new Float32Array(100))).toBe(0);
|
||||
expect(rms(speech(0.1))).toBeCloseTo(0.1, 5);
|
||||
});
|
||||
});
|
||||
|
||||
describe("LiveSegmenter", () => {
|
||||
test("pure silence never produces a segment", () => {
|
||||
const segmenter = new LiveSegmenter();
|
||||
expect(segmenter.push(silence(3))).toEqual([]);
|
||||
expect(segmenter.currentSegmentId).toBeNull();
|
||||
expect(segmenter.currentSegmentSamples).toBe(0);
|
||||
expect(segmenter.currentSegment()).toBeNull();
|
||||
expect(segmenter.flush()).toBeNull();
|
||||
});
|
||||
|
||||
test("speech followed by trailing silence commits with pre-roll", () => {
|
||||
const segmenter = new LiveSegmenter();
|
||||
// 16 frames silence, 32 frames speech, then silence until commit.
|
||||
expect(segmenter.push(silence(0.512))).toEqual([]);
|
||||
expect(segmenter.push(speech(1.024))).toEqual([]);
|
||||
expect(segmenter.currentSegmentId).not.toBeNull();
|
||||
|
||||
const committed = segmenter.push(silence(1.024));
|
||||
expect(committed).toHaveLength(1);
|
||||
const segment = committed[0]!;
|
||||
expect(segment.reason).toBe("silence");
|
||||
// Pre-roll keeps 10 frames (0.3 s / 0.032 s) of the leading silence.
|
||||
expect(segment.startSample).toBe((16 - 10) * FRAME);
|
||||
// 10 pre-roll + 32 speech + 24 trailing-silence frames (ceil(0.75/0.032)).
|
||||
expect(segment.audio.length).toBe((10 + 32 + 24) * FRAME);
|
||||
// Back to idle after the commit.
|
||||
expect(segmenter.currentSegmentId).toBeNull();
|
||||
});
|
||||
|
||||
test("near-silent blips are dropped, not committed", () => {
|
||||
const segmenter = new LiveSegmenter();
|
||||
// 4 speech frames = 0.128 s < minSpeechSeconds 0.25 s.
|
||||
expect(segmenter.push(speech(0.128))).toEqual([]);
|
||||
expect(segmenter.currentSegmentId).not.toBeNull();
|
||||
expect(segmenter.push(silence(2))).toEqual([]);
|
||||
expect(segmenter.currentSegmentId).toBeNull();
|
||||
expect(segmenter.flush()).toBeNull();
|
||||
});
|
||||
|
||||
test("continuous speech commits at the max segment length and restarts", () => {
|
||||
const segmenter = new LiveSegmenter({ maxSegmentSeconds: 2 });
|
||||
const committed = segmenter.push(speech(3));
|
||||
expect(committed).toHaveLength(1);
|
||||
expect(committed[0]!.reason).toBe("max-length");
|
||||
expect(committed[0]!.startSample).toBe(0);
|
||||
// 2 s cap → first frame count at/over 32 000 samples.
|
||||
expect(committed[0]!.audio.length).toBe(Math.ceil((2 * RATE) / FRAME) * FRAME);
|
||||
// The rest of the speech opened a fresh segment.
|
||||
expect(segmenter.currentSegmentId).not.toBeNull();
|
||||
expect(segmenter.currentSegmentSamples).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
test("flush commits the in-progress segment, including the sub-frame tail", () => {
|
||||
const segmenter = new LiveSegmenter();
|
||||
// 1 s + 100 samples: the 100-sample tail is smaller than one frame.
|
||||
segmenter.push(speech(1));
|
||||
segmenter.push(speech(100 / RATE));
|
||||
const flushed = segmenter.flush();
|
||||
expect(flushed).not.toBeNull();
|
||||
expect(flushed!.reason).toBe("flush");
|
||||
expect(flushed!.audio.length).toBe(Math.round(1 * RATE) + 100);
|
||||
expect(segmenter.currentSegmentId).toBeNull();
|
||||
});
|
||||
|
||||
test("currentSegment snapshots the in-progress audio", () => {
|
||||
const segmenter = new LiveSegmenter();
|
||||
segmenter.push(speech(1.024));
|
||||
expect(segmenter.currentSegmentSamples).toBe(32 * FRAME);
|
||||
const snapshot = segmenter.currentSegment();
|
||||
expect(snapshot).not.toBeNull();
|
||||
expect(snapshot!.audio.length).toBe(32 * FRAME);
|
||||
expect(snapshot!.startSample).toBe(0);
|
||||
expect(rms(snapshot!.audio)).toBeCloseTo(0.1, 5);
|
||||
});
|
||||
|
||||
test("frame splitting is independent of push block sizes", () => {
|
||||
const bulk = new LiveSegmenter();
|
||||
const dribble = new LiveSegmenter();
|
||||
const audio = speech(1.024);
|
||||
bulk.push(audio);
|
||||
for (let i = 0; i < audio.length; i += 100) {
|
||||
dribble.push(audio.slice(i, Math.min(i + 100, audio.length)));
|
||||
}
|
||||
expect(dribble.currentSegmentSamples).toBe(bulk.currentSegmentSamples);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// LiveTranscriber — fake engine with manually resolved passes
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
interface FakeCall {
|
||||
audio: Float32Array;
|
||||
resolve: (result: LiveTranscribeResult) => void;
|
||||
reject: (error: Error) => void;
|
||||
}
|
||||
|
||||
function fakeEngine() {
|
||||
const calls: FakeCall[] = [];
|
||||
const updates: LiveUpdate[] = [];
|
||||
let now = 0;
|
||||
return {
|
||||
calls,
|
||||
updates,
|
||||
advance(ms: number) {
|
||||
now += ms;
|
||||
},
|
||||
make(options: { tentativeIntervalMs?: number; minTentativeSeconds?: number } = {}) {
|
||||
return new LiveTranscriber({
|
||||
transcribe: (audio) =>
|
||||
new Promise<LiveTranscribeResult>((resolve, reject) => {
|
||||
calls.push({ audio, resolve, reject });
|
||||
}),
|
||||
onUpdate: (update) => updates.push(update),
|
||||
now: () => now,
|
||||
...options,
|
||||
});
|
||||
},
|
||||
last() {
|
||||
return updates[updates.length - 1]!;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
describe("LiveTranscriber", () => {
|
||||
test("runs a throttled tentative pass over the growing segment", async () => {
|
||||
const engine = fakeEngine();
|
||||
const live = engine.make();
|
||||
|
||||
live.push(speech(1.5));
|
||||
expect(engine.calls).toHaveLength(1); // ≥ minTentativeSeconds → first pass
|
||||
expect(engine.calls[0]!.audio.length).toBeGreaterThanOrEqual(1.4 * RATE);
|
||||
|
||||
// Single-flight: more audio while a pass is running starts nothing new.
|
||||
live.push(speech(0.5));
|
||||
expect(engine.calls).toHaveLength(1);
|
||||
|
||||
engine.calls[0]!.resolve({ text: " hello", chunks: [] });
|
||||
await tick();
|
||||
expect(engine.last().tentativeText).toBe(" hello");
|
||||
expect(engine.last().committedText).toBe("");
|
||||
|
||||
// Same segment, interval not elapsed → still no second pass on push.
|
||||
live.push(speech(0.2));
|
||||
expect(engine.calls).toHaveLength(1);
|
||||
|
||||
// Interval elapsed + new samples → refresh.
|
||||
engine.advance(2000);
|
||||
live.push(speech(0.2));
|
||||
expect(engine.calls).toHaveLength(2);
|
||||
});
|
||||
|
||||
test("committed segments get a final pass that freezes their text", async () => {
|
||||
const engine = fakeEngine();
|
||||
const live = engine.make();
|
||||
|
||||
live.push(speech(1.5));
|
||||
engine.calls[0]!.resolve({ text: " helo", chunks: [] });
|
||||
await tick();
|
||||
|
||||
// Trailing silence commits the segment; tentative text turns provisional.
|
||||
live.push(silence(1.5));
|
||||
expect(engine.last().committedText).toBe("helo");
|
||||
expect(engine.last().pendingFinalPasses).toBe(1);
|
||||
expect(engine.calls).toHaveLength(2); // final pass started immediately
|
||||
|
||||
engine.calls[1]!.resolve({
|
||||
text: " hello world",
|
||||
chunks: [{ text: " hello world", timestamp: [0, 1.4] }],
|
||||
});
|
||||
await tick();
|
||||
expect(engine.last().committedText).toBe("hello world");
|
||||
expect(engine.last().tentativeText).toBe("");
|
||||
expect(engine.last().pendingFinalPasses).toBe(0);
|
||||
// Speech started at sample 0 → no offset on the chunk timestamps.
|
||||
expect(engine.last().chunks[0]!.timestamp[0]).toBeCloseTo(0, 5);
|
||||
expect(engine.last().chunks[0]!.timestamp[1]!).toBeCloseTo(1.4, 5);
|
||||
});
|
||||
|
||||
test("chunk timestamps are offset by the segment's session position", async () => {
|
||||
const engine = fakeEngine();
|
||||
const live = engine.make({ minTentativeSeconds: 9999 }); // finals only
|
||||
|
||||
live.push(silence(0.512)); // 16 frames; pre-roll keeps the last 10
|
||||
live.push(speech(1));
|
||||
const stopping = live.stop();
|
||||
await tick();
|
||||
expect(engine.calls).toHaveLength(1);
|
||||
engine.calls[0]!.resolve({
|
||||
text: " offset",
|
||||
chunks: [{ text: " offset", timestamp: [0, 1] }],
|
||||
});
|
||||
const result = await stopping;
|
||||
|
||||
const offset = ((16 - 10) * FRAME) / RATE;
|
||||
expect(result.text).toBe("offset");
|
||||
expect(result.chunks[0]!.timestamp[0]).toBeCloseTo(offset, 5);
|
||||
expect(result.chunks[0]!.timestamp[1]!).toBeCloseTo(1 + offset, 5);
|
||||
expect(result.chunks[0]!.timestamp[1]).not.toBeNull();
|
||||
});
|
||||
|
||||
test("stop flushes, drains pending passes, and joins the transcript", async () => {
|
||||
const engine = fakeEngine();
|
||||
const live = engine.make({ minTentativeSeconds: 9999 });
|
||||
|
||||
live.push(speech(1));
|
||||
live.push(silence(1.5)); // segment 1 commits → final pass 1
|
||||
live.push(speech(1)); // segment 2 in progress
|
||||
const stopping = live.stop(); // flush commits segment 2
|
||||
|
||||
let resolved = false;
|
||||
void stopping.then(() => {
|
||||
resolved = true;
|
||||
});
|
||||
await tick();
|
||||
expect(resolved).toBe(false); // still draining
|
||||
|
||||
engine.calls[0]!.resolve({ text: " one.", chunks: [] });
|
||||
await tick();
|
||||
engine.calls[1]!.resolve({ text: " two.", chunks: [] });
|
||||
const result = await stopping;
|
||||
expect(result.text).toBe("one. two.");
|
||||
|
||||
// The session is over: further audio is ignored.
|
||||
live.push(speech(2));
|
||||
expect(engine.calls).toHaveLength(2);
|
||||
});
|
||||
|
||||
test("a failing pass keeps the provisional text and the session alive", async () => {
|
||||
const engine = fakeEngine();
|
||||
const live = engine.make();
|
||||
|
||||
live.push(speech(1.5));
|
||||
engine.calls[0]!.resolve({ text: " provisional", chunks: [] });
|
||||
await tick();
|
||||
live.push(silence(1.5)); // commit → final pass
|
||||
engine.calls[1]!.reject(new Error("inference exploded"));
|
||||
await tick();
|
||||
|
||||
expect(engine.last().error).toBe("inference exploded");
|
||||
expect(engine.last().committedText).toBe("provisional");
|
||||
expect(engine.last().pendingFinalPasses).toBe(0);
|
||||
|
||||
// Later segments still work.
|
||||
engine.advance(5000);
|
||||
live.push(speech(1.5));
|
||||
expect(engine.calls).toHaveLength(3);
|
||||
const stopping = live.stop();
|
||||
await tick();
|
||||
engine.calls[2]!.resolve({ text: " ignored (stale tentative)", chunks: [] });
|
||||
await tick();
|
||||
engine.calls[3]!.resolve({ text: " recovered", chunks: [] });
|
||||
const result = await stopping;
|
||||
expect(result.text).toBe("provisional recovered");
|
||||
});
|
||||
|
||||
test("a tentative result for an already-committed segment is discarded", async () => {
|
||||
const engine = fakeEngine();
|
||||
const live = engine.make();
|
||||
|
||||
live.push(speech(1.5)); // tentative pass in flight
|
||||
live.push(silence(1.5)); // segment commits while the pass runs
|
||||
expect(engine.last().committedText).toBe(""); // no text known yet
|
||||
|
||||
engine.calls[0]!.resolve({ text: " stale", chunks: [] });
|
||||
await tick();
|
||||
// Stale text was not applied as tentative; the final pass owns the text.
|
||||
expect(engine.last().tentativeText).toBe("");
|
||||
expect(engine.calls).toHaveLength(2);
|
||||
engine.calls[1]!.resolve({ text: " final", chunks: [] });
|
||||
await tick();
|
||||
expect(engine.last().committedText).toBe("final");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,51 @@
|
||||
# Manual verification: server engine (M5, D-013)
|
||||
|
||||
The dev environment has no Python package manager and no browser, so — like
|
||||
`tests/offline.md` and `tests/live.md` — the real-inference path needs a
|
||||
manual run on a machine with Python ≥ 3.9. Everything protocol- and
|
||||
HTTP-level is already covered by `bun test` (`tests/transcribe.test.ts`
|
||||
exercises the real `Bun.spawn` supervisor against
|
||||
`tests/fixtures/fake_sidecar.py`, plus the degradation paths against the real
|
||||
`server-py/sidecar.py`).
|
||||
|
||||
## 1. Install the sidecar dependencies
|
||||
|
||||
```sh
|
||||
python3 -m pip install -r server-py/requirements.txt
|
||||
```
|
||||
|
||||
## 2. Opt-in automated pass (no browser needed)
|
||||
|
||||
```sh
|
||||
FW_INTEGRATION=1 bun test tests/transcribe.test.ts
|
||||
```
|
||||
|
||||
Runs the real sidecar end to end (spawn → ready handshake → `tiny.en` load →
|
||||
transcribe → done). First run downloads ~75 MB of weights from the Hub.
|
||||
|
||||
- [ ] The `real faster-whisper sidecar` describe block passes.
|
||||
|
||||
## 3. In-browser run (both engines, same audio)
|
||||
|
||||
1. `bun run dev`, open the app, register/sign in.
|
||||
2. `GET /api/transcribe/status` (with the session cookie) returns
|
||||
`{"available": true}` — e.g. check the Network tab.
|
||||
3. Pick `Whisper Tiny (English)`; transcribe a short clean English clip
|
||||
(e.g. the JFK "ask not…" sample) **on device**. Note the transcript.
|
||||
4. Switch the engine toggle to **Server** (visible only while signed in;
|
||||
requires the M5 UI wiring — until that lands, drive it manually:
|
||||
`curl -X POST --data-binary @audio.f32 -b "sid=…" \
|
||||
"http://localhost:3000/api/transcribe?model=onnx-community/whisper-tiny.en"`
|
||||
where `audio.f32` is raw float32 mono 16 kHz PCM).
|
||||
5. Transcribe the same clip through the server engine.
|
||||
|
||||
- [ ] Segments stream in as JSON lines (not one blob at the end).
|
||||
- [ ] Both transcripts agree in content (minor punctuation/casing drift
|
||||
between Transformers.js and faster-whisper is expected).
|
||||
- [ ] Signed out, the toggle is gone and `POST /api/transcribe` answers 401.
|
||||
- [ ] Second request on the same model is visibly faster (no reload —
|
||||
warm-up strategy).
|
||||
- [ ] `AUTH_ENABLED=false bun run dev`: `/api/transcribe*` answers 404 and
|
||||
the anonymous browser flow is unaffected.
|
||||
|
||||
When this passes, check off the M5 integration item in `WorkBreakdown.md`.
|
||||
@@ -0,0 +1,448 @@
|
||||
import { afterAll, describe, expect, test } from "bun:test";
|
||||
import { mkdtemp, readdir, rm } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import {
|
||||
createSidecar,
|
||||
createTranscribeHandlers,
|
||||
type Sidecar,
|
||||
type SidecarSegment,
|
||||
type SidecarTranscribeRequest,
|
||||
} from "../src/server/transcribe";
|
||||
import { isTranscribeStreamLine, type TranscribeStreamLine } from "../src/lib/engine";
|
||||
import type { SessionUser } from "../src/server/auth";
|
||||
|
||||
// M5 server engine: HTTP handlers against a fake sidecar, and the real
|
||||
// Bun.spawn supervisor against a protocol-faithful Python stand-in
|
||||
// (tests/fixtures/fake_sidecar.py) — no faster-whisper needed. The end-to-end
|
||||
// run with real inference is manual/opt-in: see tests/server-engine.md.
|
||||
|
||||
const FIXTURE_SIDECAR = join(import.meta.dir, "fixtures/fake_sidecar.py");
|
||||
const REAL_SIDECAR = join(import.meta.dir, "../server-py/sidecar.py");
|
||||
|
||||
const tempDirs: string[] = [];
|
||||
async function makeTempDir(): Promise<string> {
|
||||
const dir = await mkdtemp(join(tmpdir(), "transcribe-test-"));
|
||||
tempDirs.push(dir);
|
||||
return dir;
|
||||
}
|
||||
afterAll(async () => {
|
||||
await Promise.all(tempDirs.map((dir) => rm(dir, { recursive: true, force: true })));
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const USER: SessionUser = { id: 1, username: "tester" };
|
||||
const sessionFromCookie = (request: Request): SessionUser | null =>
|
||||
request.headers.get("cookie") === "sid=ok" ? USER : null;
|
||||
|
||||
interface FakeSidecar extends Sidecar {
|
||||
requests: SidecarTranscribeRequest[];
|
||||
/** Bytes of each request's audio file, captured before cleanup. */
|
||||
audioPayloads: Uint8Array[];
|
||||
}
|
||||
|
||||
function fakeSidecar(
|
||||
behavior: {
|
||||
available?: boolean;
|
||||
reason?: string;
|
||||
segments?: SidecarSegment[];
|
||||
error?: string;
|
||||
waitForAbort?: boolean;
|
||||
} = {},
|
||||
): FakeSidecar {
|
||||
const segments = behavior.segments ?? [
|
||||
{ start: 0, end: 1.5, text: " Hello" },
|
||||
{ start: 1.5, end: 3, text: " world." },
|
||||
];
|
||||
const sidecar: FakeSidecar = {
|
||||
requests: [],
|
||||
audioPayloads: [],
|
||||
async status() {
|
||||
return behavior.available === false
|
||||
? { available: false, reason: behavior.reason ?? "nope" }
|
||||
: { available: true };
|
||||
},
|
||||
async transcribe(request, onSegment, signal) {
|
||||
sidecar.requests.push(request);
|
||||
sidecar.audioPayloads.push(
|
||||
new Uint8Array(await Bun.file(request.audioPath).arrayBuffer()),
|
||||
);
|
||||
if (behavior.error) throw new Error(behavior.error);
|
||||
onSegment(segments[0]!);
|
||||
if (behavior.waitForAbort) {
|
||||
await new Promise<void>((resolve) => {
|
||||
if (signal?.aborted) return resolve();
|
||||
signal?.addEventListener("abort", () => resolve(), { once: true });
|
||||
});
|
||||
return { cancelled: true };
|
||||
}
|
||||
for (const segment of segments.slice(1)) onSegment(segment);
|
||||
return { cancelled: false };
|
||||
},
|
||||
};
|
||||
return sidecar;
|
||||
}
|
||||
|
||||
function pcmBody(samples: number[]): Uint8Array<ArrayBuffer> {
|
||||
const audio = new Float32Array(samples);
|
||||
return new Uint8Array(audio.buffer as ArrayBuffer);
|
||||
}
|
||||
|
||||
function postRequest(
|
||||
query: string,
|
||||
body: BodyInit | null,
|
||||
init: { cookie?: string; signal?: AbortSignal; contentLength?: boolean } = {},
|
||||
): Request {
|
||||
const headers: Record<string, string> = {
|
||||
"Content-Type": "application/octet-stream",
|
||||
};
|
||||
if (init.cookie !== undefined) headers.cookie = init.cookie;
|
||||
return new Request(`http://localhost/api/transcribe${query}`, {
|
||||
method: "POST",
|
||||
headers,
|
||||
body,
|
||||
signal: init.signal,
|
||||
});
|
||||
}
|
||||
|
||||
async function readStreamLines(response: Response): Promise<TranscribeStreamLine[]> {
|
||||
const text = await response.text();
|
||||
return text
|
||||
.split("\n")
|
||||
.filter((line) => line.trim().length > 0)
|
||||
.map((line) => {
|
||||
const parsed: unknown = JSON.parse(line);
|
||||
if (!isTranscribeStreamLine(parsed)) throw new Error(`bad line: ${line}`);
|
||||
return parsed;
|
||||
});
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// HTTP handlers (fake sidecar)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe("GET /api/transcribe/status", () => {
|
||||
test("401 without a session", async () => {
|
||||
const handlers = createTranscribeHandlers({
|
||||
sidecar: fakeSidecar(),
|
||||
getSessionUser: sessionFromCookie,
|
||||
});
|
||||
const response = await handlers.status(
|
||||
new Request("http://localhost/api/transcribe/status"),
|
||||
);
|
||||
expect(response.status).toBe(401);
|
||||
});
|
||||
|
||||
test("passes the sidecar's availability through", async () => {
|
||||
const handlers = createTranscribeHandlers({
|
||||
sidecar: fakeSidecar({ available: false, reason: "python3 missing" }),
|
||||
getSessionUser: sessionFromCookie,
|
||||
});
|
||||
const response = await handlers.status(
|
||||
new Request("http://localhost/api/transcribe/status", {
|
||||
headers: { cookie: "sid=ok" },
|
||||
}),
|
||||
);
|
||||
expect(response.status).toBe(200);
|
||||
expect(await response.json()).toEqual({
|
||||
available: false,
|
||||
reason: "python3 missing",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("POST /api/transcribe", () => {
|
||||
test("401 without a session — the server engine is logged-in only", async () => {
|
||||
const sidecar = fakeSidecar();
|
||||
const handlers = createTranscribeHandlers({
|
||||
sidecar,
|
||||
getSessionUser: sessionFromCookie,
|
||||
});
|
||||
const response = await handlers.post(
|
||||
postRequest("?model=onnx-community/whisper-tiny.en", pcmBody([0.1])),
|
||||
);
|
||||
expect(response.status).toBe(401);
|
||||
expect(sidecar.requests).toHaveLength(0);
|
||||
});
|
||||
|
||||
test("400 for missing/unknown model, bad language, bad task", async () => {
|
||||
const handlers = createTranscribeHandlers({
|
||||
sidecar: fakeSidecar(),
|
||||
getSessionUser: sessionFromCookie,
|
||||
tempDir: await makeTempDir(),
|
||||
});
|
||||
const cases = [
|
||||
"",
|
||||
"?model=someone/other-model",
|
||||
"?model=onnx-community/whisper-tiny&language=Danish!",
|
||||
"?model=onnx-community/whisper-tiny&task=summarize",
|
||||
];
|
||||
for (const query of cases) {
|
||||
const response = await handlers.post(
|
||||
postRequest(query, pcmBody([0.1]), { cookie: "sid=ok" }),
|
||||
);
|
||||
expect(response.status).toBe(400);
|
||||
}
|
||||
});
|
||||
|
||||
test("503 with the reason when the sidecar is unavailable", async () => {
|
||||
const sidecar = fakeSidecar({ available: false, reason: "faster-whisper is not installed" });
|
||||
const handlers = createTranscribeHandlers({
|
||||
sidecar,
|
||||
getSessionUser: sessionFromCookie,
|
||||
});
|
||||
const response = await handlers.post(
|
||||
postRequest("?model=onnx-community/whisper-tiny.en", pcmBody([0.1]), {
|
||||
cookie: "sid=ok",
|
||||
}),
|
||||
);
|
||||
expect(response.status).toBe(503);
|
||||
expect(await response.json()).toEqual({ error: "faster-whisper is not installed" });
|
||||
expect(sidecar.requests).toHaveLength(0);
|
||||
});
|
||||
|
||||
test("413 over the upload limit; 400 for empty or non-float32 bodies", async () => {
|
||||
const tempDir = await makeTempDir();
|
||||
const handlers = createTranscribeHandlers({
|
||||
sidecar: fakeSidecar(),
|
||||
getSessionUser: sessionFromCookie,
|
||||
maxUploadBytes: 16,
|
||||
tempDir,
|
||||
});
|
||||
const over = await handlers.post(
|
||||
postRequest("?model=onnx-community/whisper-tiny.en", pcmBody([1, 2, 3, 4, 5]), {
|
||||
cookie: "sid=ok",
|
||||
}),
|
||||
);
|
||||
expect(over.status).toBe(413);
|
||||
|
||||
const empty = await handlers.post(
|
||||
postRequest("?model=onnx-community/whisper-tiny.en", new Uint8Array(0), {
|
||||
cookie: "sid=ok",
|
||||
}),
|
||||
);
|
||||
expect(empty.status).toBe(400);
|
||||
|
||||
const misaligned = await handlers.post(
|
||||
postRequest("?model=onnx-community/whisper-tiny.en", new Uint8Array(6), {
|
||||
cookie: "sid=ok",
|
||||
}),
|
||||
);
|
||||
expect(misaligned.status).toBe(400);
|
||||
|
||||
// Rejected uploads must not leave temp files behind.
|
||||
expect(await readdir(tempDir)).toEqual([]);
|
||||
});
|
||||
|
||||
test("streams segments then the result, maps the model, cleans up", async () => {
|
||||
const tempDir = await makeTempDir();
|
||||
const sidecar = fakeSidecar();
|
||||
const handlers = createTranscribeHandlers({
|
||||
sidecar,
|
||||
getSessionUser: sessionFromCookie,
|
||||
tempDir,
|
||||
});
|
||||
const audio = [0.25, -0.25, 0.5, -0.5];
|
||||
const response = await handlers.post(
|
||||
postRequest(
|
||||
"?model=onnx-community/whisper-base&language=da&task=translate",
|
||||
pcmBody(audio),
|
||||
{ cookie: "sid=ok" },
|
||||
),
|
||||
);
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.headers.get("content-type")).toContain("application/x-ndjson");
|
||||
|
||||
const lines = await readStreamLines(response);
|
||||
expect(lines).toEqual([
|
||||
{ type: "segment", text: " Hello", start: 0, end: 1.5 },
|
||||
{ type: "segment", text: " world.", start: 1.5, end: 3 },
|
||||
{
|
||||
type: "result",
|
||||
text: "Hello world.",
|
||||
chunks: [
|
||||
{ text: " Hello", timestamp: [0, 1.5] },
|
||||
{ text: " world.", timestamp: [1.5, 3] },
|
||||
],
|
||||
},
|
||||
]);
|
||||
|
||||
// Catalog id → faster-whisper name (D-013), options passed through.
|
||||
expect(sidecar.requests).toEqual([
|
||||
expect.objectContaining({ model: "base", language: "da", task: "translate" }),
|
||||
]);
|
||||
// The sidecar saw exactly the uploaded PCM bytes…
|
||||
expect([...new Float32Array(sidecar.audioPayloads[0]!.buffer.slice(0))]).toEqual(
|
||||
audio,
|
||||
);
|
||||
// …and the temp file is gone once the stream finished.
|
||||
expect(await readdir(tempDir)).toEqual([]);
|
||||
});
|
||||
|
||||
test("sidecar errors travel in-stream after the 200 header", async () => {
|
||||
const tempDir = await makeTempDir();
|
||||
const handlers = createTranscribeHandlers({
|
||||
sidecar: fakeSidecar({ error: "model load failed" }),
|
||||
getSessionUser: sessionFromCookie,
|
||||
tempDir,
|
||||
});
|
||||
const response = await handlers.post(
|
||||
postRequest("?model=onnx-community/whisper-tiny.en", pcmBody([0.1]), {
|
||||
cookie: "sid=ok",
|
||||
}),
|
||||
);
|
||||
expect(response.status).toBe(200);
|
||||
const lines = await readStreamLines(response);
|
||||
expect(lines).toEqual([{ type: "error", message: "model load failed" }]);
|
||||
expect(await readdir(tempDir)).toEqual([]);
|
||||
});
|
||||
|
||||
test("an aborted request cancels the sidecar run", async () => {
|
||||
const tempDir = await makeTempDir();
|
||||
const sidecar = fakeSidecar({ waitForAbort: true });
|
||||
const handlers = createTranscribeHandlers({
|
||||
sidecar,
|
||||
getSessionUser: sessionFromCookie,
|
||||
tempDir,
|
||||
});
|
||||
const controller = new AbortController();
|
||||
const response = await handlers.post(
|
||||
postRequest("?model=onnx-community/whisper-tiny.en", pcmBody([0.1]), {
|
||||
cookie: "sid=ok",
|
||||
signal: controller.signal,
|
||||
}),
|
||||
);
|
||||
expect(response.status).toBe(200);
|
||||
controller.abort();
|
||||
// Cancelled runs still terminate the stream with the partial result.
|
||||
const lines = await readStreamLines(response);
|
||||
expect(lines.at(-1)?.type).toBe("result");
|
||||
expect(await readdir(tempDir)).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// createSidecar — the real Bun.spawn supervisor against a Python child
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe("createSidecar (fake python sidecar)", () => {
|
||||
test("handshake, segment streaming, and request/option pass-through", async () => {
|
||||
const sidecar = createSidecar(["python3", FIXTURE_SIDECAR]);
|
||||
expect(await sidecar.status()).toEqual({ available: true });
|
||||
|
||||
const audioPath = join(await makeTempDir(), "audio.f32");
|
||||
await Bun.write(audioPath, pcmBody([0.1, 0.2, 0.3]));
|
||||
|
||||
const segments: SidecarSegment[] = [];
|
||||
const outcome = await sidecar.transcribe(
|
||||
{ model: "ok", audioPath, language: "da", task: "translate" },
|
||||
(segment) => segments.push(segment),
|
||||
);
|
||||
expect(outcome).toEqual({ cancelled: false });
|
||||
expect(segments).toHaveLength(2);
|
||||
expect(segments[0]).toEqual({ start: 0, end: 1.5, text: " Hello" });
|
||||
// The fixture echoes what it received (audio bytes, language, task).
|
||||
expect(segments[1]!.text).toContain("model=ok bytes=12 lang=da task=translate");
|
||||
});
|
||||
|
||||
test("sidecar 'error' lines reject that request only", async () => {
|
||||
const sidecar = createSidecar(["python3", FIXTURE_SIDECAR]);
|
||||
const audioPath = join(await makeTempDir(), "audio.f32");
|
||||
await Bun.write(audioPath, pcmBody([0.1]));
|
||||
|
||||
expect(
|
||||
sidecar.transcribe({ model: "boom", audioPath }, () => {}),
|
||||
).rejects.toThrow("kaboom");
|
||||
// The process survived; the next request works.
|
||||
const outcome = await sidecar.transcribe({ model: "ok", audioPath }, () => {});
|
||||
expect(outcome).toEqual({ cancelled: false });
|
||||
});
|
||||
|
||||
test("a crash rejects in-flight requests and the next request respawns", async () => {
|
||||
const sidecar = createSidecar(["python3", FIXTURE_SIDECAR]);
|
||||
const audioPath = join(await makeTempDir(), "audio.f32");
|
||||
await Bun.write(audioPath, pcmBody([0.1]));
|
||||
|
||||
expect(
|
||||
sidecar.transcribe({ model: "die", audioPath }, () => {}),
|
||||
).rejects.toThrow(/exited unexpectedly/);
|
||||
// Supervision: a fresh process serves the next request.
|
||||
const segments: SidecarSegment[] = [];
|
||||
const outcome = await sidecar.transcribe(
|
||||
{ model: "ok", audioPath },
|
||||
(segment) => segments.push(segment),
|
||||
);
|
||||
expect(outcome).toEqual({ cancelled: false });
|
||||
expect(segments).toHaveLength(2);
|
||||
});
|
||||
|
||||
test("abort sends a cancel and resolves {cancelled: true}", async () => {
|
||||
const sidecar = createSidecar(["python3", FIXTURE_SIDECAR]);
|
||||
const audioPath = join(await makeTempDir(), "audio.f32");
|
||||
await Bun.write(audioPath, pcmBody([0.1]));
|
||||
|
||||
const controller = new AbortController();
|
||||
const outcome = await sidecar.transcribe(
|
||||
{ model: "wait-for-cancel", audioPath },
|
||||
() => controller.abort(), // abort as soon as the first segment arrives
|
||||
controller.signal,
|
||||
);
|
||||
expect(outcome).toEqual({ cancelled: true });
|
||||
});
|
||||
});
|
||||
|
||||
describe("createSidecar (degradation, D-013)", () => {
|
||||
test("the real sidecar without faster-whisper reports a permanent reason", async () => {
|
||||
// -S/-I keep site-packages off sys.path, so this behaves like a
|
||||
// Bun-only deployment even on machines that have faster-whisper.
|
||||
const sidecar = createSidecar(["python3", "-S", "-I", REAL_SIDECAR]);
|
||||
const first = await sidecar.status();
|
||||
expect(first.available).toBe(false);
|
||||
expect(first.reason).toContain("faster-whisper is not installed");
|
||||
// Remembered as permanent — no respawn storm, same answer immediately.
|
||||
const second = await sidecar.status();
|
||||
expect(second).toEqual(first);
|
||||
});
|
||||
|
||||
test("a missing python binary reports unavailable", async () => {
|
||||
const sidecar = createSidecar(["definitely-not-a-python-binary"]);
|
||||
const status = await sidecar.status();
|
||||
expect(status.available).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Opt-in: real faster-whisper end to end (needs python3 + requirements.txt
|
||||
// installed and network/cached weights). See tests/server-engine.md.
|
||||
// FW_INTEGRATION=1 bun test tests/transcribe.test.ts
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe.skipIf(!process.env.FW_INTEGRATION)("real faster-whisper sidecar", () => {
|
||||
test(
|
||||
"transcribes PCM through the real sidecar",
|
||||
async () => {
|
||||
const sidecar = createSidecar();
|
||||
const status = await sidecar.status();
|
||||
expect(status).toEqual({ available: true });
|
||||
|
||||
// One second of a 440 Hz tone — not speech, but exercises model load,
|
||||
// numpy decode, and the full protocol round-trip.
|
||||
const samples = new Float32Array(16_000);
|
||||
for (let i = 0; i < samples.length; i++) {
|
||||
samples[i] = 0.1 * Math.sin((2 * Math.PI * 440 * i) / 16_000);
|
||||
}
|
||||
const audioPath = join(await makeTempDir(), "tone.f32");
|
||||
await Bun.write(audioPath, new Uint8Array(samples.buffer));
|
||||
|
||||
const outcome = await sidecar.transcribe(
|
||||
{ model: "tiny.en", audioPath },
|
||||
() => {},
|
||||
);
|
||||
expect(outcome.cancelled).toBe(false);
|
||||
},
|
||||
{ timeout: 300_000 },
|
||||
);
|
||||
});
|
||||
Reference in New Issue
Block a user