"""faster-whisper sidecar (Decisions.md D-013). Bun spawns and supervises this process (src/server/transcribe.ts); it is the only place server-side inference happens. The interface is JSON-lines over stdio — one JSON object per line, no framing beyond that: stdout <- {"type": "ready", "version": "1.1.1"} # startup handshake stdout <- {"type": "fatal", "message": "..."} # then exit 1 stdin -> {"type": "transcribe", "id": "1", "model": "tiny.en", "audio_path": "/tmp/...", "language": "en" | null, "task": "transcribe" | "translate", "prompt": "Maxwell, Voices of the Void" | null} stdout <- {"type": "segment", "id": "1", "start": 0.0, "end": 4.2, "text": " Hello world."} # 0..n of these stdout <- {"type": "done", "id": "1", "cancelled": false} stdout <- {"type": "error", "id": "1", "message": "..."} # instead of done stdin -> {"type": "cancel", "id": "1"} # stop between segments; still "done" stdin -> {"type": "shutdown"} # clean exit (EOF does the same) Audio arrives as a file of raw little-endian float32 mono PCM at 16 kHz — the exact representation the browser already produces for its own engine — so no container decoding happens here. Requests are processed strictly one at a time; a reader thread keeps consuming stdin meanwhile so `cancel` can interrupt the segment loop of the request in flight. The last-used model stays loaded between requests (WorkBreakdown M5 warm-up: never reload weights per request) and is replaced — not accumulated — when a request names a different model, bounding memory to one model. Environment: FASTER_WHISPER_DEVICE ctranslate2 device (default "auto") FASTER_WHISPER_COMPUTE compute type (default "int8" — the fast, low-memory CPU choice; CT2 converts on load) FASTER_WHISPER_DOWNLOAD_ROOT optional model download/cache directory Requires Python >= 3.9 and the packages in requirements.txt. This file must keep working without them up to the import: a missing dependency is reported as a "fatal" line, which the Bun side surfaces as "server engine unavailable" (the anonymous/browser path never needs any of this). """ import json import os import queue import re import sys import threading SAMPLE_RATE = 16000 # "Names and terms" vocabulary hint (D-WS2a). Twins of normalizeHint / # fitHintToBudget in src/lib/hint.ts — change both together; parity is tested # in tests/hint-parity.test.ts. Whisper's prompt budget is half of its 448 # decoder positions minus one (<|startofprev|>). PROMPT_TOKEN_BUDGET = 223 _CONTROL_CHARS = re.compile("[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]") def normalize_hint(raw): """One term per line or comma-separated -> "a, b, c"; "" means no hint.""" lines = re.split(r"\r?\n|\r", _CONTROL_CHARS.sub("", raw)) parts = [re.sub(r"\s+", " ", line).strip() for line in lines] return ", ".join(part for part in parts if part) def _prompt_sentence(text): return " " + text if re.search(r"[.?!]$", text) else " " + text + "." def fit_prompt_ids(hint, encode, budget=PROMPT_TOKEN_BUDGET): """Prompt token ids for `hint`, cut to `budget` tokens at a whole word. The prompt is a space-led sentence ending in a full stop (unless it ends in . ? ! already) - an unterminated list drops punctuation and loops. The start of the list is kept and the end dropped; a dangling ","/";" is removed; a first word longer than the budget is cut at the budget. `encode` must add no special tokens. """ text = normalize_hint(hint) if not text: return [] full = encode(_prompt_sentence(text)) if len(full) <= budget: return full words = text.split(" ") def encode_prefix(count): return encode(_prompt_sentence(re.sub(r"[,;]+$", "", " ".join(words[:count])))) low, high = 0, len(words) - 1 while low < high: mid = (low + high + 1) // 2 if len(encode_prefix(mid)) <= budget: low = mid else: high = mid - 1 if low == 0: return encode(" " + words[0])[:budget] return encode_prefix(low) def emit(message): """Write one JSON line to stdout. Only the main thread emits.""" sys.stdout.write(json.dumps(message) + "\n") sys.stdout.flush() def main(): try: import faster_whisper import numpy as np except ImportError as error: emit( { "type": "fatal", "message": "faster-whisper is not installed (%s) — " "see server-py/requirements.txt" % error, } ) return 1 device = os.environ.get("FASTER_WHISPER_DEVICE", "auto") compute_type = os.environ.get("FASTER_WHISPER_COMPUTE", "int8") download_root = os.environ.get("FASTER_WHISPER_DOWNLOAD_ROOT") or None requests = queue.Queue() cancelled = set() # request ids; GIL-atomic add/contains is enough here SHUTDOWN = object() def read_stdin(): """Reader thread: queue work, flag cancels, shut down on EOF.""" for line in sys.stdin: line = line.strip() if not line: continue try: message = json.loads(line) except ValueError: continue # a malformed line must not kill the sidecar kind = message.get("type") if kind == "transcribe": requests.put(message) elif kind == "cancel": cancelled.add(message.get("id")) elif kind == "shutdown": requests.put(SHUTDOWN) return requests.put(SHUTDOWN) # supervisor closed the pipe / died threading.Thread(target=read_stdin, daemon=True).start() cached_name = None cached_model = None emit({"type": "ready", "version": getattr(faster_whisper, "__version__", "?")}) while True: request = requests.get() if request is SHUTDOWN: return 0 request_id = request.get("id") try: model_name = request["model"] audio_path = request["audio_path"] language = request.get("language") or None task = request.get("task") or "transcribe" prompt = request.get("prompt") or "" if model_name != cached_name: cached_model = None # release before loading the replacement cached_model = faster_whisper.WhisperModel( model_name, device=device, compute_type=compute_type, download_root=download_root, ) cached_name = model_name audio = np.fromfile(audio_path, dtype=np.float32) options = {} prompt_ids = fit_prompt_ids( prompt, lambda text: cached_model.hf_tokenizer.encode( text, add_special_tokens=False ).ids, ) if prompt_ids: # D-WS2a: token ids, already cut to the budget by the same # rule as the browser engine. options["initial_prompt"] = prompt_ids segments, _info = cached_model.transcribe( audio, language=language, task=task, **options ) was_cancelled = False # `segments` is a generator: inference happens during iteration, # so each segment can stream out (and cancel can cut in) while the # next one is still being decoded. for segment in segments: if request_id in cancelled: was_cancelled = True break emit( { "type": "segment", "id": request_id, "start": round(float(segment.start), 3), "end": round(float(segment.end), 3), "text": segment.text, } ) emit({"type": "done", "id": request_id, "cancelled": was_cancelled}) except Exception as error: # noqa: BLE001 — one bad request must not exit emit({"type": "error", "id": request_id, "message": str(error)}) finally: cancelled.discard(request_id) if __name__ == "__main__": sys.exit(main())