224 lines
8.3 KiB
Python
224 lines
8.3 KiB
Python
"""faster-whisper sidecar (Decisions.md D-013).
|
|
|
|
Bun spawns and supervises this process (src/server/transcribe.ts); it is the
|
|
only place server-side inference happens. The interface is JSON-lines over
|
|
stdio — one JSON object per line, no framing beyond that:
|
|
|
|
stdout <- {"type": "ready", "version": "1.1.1"} # startup handshake
|
|
stdout <- {"type": "fatal", "message": "..."} # then exit 1
|
|
|
|
stdin -> {"type": "transcribe", "id": "1", "model": "tiny.en",
|
|
"audio_path": "/tmp/...", "language": "en" | null,
|
|
"task": "transcribe" | "translate",
|
|
"prompt": "Maxwell, Voices of the Void" | null}
|
|
stdout <- {"type": "segment", "id": "1", "start": 0.0, "end": 4.2,
|
|
"text": " Hello world."} # 0..n of these
|
|
stdout <- {"type": "done", "id": "1", "cancelled": false}
|
|
stdout <- {"type": "error", "id": "1", "message": "..."} # instead of done
|
|
|
|
stdin -> {"type": "cancel", "id": "1"} # stop between segments; still "done"
|
|
stdin -> {"type": "shutdown"} # clean exit (EOF does the same)
|
|
|
|
Audio arrives as a file of raw little-endian float32 mono PCM at 16 kHz — the
|
|
exact representation the browser already produces for its own engine — so no
|
|
container decoding happens here. Requests are processed strictly one at a
|
|
time; a reader thread keeps consuming stdin meanwhile so `cancel` can
|
|
interrupt the segment loop of the request in flight.
|
|
|
|
The last-used model stays loaded between requests (WorkBreakdown M5 warm-up:
|
|
never reload weights per request) and is replaced — not accumulated — when a
|
|
request names a different model, bounding memory to one model.
|
|
|
|
Environment:
|
|
FASTER_WHISPER_DEVICE ctranslate2 device (default "auto")
|
|
FASTER_WHISPER_COMPUTE compute type (default "int8" — the fast,
|
|
low-memory CPU choice; CT2 converts on load)
|
|
FASTER_WHISPER_DOWNLOAD_ROOT optional model download/cache directory
|
|
|
|
Requires Python >= 3.9 and the packages in requirements.txt. This file must
|
|
keep working without them up to the import: a missing dependency is reported
|
|
as a "fatal" line, which the Bun side surfaces as "server engine unavailable"
|
|
(the anonymous/browser path never needs any of this).
|
|
"""
|
|
|
|
import json
|
|
import os
|
|
import queue
|
|
import re
|
|
import sys
|
|
import threading
|
|
|
|
SAMPLE_RATE = 16000
|
|
|
|
# "Names and terms" vocabulary hint (D-WS2a). Twins of normalizeHint /
|
|
# fitHintToBudget in src/lib/hint.ts — change both together; parity is tested
|
|
# in tests/hint-parity.test.ts. Whisper's prompt budget is half of its 448
|
|
# decoder positions minus one (<|startofprev|>).
|
|
PROMPT_TOKEN_BUDGET = 223
|
|
_CONTROL_CHARS = re.compile("[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]")
|
|
|
|
|
|
def normalize_hint(raw):
|
|
"""One term per line or comma-separated -> "a, b, c"; "" means no hint."""
|
|
lines = re.split(r"\r?\n|\r", _CONTROL_CHARS.sub("", raw))
|
|
parts = [re.sub(r"\s+", " ", line).strip() for line in lines]
|
|
return ", ".join(part for part in parts if part)
|
|
|
|
|
|
def _prompt_sentence(text):
|
|
return " " + text if re.search(r"[.?!]$", text) else " " + text + "."
|
|
|
|
|
|
def fit_prompt_ids(hint, encode, budget=PROMPT_TOKEN_BUDGET):
|
|
"""Prompt token ids for `hint`, cut to `budget` tokens at a whole word.
|
|
|
|
The prompt is a space-led sentence ending in a full stop (unless it ends
|
|
in . ? ! already) - an unterminated list drops punctuation and loops.
|
|
The start of the list is kept and the end dropped; a dangling ","/";" is
|
|
removed; a first word longer than the budget is cut at the budget.
|
|
`encode` must add no special tokens.
|
|
"""
|
|
text = normalize_hint(hint)
|
|
if not text:
|
|
return []
|
|
full = encode(_prompt_sentence(text))
|
|
if len(full) <= budget:
|
|
return full
|
|
words = text.split(" ")
|
|
|
|
def encode_prefix(count):
|
|
return encode(_prompt_sentence(re.sub(r"[,;]+$", "", " ".join(words[:count]))))
|
|
|
|
low, high = 0, len(words) - 1
|
|
while low < high:
|
|
mid = (low + high + 1) // 2
|
|
if len(encode_prefix(mid)) <= budget:
|
|
low = mid
|
|
else:
|
|
high = mid - 1
|
|
if low == 0:
|
|
return encode(" " + words[0])[:budget]
|
|
return encode_prefix(low)
|
|
|
|
|
|
def emit(message):
|
|
"""Write one JSON line to stdout. Only the main thread emits."""
|
|
sys.stdout.write(json.dumps(message) + "\n")
|
|
sys.stdout.flush()
|
|
|
|
|
|
def main():
|
|
try:
|
|
import faster_whisper
|
|
import numpy as np
|
|
except ImportError as error:
|
|
emit(
|
|
{
|
|
"type": "fatal",
|
|
"message": "faster-whisper is not installed (%s) — "
|
|
"see server-py/requirements.txt" % error,
|
|
}
|
|
)
|
|
return 1
|
|
|
|
device = os.environ.get("FASTER_WHISPER_DEVICE", "auto")
|
|
compute_type = os.environ.get("FASTER_WHISPER_COMPUTE", "int8")
|
|
download_root = os.environ.get("FASTER_WHISPER_DOWNLOAD_ROOT") or None
|
|
|
|
requests = queue.Queue()
|
|
cancelled = set() # request ids; GIL-atomic add/contains is enough here
|
|
SHUTDOWN = object()
|
|
|
|
def read_stdin():
|
|
"""Reader thread: queue work, flag cancels, shut down on EOF."""
|
|
for line in sys.stdin:
|
|
line = line.strip()
|
|
if not line:
|
|
continue
|
|
try:
|
|
message = json.loads(line)
|
|
except ValueError:
|
|
continue # a malformed line must not kill the sidecar
|
|
kind = message.get("type")
|
|
if kind == "transcribe":
|
|
requests.put(message)
|
|
elif kind == "cancel":
|
|
cancelled.add(message.get("id"))
|
|
elif kind == "shutdown":
|
|
requests.put(SHUTDOWN)
|
|
return
|
|
requests.put(SHUTDOWN) # supervisor closed the pipe / died
|
|
|
|
threading.Thread(target=read_stdin, daemon=True).start()
|
|
|
|
cached_name = None
|
|
cached_model = None
|
|
|
|
emit({"type": "ready", "version": getattr(faster_whisper, "__version__", "?")})
|
|
|
|
while True:
|
|
request = requests.get()
|
|
if request is SHUTDOWN:
|
|
return 0
|
|
|
|
request_id = request.get("id")
|
|
try:
|
|
model_name = request["model"]
|
|
audio_path = request["audio_path"]
|
|
language = request.get("language") or None
|
|
task = request.get("task") or "transcribe"
|
|
prompt = request.get("prompt") or ""
|
|
|
|
if model_name != cached_name:
|
|
cached_model = None # release before loading the replacement
|
|
cached_model = faster_whisper.WhisperModel(
|
|
model_name,
|
|
device=device,
|
|
compute_type=compute_type,
|
|
download_root=download_root,
|
|
)
|
|
cached_name = model_name
|
|
|
|
audio = np.fromfile(audio_path, dtype=np.float32)
|
|
options = {}
|
|
prompt_ids = fit_prompt_ids(
|
|
prompt,
|
|
lambda text: cached_model.hf_tokenizer.encode(
|
|
text, add_special_tokens=False
|
|
).ids,
|
|
)
|
|
if prompt_ids:
|
|
# D-WS2a: token ids, already cut to the budget by the same
|
|
# rule as the browser engine.
|
|
options["initial_prompt"] = prompt_ids
|
|
segments, _info = cached_model.transcribe(
|
|
audio, language=language, task=task, **options
|
|
)
|
|
|
|
was_cancelled = False
|
|
# `segments` is a generator: inference happens during iteration,
|
|
# so each segment can stream out (and cancel can cut in) while the
|
|
# next one is still being decoded.
|
|
for segment in segments:
|
|
if request_id in cancelled:
|
|
was_cancelled = True
|
|
break
|
|
emit(
|
|
{
|
|
"type": "segment",
|
|
"id": request_id,
|
|
"start": round(float(segment.start), 3),
|
|
"end": round(float(segment.end), 3),
|
|
"text": segment.text,
|
|
}
|
|
)
|
|
emit({"type": "done", "id": request_id, "cancelled": was_cancelled})
|
|
except Exception as error: # noqa: BLE001 — one bad request must not exit
|
|
emit({"type": "error", "id": request_id, "message": str(error)})
|
|
finally:
|
|
cancelled.discard(request_id)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|