/** * Curated Whisper model catalog (Decisions.md D-006). * * Pure, typed data — no browser globals — so it is unit-testable with * `bun test` and shareable with the server engine (M5). ONNX conversions come * from the `onnx-community` Hub org (the recommended source for * Transformers.js v3+). * * dtype defaults per device follow the community-known-good combinations: * - WASM: q8 for small models, q4 for the large ones (download size). * - WebGPU: fp16 encoder + q4 merged decoder for the big models; fp32 for * tiny/base where fp16 has known accuracy issues. * `verified` is only true for combinations we have actually run in a browser; * the rest are pending manual testing (WorkBreakdown M2). */ import type { DevicePreference, DtypeSpec, InferenceDevice, WhisperDtype, WhisperTask, } from "../worker/protocol"; // --------------------------------------------------------------------------- // Catalog types // --------------------------------------------------------------------------- /** Dtype default for one device, with verification status (D-006). */ export interface DtypeDefault { dtype: DtypeSpec; /** True only if this exact combination has been run in a real browser. */ verified: boolean; /** Why this combination was chosen / caveats. */ note: string; } export interface WhisperModel { /** Hugging Face Hub id, e.g. "onnx-community/whisper-tiny.en". */ id: string; /** Human-readable name for the picker. */ displayName: string; /** False for `.en` (English-only) variants. */ multilingual: boolean; /** Encoder parameter count, in millions (drives size estimates). */ encoderParamsM: number; /** Decoder parameter count, in millions (drives size estimates). */ decoderParamsM: number; /** Per-device dtype defaults (D-004 / D-006). */ defaults: Record; /** * Model name the server engine passes to faster-whisper (D-013). These are * faster-whisper's own aliases, which it resolves to the official * CTranslate2 conversions on the Hub (`Systran/faster-whisper-*`; * large-v3-turbo → `mobiuslabsgmbh/faster-whisper-large-v3-turbo`), so the * pinned library version stays the single authority on the exact repo. */ fasterWhisperModel: string; } // --------------------------------------------------------------------------- // Size estimation (approximate, shown before download — D-006) // --------------------------------------------------------------------------- /** Approximate stored bytes per parameter for each quantization level. */ export const BYTES_PER_PARAM: Record = { fp32: 4, fp16: 2, q8: 1.05, // int8 weights + scales/zero-points overhead q4: 0.6, // 4-bit blocks + per-block scales; some tensors stay higher precision }; /** Tokenizer + configs + preprocessor, roughly constant per model. */ const AUX_FILES_BYTES = 3_000_000; /** Approximate total download size for a model at a given dtype spec. */ export function estimateDownloadBytes( model: WhisperModel, dtype: DtypeSpec, ): number { const encDtype = typeof dtype === "string" ? dtype : dtype.encoder_model; const decDtype = typeof dtype === "string" ? dtype : dtype.decoder_model_merged; return Math.round( model.encoderParamsM * 1e6 * BYTES_PER_PARAM[encDtype] + model.decoderParamsM * 1e6 * BYTES_PER_PARAM[decDtype] + AUX_FILES_BYTES, ); } /** "487 MB" / "1.6 GB" style formatting for size estimates. */ export function formatModelSize(bytes: number): string { if (bytes >= 1e9) return `${(bytes / 1e9).toFixed(1)} GB`; return `${Math.round(bytes / 1e6)} MB`; } /** * Approximate peak transient memory a browser-engine load needs (D-027): for * whichever of the encoder/decoder weight files is larger, transformers.js * buffers the full download into one JS-heap `Uint8Array`, then * onnxruntime-web mallocs a second full-size copy inside WASM linear memory * to parse it at session-creation time — a per-file ~2x peak. The * sequential-session-load patch (`patches/`) keeps the encoder and decoder * from both being resident at once, so the two files' peaks don't add * together; only the larger one sets the peak. Approximate — real browser/ * runtime overhead (page baseline, Cache Storage, other tabs) adds more. */ export function estimatePeakLoadBytes(model: WhisperModel, dtype: DtypeSpec): number { const encDtype = typeof dtype === "string" ? dtype : dtype.encoder_model; const decDtype = typeof dtype === "string" ? dtype : dtype.decoder_model_merged; const encoderBytes = model.encoderParamsM * 1e6 * BYTES_PER_PARAM[encDtype]; const decoderBytes = model.decoderParamsM * 1e6 * BYTES_PER_PARAM[decDtype]; return Math.round(2 * Math.max(encoderBytes, decoderBytes)); } // --------------------------------------------------------------------------- // dtype defaults (community-known-good; see file header) // --------------------------------------------------------------------------- const WASM_Q8: DtypeDefault = { dtype: "q8", verified: false, note: "q8 is the Transformers.js WASM default; good quality/size trade-off.", }; const WASM_Q4: DtypeDefault = { dtype: "q4", verified: false, note: "q4 keeps the download manageable for larger models on WASM.", }; /** fp16 has known accuracy issues for the small Whisper encoders on WebGPU. */ const WEBGPU_FP32: DtypeDefault = { dtype: { encoder_model: "fp32", decoder_model_merged: "fp32" }, verified: false, note: "fp32 both parts: fp16 encoders are known-bad for tiny/base on WebGPU.", }; const WEBGPU_FP16_Q4: DtypeDefault = { dtype: { encoder_model: "fp16", decoder_model_merged: "q4" }, verified: false, note: "fp16 encoder + q4 merged decoder — the common working WebGPU setup.", }; const WEBNN_FP16: DtypeDefault = { dtype: "fp16", verified: false, note: "fp16 is commonly used with WebNN.", }; /** * The medium fp16 conversions are unusable (D-022, verified 2026-07-15): the * ONNX files crash onnxruntime's graph optimizer (SimplifiedLayerNormFusion * chokes on their InsertedPrecisionFreeCast nodes — reproduced on both * ort-node and ort-web 1.26.0-dev), and Whisper medium/large encoder * activations are the classic fp16 overflow case besides. In the browser the * symptom was a degenerate one-word transcript with no error. q4 is verified * correct for medium on the WASM runtime; a backend that cannot run it * throws at session creation and falls down the D-017 device chain instead * of silently producing garbage. */ const ACCEL_Q4: DtypeDefault = { dtype: "q4", verified: false, note: "q4: the medium fp16 conversions fail to load / overflow (D-022).", }; // --------------------------------------------------------------------------- // The catalog (D-006: curated, not exhaustive) // --------------------------------------------------------------------------- export const MODELS: readonly WhisperModel[] = [ { id: "onnx-community/whisper-tiny.en", fasterWhisperModel: "tiny.en", displayName: "Whisper Tiny (English)", multilingual: false, encoderParamsM: 8, decoderParamsM: 31, defaults: { // Verified in-browser during M1 (WASM, Transformers.js default q8). wasm: { ...WASM_Q8, verified: true }, webgpu: WEBGPU_FP32, webnn: WEBNN_FP16, }, }, { id: "onnx-community/whisper-tiny", fasterWhisperModel: "tiny", displayName: "Whisper Tiny", multilingual: true, encoderParamsM: 8, decoderParamsM: 31, defaults: { wasm: WASM_Q8, webgpu: WEBGPU_FP32, webnn: WEBNN_FP16 }, }, { id: "onnx-community/whisper-base.en", fasterWhisperModel: "base.en", displayName: "Whisper Base (English)", multilingual: false, encoderParamsM: 20, decoderParamsM: 54, defaults: { wasm: WASM_Q8, webgpu: WEBGPU_FP32, webnn: WEBNN_FP16 }, }, { id: "onnx-community/whisper-base", fasterWhisperModel: "base", displayName: "Whisper Base", multilingual: true, encoderParamsM: 20, decoderParamsM: 54, defaults: { wasm: WASM_Q8, webgpu: WEBGPU_FP32, webnn: WEBNN_FP16 }, }, { id: "onnx-community/whisper-small.en", fasterWhisperModel: "small.en", displayName: "Whisper Small (English)", multilingual: false, encoderParamsM: 88, decoderParamsM: 156, defaults: { wasm: WASM_Q8, webgpu: WEBGPU_FP16_Q4, webnn: WEBNN_FP16 }, }, { id: "onnx-community/whisper-small", fasterWhisperModel: "small", displayName: "Whisper Small", multilingual: true, encoderParamsM: 88, decoderParamsM: 156, defaults: { wasm: WASM_Q8, webgpu: WEBGPU_FP16_Q4, webnn: WEBNN_FP16 }, }, { // onnx-community has no plain "whisper-medium.en"/"whisper-medium" repos // (the Hub answers 401 "Unauthorized" for missing repos) — the medium // conversions live under these names instead. Verified to carry the full // q4/q8/fp16 encoder + merged-decoder variant set. id: "onnx-community/whisper-medium.en_timestamped", fasterWhisperModel: "medium.en", displayName: "Whisper Medium (English)", multilingual: false, encoderParamsM: 306, decoderParamsM: 463, defaults: { wasm: WASM_Q4, webgpu: ACCEL_Q4, webnn: ACCEL_Q4 }, }, { id: "onnx-community/whisper-medium-ONNX", fasterWhisperModel: "medium", displayName: "Whisper Medium", multilingual: true, encoderParamsM: 306, decoderParamsM: 463, defaults: { wasm: WASM_Q4, webgpu: ACCEL_Q4, webnn: ACCEL_Q4 }, }, { id: "onnx-community/whisper-large-v3-turbo", fasterWhisperModel: "large-v3-turbo", displayName: "Whisper Large v3 Turbo", multilingual: true, encoderParamsM: 635, decoderParamsM: 174, defaults: { wasm: WASM_Q4, webgpu: WEBGPU_FP16_Q4, webnn: WEBNN_FP16 }, }, ]; export function getModel(id: string): WhisperModel | undefined { return MODELS.find((model) => model.id === id); } /** * faster-whisper model name for a catalog id (server engine, D-013), or * undefined for ids outside the catalog — the transcribe endpoint answers 400 * rather than forwarding arbitrary strings to the sidecar. */ export function getFasterWhisperModel(id: string): string | undefined { return getModel(id)?.fasterWhisperModel; } /** Default dtype for a model on a given device. Unknown ids fall back to q8. */ export function resolveDefaultDtype( modelId: string, device: InferenceDevice, ): DtypeSpec { return getModel(modelId)?.defaults[device].dtype ?? "q8"; } /** Approximate download size using the model's default dtype for `device`. */ export function estimateDefaultDownloadBytes( model: WhisperModel, device: InferenceDevice, ): number { return estimateDownloadBytes(model, model.defaults[device].dtype); } // --------------------------------------------------------------------------- // Defaults & language list // --------------------------------------------------------------------------- export const DEFAULT_MODEL_ID = "onnx-community/whisper-tiny.en"; /** UI default: worker probes WebNN → WebGPU → WASM in order (D-004, D-017). */ export const DEFAULT_DEVICE_PREFERENCE: DevicePreference = "auto"; export const DEFAULT_TASK: WhisperTask = "transcribe"; /** Sentinel for "let Whisper detect the language". */ export const AUTO_LANGUAGE = "auto"; /** * Curated language list for multilingual models (Whisper language codes). * Not exhaustive — Whisper supports ~100; extend on demand. */ export const WHISPER_LANGUAGES: readonly { code: string; label: string }[] = [ { code: "en", label: "English" }, { code: "da", label: "Danish" }, { code: "de", label: "German" }, { code: "es", label: "Spanish" }, { code: "fr", label: "French" }, { code: "it", label: "Italian" }, { code: "nl", label: "Dutch" }, { code: "no", label: "Norwegian" }, { code: "sv", label: "Swedish" }, { code: "fi", label: "Finnish" }, { code: "is", label: "Icelandic" }, { code: "pl", label: "Polish" }, { code: "pt", label: "Portuguese" }, { code: "ro", label: "Romanian" }, { code: "cs", label: "Czech" }, { code: "el", label: "Greek" }, { code: "hu", label: "Hungarian" }, { code: "uk", label: "Ukrainian" }, { code: "ru", label: "Russian" }, { code: "tr", label: "Turkish" }, { code: "ar", label: "Arabic" }, { code: "he", label: "Hebrew" }, { code: "hi", label: "Hindi" }, { code: "id", label: "Indonesian" }, { code: "vi", label: "Vietnamese" }, { code: "th", label: "Thai" }, { code: "zh", label: "Chinese" }, { code: "ja", label: "Japanese" }, { code: "ko", label: "Korean" }, ];