323 lines
11 KiB
TypeScript
323 lines
11 KiB
TypeScript
/**
|
|
* Curated Whisper model catalog (Decisions.md D-006).
|
|
*
|
|
* Pure, typed data — no browser globals — so it is unit-testable with
|
|
* `bun test` and shareable with the server engine (M5). ONNX conversions come
|
|
* from the `onnx-community` Hub org (the recommended source for
|
|
* Transformers.js v3+).
|
|
*
|
|
* dtype defaults per device follow the community-known-good combinations:
|
|
* - WASM: q8 for small models, q4 for the large ones (download size).
|
|
* - WebGPU: fp16 encoder + q4 merged decoder for the big models; fp32 for
|
|
* tiny/base where fp16 has known accuracy issues.
|
|
* `verified` is only true for combinations we have actually run in a browser;
|
|
* the rest are pending manual testing (WorkBreakdown M2).
|
|
*/
|
|
|
|
import type {
|
|
DevicePreference,
|
|
DtypeSpec,
|
|
InferenceDevice,
|
|
WhisperDtype,
|
|
WhisperTask,
|
|
} from "../worker/protocol";
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Catalog types
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/** Dtype default for one device, with verification status (D-006). */
|
|
export interface DtypeDefault {
|
|
dtype: DtypeSpec;
|
|
/** True only if this exact combination has been run in a real browser. */
|
|
verified: boolean;
|
|
/** Why this combination was chosen / caveats. */
|
|
note: string;
|
|
}
|
|
|
|
export interface WhisperModel {
|
|
/** Hugging Face Hub id, e.g. "onnx-community/whisper-tiny.en". */
|
|
id: string;
|
|
/** Human-readable name for the picker. */
|
|
displayName: string;
|
|
/** False for `.en` (English-only) variants. */
|
|
multilingual: boolean;
|
|
/** Encoder parameter count, in millions (drives size estimates). */
|
|
encoderParamsM: number;
|
|
/** Decoder parameter count, in millions (drives size estimates). */
|
|
decoderParamsM: number;
|
|
/** Per-device dtype defaults (D-004 / D-006). */
|
|
defaults: Record<InferenceDevice, DtypeDefault>;
|
|
/**
|
|
* Model name the server engine passes to faster-whisper (D-013). These are
|
|
* faster-whisper's own aliases, which it resolves to the official
|
|
* CTranslate2 conversions on the Hub (`Systran/faster-whisper-*`;
|
|
* large-v3-turbo → `mobiuslabsgmbh/faster-whisper-large-v3-turbo`), so the
|
|
* pinned library version stays the single authority on the exact repo.
|
|
*/
|
|
fasterWhisperModel: string;
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Size estimation (approximate, shown before download — D-006)
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/** Approximate stored bytes per parameter for each quantization level. */
|
|
export const BYTES_PER_PARAM: Record<WhisperDtype, number> = {
|
|
fp32: 4,
|
|
fp16: 2,
|
|
q8: 1.05, // int8 weights + scales/zero-points overhead
|
|
q4: 0.6, // 4-bit blocks + per-block scales; some tensors stay higher precision
|
|
};
|
|
|
|
/** Tokenizer + configs + preprocessor, roughly constant per model. */
|
|
const AUX_FILES_BYTES = 3_000_000;
|
|
|
|
/** Approximate total download size for a model at a given dtype spec. */
|
|
export function estimateDownloadBytes(
|
|
model: WhisperModel,
|
|
dtype: DtypeSpec,
|
|
): number {
|
|
const encDtype = typeof dtype === "string" ? dtype : dtype.encoder_model;
|
|
const decDtype = typeof dtype === "string" ? dtype : dtype.decoder_model_merged;
|
|
return Math.round(
|
|
model.encoderParamsM * 1e6 * BYTES_PER_PARAM[encDtype] +
|
|
model.decoderParamsM * 1e6 * BYTES_PER_PARAM[decDtype] +
|
|
AUX_FILES_BYTES,
|
|
);
|
|
}
|
|
|
|
/** "487 MB" / "1.6 GB" style formatting for size estimates. */
|
|
export function formatModelSize(bytes: number): string {
|
|
if (bytes >= 1e9) return `${(bytes / 1e9).toFixed(1)} GB`;
|
|
return `${Math.round(bytes / 1e6)} MB`;
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// dtype defaults (community-known-good; see file header)
|
|
// ---------------------------------------------------------------------------
|
|
|
|
const WASM_Q8: DtypeDefault = {
|
|
dtype: "q8",
|
|
verified: false,
|
|
note: "q8 is the Transformers.js WASM default; good quality/size trade-off.",
|
|
};
|
|
|
|
const WASM_Q4: DtypeDefault = {
|
|
dtype: "q4",
|
|
verified: false,
|
|
note: "q4 keeps the download manageable for larger models on WASM.",
|
|
};
|
|
|
|
/** fp16 has known accuracy issues for the small Whisper encoders on WebGPU. */
|
|
const WEBGPU_FP32: DtypeDefault = {
|
|
dtype: { encoder_model: "fp32", decoder_model_merged: "fp32" },
|
|
verified: false,
|
|
note: "fp32 both parts: fp16 encoders are known-bad for tiny/base on WebGPU.",
|
|
};
|
|
|
|
const WEBGPU_FP16_Q4: DtypeDefault = {
|
|
dtype: { encoder_model: "fp16", decoder_model_merged: "q4" },
|
|
verified: false,
|
|
note: "fp16 encoder + q4 merged decoder — the common working WebGPU setup.",
|
|
};
|
|
|
|
const WEBNN_FP16: DtypeDefault = {
|
|
dtype: "fp16",
|
|
verified: false,
|
|
note: "fp16 is commonly used with WebNN.",
|
|
};
|
|
|
|
/**
|
|
* The medium fp16 conversions are unusable (D-022, verified 2026-07-15): the
|
|
* ONNX files crash onnxruntime's graph optimizer (SimplifiedLayerNormFusion
|
|
* chokes on their InsertedPrecisionFreeCast nodes — reproduced on both
|
|
* ort-node and ort-web 1.26.0-dev), and Whisper medium/large encoder
|
|
* activations are the classic fp16 overflow case besides. In the browser the
|
|
* symptom was a degenerate one-word transcript with no error. q4 is verified
|
|
* correct for medium on the WASM runtime; a backend that cannot run it
|
|
* throws at session creation and falls down the D-017 device chain instead
|
|
* of silently producing garbage.
|
|
*/
|
|
const ACCEL_Q4: DtypeDefault = {
|
|
dtype: "q4",
|
|
verified: false,
|
|
note: "q4: the medium fp16 conversions fail to load / overflow (D-022).",
|
|
};
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// The catalog (D-006: curated, not exhaustive)
|
|
// ---------------------------------------------------------------------------
|
|
|
|
export const MODELS: readonly WhisperModel[] = [
|
|
{
|
|
id: "onnx-community/whisper-tiny.en",
|
|
fasterWhisperModel: "tiny.en",
|
|
displayName: "Whisper Tiny (English)",
|
|
multilingual: false,
|
|
encoderParamsM: 8,
|
|
decoderParamsM: 31,
|
|
defaults: {
|
|
// Verified in-browser during M1 (WASM, Transformers.js default q8).
|
|
wasm: { ...WASM_Q8, verified: true },
|
|
webgpu: WEBGPU_FP32,
|
|
webnn: WEBNN_FP16,
|
|
},
|
|
},
|
|
{
|
|
id: "onnx-community/whisper-tiny",
|
|
fasterWhisperModel: "tiny",
|
|
displayName: "Whisper Tiny",
|
|
multilingual: true,
|
|
encoderParamsM: 8,
|
|
decoderParamsM: 31,
|
|
defaults: { wasm: WASM_Q8, webgpu: WEBGPU_FP32, webnn: WEBNN_FP16 },
|
|
},
|
|
{
|
|
id: "onnx-community/whisper-base.en",
|
|
fasterWhisperModel: "base.en",
|
|
displayName: "Whisper Base (English)",
|
|
multilingual: false,
|
|
encoderParamsM: 20,
|
|
decoderParamsM: 54,
|
|
defaults: { wasm: WASM_Q8, webgpu: WEBGPU_FP32, webnn: WEBNN_FP16 },
|
|
},
|
|
{
|
|
id: "onnx-community/whisper-base",
|
|
fasterWhisperModel: "base",
|
|
displayName: "Whisper Base",
|
|
multilingual: true,
|
|
encoderParamsM: 20,
|
|
decoderParamsM: 54,
|
|
defaults: { wasm: WASM_Q8, webgpu: WEBGPU_FP32, webnn: WEBNN_FP16 },
|
|
},
|
|
{
|
|
id: "onnx-community/whisper-small.en",
|
|
fasterWhisperModel: "small.en",
|
|
displayName: "Whisper Small (English)",
|
|
multilingual: false,
|
|
encoderParamsM: 88,
|
|
decoderParamsM: 156,
|
|
defaults: { wasm: WASM_Q8, webgpu: WEBGPU_FP16_Q4, webnn: WEBNN_FP16 },
|
|
},
|
|
{
|
|
id: "onnx-community/whisper-small",
|
|
fasterWhisperModel: "small",
|
|
displayName: "Whisper Small",
|
|
multilingual: true,
|
|
encoderParamsM: 88,
|
|
decoderParamsM: 156,
|
|
defaults: { wasm: WASM_Q8, webgpu: WEBGPU_FP16_Q4, webnn: WEBNN_FP16 },
|
|
},
|
|
{
|
|
// onnx-community has no plain "whisper-medium.en"/"whisper-medium" repos
|
|
// (the Hub answers 401 "Unauthorized" for missing repos) — the medium
|
|
// conversions live under these names instead. Verified to carry the full
|
|
// q4/q8/fp16 encoder + merged-decoder variant set.
|
|
id: "onnx-community/whisper-medium.en_timestamped",
|
|
fasterWhisperModel: "medium.en",
|
|
displayName: "Whisper Medium (English)",
|
|
multilingual: false,
|
|
encoderParamsM: 306,
|
|
decoderParamsM: 463,
|
|
defaults: { wasm: WASM_Q4, webgpu: ACCEL_Q4, webnn: ACCEL_Q4 },
|
|
},
|
|
{
|
|
id: "onnx-community/whisper-medium-ONNX",
|
|
fasterWhisperModel: "medium",
|
|
displayName: "Whisper Medium",
|
|
multilingual: true,
|
|
encoderParamsM: 306,
|
|
decoderParamsM: 463,
|
|
defaults: { wasm: WASM_Q4, webgpu: ACCEL_Q4, webnn: ACCEL_Q4 },
|
|
},
|
|
{
|
|
id: "onnx-community/whisper-large-v3-turbo",
|
|
fasterWhisperModel: "large-v3-turbo",
|
|
displayName: "Whisper Large v3 Turbo",
|
|
multilingual: true,
|
|
encoderParamsM: 635,
|
|
decoderParamsM: 174,
|
|
defaults: { wasm: WASM_Q4, webgpu: WEBGPU_FP16_Q4, webnn: WEBNN_FP16 },
|
|
},
|
|
];
|
|
|
|
export function getModel(id: string): WhisperModel | undefined {
|
|
return MODELS.find((model) => model.id === id);
|
|
}
|
|
|
|
/**
|
|
* faster-whisper model name for a catalog id (server engine, D-013), or
|
|
* undefined for ids outside the catalog — the transcribe endpoint answers 400
|
|
* rather than forwarding arbitrary strings to the sidecar.
|
|
*/
|
|
export function getFasterWhisperModel(id: string): string | undefined {
|
|
return getModel(id)?.fasterWhisperModel;
|
|
}
|
|
|
|
/** Default dtype for a model on a given device. Unknown ids fall back to q8. */
|
|
export function resolveDefaultDtype(
|
|
modelId: string,
|
|
device: InferenceDevice,
|
|
): DtypeSpec {
|
|
return getModel(modelId)?.defaults[device].dtype ?? "q8";
|
|
}
|
|
|
|
/** Approximate download size using the model's default dtype for `device`. */
|
|
export function estimateDefaultDownloadBytes(
|
|
model: WhisperModel,
|
|
device: InferenceDevice,
|
|
): number {
|
|
return estimateDownloadBytes(model, model.defaults[device].dtype);
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Defaults & language list
|
|
// ---------------------------------------------------------------------------
|
|
|
|
export const DEFAULT_MODEL_ID = "onnx-community/whisper-tiny.en";
|
|
|
|
/** UI default: worker probes WebNN → WebGPU → WASM in order (D-004, D-017). */
|
|
export const DEFAULT_DEVICE_PREFERENCE: DevicePreference = "auto";
|
|
|
|
export const DEFAULT_TASK: WhisperTask = "transcribe";
|
|
|
|
/** Sentinel for "let Whisper detect the language". */
|
|
export const AUTO_LANGUAGE = "auto";
|
|
|
|
/**
|
|
* Curated language list for multilingual models (Whisper language codes).
|
|
* Not exhaustive — Whisper supports ~100; extend on demand.
|
|
*/
|
|
export const WHISPER_LANGUAGES: readonly { code: string; label: string }[] = [
|
|
{ code: "en", label: "English" },
|
|
{ code: "da", label: "Danish" },
|
|
{ code: "de", label: "German" },
|
|
{ code: "es", label: "Spanish" },
|
|
{ code: "fr", label: "French" },
|
|
{ code: "it", label: "Italian" },
|
|
{ code: "nl", label: "Dutch" },
|
|
{ code: "no", label: "Norwegian" },
|
|
{ code: "sv", label: "Swedish" },
|
|
{ code: "fi", label: "Finnish" },
|
|
{ code: "is", label: "Icelandic" },
|
|
{ code: "pl", label: "Polish" },
|
|
{ code: "pt", label: "Portuguese" },
|
|
{ code: "ro", label: "Romanian" },
|
|
{ code: "cs", label: "Czech" },
|
|
{ code: "el", label: "Greek" },
|
|
{ code: "hu", label: "Hungarian" },
|
|
{ code: "uk", label: "Ukrainian" },
|
|
{ code: "ru", label: "Russian" },
|
|
{ code: "tr", label: "Turkish" },
|
|
{ code: "ar", label: "Arabic" },
|
|
{ code: "he", label: "Hebrew" },
|
|
{ code: "hi", label: "Hindi" },
|
|
{ code: "id", label: "Indonesian" },
|
|
{ code: "vi", label: "Vietnamese" },
|
|
{ code: "th", label: "Thai" },
|
|
{ code: "zh", label: "Chinese" },
|
|
{ code: "ja", label: "Japanese" },
|
|
{ code: "ko", label: "Korean" },
|
|
];
|