lucasw89/whisper_web
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Prefer WASM on firefox cause WebGPU is incredibly slow there.

Browse Source
This commit is contained in:
Lucas Winther
2026-07-17 16:35:49 +02:00
parent 74858f63fc
commit f6019b3a42
7 changed files with 486 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/App.tsx
+161 -11
View File
@@ -78,6 +78,22 @@ export function App() {
// audio to the (fresh) worker. Decoding itself cannot be aborted.
const fileRun = useRef(0);
// Accelerator failures learned this session (D-025), keyed
// "modelId::preference" → the explicit device to load with instead, so
// repeat loads skip the doomed accelerator attempt (and its restart)
// entirely. Session-only on purpose: a browser update may fix the backend.
const deviceOverrides = useRef(new Map<string, InferenceDevice>());
// Resolves when the current browser-engine load reports ready (rejects on
// load failure/cancel). Transcribe posts await it, so no request is ever
// queued inside a worker that a load-restart (D-025) is about to terminate
// — its audio buffer would die with the worker.
const loadGate = useRef<{
promise: Promise<void>;
resolve: () => void;
reject: (error: Error) => void;
} | null>(null);
// Engine plumbing (M5, D-010). The BrowserEngine is bound to the worker's
// lifetime (recreated together on cancel); the ServerEngine is stateless
// and lives for the app. A server run's fetch is cancelled via abortRef.
@@ -113,6 +129,8 @@ export function App() {
DEFAULT_DEVICE_PREFERENCE,
);
const [activeDevice, setActiveDevice] = useState<InferenceDevice | null>(null);
// WASM thread count ort actually got (D-025 observability); null until ready.
const [activeThreads, setActiveThreads] = useState<number | null>(null);
const [language, setLanguage] = useState(AUTO_LANGUAGE);
const [task, setTask] = useState<WhisperTask>(DEFAULT_TASK);
const [downloaded, setDownloaded] = useState<ReadonlySet<string>>(new Set());
@@ -173,6 +191,36 @@ export function App() {
};
}, [auth.user]);
function overrideKey(forModelId: string, preference: DevicePreference): string {
return `${forModelId}::${preference}`;
}
/** Device to actually load with — the preference unless D-025 learned better. */
function effectiveDevice(
forModelId: string,
preference: DevicePreference,
): DevicePreference {
return deviceOverrides.current.get(overrideKey(forModelId, preference)) ?? preference;
}
function newLoadGate(): void {
let resolve!: () => void;
let reject!: (error: Error) => void;
const promise = new Promise<void>((res, rej) => {
resolve = res;
reject = rej;
});
// Waiters attach their own handlers; this swallows the "unhandled
// rejection" a gate rejected with nobody waiting would otherwise raise.
promise.catch(() => {});
loadGate.current = { promise, resolve, reject };
}
/** Wait until the browser pipeline is ready (or its load failed/cancelled). */
function browserModelReady(): Promise<void> {
return loadGate.current?.promise ?? Promise.resolve();
}
function takeLiveRequest(requestId: string | undefined) {
if (requestId === undefined) return undefined;
const pending = pendingLive.current.get(requestId);
@@ -197,7 +245,10 @@ export function App() {
// Surface the backend the worker actually selected (D-004) and
// refresh the downloaded badges now that weights are cached.
setActiveDevice(message.device);
setActiveThreads(message.threads ?? null);
refreshDownloaded();
// Release any transcribe posts waiting on this load (D-025).
loadGate.current?.resolve();
}
break;
case "partial":
@@ -235,6 +286,9 @@ export function App() {
} else if (message.requestId === undefined) {
// Load-stage failures and legacy id-less passes. Engine-routed
// file passes carry an id and surface through the engine promise.
if (message.stage === "load") {
loadGate.current?.reject(new Error(message.message));
}
setError(message.message);
setTranscribeProgress(null);
setBusy(false);
@@ -242,6 +296,42 @@ export function App() {
}
break;
}
case "load-restart": {
// The accelerator attempt failed after it had already initialized
// onnxruntime's WASM runtime single-threaded (D-025). This worker
// can never run multithreaded WASM again, so: remember the outcome
// for this session, replace the worker, and reload with the
// explicit fallback device. The load gate stays pending — the fresh
// worker's "ready" releases the waiting transcribe posts, which
// then flow to the new worker.
deviceOverrides.current.set(
overrideKey(message.modelId, message.preference),
message.device,
);
// Defensive: the gate means nothing should be in flight, but never
// leave a promise awaiting a worker we are about to terminate.
const restart = new Error(
"Inference restarted to use all CPU cores — this pass was skipped.",
);
browserEngineRef.current?.failAll(restart);
for (const pendingPass of pendingLive.current.values()) {
pendingPass.reject(restart);
}
pendingLive.current.clear();
workerRef.current?.terminate();
workerRef.current = null;
browserEngineRef.current = null;
setLoadProgress(initialLoadProgress);
setActiveDevice(null);
setActiveThreads(null);
const fresh = ensureWorker();
postToWorker(fresh, {
type: "load",
modelId: message.modelId,
device: message.device,
});
break;
}
}
});
worker.addEventListener("error", (event) => {
@@ -251,6 +341,7 @@ export function App() {
// Unblock any live passes so a live session can drain and stop.
for (const pending of pendingLive.current.values()) pending.reject(failure);
pendingLive.current.clear();
loadGate.current?.reject(failure);
setError(failure.message);
setBusy(false);
});
@@ -275,7 +366,12 @@ export function App() {
setLoadProgress(initialLoadProgress);
setLoadStarted(true);
setError(null);
postToWorker(worker, { type: "load", modelId: nextModelId, device: preference });
newLoadGate();
postToWorker(worker, {
type: "load",
modelId: nextModelId,
device: effectiveDevice(nextModelId, preference),
});
return worker;
}
@@ -283,6 +379,7 @@ export function App() {
if (busy || liveStatus !== "idle" || nextModelId === modelId) return;
setModelId(nextModelId);
setActiveDevice(null);
setActiveThreads(null);
setPartialText("");
setTranscribeProgress(null);
setResult(null);
@@ -292,15 +389,18 @@ export function App() {
// reports fresh progress for this model.
setLoadStarted(false);
setLoadProgress(initialLoadProgress);
loadGate.current = null;
}
function handleDevicePreference(preference: DevicePreference) {
if (busy || liveStatus !== "idle" || preference === devicePreference) return;
setDevicePreference(preference);
setActiveDevice(null);
setActiveThreads(null);
// No eager reload (D-020) — the next load picks up the new preference.
setLoadStarted(false);
setLoadProgress(initialLoadProgress);
loadGate.current = null;
}
/** Explicit model download (D-020) — pre-caches weights for offline use. */
@@ -334,9 +434,13 @@ export function App() {
workerRef.current = null;
// The worker (and any pipeline / partial download) is gone — reset the
// load state so the next transcription reloads from a clean slate.
// Rejecting the gate unblocks a transcribe post still waiting on it.
loadGate.current?.reject(cancelled);
loadGate.current = null;
setLoadStarted(false);
setLoadProgress(initialLoadProgress);
setActiveDevice(null);
setActiveThreads(null);
}
// The file flow never has live passes pending, but if anything is in the
// map (defensive), unblock it so nothing awaits a dead worker forever.
@@ -368,13 +472,15 @@ export function App() {
setResult(null);
setFileName(file.name);
let engine: Engine;
let engine: Engine | null = null;
if (useServer) {
engine = serverEngineRef.current ??= new ServerEngine();
} else {
// Kick off the (idempotent) model load immediately so the download runs
// while we decode the audio locally. Server runs skip this — they don't
// need the browser pipeline.
// need the browser pipeline. The engine itself is resolved after the
// load gate below: a load-restart (D-025) may replace the worker (and
// its BrowserEngine) while the audio decodes.
ensureWorker();
if (!loadStarted) {
startLoad(modelId, devicePreference);
@@ -382,10 +488,9 @@ export function App() {
postToWorker(workerRef.current!, {
type: "load",
modelId,
device: devicePreference,
device: effectiveDevice(modelId, devicePreference),
});
}
engine = browserEngineRef.current!;
}
const multilingual = getModel(modelId)?.multilingual ?? false;
@@ -411,6 +516,24 @@ export function App() {
}
if (fileRun.current !== run) return; // cancelled while decoding (D-018)
if (engine === null) {
// Browser run: wait for the pipeline before posting the pass, so the
// audio never sits queued in a worker a load-restart may terminate
// (D-025). The worker serialized the pass on the load anyway, so this
// changes nothing about when inference actually starts.
try {
await browserModelReady();
} catch (loadError) {
if (fileRun.current !== run) return; // cancelled while loading
setError(loadError instanceof Error ? loadError.message : String(loadError));
setBusy(false);
setRunEngine(null);
return;
}
if (fileRun.current !== run) return;
engine = browserEngineRef.current!;
}
// Server runs cancel by aborting the fetch (D-018 amendment); browser
// runs cancel by terminating the worker, which rejects the promise via
// BrowserEngine.failAll.
@@ -457,11 +580,17 @@ export function App() {
// -------------------------------------------------------------------------
/** One inference pass over a live segment, correlated via requestId. */
function transcribeLiveSegment(
worker: Worker,
async function transcribeLiveSegment(
audio: Float32Array,
options: { language?: string; task?: WhisperTask },
): Promise<LiveTranscribeResult> {
// Wait for the pipeline before posting: a load-restart (D-025) may swap
// the worker while the model loads, and a pass queued in the old worker
// would die with it (its audio buffer is transferred, so it cannot be
// re-posted). The worker is re-read after the wait for the same reason.
await browserModelReady();
const worker = workerRef.current;
if (!worker) throw new Error("The inference worker is not running.");
const requestId = `live-${++nextLiveRequest.current}`;
return new Promise((resolve, reject) => {
pendingLive.current.set(requestId, { resolve, reject });
@@ -561,17 +690,21 @@ export function App() {
}
// Idempotent model load, like the file flow — a pending download runs
// while the mic opens; segment passes queue in the worker until ready.
// while the mic opens; segment passes wait on the load gate until ready.
const worker = ensureWorker();
if (!loadStarted) {
startLoad(modelId, devicePreference);
} else {
postToWorker(worker, { type: "load", modelId, device: devicePreference });
postToWorker(worker, {
type: "load",
modelId,
device: effectiveDevice(modelId, devicePreference),
});
}
const transcriber = new LiveTranscriber({
...LIVE_TUNING,
transcribe: (audio) => transcribeLiveSegment(worker, audio, options),
transcribe: (audio) => transcribeLiveSegment(audio, options),
onUpdate: (update) => {
setLiveCommitted(update.committedText);
setLiveTentative(update.tentativeText);
@@ -639,6 +772,7 @@ export function App() {
if (!allowed.includes(modelId) && !busy && liveStatus === "idle") {
setModelId("onnx-community/whisper-large-v3-turbo");
setActiveDevice(null);
setActiveThreads(null);
setPartialText("");
setTranscribeProgress(null);
setResult(null);
@@ -713,6 +847,18 @@ export function App() {
disabled={busy}
/>
{/* Live on the WASM backend cannot keep up with real time for the big
models — every pass pads to a full 30 s Whisper window (D-024), so
tentative text lags and the stop drain grows. Say so instead of
looking frozen. */}
{liveActive && activeDevice === "wasm" && (model?.encoderParamsM ?? 0) >= 300 && (
<p className="live-hint" role="status">
{model?.displayName} runs slower than real time on the WASM backend, so
live text will lag behind speech. A smaller model (Tiny, Base or Small)
keeps up better on this device.
</p>
)}
{fileName && (
<p className="file-name">
File: <strong>{fileName}</strong>
@@ -797,7 +943,11 @@ export function App() {
: activeDevice === "webnn"
? "WebNN (on-device)"
: activeDevice === "wasm"
? "WASM (on-device)"
? `WASM (on-device${
activeThreads !== null
? `, ${activeThreads} thread${activeThreads === 1 ? "" : "s"}`
: ""
})`
: "on-device — backend selected on first load"}
</p>
</footer>
src/worker/inference.worker.ts
+99 -12
View File
@@ -18,6 +18,7 @@ import {
import { resolveDefaultDtype } from "../lib/models";
import {
deviceAttempts,
dtypeRequiresF16,
isUiToWorkerMessage,
TranscribeProgressTracker,
type DeviceCapabilities,
@@ -54,13 +55,27 @@ if (env.backends.onnx.wasm) {
* therefore may run single-threaded — correct, just slower; the next worker
* respawn (cancel does one, D-018) picks the full count up again.
*/
function desiredWasmThreads(): number {
return globalThis.crossOriginIsolated
? Math.max(1, Math.min((navigator.hardwareConcurrency || 4) - 1, 12))
: 1;
}
function configureWasmThreads(device: InferenceDevice): void {
const wasm = env.backends.onnx.wasm;
if (!wasm) return;
wasm.numThreads =
device === "wasm" && globalThis.crossOriginIsolated
? Math.max(1, Math.min((navigator.hardwareConcurrency || 4) - 1, 12))
: 1;
wasm.numThreads = device === "wasm" ? desiredWasmThreads() : 1;
}
/**
* Thrown to abort a load whose WASM fallback would be stuck single-threaded
* (D-025). The worker has already posted `load-restart`; the UI terminates
* this worker and retries in a fresh one, so nothing is reported as an error.
*/
class RestartRequested extends Error {
constructor() {
super("load aborted pending worker restart (D-025)");
}
}
// `lib: ["DOM"]` types `self` as Window; give it the worker-scope shape we use.
@@ -117,8 +132,22 @@ function forwardProgress(info: ProgressInfo): void {
// fallback policy live in the pure `deviceAttempts` (protocol.ts, unit-tested).
// ---------------------------------------------------------------------------
interface GpuAdapterLike {
features?: { has(name: string): boolean };
/** Pre-2024 spec location of the software-adapter flag. */
isFallbackAdapter?: boolean;
/** Current spec location (GPUAdapterInfo). */
info?: { isFallbackAdapter?: boolean };
}
/** True for software adapters (SwiftShader, llvmpipe, WARP) — GPU-shaped but
* slower than multithreaded WASM, so never worth an attempt (D-025). */
function isSoftwareAdapter(adapter: GpuAdapterLike): boolean {
return Boolean(adapter.isFallbackAdapter ?? adapter.info?.isFallbackAdapter);
}
interface NavigatorWithHardware {
gpu?: { requestAdapter(): Promise<unknown | null> };
gpu?: { requestAdapter(): Promise<GpuAdapterLike | null> };
ml?: unknown;
}
@@ -134,9 +163,30 @@ async function probeCapabilities(
// Probing threw — treat WebNN as unavailable.
}
}
if (preference === "auto" || preference === "webgpu") {
if (preference === "auto" && /Firefox\//.test(navigator.userAgent)) {
// Firefox creates ort-web WebGPU sessions fine but runs them far slower
// than this machine's multithreaded WASM (field-verified 2026-07-17,
// D-026) — and a *successful* slow session is invisible to the
// failure-driven fallback chain. Never auto-pick WebGPU on Firefox; an
// explicit selection is honored (the retest hatch for future ort/Firefox).
console.info(
"[whisper] Firefox: skipping WebGPU for \"auto\" — onnxruntime-web's WebGPU backend " +
"runs much slower than multithreaded WASM on Firefox (D-026). " +
"Select WebGPU explicitly to retest.",
);
} else if (preference === "auto" || preference === "webgpu") {
try {
capabilities.webgpu = Boolean(await hardware.gpu?.requestAdapter());
const adapter = (await hardware.gpu?.requestAdapter()) ?? null;
if (adapter && isSoftwareAdapter(adapter)) {
// A fallback (software-rendered) adapter would "succeed" and then run
// far slower than multithreaded WASM — never attempt it (D-025).
console.info("[whisper] WebGPU adapter is software-rendered — skipping WebGPU.");
} else {
capabilities.webgpu = adapter !== null;
// fp16 weights need the "shader-f16" feature (D-025); Firefox ships
// WebGPU without it, so createPipeline gates on this.
capabilities.webgpuF16 = Boolean(adapter?.features?.has("shader-f16"));
}
} catch {
// Probing threw — treat WebGPU as unavailable.
}
@@ -167,17 +217,42 @@ function loadKey(message: LoadMessage): string {
async function createPipeline(
message: LoadMessage,
): Promise<AutomaticSpeechRecognitionPipeline> {
const capabilities = await probeCapabilities(message.device);
// WebGPU without "shader-f16" (Firefox) cannot run fp16 weights — skip the
// doomed attempt instead of failing into a degraded-thread fallback (D-025).
if (capabilities.webgpu && !capabilities.webgpuF16) {
const webgpuDtype =
message.dtype ?? resolveDefaultDtype(message.modelId, "webgpu");
if (dtypeRequiresF16(webgpuDtype)) {
capabilities.webgpu = false;
console.info(
"[whisper] WebGPU adapter lacks shader-f16 but the model dtype needs fp16 — skipping WebGPU.",
);
}
}
// Ordered devices to try (D-004/D-017): the first is the selected device;
// if pipeline creation throws there (driver quirks, unsupported ops,
// out-of-memory), the next one is tried automatically. For "auto" the chain
// is WebNN → WebGPU → WASM (filtered by the probe); WASM always terminates it.
const attempts: InferenceDevice[] = deviceAttempts(
message.device,
await probeCapabilities(message.device),
);
const attempts: InferenceDevice[] = deviceAttempts(message.device, capabilities);
let lastError: unknown;
let acceleratorAttempted = false;
for (const device of attempts) {
// A failed WebGPU/WebNN attempt has already initialized onnxruntime's
// WASM runtime single-threaded, and the thread pool is fixed at first
// init (D-023) — a WASM session created now would run on one thread for
// this worker's whole life (measured 5.5× slower). When multithreading is
// actually available, hand the fallback to a fresh worker instead (D-025).
if (device === "wasm" && acceleratorAttempted && desiredWasmThreads() > 1) {
post({
type: "load-restart",
modelId: message.modelId,
preference: message.device,
device: "wasm",
});
throw new RestartRequested();
}
configureWasmThreads(device);
const dtype = message.dtype ?? resolveDefaultDtype(message.modelId, device);
try {
@@ -194,10 +269,18 @@ async function createPipeline(
}),
progress_callback: forwardProgress,
});
post({ type: "load-progress", status: "ready", device });
// What ort actually ended up with (it clamps to 1 without isolation) —
// surfaced to the UI so degraded threading is visible in the field.
const threads = env.backends.onnx.wasm?.numThreads ?? 1;
console.info(`[whisper] pipeline ready: device=${device} wasmThreads=${threads}`);
post({ type: "load-progress", status: "ready", device, threads });
return asr;
} catch (error) {
lastError = error;
if (device !== "wasm") acceleratorAttempted = true;
console.warn(
`[whisper] pipeline creation failed on ${device}: ${errorMessage(error)}`,
);
}
}
throw lastError;
@@ -350,11 +433,15 @@ scope.addEventListener("message", (event) => {
switch (data.type) {
case "load":
load(data).catch((error: unknown) => {
// A restart-aborted load is not an error — the UI already got the
// load-restart message and is respawning this worker (D-025).
if (error instanceof RestartRequested) return;
post({ type: "error", stage: "load", message: errorMessage(error) });
});
break;
case "transcribe":
transcribe(data).catch((error: unknown) => {
if (error instanceof RestartRequested) return;
post({
type: "error",
stage: "transcribe",
src/worker/protocol.ts
+60 -3
View File
@@ -106,7 +106,18 @@ export type LoadProgressMessage =
progress: number;
}
| { type: "load-progress"; status: "done"; file: string }
| { type: "load-progress"; status: "ready"; device: InferenceDevice };
| {
type: "load-progress";
status: "ready";
device: InferenceDevice;
/**
* WASM thread count onnxruntime actually ended up with (D-025) — ort
* clamps to 1 when cross-origin isolation is missing, so this is field
* observability for "is multithreading really on?". Accelerator devices
* report 1 (their WASM side must be single-threaded, D-023).
*/
threads?: number;
};
/** Incremental transcript fragment — append to the text streamed so far. */
export interface PartialMessage {
@@ -155,12 +166,32 @@ export interface ErrorMessage {
requestId?: string;
}
/**
* The worker cannot finish this load usefully in-place and asks the UI to
* respawn it (D-025). Posted when an accelerator attempt already initialized
* onnxruntime's WASM runtime — whose thread pool is fixed at first init —
* single-threaded, so the WASM fallback in *this* worker would run on one
* thread for the worker's whole life. The UI terminates the worker, spawns a
* fresh one, and re-issues the load with the explicit `device`, remembering
* `modelId`+`preference` so later loads skip the doomed attempt entirely.
*/
export interface LoadRestartMessage {
type: "load-restart";
/** Echo of the aborted load's model id. */
modelId: string;
/** Echo of the aborted load's device preference. */
preference: DevicePreference;
/** Explicit device the fresh worker should load with. */
device: InferenceDevice;
}
export type WorkerToUiMessage =
| LoadProgressMessage
| PartialMessage
| TranscribeProgressMessage
| ResultMessage
| ErrorMessage;
| ErrorMessage
| LoadRestartMessage;
// ---------------------------------------------------------------------------
// Device selection policy (D-004, D-017) — pure so it is unit-testable; the
@@ -173,6 +204,23 @@ export interface DeviceCapabilities {
webnn: boolean;
/** `navigator.gpu.requestAdapter()` returned an adapter (WebGPU). */
webgpu: boolean;
/**
* The WebGPU adapter supports the "shader-f16" feature. Firefox ships
* WebGPU without it, so fp16 weights cannot run there (D-025); the worker
* skips the WebGPU attempt when the resolved dtype needs fp16 and this is
* false. Optional so `deviceAttempts` callers that don't probe it (tests)
* stay valid.
*/
webgpuF16?: boolean;
}
/**
* Whether a dtype spec involves fp16 weights anywhere — such a model cannot
* run on a WebGPU adapter without the "shader-f16" feature (D-025).
*/
export function dtypeRequiresF16(dtype: DtypeSpec): boolean {
if (typeof dtype === "string") return dtype === "fp16";
return dtype.encoder_model === "fp16" || dtype.decoder_model_merged === "fp16";
}
/**
@@ -284,7 +332,10 @@ export function isWorkerToUiMessage(value: unknown): value is WorkerToUiMessage
typeof value.progress === "number"
);
case "ready":
return isInferenceDevice(value.device);
return (
isInferenceDevice(value.device) &&
(value.threads === undefined || typeof value.threads === "number")
);
default:
return false;
}
@@ -308,6 +359,12 @@ export function isWorkerToUiMessage(value: unknown): value is WorkerToUiMessage
(value.stage === "load" || value.stage === "transcribe") &&
isOptionalString(value.requestId)
);
case "load-restart":
return (
typeof value.modelId === "string" &&
isDevicePreference(value.preference) &&
isInferenceDevice(value.device)
);
default:
return false;
}