Run browser file transcription through the long-form pipeline when asked and the VAD model is available

Adds an Advanced "Skip silence in uploaded files" option (off by default
pending measurement), the vad/longForm protocol fields, the old-path
fallback when the VAD model is missing, and a plain message for files
with no speech.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
Lucas Winther
2026-10-29 22:02:47 +01:00
co-authored by Claude Opus 5.5
parent 0c3a83c23c
commit f55ea6ecf8
14 changed files with 363 additions and 8 deletions
+24
View File
@@ -1,6 +1,7 @@
import { useEffect, useRef, useState } from "react";
import "./index.css";
import { AdvancedOptions } from "./components/AdvancedOptions";
import { AuthForms } from "./components/AuthForms";
import { EngineToggle } from "./components/EngineToggle";
import { FileDropzone } from "./components/FileDropzone";
@@ -25,6 +26,7 @@ import {
import { useAuth } from "./lib/useAuth";
import { LIVE_TUNING, LiveTranscriber, type LiveTranscribeResult } from "./lib/live";
import { startMicCapture, type MicCapture } from "./lib/mic";
import { loadLongFormChoice, saveLongFormChoice } from "./lib/longForm";
import { getDownloadedModels } from "./lib/modelCache";
import { registerServiceWorker } from "./pwa/register";
import {
@@ -179,6 +181,11 @@ export function App() {
const auth = useAuth();
const [engineChoice, setEngineChoice] = useState<EngineKind>(() => loadEngineChoice());
const [serverStatus, setServerStatus] = useState<ServerEngineStatus | null>(null);
// "Skip silence" (long-form pipeline, D-WS3a): remembered per browser. A
// ref mirrors it for the worker-message handler set up once per worker.
const [longForm, setLongForm] = useState(() => loadLongFormChoice());
const longFormRef = useRef(longForm);
longFormRef.current = longForm;
function refreshDownloaded() {
// Downloaded badges (D-005): query Cache Storage for cached weights.
@@ -432,6 +439,7 @@ export function App() {
modelId: message.modelId,
device: message.device,
sequentialSessionLoad: isMobileLoadRisk(message.modelId, message.device),
vad: longFormRef.current,
});
break;
}
@@ -492,6 +500,9 @@ export function App() {
// use) only where the peak-memory crash risk is real; everywhere else
// (desktop, low-risk models) the worker loads them in parallel.
sequentialSessionLoad: isMobileLoadRisk(nextModelId, guessDevice(device)),
// D-WS3b: the speech-detection model rides along with the same
// user-initiated load when "Skip silence" is on.
vad: longFormRef.current,
});
return worker;
}
@@ -621,6 +632,7 @@ export function App() {
modelId,
device,
sequentialSessionLoad: isMobileLoadRisk(modelId, guessDevice(device)),
vad: longFormRef.current,
});
}
}
@@ -676,6 +688,9 @@ export function App() {
// take neither. The browser engine ignores modelId (worker holds it).
modelId,
...(multilingual && { language, task }),
// Browser runs only; the worker falls back to the chunked path when
// the VAD model isn't available (D-WS3a/D-WS3b).
...(!useServer && { longForm }),
onPartial: (fragment) => {
if (fileRun.current === run) setPartialText((text) => text + fragment);
},
@@ -1042,6 +1057,15 @@ export function App() {
/>
)}
<AdvancedOptions
skipSilence={longForm}
onSkipSilenceChange={(enabled) => {
setLongForm(enabled);
saveLongFormChoice(enabled);
}}
disabled={busy || liveActive}
/>
{/* Engine toggle (M5, D-010): signed-in users choose where uploaded
files are transcribed. Live mode always stays on-device (D-016). */}
{auth.user && (
+37
View File
@@ -0,0 +1,37 @@
interface AdvancedOptionsProps {
skipSilence: boolean;
onSkipSilenceChange: (enabled: boolean) => void;
disabled?: boolean;
}
/**
* Technical file-transcription controls, collapsed by default (PRODUCT.md:
* every control stays available without crowding the primary flow).
* "Skip silence" turns on the long-form pipeline (D-WS3a), which needs a
* small speech-detection model downloaded once (D-WS3b).
*/
export function AdvancedOptions({
skipSilence,
onSkipSilenceChange,
disabled = false,
}: AdvancedOptionsProps) {
return (
<details className="advanced-options">
<summary>Advanced</summary>
<label>
<input
type="checkbox"
checked={skipSilence}
disabled={disabled}
onChange={(event) => onSkipSilenceChange(event.target.checked)}
/>
Skip silence in uploaded files
</label>
<p className="advanced-hint">
Finds the speech first and transcribes only that, which can be faster for long recordings
with pauses. Downloads a small extra model (2 MB) the first time; after that it works
offline. Without it, files are transcribed the usual way.
</p>
</details>
);
}
+7 -2
View File
@@ -1,5 +1,6 @@
import { useEffect, useRef, useState } from "react";
import {
emptyTranscriptNotice,
exportFilename,
transcriptToSrt,
transcriptToTimestampedTxt,
@@ -90,7 +91,11 @@ export function TranscriptView({
prevResultRef.current = result;
}, [result]);
if (!transcribing && !text && !tentativeText) return null;
// A finished run that found no words (silence/music-only file) says so
// instead of vanishing (plan 001-0 Review Focus 4).
const notice = tentativeText ? null : emptyTranscriptNotice(result);
if (!transcribing && !text && !tentativeText && !notice) return null;
async function copy() {
if (!copyText) return;
@@ -181,7 +186,7 @@ export function TranscriptView({
</div>
)}
<p className="transcript-text" aria-live="polite">
{text || (tentativeText ? "" : placeholder)}
{text || notice || (tentativeText ? "" : placeholder)}
{tentativeText && (
<span className="transcript-tentative">{tentativeText}</span>
)}
+26
View File
@@ -519,6 +519,32 @@ body {
flex-wrap: wrap;
}
/* --- AdvancedOptions (long-form "Skip silence", D-WS3a) --- */
.advanced-options summary {
cursor: pointer;
color: var(--muted);
font-size: 0.85rem;
}
.advanced-options label {
display: flex;
align-items: center;
gap: 0.5rem;
margin-top: 0.5rem;
font-size: 0.9rem;
}
.advanced-options input[type="checkbox"] {
accent-color: var(--accent);
}
.advanced-hint {
margin: 0.25rem 0 0;
color: var(--muted);
font-size: 0.85rem;
max-width: 60ch;
}
/* --- EngineToggle (M5, D-010) --- */
.engine-toggle {
+6
View File
@@ -54,6 +54,11 @@ export interface EngineTranscribeOptions {
* is worker termination via `BrowserEngine.failAll` (D-018).
*/
signal?: AbortSignal;
/**
* Browser engine only: use the long-form pipeline when the worker has the
* VAD model (D-WS3a). The server engine runs faster-whisper's own pipeline.
*/
longForm?: boolean;
}
export interface EngineTranscribeResult {
@@ -140,6 +145,7 @@ export class BrowserEngine implements Engine {
...(options.language !== undefined &&
options.language !== "auto" && { language: options.language }),
...(options.task !== undefined && { task: options.task }),
...(options.longForm !== undefined && { longForm: options.longForm }),
};
// Ownership of the audio moves to the worker — zero-copy transfer.
this.worker.postMessage(message, [audio.buffer as ArrayBuffer]);
+10
View File
@@ -14,6 +14,16 @@ import type { TranscriptChunk } from "../worker/protocol";
*/
const FALLBACK_CUE_SECONDS = 2;
/**
* What to show instead of a blank transcript when a finished run found no
* words — silence- or music-only files (plan 001-0 Review Focus 4). Null
* while running (no result yet) or when there is text.
*/
export function emptyTranscriptNotice(result: { text: string } | null): string | null {
if (result === null || result.text.trim().length > 0) return null;
return "No speech was found in this recording.";
}
/** Final transcript as plain text: trimmed, single trailing newline. */
export function transcriptToTxt(result: { text: string }): string {
return `${result.text.trim()}\n`;
+43
View File
@@ -0,0 +1,43 @@
/**
* The "Skip silence" file-transcription setting (long-form pipeline,
* D-WS3a): remembered in localStorage like the engine choice.
*/
import type { StorageLike } from "./engine";
export const LONG_FORM_KEY = "whisper-long-form";
/**
* Off by default: the spec's rollout rule makes the long-form pipeline the
* default only if it is faster and not less accurate on every measurement
* clip — see D-WS3a for the numbers that decided this.
*/
export const LONG_FORM_DEFAULT = false;
function defaultStorage(): StorageLike | null {
try {
return (globalThis as { localStorage?: StorageLike }).localStorage ?? null;
} catch {
return null; // storage access can throw (privacy modes)
}
}
export function loadLongFormChoice(storage: StorageLike | null = defaultStorage()): boolean {
try {
const value = storage?.getItem(LONG_FORM_KEY);
return value === "on" ? true : value === "off" ? false : LONG_FORM_DEFAULT;
} catch {
return LONG_FORM_DEFAULT;
}
}
export function saveLongFormChoice(
enabled: boolean,
storage: StorageLike | null = defaultStorage(),
): void {
try {
storage?.setItem(LONG_FORM_KEY, enabled ? "on" : "off");
} catch {
// Not persistable (private mode / quota) — the choice just won't stick.
}
}
+86 -3
View File
@@ -17,7 +17,10 @@ import {
type ProgressInfo,
} from "@huggingface/transformers";
import { resolveDefaultDtype } from "../lib/models";
import { LONG_FORM_DEFAULTS, transcribeLongForm } from "./longform";
import { createWhisperDeps, loadSileroVad, type SileroVad } from "./longform/transformers";
import {
choosePipeline,
deviceAttempts,
dtypeRequiresF16,
isUiToWorkerMessage,
@@ -321,6 +324,36 @@ function load(message: LoadMessage): Promise<AutomaticSpeechRecognitionPipeline>
return promise;
}
/**
* Silero VAD for long-form file passes (D-WS3a/D-WS3b). Loaded on request
* (`load.vad`) once the Whisper pipeline is up — after it, so the Whisper
* attempt decides onnxruntime's thread pool (D-023/D-025). Offline it is
* used only if already cached: no network request (Review Focus 3). A failed
* load resolves to null (the old path runs) and is retried on a later load.
*/
let vad: Promise<SileroVad | null> | null = null;
function ensureVad(whisper: Promise<unknown>): void {
if (vad !== null) return;
const attempt = (async () => {
await whisper.catch(() => {});
try {
const model = await loadSileroVad({ localOnly: navigator.onLine === false, device: "wasm" });
console.info("[whisper] speech-detection (VAD) model ready — long-form file pipeline available");
return model;
} catch (error) {
console.info(
`[whisper] speech-detection (VAD) model unavailable, using the chunked file path: ${errorMessage(error)}`,
);
return null;
}
})();
vad = attempt;
void attempt.then((model) => {
if (model === null && vad === attempt) vad = null;
});
}
// ---------------------------------------------------------------------------
// Transcription
// ---------------------------------------------------------------------------
@@ -343,6 +376,17 @@ async function transcribe(message: TranscribeMessage): Promise<void> {
}
const asr = await active.promise;
const streaming = message.stream ?? message.requestId === undefined;
// Live passes (not streaming) always keep the chunked path (D-016/D-024).
const vadModel = message.longForm && streaming && vad ? await vad : null;
if (
choosePipeline({ longForm: message.longForm && streaming, vadReady: vadModel !== null }) ===
"longform"
) {
await transcribeLongFormPass(message, asr, vadModel!);
return;
}
const options: Record<string, unknown> = {
chunk_length_s: CHUNK_LENGTH_S,
stride_length_s: STRIDE_LENGTH_S,
@@ -360,7 +404,7 @@ async function transcribe(message: TranscribeMessage): Promise<void> {
// than any real 30 s window produces.
max_new_tokens: 440,
};
if (message.stream ?? message.requestId === undefined) {
if (streaming) {
// File-flow pass (id-less, or routed through BrowserEngine with an
// explicit stream flag, D-010): stream partial tokens and report audio
// progress. The streamer's chunk timestamps are window-relative; the
@@ -426,6 +470,42 @@ async function transcribe(message: TranscribeMessage): Promise<void> {
post(result);
}
/**
* A file pass through the long-form pipeline (D-WS3a): same partial /
* transcribe-progress / result messages as the chunked path.
*/
async function transcribeLongFormPass(
message: TranscribeMessage,
asr: AutomaticSpeechRecognitionPipeline,
vadModel: SileroVad,
): Promise<void> {
const multilingual = Boolean(
(asr.model as unknown as { generation_config?: { is_multilingual?: boolean } })
.generation_config?.is_multilingual,
);
const result = await transcribeLongForm(
message.audio,
{
speechProbabilities: (audio) => vadModel.speechProbabilities(audio),
...createWhisperDeps(asr, { multilingual }),
},
{
...LONG_FORM_DEFAULTS,
...(multilingual && message.language !== undefined && { language: message.language }),
...(multilingual && message.task !== undefined && { task: message.task }),
onPartial: (text) => post({ type: "partial", text, requestId: message.requestId }),
onProgress: (processedSeconds, totalSeconds) =>
post({
type: "transcribe-progress",
processedSeconds,
totalSeconds,
requestId: message.requestId,
}),
},
);
post({ type: "result", text: result.text, chunks: result.chunks, requestId: message.requestId });
}
scope.addEventListener("message", (event) => {
const data = event.data;
if (!isUiToWorkerMessage(data)) {
@@ -437,14 +517,17 @@ scope.addEventListener("message", (event) => {
return;
}
switch (data.type) {
case "load":
load(data).catch((error: unknown) => {
case "load": {
const whisper = load(data);
if (data.vad) ensureVad(whisper);
whisper.catch((error: unknown) => {
// A restart-aborted load is not an error — the UI already got the
// load-restart message and is respawning this worker (D-025).
if (error instanceof RestartRequested) return;
post({ type: "error", stage: "load", message: errorMessage(error) });
});
break;
}
case "transcribe":
transcribe(data).catch((error: unknown) => {
if (error instanceof RestartRequested) return;
+1 -1
View File
@@ -1,5 +1,5 @@
/** Long-form file transcription pipeline (D-WS3a). See pipeline.ts. */
export { DEFAULT_GUARD, transcribeLongForm } from "./pipeline";
export { DEFAULT_GUARD, LONG_FORM_DEFAULTS, transcribeLongForm } from "./pipeline";
export { DEFAULT_FALLBACK, type FallbackOptions } from "./quality";
export type {
DecodeRequest,
+6
View File
@@ -37,6 +37,12 @@ const MAX_CHUNK_SECONDS = 30;
/** No guard: Whisper's own decoding (the measured default, D-WS3a). */
export const DEFAULT_GUARD: RepetitionGuard = { noRepeatNgramSize: 0, repetitionPenalty: 1 };
/** The tuning the worker runs with — chosen by measurement (D-WS3a). */
export const LONG_FORM_DEFAULTS: Pick<LongFormOptions, "guard" | "fallback" | "vad"> = {
guard: DEFAULT_GUARD,
fallback: false,
};
export async function transcribeLongForm(
audio: Float32Array,
deps: LongFormDeps,
+34 -2
View File
@@ -57,6 +57,13 @@ export interface LoadMessage {
* in App.tsx); omitted/false loads in parallel.
*/
sequentialSessionLoad?: boolean;
/**
* Also make the Silero VAD model (2.2 MB) available for long-form file
* transcription (D-WS3a/D-WS3b): cache-first, fetched from the Hub only
* when online — offline it is used only if already cached. Sent with the
* same user-initiated loads as the Whisper weights (D-020).
*/
vad?: boolean;
}
export interface TranscribeMessage {
@@ -87,6 +94,12 @@ export interface TranscribeMessage {
* engine (which correlates via requestId, D-010) still streams.
*/
stream?: boolean;
/**
* Run this file pass through the long-form pipeline (VAD-cut chunks,
* language pinned once, D-WS3a). Honoured only when the VAD model loaded;
* otherwise the classic 30 s/5 s chunked path runs (`choosePipeline`).
*/
longForm?: boolean;
}
export type UiToWorkerMessage = LoadMessage | TranscribeMessage;
@@ -267,6 +280,19 @@ export function deviceAttempts(
}
}
/**
* Which file-transcription path a pass takes (D-WS3a/D-WS3b): long-form only
* when the pass asked for it and the VAD model is loaded. A missing VAD model
* (offline, never downloaded) silently keeps the classic chunked path — it is
* a fallback, never an error.
*/
export function choosePipeline(input: {
longForm: boolean | undefined;
vadReady: boolean;
}): "longform" | "chunked" {
return input.longForm === true && input.vadReady ? "longform" : "chunked";
}
// ---------------------------------------------------------------------------
// Runtime guards (postMessage data arrives untyped)
// ---------------------------------------------------------------------------
@@ -279,6 +305,10 @@ function isOptionalString(value: unknown): value is string | undefined {
return value === undefined || typeof value === "string";
}
function isOptionalBoolean(value: unknown): value is boolean | undefined {
return value === undefined || typeof value === "boolean";
}
const DEVICES: readonly string[] = ["webgpu", "wasm", "webnn"];
const DTYPES: readonly string[] = ["fp32", "fp16", "q8", "q4"];
@@ -308,7 +338,8 @@ export function isUiToWorkerMessage(value: unknown): value is UiToWorkerMessage
return (
typeof value.modelId === "string" &&
isDevicePreference(value.device) &&
(value.dtype === undefined || isDtypeSpec(value.dtype))
(value.dtype === undefined || isDtypeSpec(value.dtype)) &&
isOptionalBoolean(value.vad)
);
case "transcribe":
return (
@@ -318,7 +349,8 @@ export function isUiToWorkerMessage(value: unknown): value is UiToWorkerMessage
value.task === "transcribe" ||
value.task === "translate") &&
isOptionalString(value.requestId) &&
(value.stream === undefined || typeof value.stream === "boolean")
isOptionalBoolean(value.stream) &&
isOptionalBoolean(value.longForm)
);
default:
return false;
+9
View File
@@ -77,6 +77,15 @@ describe("BrowserEngine", () => {
expect("modelId" in message).toBe(false);
});
test("forwards the long-form request only when set (D-WS3a)", () => {
const worker = new FakeWorker();
const engine = new BrowserEngine(worker);
void engine.transcribe(new Float32Array(4), { longForm: true });
void engine.transcribe(new Float32Array(4));
expect(worker.posted[0]!.message.longForm).toBe(true);
expect("longForm" in worker.posted[1]!.message).toBe(false);
});
test("asks the worker to stream only when a consumer listens", () => {
const worker = new FakeWorker();
const engine = new BrowserEngine(worker);
+55
View File
@@ -0,0 +1,55 @@
import { describe, expect, test } from "bun:test";
import { emptyTranscriptNotice } from "../src/lib/export";
import {
LONG_FORM_DEFAULT,
LONG_FORM_KEY,
loadLongFormChoice,
saveLongFormChoice,
} from "../src/lib/longForm";
function memoryStorage(initial: Record<string, string> = {}) {
const data = { ...initial };
return {
data,
getItem: (key: string) => data[key] ?? null,
setItem: (key: string, value: string) => {
data[key] = value;
},
};
}
describe("long-form setting persistence (D-WS3a)", () => {
test("defaults when nothing is stored or storage is unavailable", () => {
expect(loadLongFormChoice(memoryStorage())).toBe(LONG_FORM_DEFAULT);
expect(loadLongFormChoice(null)).toBe(LONG_FORM_DEFAULT);
const throwing = {
getItem: () => {
throw new Error("denied");
},
setItem: () => {
throw new Error("denied");
},
};
expect(loadLongFormChoice(throwing)).toBe(LONG_FORM_DEFAULT);
expect(() => saveLongFormChoice(true, throwing)).not.toThrow();
});
test("round-trips an explicit choice", () => {
const storage = memoryStorage();
saveLongFormChoice(!LONG_FORM_DEFAULT, storage);
expect(storage.data[LONG_FORM_KEY]).toBeDefined();
expect(loadLongFormChoice(storage)).toBe(!LONG_FORM_DEFAULT);
});
});
describe("emptyTranscriptNotice (plan 001-0 Review Focus 4)", () => {
test("a finished transcript with no words says so plainly", () => {
expect(emptyTranscriptNotice({ text: "" })).toBe("No speech was found in this recording.");
expect(emptyTranscriptNotice({ text: " \n" })).toBe("No speech was found in this recording.");
});
test("no notice while running or when there is text", () => {
expect(emptyTranscriptNotice(null)).toBeNull();
expect(emptyTranscriptNotice({ text: " Hello." })).toBeNull();
});
});
+19
View File
@@ -4,6 +4,7 @@ import {
deviceAttempts,
dtypeRequiresF16,
initialLoadProgress,
choosePipeline,
isUiToWorkerMessage,
isWorkerToUiMessage,
TranscribeProgressTracker,
@@ -35,7 +36,9 @@ describe("isUiToWorkerMessage", () => {
{ type: "transcribe", audio: new Float32Array(8), requestId: "live-3" },
{ type: "transcribe", audio: new Float32Array(8), requestId: "engine-1-1", stream: true },
{ type: "transcribe", audio: new Float32Array(8), stream: false },
{ type: "transcribe", audio: new Float32Array(8), stream: true, longForm: true },
];
loads.push({ type: "load", modelId: "onnx-community/whisper-base", device: "auto", vad: true });
for (const message of [...loads, ...transcribes]) {
expect(isUiToWorkerMessage(message)).toBe(true);
}
@@ -65,10 +68,26 @@ describe("isUiToWorkerMessage", () => {
expect(
isUiToWorkerMessage({ type: "transcribe", audio: new Float32Array(4), stream: "yes" }),
).toBe(false);
expect(
isUiToWorkerMessage({ type: "transcribe", audio: new Float32Array(4), longForm: 1 }),
).toBe(false);
expect(isUiToWorkerMessage({ type: "load", modelId: "x", device: "auto", vad: "yes" })).toBe(
false,
);
expect(isUiToWorkerMessage({ type: "result", text: "", chunks: [] })).toBe(false);
});
});
describe("choosePipeline (D-WS3a/D-WS3b)", () => {
test("long-form only when asked for and the VAD model loaded", () => {
expect(choosePipeline({ longForm: true, vadReady: true })).toBe("longform");
// Offline / never downloaded: the old path, never an error (Review Focus 3).
expect(choosePipeline({ longForm: true, vadReady: false })).toBe("chunked");
expect(choosePipeline({ longForm: false, vadReady: true })).toBe("chunked");
expect(choosePipeline({ longForm: undefined, vadReady: true })).toBe("chunked");
});
});
describe("isWorkerToUiMessage", () => {
test("accepts every worker → UI message shape", () => {
const messages: WorkerToUiMessage[] = [