diff --git a/src/App.tsx b/src/App.tsx index e282021..4a2e979 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -1,6 +1,7 @@ import { useEffect, useRef, useState } from "react"; import "./index.css"; +import { AdvancedOptions } from "./components/AdvancedOptions"; import { AuthForms } from "./components/AuthForms"; import { EngineToggle } from "./components/EngineToggle"; import { FileDropzone } from "./components/FileDropzone"; @@ -25,6 +26,7 @@ import { import { useAuth } from "./lib/useAuth"; import { LIVE_TUNING, LiveTranscriber, type LiveTranscribeResult } from "./lib/live"; import { startMicCapture, type MicCapture } from "./lib/mic"; +import { loadLongFormChoice, saveLongFormChoice } from "./lib/longForm"; import { getDownloadedModels } from "./lib/modelCache"; import { registerServiceWorker } from "./pwa/register"; import { @@ -179,6 +181,11 @@ export function App() { const auth = useAuth(); const [engineChoice, setEngineChoice] = useState(() => loadEngineChoice()); const [serverStatus, setServerStatus] = useState(null); + // "Skip silence" (long-form pipeline, D-WS3a): remembered per browser. A + // ref mirrors it for the worker-message handler set up once per worker. + const [longForm, setLongForm] = useState(() => loadLongFormChoice()); + const longFormRef = useRef(longForm); + longFormRef.current = longForm; function refreshDownloaded() { // Downloaded badges (D-005): query Cache Storage for cached weights. @@ -432,6 +439,7 @@ export function App() { modelId: message.modelId, device: message.device, sequentialSessionLoad: isMobileLoadRisk(message.modelId, message.device), + vad: longFormRef.current, }); break; } @@ -492,6 +500,9 @@ export function App() { // use) only where the peak-memory crash risk is real; everywhere else // (desktop, low-risk models) the worker loads them in parallel. sequentialSessionLoad: isMobileLoadRisk(nextModelId, guessDevice(device)), + // D-WS3b: the speech-detection model rides along with the same + // user-initiated load when "Skip silence" is on. + vad: longFormRef.current, }); return worker; } @@ -621,6 +632,7 @@ export function App() { modelId, device, sequentialSessionLoad: isMobileLoadRisk(modelId, guessDevice(device)), + vad: longFormRef.current, }); } } @@ -676,6 +688,9 @@ export function App() { // take neither. The browser engine ignores modelId (worker holds it). modelId, ...(multilingual && { language, task }), + // Browser runs only; the worker falls back to the chunked path when + // the VAD model isn't available (D-WS3a/D-WS3b). + ...(!useServer && { longForm }), onPartial: (fragment) => { if (fileRun.current === run) setPartialText((text) => text + fragment); }, @@ -1042,6 +1057,15 @@ export function App() { /> )} + { + setLongForm(enabled); + saveLongFormChoice(enabled); + }} + disabled={busy || liveActive} + /> + {/* Engine toggle (M5, D-010): signed-in users choose where uploaded files are transcribed. Live mode always stays on-device (D-016). */} {auth.user && ( diff --git a/src/components/AdvancedOptions.tsx b/src/components/AdvancedOptions.tsx new file mode 100644 index 0000000..870ae96 --- /dev/null +++ b/src/components/AdvancedOptions.tsx @@ -0,0 +1,37 @@ +interface AdvancedOptionsProps { + skipSilence: boolean; + onSkipSilenceChange: (enabled: boolean) => void; + disabled?: boolean; +} + +/** + * Technical file-transcription controls, collapsed by default (PRODUCT.md: + * every control stays available without crowding the primary flow). + * "Skip silence" turns on the long-form pipeline (D-WS3a), which needs a + * small speech-detection model downloaded once (D-WS3b). + */ +export function AdvancedOptions({ + skipSilence, + onSkipSilenceChange, + disabled = false, +}: AdvancedOptionsProps) { + return ( +
+ Advanced + +

+ Finds the speech first and transcribes only that, which can be faster for long recordings + with pauses. Downloads a small extra model (2 MB) the first time; after that it works + offline. Without it, files are transcribed the usual way. +

+
+ ); +} diff --git a/src/components/TranscriptView.tsx b/src/components/TranscriptView.tsx index 188b832..69eef49 100644 --- a/src/components/TranscriptView.tsx +++ b/src/components/TranscriptView.tsx @@ -1,5 +1,6 @@ import { useEffect, useRef, useState } from "react"; import { + emptyTranscriptNotice, exportFilename, transcriptToSrt, transcriptToTimestampedTxt, @@ -90,7 +91,11 @@ export function TranscriptView({ prevResultRef.current = result; }, [result]); - if (!transcribing && !text && !tentativeText) return null; + // A finished run that found no words (silence/music-only file) says so + // instead of vanishing (plan 001-0 Review Focus 4). + const notice = tentativeText ? null : emptyTranscriptNotice(result); + + if (!transcribing && !text && !tentativeText && !notice) return null; async function copy() { if (!copyText) return; @@ -181,7 +186,7 @@ export function TranscriptView({ )}

- {text || (tentativeText ? "" : placeholder)} + {text || notice || (tentativeText ? "" : placeholder)} {tentativeText && ( {tentativeText} )} diff --git a/src/index.css b/src/index.css index 999d03c..4ea703d 100644 --- a/src/index.css +++ b/src/index.css @@ -519,6 +519,32 @@ body { flex-wrap: wrap; } +/* --- AdvancedOptions (long-form "Skip silence", D-WS3a) --- */ +.advanced-options summary { + cursor: pointer; + color: var(--muted); + font-size: 0.85rem; +} + +.advanced-options label { + display: flex; + align-items: center; + gap: 0.5rem; + margin-top: 0.5rem; + font-size: 0.9rem; +} + +.advanced-options input[type="checkbox"] { + accent-color: var(--accent); +} + +.advanced-hint { + margin: 0.25rem 0 0; + color: var(--muted); + font-size: 0.85rem; + max-width: 60ch; +} + /* --- EngineToggle (M5, D-010) --- */ .engine-toggle { diff --git a/src/lib/engine.ts b/src/lib/engine.ts index 5c60e6b..0ab07c6 100644 --- a/src/lib/engine.ts +++ b/src/lib/engine.ts @@ -54,6 +54,11 @@ export interface EngineTranscribeOptions { * is worker termination via `BrowserEngine.failAll` (D-018). */ signal?: AbortSignal; + /** + * Browser engine only: use the long-form pipeline when the worker has the + * VAD model (D-WS3a). The server engine runs faster-whisper's own pipeline. + */ + longForm?: boolean; } export interface EngineTranscribeResult { @@ -140,6 +145,7 @@ export class BrowserEngine implements Engine { ...(options.language !== undefined && options.language !== "auto" && { language: options.language }), ...(options.task !== undefined && { task: options.task }), + ...(options.longForm !== undefined && { longForm: options.longForm }), }; // Ownership of the audio moves to the worker — zero-copy transfer. this.worker.postMessage(message, [audio.buffer as ArrayBuffer]); diff --git a/src/lib/export.ts b/src/lib/export.ts index 23e0d4d..1d6ead5 100644 --- a/src/lib/export.ts +++ b/src/lib/export.ts @@ -14,6 +14,16 @@ import type { TranscriptChunk } from "../worker/protocol"; */ const FALLBACK_CUE_SECONDS = 2; +/** + * What to show instead of a blank transcript when a finished run found no + * words — silence- or music-only files (plan 001-0 Review Focus 4). Null + * while running (no result yet) or when there is text. + */ +export function emptyTranscriptNotice(result: { text: string } | null): string | null { + if (result === null || result.text.trim().length > 0) return null; + return "No speech was found in this recording."; +} + /** Final transcript as plain text: trimmed, single trailing newline. */ export function transcriptToTxt(result: { text: string }): string { return `${result.text.trim()}\n`; diff --git a/src/lib/longForm.ts b/src/lib/longForm.ts new file mode 100644 index 0000000..417e910 --- /dev/null +++ b/src/lib/longForm.ts @@ -0,0 +1,43 @@ +/** + * The "Skip silence" file-transcription setting (long-form pipeline, + * D-WS3a): remembered in localStorage like the engine choice. + */ + +import type { StorageLike } from "./engine"; + +export const LONG_FORM_KEY = "whisper-long-form"; + +/** + * Off by default: the spec's rollout rule makes the long-form pipeline the + * default only if it is faster and not less accurate on every measurement + * clip — see D-WS3a for the numbers that decided this. + */ +export const LONG_FORM_DEFAULT = false; + +function defaultStorage(): StorageLike | null { + try { + return (globalThis as { localStorage?: StorageLike }).localStorage ?? null; + } catch { + return null; // storage access can throw (privacy modes) + } +} + +export function loadLongFormChoice(storage: StorageLike | null = defaultStorage()): boolean { + try { + const value = storage?.getItem(LONG_FORM_KEY); + return value === "on" ? true : value === "off" ? false : LONG_FORM_DEFAULT; + } catch { + return LONG_FORM_DEFAULT; + } +} + +export function saveLongFormChoice( + enabled: boolean, + storage: StorageLike | null = defaultStorage(), +): void { + try { + storage?.setItem(LONG_FORM_KEY, enabled ? "on" : "off"); + } catch { + // Not persistable (private mode / quota) — the choice just won't stick. + } +} diff --git a/src/worker/inference.worker.ts b/src/worker/inference.worker.ts index 2d5b4b2..4ff1f65 100644 --- a/src/worker/inference.worker.ts +++ b/src/worker/inference.worker.ts @@ -17,7 +17,10 @@ import { type ProgressInfo, } from "@huggingface/transformers"; import { resolveDefaultDtype } from "../lib/models"; +import { LONG_FORM_DEFAULTS, transcribeLongForm } from "./longform"; +import { createWhisperDeps, loadSileroVad, type SileroVad } from "./longform/transformers"; import { + choosePipeline, deviceAttempts, dtypeRequiresF16, isUiToWorkerMessage, @@ -321,6 +324,36 @@ function load(message: LoadMessage): Promise return promise; } +/** + * Silero VAD for long-form file passes (D-WS3a/D-WS3b). Loaded on request + * (`load.vad`) once the Whisper pipeline is up — after it, so the Whisper + * attempt decides onnxruntime's thread pool (D-023/D-025). Offline it is + * used only if already cached: no network request (Review Focus 3). A failed + * load resolves to null (the old path runs) and is retried on a later load. + */ +let vad: Promise | null = null; + +function ensureVad(whisper: Promise): void { + if (vad !== null) return; + const attempt = (async () => { + await whisper.catch(() => {}); + try { + const model = await loadSileroVad({ localOnly: navigator.onLine === false, device: "wasm" }); + console.info("[whisper] speech-detection (VAD) model ready — long-form file pipeline available"); + return model; + } catch (error) { + console.info( + `[whisper] speech-detection (VAD) model unavailable, using the chunked file path: ${errorMessage(error)}`, + ); + return null; + } + })(); + vad = attempt; + void attempt.then((model) => { + if (model === null && vad === attempt) vad = null; + }); +} + // --------------------------------------------------------------------------- // Transcription // --------------------------------------------------------------------------- @@ -343,6 +376,17 @@ async function transcribe(message: TranscribeMessage): Promise { } const asr = await active.promise; + const streaming = message.stream ?? message.requestId === undefined; + // Live passes (not streaming) always keep the chunked path (D-016/D-024). + const vadModel = message.longForm && streaming && vad ? await vad : null; + if ( + choosePipeline({ longForm: message.longForm && streaming, vadReady: vadModel !== null }) === + "longform" + ) { + await transcribeLongFormPass(message, asr, vadModel!); + return; + } + const options: Record = { chunk_length_s: CHUNK_LENGTH_S, stride_length_s: STRIDE_LENGTH_S, @@ -360,7 +404,7 @@ async function transcribe(message: TranscribeMessage): Promise { // than any real 30 s window produces. max_new_tokens: 440, }; - if (message.stream ?? message.requestId === undefined) { + if (streaming) { // File-flow pass (id-less, or routed through BrowserEngine with an // explicit stream flag, D-010): stream partial tokens and report audio // progress. The streamer's chunk timestamps are window-relative; the @@ -426,6 +470,42 @@ async function transcribe(message: TranscribeMessage): Promise { post(result); } +/** + * A file pass through the long-form pipeline (D-WS3a): same partial / + * transcribe-progress / result messages as the chunked path. + */ +async function transcribeLongFormPass( + message: TranscribeMessage, + asr: AutomaticSpeechRecognitionPipeline, + vadModel: SileroVad, +): Promise { + const multilingual = Boolean( + (asr.model as unknown as { generation_config?: { is_multilingual?: boolean } }) + .generation_config?.is_multilingual, + ); + const result = await transcribeLongForm( + message.audio, + { + speechProbabilities: (audio) => vadModel.speechProbabilities(audio), + ...createWhisperDeps(asr, { multilingual }), + }, + { + ...LONG_FORM_DEFAULTS, + ...(multilingual && message.language !== undefined && { language: message.language }), + ...(multilingual && message.task !== undefined && { task: message.task }), + onPartial: (text) => post({ type: "partial", text, requestId: message.requestId }), + onProgress: (processedSeconds, totalSeconds) => + post({ + type: "transcribe-progress", + processedSeconds, + totalSeconds, + requestId: message.requestId, + }), + }, + ); + post({ type: "result", text: result.text, chunks: result.chunks, requestId: message.requestId }); +} + scope.addEventListener("message", (event) => { const data = event.data; if (!isUiToWorkerMessage(data)) { @@ -437,14 +517,17 @@ scope.addEventListener("message", (event) => { return; } switch (data.type) { - case "load": - load(data).catch((error: unknown) => { + case "load": { + const whisper = load(data); + if (data.vad) ensureVad(whisper); + whisper.catch((error: unknown) => { // A restart-aborted load is not an error — the UI already got the // load-restart message and is respawning this worker (D-025). if (error instanceof RestartRequested) return; post({ type: "error", stage: "load", message: errorMessage(error) }); }); break; + } case "transcribe": transcribe(data).catch((error: unknown) => { if (error instanceof RestartRequested) return; diff --git a/src/worker/longform/index.ts b/src/worker/longform/index.ts index 4c7094d..8113b68 100644 --- a/src/worker/longform/index.ts +++ b/src/worker/longform/index.ts @@ -1,5 +1,5 @@ /** Long-form file transcription pipeline (D-WS3a). See pipeline.ts. */ -export { DEFAULT_GUARD, transcribeLongForm } from "./pipeline"; +export { DEFAULT_GUARD, LONG_FORM_DEFAULTS, transcribeLongForm } from "./pipeline"; export { DEFAULT_FALLBACK, type FallbackOptions } from "./quality"; export type { DecodeRequest, diff --git a/src/worker/longform/pipeline.ts b/src/worker/longform/pipeline.ts index 3e3417a..b3f43c6 100644 --- a/src/worker/longform/pipeline.ts +++ b/src/worker/longform/pipeline.ts @@ -37,6 +37,12 @@ const MAX_CHUNK_SECONDS = 30; /** No guard: Whisper's own decoding (the measured default, D-WS3a). */ export const DEFAULT_GUARD: RepetitionGuard = { noRepeatNgramSize: 0, repetitionPenalty: 1 }; +/** The tuning the worker runs with — chosen by measurement (D-WS3a). */ +export const LONG_FORM_DEFAULTS: Pick = { + guard: DEFAULT_GUARD, + fallback: false, +}; + export async function transcribeLongForm( audio: Float32Array, deps: LongFormDeps, diff --git a/src/worker/protocol.ts b/src/worker/protocol.ts index 7cdcd9b..106de9a 100644 --- a/src/worker/protocol.ts +++ b/src/worker/protocol.ts @@ -57,6 +57,13 @@ export interface LoadMessage { * in App.tsx); omitted/false loads in parallel. */ sequentialSessionLoad?: boolean; + /** + * Also make the Silero VAD model (2.2 MB) available for long-form file + * transcription (D-WS3a/D-WS3b): cache-first, fetched from the Hub only + * when online — offline it is used only if already cached. Sent with the + * same user-initiated loads as the Whisper weights (D-020). + */ + vad?: boolean; } export interface TranscribeMessage { @@ -87,6 +94,12 @@ export interface TranscribeMessage { * engine (which correlates via requestId, D-010) still streams. */ stream?: boolean; + /** + * Run this file pass through the long-form pipeline (VAD-cut chunks, + * language pinned once, D-WS3a). Honoured only when the VAD model loaded; + * otherwise the classic 30 s/5 s chunked path runs (`choosePipeline`). + */ + longForm?: boolean; } export type UiToWorkerMessage = LoadMessage | TranscribeMessage; @@ -267,6 +280,19 @@ export function deviceAttempts( } } +/** + * Which file-transcription path a pass takes (D-WS3a/D-WS3b): long-form only + * when the pass asked for it and the VAD model is loaded. A missing VAD model + * (offline, never downloaded) silently keeps the classic chunked path — it is + * a fallback, never an error. + */ +export function choosePipeline(input: { + longForm: boolean | undefined; + vadReady: boolean; +}): "longform" | "chunked" { + return input.longForm === true && input.vadReady ? "longform" : "chunked"; +} + // --------------------------------------------------------------------------- // Runtime guards (postMessage data arrives untyped) // --------------------------------------------------------------------------- @@ -279,6 +305,10 @@ function isOptionalString(value: unknown): value is string | undefined { return value === undefined || typeof value === "string"; } +function isOptionalBoolean(value: unknown): value is boolean | undefined { + return value === undefined || typeof value === "boolean"; +} + const DEVICES: readonly string[] = ["webgpu", "wasm", "webnn"]; const DTYPES: readonly string[] = ["fp32", "fp16", "q8", "q4"]; @@ -308,7 +338,8 @@ export function isUiToWorkerMessage(value: unknown): value is UiToWorkerMessage return ( typeof value.modelId === "string" && isDevicePreference(value.device) && - (value.dtype === undefined || isDtypeSpec(value.dtype)) + (value.dtype === undefined || isDtypeSpec(value.dtype)) && + isOptionalBoolean(value.vad) ); case "transcribe": return ( @@ -318,7 +349,8 @@ export function isUiToWorkerMessage(value: unknown): value is UiToWorkerMessage value.task === "transcribe" || value.task === "translate") && isOptionalString(value.requestId) && - (value.stream === undefined || typeof value.stream === "boolean") + isOptionalBoolean(value.stream) && + isOptionalBoolean(value.longForm) ); default: return false; diff --git a/tests/engine.test.ts b/tests/engine.test.ts index d174aa7..9a2344d 100644 --- a/tests/engine.test.ts +++ b/tests/engine.test.ts @@ -77,6 +77,15 @@ describe("BrowserEngine", () => { expect("modelId" in message).toBe(false); }); + test("forwards the long-form request only when set (D-WS3a)", () => { + const worker = new FakeWorker(); + const engine = new BrowserEngine(worker); + void engine.transcribe(new Float32Array(4), { longForm: true }); + void engine.transcribe(new Float32Array(4)); + expect(worker.posted[0]!.message.longForm).toBe(true); + expect("longForm" in worker.posted[1]!.message).toBe(false); + }); + test("asks the worker to stream only when a consumer listens", () => { const worker = new FakeWorker(); const engine = new BrowserEngine(worker); diff --git a/tests/longform-setting.test.ts b/tests/longform-setting.test.ts new file mode 100644 index 0000000..4c52374 --- /dev/null +++ b/tests/longform-setting.test.ts @@ -0,0 +1,55 @@ +import { describe, expect, test } from "bun:test"; +import { emptyTranscriptNotice } from "../src/lib/export"; +import { + LONG_FORM_DEFAULT, + LONG_FORM_KEY, + loadLongFormChoice, + saveLongFormChoice, +} from "../src/lib/longForm"; + +function memoryStorage(initial: Record = {}) { + const data = { ...initial }; + return { + data, + getItem: (key: string) => data[key] ?? null, + setItem: (key: string, value: string) => { + data[key] = value; + }, + }; +} + +describe("long-form setting persistence (D-WS3a)", () => { + test("defaults when nothing is stored or storage is unavailable", () => { + expect(loadLongFormChoice(memoryStorage())).toBe(LONG_FORM_DEFAULT); + expect(loadLongFormChoice(null)).toBe(LONG_FORM_DEFAULT); + const throwing = { + getItem: () => { + throw new Error("denied"); + }, + setItem: () => { + throw new Error("denied"); + }, + }; + expect(loadLongFormChoice(throwing)).toBe(LONG_FORM_DEFAULT); + expect(() => saveLongFormChoice(true, throwing)).not.toThrow(); + }); + + test("round-trips an explicit choice", () => { + const storage = memoryStorage(); + saveLongFormChoice(!LONG_FORM_DEFAULT, storage); + expect(storage.data[LONG_FORM_KEY]).toBeDefined(); + expect(loadLongFormChoice(storage)).toBe(!LONG_FORM_DEFAULT); + }); +}); + +describe("emptyTranscriptNotice (plan 001-0 Review Focus 4)", () => { + test("a finished transcript with no words says so plainly", () => { + expect(emptyTranscriptNotice({ text: "" })).toBe("No speech was found in this recording."); + expect(emptyTranscriptNotice({ text: " \n" })).toBe("No speech was found in this recording."); + }); + + test("no notice while running or when there is text", () => { + expect(emptyTranscriptNotice(null)).toBeNull(); + expect(emptyTranscriptNotice({ text: " Hello." })).toBeNull(); + }); +}); diff --git a/tests/protocol.test.ts b/tests/protocol.test.ts index f5dde6e..af122c6 100644 --- a/tests/protocol.test.ts +++ b/tests/protocol.test.ts @@ -4,6 +4,7 @@ import { deviceAttempts, dtypeRequiresF16, initialLoadProgress, + choosePipeline, isUiToWorkerMessage, isWorkerToUiMessage, TranscribeProgressTracker, @@ -35,7 +36,9 @@ describe("isUiToWorkerMessage", () => { { type: "transcribe", audio: new Float32Array(8), requestId: "live-3" }, { type: "transcribe", audio: new Float32Array(8), requestId: "engine-1-1", stream: true }, { type: "transcribe", audio: new Float32Array(8), stream: false }, + { type: "transcribe", audio: new Float32Array(8), stream: true, longForm: true }, ]; + loads.push({ type: "load", modelId: "onnx-community/whisper-base", device: "auto", vad: true }); for (const message of [...loads, ...transcribes]) { expect(isUiToWorkerMessage(message)).toBe(true); } @@ -65,10 +68,26 @@ describe("isUiToWorkerMessage", () => { expect( isUiToWorkerMessage({ type: "transcribe", audio: new Float32Array(4), stream: "yes" }), ).toBe(false); + expect( + isUiToWorkerMessage({ type: "transcribe", audio: new Float32Array(4), longForm: 1 }), + ).toBe(false); + expect(isUiToWorkerMessage({ type: "load", modelId: "x", device: "auto", vad: "yes" })).toBe( + false, + ); expect(isUiToWorkerMessage({ type: "result", text: "", chunks: [] })).toBe(false); }); }); +describe("choosePipeline (D-WS3a/D-WS3b)", () => { + test("long-form only when asked for and the VAD model loaded", () => { + expect(choosePipeline({ longForm: true, vadReady: true })).toBe("longform"); + // Offline / never downloaded: the old path, never an error (Review Focus 3). + expect(choosePipeline({ longForm: true, vadReady: false })).toBe("chunked"); + expect(choosePipeline({ longForm: false, vadReady: true })).toBe("chunked"); + expect(choosePipeline({ longForm: undefined, vadReady: true })).toBe("chunked"); + }); +}); + describe("isWorkerToUiMessage", () => { test("accepts every worker → UI message shape", () => { const messages: WorkerToUiMessage[] = [