Add cancel + progress UI, EngineToggle, and Docker deployment

This commit is contained in:
Lucas Winther
2026-07-14 19:32:10 +02:00
parent be82ecfe57
commit 0084e0435a
16 changed files with 972 additions and 60 deletions
+220 -43
View File
@@ -2,6 +2,7 @@ import { useEffect, useRef, useState } from "react";
import "./index.css";
import { AuthForms } from "./components/AuthForms";
import { EngineToggle } from "./components/EngineToggle";
import { FileDropzone } from "./components/FileDropzone";
import { LiveControls, type LiveStatus } from "./components/LiveControls";
import { ModelLoadProgress } from "./components/ModelLoadProgress";
@@ -9,6 +10,17 @@ import { ModelPicker } from "./components/ModelPicker";
import { TranscriptView } from "./components/TranscriptView";
import { TranscriptionOptions } from "./components/TranscriptionOptions";
import { decodeAudioFile } from "./lib/audio";
import {
BrowserEngine,
ServerEngine,
fetchServerEngineStatus,
loadEngineChoice,
saveEngineChoice,
type Engine,
type EngineKind,
type ServerEngineStatus,
} from "./lib/engine";
import { useAuth } from "./lib/useAuth";
import { LIVE_TUNING, LiveTranscriber, type LiveTranscribeResult } from "./lib/live";
import { startMicCapture, type MicCapture } from "./lib/mic";
import { getDownloadedModels } from "./lib/modelCache";
@@ -64,11 +76,25 @@ export function App() {
// audio to the (fresh) worker. Decoding itself cannot be aborted.
const fileRun = useRef(0);
// Engine plumbing (M5, D-010). The BrowserEngine is bound to the worker's
// lifetime (recreated together on cancel); the ServerEngine is stateless
// and lives for the app. A server run's fetch is cancelled via abortRef.
const browserEngineRef = useRef<BrowserEngine | null>(null);
const serverEngineRef = useRef<ServerEngine | null>(null);
const abortRef = useRef<AbortController | null>(null);
const [busy, setBusy] = useState(false);
// Which engine the current file run uses; null while idle. Server runs
// don't need the browser model, so parts of the UI key off this.
const [runEngine, setRunEngine] = useState<EngineKind | null>(null);
const [decoding, setDecoding] = useState(false);
const [loadStarted, setLoadStarted] = useState(false);
const [loadProgress, setLoadProgress] = useState(initialLoadProgress);
const [partialText, setPartialText] = useState("");
const [transcribeProgress, setTranscribeProgress] = useState<{
processedSeconds: number;
totalSeconds: number;
} | null>(null);
const [result, setResult] = useState<TranscriptResult | null>(null);
const [error, setError] = useState<string | null>(null);
const [fileName, setFileName] = useState<string | null>(null);
@@ -91,6 +117,15 @@ export function App() {
const [swControlled, setSwControlled] = useState(false);
const [applyUpdate, setApplyUpdate] = useState<(() => void) | null>(null);
// Session + engine choice (M4/M5). One useAuth for the whole app — the
// header forms and the EngineToggle must agree on the session. The choice
// persists across reloads; it only takes effect while signed in AND the
// deployment reports the server engine available (anonymous-first: logged
// out, everything runs on-device exactly as before).
const auth = useAuth();
const [engineChoice, setEngineChoice] = useState<EngineKind>(() => loadEngineChoice());
const [serverStatus, setServerStatus] = useState<ServerEngineStatus | null>(null);
function refreshDownloaded() {
// Downloaded badges (D-005): query Cache Storage for cached weights.
getDownloadedModels(MODEL_IDS).then(setDownloaded, () => {});
@@ -109,9 +144,28 @@ export function App() {
liveRef.current = null;
workerRef.current?.terminate();
workerRef.current = null;
browserEngineRef.current = null;
abortRef.current?.abort();
abortRef.current = null;
};
}, []);
// Probe server-engine availability once a session exists (the endpoint is
// auth-guarded, so probing while anonymous is pointless). Never rejects.
useEffect(() => {
if (!auth.user) {
setServerStatus(null);
return;
}
let cancelled = false;
fetchServerEngineStatus().then((status) => {
if (!cancelled) setServerStatus(status);
});
return () => {
cancelled = true;
};
}, [auth.user]);
function takeLiveRequest(requestId: string | undefined) {
if (requestId === undefined) return undefined;
const pending = pendingLive.current.get(requestId);
@@ -146,13 +200,24 @@ export function App() {
setPartialText((text) => text + message.text);
}
break;
case "transcribe-progress":
// File flow only — live passes never carry progress (D-016).
if (message.requestId === undefined) {
setTranscribeProgress({
processedSeconds: message.processedSeconds,
totalSeconds: message.totalSeconds,
});
}
break;
case "result": {
const pending = takeLiveRequest(message.requestId);
if (pending) {
pending.resolve({ text: message.text, chunks: message.chunks });
} else if (message.requestId === undefined) {
setResult({ text: message.text, chunks: message.chunks });
setTranscribeProgress(null);
setBusy(false);
setRunEngine(null);
}
break;
}
@@ -160,9 +225,13 @@ export function App() {
const pending = takeLiveRequest(message.requestId);
if (pending) {
pending.reject(new Error(message.message));
} else {
} else if (message.requestId === undefined) {
// Load-stage failures and legacy id-less passes. Engine-routed
// file passes carry an id and surface through the engine promise.
setError(message.message);
setTranscribeProgress(null);
setBusy(false);
setRunEngine(null);
}
break;
}
@@ -179,6 +248,9 @@ export function App() {
setBusy(false);
});
workerRef.current = worker;
// The BrowserEngine shares this worker and only sees its own request ids
// (engine-*), so it coexists with the live loop's live-* ids above.
browserEngineRef.current = new BrowserEngine(worker);
return worker;
}
@@ -205,6 +277,7 @@ export function App() {
setModelId(nextModelId);
setActiveDevice(null);
setPartialText("");
setTranscribeProgress(null);
setResult(null);
startLoad(nextModelId, devicePreference);
}
@@ -219,78 +292,146 @@ export function App() {
}
/**
* Cancel the in-flight file transcription (D-018). Transformers.js has no
* abort API for a running `asr()` call — and on the WASM backend the pass
* Cancel the in-flight file transcription (D-018). Browser runs: no abort
* API exists for a running `asr()` call — and on the WASM backend the pass
* blocks the worker's event loop, so even a "cancel" message would sit
* unread until the pass finished. Terminating the worker is the only
* reliable preemption; it also aborts an in-flight model download. The next
* run spawns a fresh worker and reloads the pipeline — cheap, because the
* weights stay in the browser Cache API (D-005).
* weights stay in the browser Cache API (D-005). Server runs: abort the
* `/api/transcribe` fetch (the server cancels the sidecar pass, D-011) and
* leave the worker — and its loaded pipeline — untouched.
*/
function cancelTranscription() {
if (!busy) return;
fileRun.current++; // any decode still running is now stale
workerRef.current?.terminate();
workerRef.current = null;
const cancelled = new Error("Transcription cancelled.");
if (runEngine === "server") {
abortRef.current?.abort();
abortRef.current = null;
} else {
browserEngineRef.current?.failAll(cancelled);
browserEngineRef.current = null;
workerRef.current?.terminate();
workerRef.current = null;
// The worker (and any pipeline / partial download) is gone — reset the
// load state so the next transcription reloads from a clean slate.
setLoadStarted(false);
setLoadProgress(initialLoadProgress);
setActiveDevice(null);
}
// The file flow never has live passes pending, but if anything is in the
// map (defensive), unblock it so nothing awaits a dead worker forever.
const cancelled = new Error("Transcription cancelled.");
for (const pending of pendingLive.current.values()) pending.reject(cancelled);
pendingLive.current.clear();
setBusy(false);
setRunEngine(null);
setPartialText("");
setTranscribeProgress(null);
setResult(null);
setFileName(null);
setError(null);
// The worker (and any pipeline / partial download) is gone — reset the
// load state so the next transcription reloads from a clean slate.
setLoadStarted(false);
setLoadProgress(initialLoadProgress);
setActiveDevice(null);
}
async function handleFile(file: File) {
if (busy || liveStatus !== "idle") return;
const run = ++fileRun.current;
// The toggle's choice only takes effect with a session and an available
// server engine (D-010); everything else runs on-device as always.
const useServer =
engineChoice === "server" &&
auth.user !== null &&
serverStatus?.available === true;
setBusy(true);
setRunEngine(useServer ? "server" : "browser");
setError(null);
setPartialText("");
setTranscribeProgress(null);
setResult(null);
setFileName(file.name);
// Kick off the (idempotent) model load immediately so the download runs
// while we decode the audio locally.
const worker = ensureWorker();
if (!loadStarted) {
startLoad(modelId, devicePreference);
let engine: Engine;
if (useServer) {
engine = serverEngineRef.current ??= new ServerEngine();
} else {
postToWorker(worker, { type: "load", modelId, device: devicePreference });
// Kick off the (idempotent) model load immediately so the download runs
// while we decode the audio locally. Server runs skip this — they don't
// need the browser pipeline.
ensureWorker();
if (!loadStarted) {
startLoad(modelId, devicePreference);
} else {
postToWorker(workerRef.current!, {
type: "load",
modelId,
device: devicePreference,
});
}
engine = browserEngineRef.current!;
}
const multilingual = getModel(modelId)?.multilingual ?? false;
// Decoded entirely on-device via Web Audio (D-012); for browser runs the
// audio never touches the network (Architecture.md §5) — server runs
// upload the decoded PCM, which is the point of the toggle.
setDecoding(true);
let audio: Float32Array;
try {
// Decoded entirely on-device via Web Audio (D-012); the audio never
// touches the network (Architecture.md §5).
const audio = await decodeAudioFile(file);
if (fileRun.current !== run) return; // cancelled while decoding (D-018)
// Transfer the buffer to the worker — zero copy. Language/task only
// apply to multilingual models; `.en` variants take neither.
postToWorker(
worker,
multilingual
? { type: "transcribe", audio, language, task }
: { type: "transcribe", audio },
[audio.buffer as ArrayBuffer],
);
audio = await decodeAudioFile(file);
} catch (decodeError) {
if (fileRun.current !== run) return; // this run was already cancelled
setError(decodeError instanceof Error ? decodeError.message : String(decodeError));
setBusy(false);
if (fileRun.current === run) {
setError(
decodeError instanceof Error ? decodeError.message : String(decodeError),
);
setBusy(false);
setRunEngine(null);
}
return;
} finally {
setDecoding(false);
}
if (fileRun.current !== run) return; // cancelled while decoding (D-018)
// Server runs cancel by aborting the fetch (D-018 amendment); browser
// runs cancel by terminating the worker, which rejects the promise via
// BrowserEngine.failAll.
const controller = useServer ? new AbortController() : null;
abortRef.current = controller;
try {
const outcome = await engine.transcribe(audio, {
// Language/task only apply to multilingual models; `.en` variants
// take neither. The browser engine ignores modelId (worker holds it).
modelId,
...(multilingual && { language, task }),
onPartial: (fragment) => {
if (fileRun.current === run) setPartialText((text) => text + fragment);
},
...(!useServer && {
onProgress: (progress: { processedSeconds: number; totalSeconds: number }) => {
if (fileRun.current === run) setTranscribeProgress(progress);
},
}),
...(controller && { signal: controller.signal }),
});
if (fileRun.current !== run) return; // cancelled mid-pass
setResult(outcome);
setTranscribeProgress(null);
setBusy(false);
setRunEngine(null);
} catch (transcribeError) {
if (fileRun.current !== run) return; // cancellation rejections are expected
setError(
transcribeError instanceof Error
? transcribeError.message
: String(transcribeError),
);
setTranscribeProgress(null);
setBusy(false);
setRunEngine(null);
} finally {
if (abortRef.current === controller) abortRef.current = null;
}
}
// -------------------------------------------------------------------------
@@ -381,8 +522,13 @@ export function App() {
const model = getModel(modelId);
const modelReady = loadProgress.ready;
const showModelProgress = loadStarted && !modelReady && !error;
const transcribing = busy && !decoding && modelReady;
// Server runs never load the browser model, so they are "transcribing" the
// moment decoding ends.
const transcribing = busy && !decoding && (runEngine === "server" || modelReady);
const liveActive = liveStatus !== "idle";
// What the next file run will use (mirrors handleFile's decision).
const serverSelected =
engineChoice === "server" && auth.user !== null && serverStatus?.available === true;
return (
<main className="app">
@@ -398,7 +544,7 @@ export function App() {
unobtrusive "Sign in" button — everything else is unchanged.
Renders nothing until the /me probe confirms the server has auth
enabled (runtime flag, D-015). */}
<AuthForms />
<AuthForms auth={auth} />
</header>
<ModelPicker
@@ -423,6 +569,20 @@ export function App() {
/>
)}
{/* Engine toggle (M5, D-010): signed-in users choose where uploaded
files are transcribed. Live mode always stays on-device (D-016). */}
{auth.user && (
<EngineToggle
choice={engineChoice}
onChoice={(choice) => {
setEngineChoice(choice);
saveEngineChoice(choice);
}}
status={serverStatus}
disabled={busy || liveActive}
/>
)}
<FileDropzone onFile={handleFile} disabled={busy || liveActive} />
<LiveControls
@@ -436,6 +596,17 @@ export function App() {
<p className="file-name">
File: <strong>{fileName}</strong>
{decoding && " — decoding audio…"}
{busy && (
// Cancel the whole run (D-018): decode, pending model download,
// or the transcription pass itself.
<button
type="button"
className="cancel-button"
onClick={cancelTranscription}
>
Cancel
</button>
)}
</p>
)}
@@ -454,6 +625,10 @@ export function App() {
tentativeText={liveActive ? liveTentative : ""}
result={result}
transcribing={liveActive || transcribing}
progress={liveActive ? null : transcribeProgress}
placeholder={
liveActive ? "Listening for the first words…" : "Processing audio…"
}
activeLabel={
liveStatus === "stopping"
? "Finishing up…"
@@ -482,13 +657,15 @@ export function App() {
<footer className="app-footer">
<p>
Model: <code>{modelId}</code> · Engine:{" "}
{activeDevice === "webgpu"
? "WebGPU (on-device)"
: activeDevice === "webnn"
? "WebNN (on-device)"
: activeDevice === "wasm"
? "WASM (on-device)"
: "on-device — backend selected on first load"}
{serverSelected
? "server (faster-whisper) — files upload to this deployment"
: activeDevice === "webgpu"
? "WebGPU (on-device)"
: activeDevice === "webnn"
? "WebNN (on-device)"
: activeDevice === "wasm"
? "WASM (on-device)"
: "on-device — backend selected on first load"}
</p>
</footer>
</main>
+6 -3
View File
@@ -1,6 +1,6 @@
import { useState, type FormEvent, type KeyboardEvent } from "react";
import { validatePassword, validateUsername } from "../lib/authClient";
import { useAuth } from "../lib/useAuth";
import type { AuthState } from "../lib/useAuth";
type Mode = "login" | "register";
@@ -10,11 +10,14 @@ type Mode = "login" | "register";
* that opens a compact login/register popover; once signed in, the username
* and a "Sign out" button.
*
* The session state comes from `App`'s single `useAuth()` — the EngineToggle
* (M5) reads the same state, so both always agree on who is signed in.
*
* Validation mirrors the server rules for instant feedback, but the server
* stays authoritative: its `{ error }` messages are shown verbatim on 4xx.
*/
export function AuthForms() {
const { enabled, user, login, register, logout } = useAuth();
export function AuthForms({ auth }: { auth: AuthState }) {
const { enabled, user, login, register, logout } = auth;
const [open, setOpen] = useState(false);
const [mode, setMode] = useState<Mode>("login");
+62
View File
@@ -0,0 +1,62 @@
import type { EngineKind, ServerEngineStatus } from "../lib/engine";
interface EngineToggleProps {
/** The persisted choice (src/lib/engine.ts helpers). */
choice: EngineKind;
onChoice: (choice: EngineKind) => void;
/** GET /api/transcribe/status result; null while the probe is in flight. */
status: ServerEngineStatus | null;
/** Locked while a run or live session is active (like the other options). */
disabled?: boolean;
}
/**
* Where should file transcription run (M5, D-010)? Rendered by `App` only for
* a signed-in session — anonymous users never see it (anonymous-first,
* Architecture.md §5). The server option stays visible but disabled when the
* deployment reports the engine unavailable (e.g. no faster-whisper sidecar,
* D-013), showing the probe's reason — hiding it entirely would read as a
* broken account feature. Live mode always stays on-device (D-016).
*/
export function EngineToggle({
choice,
onChoice,
status,
disabled = false,
}: EngineToggleProps) {
const serverAvailable = status?.available === true;
const effective: EngineKind = serverAvailable ? choice : "browser";
function option(kind: EngineKind, label: string, optionDisabled: boolean) {
return (
<label className="engine-option">
<input
type="radio"
name="engine"
value={kind}
checked={effective === kind}
disabled={disabled || optionDisabled}
onChange={() => onChoice(kind)}
/>
{label}
</label>
);
}
return (
<section className="engine-toggle" aria-label="Transcription engine">
<span className="engine-toggle-label">Transcribe on</span>
{option("browser", "This device", false)}
{option("server", "Server", !serverAvailable)}
<span className="engine-toggle-hint">
{status === null
? "Checking server availability…"
: serverAvailable
? effective === "server"
? "Uploaded files are transcribed on the server (faster-whisper). Live mode always stays on this device."
: "Your audio stays on this device."
: (status.reason ?? "The server engine is unavailable.")}
</span>
</section>
);
}
+34 -2
View File
@@ -15,6 +15,14 @@ interface TranscriptViewProps {
tentativeText?: string;
/** Heading while running; defaults to "Transcribing…". */
activeLabel?: string;
/**
* How far the pass has progressed through the audio (file flow only) —
* renders a percent in the heading plus a thin progress bar. Live mode
* passes null; its feedback is the committed/tentative text itself.
*/
progress?: { processedSeconds: number; totalSeconds: number } | null;
/** Text shown before the first words arrive. */
placeholder?: string;
}
function formatTimestamp(seconds: number): string {
@@ -30,10 +38,17 @@ export function TranscriptView({
transcribing,
tentativeText = "",
activeLabel = "Transcribing…",
progress = null,
placeholder = "Listening for the first words…",
}: TranscriptViewProps) {
const [copied, setCopied] = useState(false);
const text = result?.text ?? partialText;
const copyText = `${text}${tentativeText}`.trim();
const showProgress =
transcribing && !result && progress !== null && progress.totalSeconds > 0;
const percent = showProgress
? Math.min(100, Math.round((progress.processedSeconds / progress.totalSeconds) * 100))
: null;
useEffect(() => {
if (!copied) return;
@@ -52,7 +67,12 @@ export function TranscriptView({
return (
<section className="transcript">
<div className="transcript-header">
<h2>{result ? "Transcript" : activeLabel}</h2>
<h2>
{result ? "Transcript" : activeLabel}
{percent !== null && (
<span className="transcript-percent"> {percent}%</span>
)}
</h2>
<button
type="button"
className="copy-button"
@@ -62,8 +82,20 @@ export function TranscriptView({
{copied ? "Copied!" : "Copy"}
</button>
</div>
{showProgress && (
<div className="transcript-progress">
<progress
max={progress.totalSeconds}
value={Math.min(progress.processedSeconds, progress.totalSeconds)}
/>
<span className="transcript-progress-time">
{formatTimestamp(progress.processedSeconds)} /{" "}
{formatTimestamp(progress.totalSeconds)}
</span>
</div>
)}
<p className="transcript-text" aria-live="polite">
{text || (tentativeText ? "" : "Listening for the first words…")}
{text || (tentativeText ? "" : placeholder)}
{tentativeText && (
<span className="transcript-tentative">{tentativeText}</span>
)}
+78
View File
@@ -357,6 +357,38 @@ body {
flex-wrap: wrap;
}
/* --- EngineToggle (M5, D-010) --- */
.engine-toggle {
display: flex;
align-items: center;
gap: 1rem;
flex-wrap: wrap;
font-size: 0.9rem;
}
.engine-toggle-label {
font-weight: 600;
}
.engine-option {
display: inline-flex;
align-items: center;
gap: 0.35rem;
cursor: pointer;
}
.engine-option:has(input:disabled) {
color: var(--muted);
cursor: default;
}
.engine-toggle-hint {
flex-basis: 100%;
color: var(--muted);
font-size: 0.85rem;
}
/* --- FileDropzone --- */
.dropzone {
@@ -458,6 +490,25 @@ body {
font-size: 0.95rem;
}
/* Cancel for the in-flight file run (D-018) — styled like .copy-button, with
a warning accent on hover. */
.cancel-button {
margin-left: 0.75rem;
border: 1px solid var(--border);
background: transparent;
color: inherit;
border-radius: 8px;
padding: 0.3rem 0.9rem;
font-size: 0.85rem;
cursor: pointer;
transition: border-color 0.15s, color 0.15s;
}
.cancel-button:hover {
border-color: #8b3a3a;
color: #f2b8b5;
}
.error {
margin: 0;
padding: 0.75rem 1rem;
@@ -561,6 +612,33 @@ body {
cursor: default;
}
.transcript-percent {
color: var(--muted);
font-weight: normal;
font-variant-numeric: tabular-nums;
}
/* Audio progress of the in-flight file pass (transcribe-progress messages). */
.transcript-progress {
display: flex;
align-items: center;
gap: 0.75rem;
margin-top: 0.6rem;
}
.transcript-progress progress {
flex: 1;
height: 6px;
accent-color: var(--accent);
}
.transcript-progress-time {
color: var(--muted);
font-size: 0.85rem;
white-space: nowrap;
font-variant-numeric: tabular-nums;
}
.transcript-text {
margin: 0.75rem 0 0;
white-space: pre-wrap;
+24 -2
View File
@@ -42,9 +42,16 @@ export interface EngineTranscribeOptions {
* Browser engine: token fragments; server engine: whole segments.
*/
onPartial?: (fragment: string) => void;
/**
* How far the pass has progressed through the audio. Browser engine only —
* faster-whisper's stream carries no window boundaries, so the server
* engine never calls it (the UI degrades to segments-as-feedback).
*/
onProgress?: (progress: { processedSeconds: number; totalSeconds: number }) => void;
/**
* Abort the run. Honored by the server engine (the request is cancelled
* server-side too); the browser worker cannot abort a pass yet (M6).
* server-side too); the browser worker cannot abort a pass — cancel there
* is worker termination via `BrowserEngine.failAll` (D-018).
*/
signal?: AbortSignal;
}
@@ -83,6 +90,7 @@ interface PendingRequest {
resolve: (result: EngineTranscribeResult) => void;
reject: (error: Error) => void;
onPartial?: (fragment: string) => void;
onProgress?: (progress: { processedSeconds: number; totalSeconds: number }) => void;
}
/**
@@ -116,11 +124,19 @@ export class BrowserEngine implements Engine {
): Promise<EngineTranscribeResult> {
const requestId = `engine-${this.instance}-${++this.nextRequestId}`;
return new Promise((resolve, reject) => {
this.pending.set(requestId, { resolve, reject, onPartial: options.onPartial });
this.pending.set(requestId, {
resolve,
reject,
onPartial: options.onPartial,
onProgress: options.onProgress,
});
const message: TranscribeMessage = {
type: "transcribe",
audio,
requestId,
// Id-carrying requests default to no streaming in the worker (the
// live-pass rule) — ask for it explicitly when a consumer listens.
...((options.onPartial || options.onProgress) && { stream: true }),
...(options.language !== undefined &&
options.language !== "auto" && { language: options.language }),
...(options.task !== undefined && { task: options.task }),
@@ -146,6 +162,12 @@ export class BrowserEngine implements Engine {
case "partial":
request.onPartial?.(data.text);
break;
case "transcribe-progress":
request.onProgress?.({
processedSeconds: data.processedSeconds,
totalSeconds: data.totalSeconds,
});
break;
case "result":
this.pending.delete(data.requestId);
request.resolve({ text: data.text, chunks: data.chunks });
+41 -3
View File
@@ -19,6 +19,7 @@ import { resolveDefaultDtype } from "../lib/models";
import {
deviceAttempts,
isUiToWorkerMessage,
TranscribeProgressTracker,
type DeviceCapabilities,
type DevicePreference,
type InferenceDevice,
@@ -209,6 +210,12 @@ function load(message: LoadMessage): Promise<AutomaticSpeechRecognitionPipeline>
// Transcription
// ---------------------------------------------------------------------------
/** Audio sample rate the whole app standardizes on (D-012). */
const SAMPLE_RATE = 16_000;
/** Chunked-pipeline window / overlap, in seconds (Whisper's usual 30/5). */
const CHUNK_LENGTH_S = 30;
const STRIDE_LENGTH_S = 5;
async function transcribe(message: TranscribeMessage): Promise<void> {
if (active === null) {
post({
@@ -222,11 +229,39 @@ async function transcribe(message: TranscribeMessage): Promise<void> {
const asr = await active.promise;
const options: Record<string, unknown> = {
chunk_length_s: 30,
stride_length_s: 5,
chunk_length_s: CHUNK_LENGTH_S,
stride_length_s: STRIDE_LENGTH_S,
return_timestamps: true,
};
if (message.requestId === undefined) {
if (message.stream ?? message.requestId === undefined) {
// File-flow pass (id-less, or routed through BrowserEngine with an
// explicit stream flag, D-010): stream partial tokens and report audio
// progress. The streamer's chunk timestamps are window-relative; the
// tracker maps them to monotone absolute seconds (see protocol.ts).
// Live passes skip both — they resolve wholesale via "result" (D-016).
const totalSeconds = message.audio.length / SAMPLE_RATE;
const tracker = new TranscribeProgressTracker(
totalSeconds,
CHUNK_LENGTH_S,
CHUNK_LENGTH_S - 2 * STRIDE_LENGTH_S,
);
const postProgress = (processedSeconds: number | null) => {
if (processedSeconds === null) return;
post({
type: "transcribe-progress",
processedSeconds,
totalSeconds,
requestId: message.requestId,
});
};
// Immediate 0/total so the UI shows the audio length before the first
// (possibly slow) token arrives.
post({
type: "transcribe-progress",
processedSeconds: 0,
totalSeconds,
requestId: message.requestId,
});
options.streamer = new WhisperTextStreamer(
// The ASR pipeline for Whisper models always carries a WhisperTokenizer.
asr.tokenizer as ConstructorParameters<typeof WhisperTextStreamer>[0],
@@ -234,6 +269,9 @@ async function transcribe(message: TranscribeMessage): Promise<void> {
skip_prompt: true,
callback_function: (text: string) =>
post({ type: "partial", text, requestId: message.requestId }),
on_chunk_start: (time: number) => postProgress(tracker.timestamp(time)),
on_chunk_end: (time: number) => postProgress(tracker.timestamp(time)),
on_finalize: () => postProgress(tracker.finalizeWindow()),
},
);
}
+86 -1
View File
@@ -69,6 +69,14 @@ export interface TranscribeMessage {
* responses to segment passes; the file flow omits it and is unchanged.
*/
requestId?: string;
/**
* Stream partial tokens and transcribe-progress for this request. Defaults
* to `requestId === undefined` (the historical rule: id-less file-flow
* passes stream, live segment passes resolve wholesale via "result").
* `BrowserEngine` sets it explicitly so a file pass routed through the
* engine (which correlates via requestId, D-010) still streams.
*/
stream?: boolean;
}
export type UiToWorkerMessage = LoadMessage | TranscribeMessage;
@@ -108,6 +116,22 @@ export interface PartialMessage {
requestId?: string;
}
/**
* How far a file-flow pass has progressed through the audio, in seconds.
* Posted once at pass start (0 / total) and then on every Whisper segment
* boundary, so slow models/backends show movement long before the first
* partial token. Values are monotone and clamped to [0, totalSeconds]
* (see TranscribeProgressTracker). Live passes (requestId set) don't report
* progress — the live UI has its own committed/tentative feedback (D-016).
*/
export interface TranscribeProgressMessage {
type: "transcribe-progress";
processedSeconds: number;
totalSeconds: number;
/** Echo of the originating transcribe request's id, if it carried one. */
requestId?: string;
}
/** One timestamped segment of the final transcript. */
export interface TranscriptChunk {
text: string;
@@ -134,6 +158,7 @@ export interface ErrorMessage {
export type WorkerToUiMessage =
| LoadProgressMessage
| PartialMessage
| TranscribeProgressMessage
| ResultMessage
| ErrorMessage;
@@ -234,7 +259,8 @@ export function isUiToWorkerMessage(value: unknown): value is UiToWorkerMessage
(value.task === undefined ||
value.task === "transcribe" ||
value.task === "translate") &&
isOptionalString(value.requestId)
isOptionalString(value.requestId) &&
(value.stream === undefined || typeof value.stream === "boolean")
);
default:
return false;
@@ -264,6 +290,12 @@ export function isWorkerToUiMessage(value: unknown): value is WorkerToUiMessage
}
case "partial":
return typeof value.text === "string" && isOptionalString(value.requestId);
case "transcribe-progress":
return (
typeof value.processedSeconds === "number" &&
typeof value.totalSeconds === "number" &&
isOptionalString(value.requestId)
);
case "result":
return (
typeof value.text === "string" &&
@@ -331,3 +363,56 @@ export function applyLoadProgress(
}
return { ...state, files: { ...state.files, [message.file]: next } };
}
// ---------------------------------------------------------------------------
// Transcription progress accumulation (pure; drives the in-progress UI)
// ---------------------------------------------------------------------------
/**
* Maps the streamer's window-relative timestamps to absolute audio progress.
*
* The chunked ASR pipeline slices audio into windows of `windowSeconds`,
* advancing by `jumpSeconds` (window − 2·stride) per window, and runs one
* generate pass per window. Whisper's timestamp tokens — surfaced via
* `WhisperTextStreamer`'s `on_chunk_start`/`on_chunk_end` — are relative to
* the current window's start, and `on_finalize` fires when a window's pass
* ends. This tracker turns those into absolute seconds that are monotone
* (windows overlap, so a later window's early timestamps can precede an
* earlier window's late ones — regressions are swallowed) and clamped to
* [0, totalSeconds].
*/
export class TranscribeProgressTracker {
private windowIndex = 0;
private reported = 0;
constructor(
private readonly totalSeconds: number,
private readonly windowSeconds: number,
private readonly jumpSeconds: number,
) {}
/**
* Absolute progress for a window-relative timestamp, or null when it does
* not advance past what was already reported.
*/
timestamp(withinWindowSeconds: number): number | null {
const clamped = Math.min(Math.max(withinWindowSeconds, 0), this.windowSeconds);
return this.advance(this.windowIndex * this.jumpSeconds + clamped);
}
/**
* The current window's generate pass finished; progress moves to the start
* of the next window (the amount of audio no later window revisits).
*/
finalizeWindow(): number | null {
this.windowIndex += 1;
return this.advance(this.windowIndex * this.jumpSeconds);
}
private advance(processedSeconds: number): number | null {
const clamped = Math.min(processedSeconds, this.totalSeconds);
if (clamped <= this.reported) return null;
this.reported = clamped;
return clamped;
}
}