Add cancel + progress UI, EngineToggle, and Docker deployment

This commit is contained in:
Lucas Winther
2026-07-14 19:32:10 +02:00
parent be82ecfe57
commit 0084e0435a
16 changed files with 972 additions and 60 deletions
+41 -3
View File
@@ -19,6 +19,7 @@ import { resolveDefaultDtype } from "../lib/models";
import {
deviceAttempts,
isUiToWorkerMessage,
TranscribeProgressTracker,
type DeviceCapabilities,
type DevicePreference,
type InferenceDevice,
@@ -209,6 +210,12 @@ function load(message: LoadMessage): Promise<AutomaticSpeechRecognitionPipeline>
// Transcription
// ---------------------------------------------------------------------------
/** Audio sample rate the whole app standardizes on (D-012). */
const SAMPLE_RATE = 16_000;
/** Chunked-pipeline window / overlap, in seconds (Whisper's usual 30/5). */
const CHUNK_LENGTH_S = 30;
const STRIDE_LENGTH_S = 5;
async function transcribe(message: TranscribeMessage): Promise<void> {
if (active === null) {
post({
@@ -222,11 +229,39 @@ async function transcribe(message: TranscribeMessage): Promise<void> {
const asr = await active.promise;
const options: Record<string, unknown> = {
chunk_length_s: 30,
stride_length_s: 5,
chunk_length_s: CHUNK_LENGTH_S,
stride_length_s: STRIDE_LENGTH_S,
return_timestamps: true,
};
if (message.requestId === undefined) {
if (message.stream ?? message.requestId === undefined) {
// File-flow pass (id-less, or routed through BrowserEngine with an
// explicit stream flag, D-010): stream partial tokens and report audio
// progress. The streamer's chunk timestamps are window-relative; the
// tracker maps them to monotone absolute seconds (see protocol.ts).
// Live passes skip both — they resolve wholesale via "result" (D-016).
const totalSeconds = message.audio.length / SAMPLE_RATE;
const tracker = new TranscribeProgressTracker(
totalSeconds,
CHUNK_LENGTH_S,
CHUNK_LENGTH_S - 2 * STRIDE_LENGTH_S,
);
const postProgress = (processedSeconds: number | null) => {
if (processedSeconds === null) return;
post({
type: "transcribe-progress",
processedSeconds,
totalSeconds,
requestId: message.requestId,
});
};
// Immediate 0/total so the UI shows the audio length before the first
// (possibly slow) token arrives.
post({
type: "transcribe-progress",
processedSeconds: 0,
totalSeconds,
requestId: message.requestId,
});
options.streamer = new WhisperTextStreamer(
// The ASR pipeline for Whisper models always carries a WhisperTokenizer.
asr.tokenizer as ConstructorParameters<typeof WhisperTextStreamer>[0],
@@ -234,6 +269,9 @@ async function transcribe(message: TranscribeMessage): Promise<void> {
skip_prompt: true,
callback_function: (text: string) =>
post({ type: "partial", text, requestId: message.requestId }),
on_chunk_start: (time: number) => postProgress(tracker.timestamp(time)),
on_chunk_end: (time: number) => postProgress(tracker.timestamp(time)),
on_finalize: () => postProgress(tracker.finalizeWindow()),
},
);
}
+86 -1
View File
@@ -69,6 +69,14 @@ export interface TranscribeMessage {
* responses to segment passes; the file flow omits it and is unchanged.
*/
requestId?: string;
/**
* Stream partial tokens and transcribe-progress for this request. Defaults
* to `requestId === undefined` (the historical rule: id-less file-flow
* passes stream, live segment passes resolve wholesale via "result").
* `BrowserEngine` sets it explicitly so a file pass routed through the
* engine (which correlates via requestId, D-010) still streams.
*/
stream?: boolean;
}
export type UiToWorkerMessage = LoadMessage | TranscribeMessage;
@@ -108,6 +116,22 @@ export interface PartialMessage {
requestId?: string;
}
/**
* How far a file-flow pass has progressed through the audio, in seconds.
* Posted once at pass start (0 / total) and then on every Whisper segment
* boundary, so slow models/backends show movement long before the first
* partial token. Values are monotone and clamped to [0, totalSeconds]
* (see TranscribeProgressTracker). Live passes (requestId set) don't report
* progress — the live UI has its own committed/tentative feedback (D-016).
*/
export interface TranscribeProgressMessage {
type: "transcribe-progress";
processedSeconds: number;
totalSeconds: number;
/** Echo of the originating transcribe request's id, if it carried one. */
requestId?: string;
}
/** One timestamped segment of the final transcript. */
export interface TranscriptChunk {
text: string;
@@ -134,6 +158,7 @@ export interface ErrorMessage {
export type WorkerToUiMessage =
| LoadProgressMessage
| PartialMessage
| TranscribeProgressMessage
| ResultMessage
| ErrorMessage;
@@ -234,7 +259,8 @@ export function isUiToWorkerMessage(value: unknown): value is UiToWorkerMessage
(value.task === undefined ||
value.task === "transcribe" ||
value.task === "translate") &&
isOptionalString(value.requestId)
isOptionalString(value.requestId) &&
(value.stream === undefined || typeof value.stream === "boolean")
);
default:
return false;
@@ -264,6 +290,12 @@ export function isWorkerToUiMessage(value: unknown): value is WorkerToUiMessage
}
case "partial":
return typeof value.text === "string" && isOptionalString(value.requestId);
case "transcribe-progress":
return (
typeof value.processedSeconds === "number" &&
typeof value.totalSeconds === "number" &&
isOptionalString(value.requestId)
);
case "result":
return (
typeof value.text === "string" &&
@@ -331,3 +363,56 @@ export function applyLoadProgress(
}
return { ...state, files: { ...state.files, [message.file]: next } };
}
// ---------------------------------------------------------------------------
// Transcription progress accumulation (pure; drives the in-progress UI)
// ---------------------------------------------------------------------------
/**
* Maps the streamer's window-relative timestamps to absolute audio progress.
*
* The chunked ASR pipeline slices audio into windows of `windowSeconds`,
* advancing by `jumpSeconds` (window − 2·stride) per window, and runs one
* generate pass per window. Whisper's timestamp tokens — surfaced via
* `WhisperTextStreamer`'s `on_chunk_start`/`on_chunk_end` — are relative to
* the current window's start, and `on_finalize` fires when a window's pass
* ends. This tracker turns those into absolute seconds that are monotone
* (windows overlap, so a later window's early timestamps can precede an
* earlier window's late ones — regressions are swallowed) and clamped to
* [0, totalSeconds].
*/
export class TranscribeProgressTracker {
private windowIndex = 0;
private reported = 0;
constructor(
private readonly totalSeconds: number,
private readonly windowSeconds: number,
private readonly jumpSeconds: number,
) {}
/**
* Absolute progress for a window-relative timestamp, or null when it does
* not advance past what was already reported.
*/
timestamp(withinWindowSeconds: number): number | null {
const clamped = Math.min(Math.max(withinWindowSeconds, 0), this.windowSeconds);
return this.advance(this.windowIndex * this.jumpSeconds + clamped);
}
/**
* The current window's generate pass finished; progress moves to the start
* of the next window (the amount of audio no later window revisits).
*/
finalizeWindow(): number | null {
this.windowIndex += 1;
return this.advance(this.windowIndex * this.jumpSeconds);
}
private advance(processedSeconds: number): number | null {
const clamped = Math.min(processedSeconds, this.totalSeconds);
if (clamped <= this.reported) return null;
this.reported = clamped;
return clamped;
}
}