From 19318a56f248a1b8c6ac01daf5563cd92c50f063 Mon Sep 17 00:00:00 2001 From: Lucas Winther Date: Thu, 29 Oct 2026 21:46:16 +0100 Subject: [PATCH] Add the long-form transcription pipeline behind transcribeLongForm Co-Authored-By: Claude Opus 5.5 --- src/worker/longform/index.ts | 12 ++ src/worker/longform/pipeline.ts | 166 +++++++++++++++++++++++ src/worker/longform/types.ts | 83 ++++++++++++ tests/longform-pipeline.test.ts | 230 ++++++++++++++++++++++++++++++++ 4 files changed, 491 insertions(+) create mode 100644 src/worker/longform/index.ts create mode 100644 src/worker/longform/pipeline.ts create mode 100644 src/worker/longform/types.ts create mode 100644 tests/longform-pipeline.test.ts diff --git a/src/worker/longform/index.ts b/src/worker/longform/index.ts new file mode 100644 index 0000000..4c7094d --- /dev/null +++ b/src/worker/longform/index.ts @@ -0,0 +1,12 @@ +/** Long-form file transcription pipeline (D-WS3a). See pipeline.ts. */ +export { DEFAULT_GUARD, transcribeLongForm } from "./pipeline"; +export { DEFAULT_FALLBACK, type FallbackOptions } from "./quality"; +export type { + DecodeRequest, + DecodeResult, + LongFormDeps, + LongFormOptions, + LongFormResult, + RepetitionGuard, +} from "./types"; +export { DEFAULT_VAD_OPTIONS, type VadOptions } from "./vad"; diff --git a/src/worker/longform/pipeline.ts b/src/worker/longform/pipeline.ts new file mode 100644 index 0000000..3e3417a --- /dev/null +++ b/src/worker/longform/pipeline.ts @@ -0,0 +1,166 @@ +/** + * Long-form file transcription (D-WS3a), mirroring faster-whisper's batched + * pipeline without the batching: Silero VAD drops silence and cuts the audio + * on speech boundaries into chunks ≤ 30 s; each chunk is one Whisper decode + * (no overlapping windows to stitch); the language is detected once on the + * first speech chunk and pinned; an optional quality check retries a bad + * decode at higher temperatures. + * + * Memory (plan 001-0 Review Focus 1): VAD reads the caller's buffer in + * place; only the chunk being decoded is copied (≤ 30 s), and the only state + * that grows with the file is the transcript itself. + */ + +import type { ProgressWindow, TranscriptChunk } from "../protocol"; +import { TranscribeProgressTracker } from "../protocol"; +import { compressionRatio, DEFAULT_FALLBACK, needsFallback, type FallbackOptions } from "./quality"; +import type { + DecodeRequest, + DecodeResult, + LongFormDeps, + LongFormOptions, + LongFormResult, + RepetitionGuard, +} from "./types"; +import { + chunkAudio, + chunkTimeToOriginal, + collectChunks, + getSpeechTimestamps, + SAMPLE_RATE, + type SpeechChunk, +} from "./vad"; + +/** Whisper's window: no chunk is longer. */ +const MAX_CHUNK_SECONDS = 30; + +/** No guard: Whisper's own decoding (the measured default, D-WS3a). */ +export const DEFAULT_GUARD: RepetitionGuard = { noRepeatNgramSize: 0, repetitionPenalty: 1 }; + +export async function transcribeLongForm( + audio: Float32Array, + deps: LongFormDeps, + options: LongFormOptions = {}, +): Promise { + const totalSeconds = audio.length / SAMPLE_RATE; + options.onProgress?.(0, totalSeconds); + + const probs = await deps.speechProbabilities(audio); + const spans = getSpeechTimestamps(probs, audio.length, { + ...options.vad, + maxSpeechDurationS: Math.min(options.vad?.maxSpeechDurationS ?? MAX_CHUNK_SECONDS, MAX_CHUNK_SECONDS), + }); + const chunks = collectChunks(spans, MAX_CHUNK_SECONDS * SAMPLE_RATE); + + const tracker = TranscribeProgressTracker.forWindows(totalSeconds, chunks.map(progressWindow(chunks, totalSeconds))); + const report = (processed: number | null) => { + if (processed !== null) options.onProgress?.(processed, totalSeconds); + }; + + const guard = options.guard ?? DEFAULT_GUARD; + const fallback: FallbackOptions | null = + options.fallback === false || options.fallback === undefined + ? null + : { ...DEFAULT_FALLBACK, ...options.fallback }; + let language = + options.language === undefined || options.language === "auto" ? undefined : options.language; + + const texts: string[] = []; + const outChunks: TranscriptChunk[] = []; + let decodeCalls = 0; + let generatedTokens = 0; + let speechSamples = 0; + + for (const chunk of chunks) { + const chunkPcm = chunkAudio(audio, chunk); + speechSamples += chunk.samples; + if (language === undefined && deps.detectLanguage) { + language = await deps.detectLanguage(chunkPcm); + } + const base: DecodeRequest = { + ...(language !== undefined && { language }), + ...(options.task !== undefined && { task: options.task }), + ...(options.prompt !== undefined && { prompt: options.prompt }), + temperature: 0, + noRepeatNgramSize: guard.noRepeatNgramSize, + repetitionPenalty: guard.repetitionPenalty, + ...(options.onPartial && { onPartial: options.onPartial }), + onTimestamp: (t) => report(tracker.timestamp(t)), + }; + const attempts = fallback?.temperatures ?? [0]; + const tried: { result: DecodeResult; ratio: number }[] = []; + let chosen: DecodeResult | null = null; + for (const temperature of attempts) { + const result = await deps.decode(chunkPcm, { ...base, temperature }); + decodeCalls += 1; + generatedTokens += result.generatedTokens; + if (!fallback) { + chosen = result; + break; + } + const ratio = await compressionRatio(result.text); + if (!needsFallback({ compressionRatio: ratio, avgLogProb: result.avgLogProb }, fallback)) { + chosen = result; + break; + } + tried.push({ result, ratio }); + } + chosen ??= bestAttempt(tried, fallback!); + + const text = chosen.text.trim(); + if (text.length > 0) texts.push(text); + for (const segment of chosen.chunks) { + const [start, end] = segment.timestamp; + outChunks.push({ + text: segment.text, + timestamp: [ + chunkTimeToOriginal(chunk, start), + end === null ? null : chunkTimeToOriginal(chunk, end), + ], + }); + } + report(tracker.finalizeWindow()); + } + if (chunks.length === 0) report(tracker.finalizeWindow()); + + return { + text: texts.join(" "), + chunks: outChunks, + ...(language !== undefined && { language }), + noSpeech: chunks.length === 0, + speechSeconds: speechSamples / SAMPLE_RATE, + decodeCalls, + generatedTokens, + }; +} + +/** + * Every temperature failed: like faster-whisper, prefer attempts under the + * compression threshold, then the highest average log-prob (unknown counts + * as lowest), then the lowest compression ratio. + */ +function bestAttempt( + tried: { result: DecodeResult; ratio: number }[], + fallback: FallbackOptions, +): DecodeResult { + const underThreshold = tried.filter((t) => t.ratio <= fallback.compressionRatioThreshold); + const pool = underThreshold.length > 0 ? underThreshold : tried; + return pool.reduce((best, t) => { + const a = best.result.avgLogProb ?? -Infinity; + const b = t.result.avgLogProb ?? -Infinity; + if (b > a || (b === a && t.ratio < best.ratio)) return t; + return best; + }).result; +} + +function progressWindow(chunks: SpeechChunk[], totalSeconds: number) { + return (chunk: SpeechChunk, index: number): ProgressWindow => { + const next = chunks[index + 1]; + const last = chunk.spans[chunk.spans.length - 1]!; + return { + toAbsolute: (t) => chunkTimeToOriginal(chunk, t), + endSeconds: last.end / SAMPLE_RATE, + nextStartSeconds: next ? next.spans[0]!.start / SAMPLE_RATE : totalSeconds, + }; + }; +} diff --git a/src/worker/longform/types.ts b/src/worker/longform/types.ts new file mode 100644 index 0000000..b052486 --- /dev/null +++ b/src/worker/longform/types.ts @@ -0,0 +1,83 @@ +/** + * The long-form pipeline's interface (D-WS3a, coordination plan 001-0). The + * pipeline is model-agnostic: everything that runs a model comes in through + * `LongFormDeps`, so a different implementation (a future Rust→WASM build, + * or batched decoding) can replace either side without touching the other. + */ + +import type { TranscriptChunk, WhisperTask } from "../protocol"; +import type { FallbackOptions } from "./quality"; +import type { VadOptions } from "./vad"; + +/** One Whisper decode of a ≤ 30 s chunk. */ +export interface DecodeRequest { + /** Pinned language code; undefined for English-only models. */ + language?: string; + task?: WhisperTask; + /** The user's names-and-terms hint (workstream 2); undefined = none. */ + prompt?: string; + /** 0 = greedy; > 0 = sampling (temperature fallback). */ + temperature: number; + /** 0 = off. */ + noRepeatNgramSize: number; + /** 1 = off. */ + repetitionPenalty: number; + /** Streamed text fragments. */ + onPartial?(text: string): void; + /** Whisper timestamp tokens as they are generated, chunk-relative seconds. */ + onTimestamp?(chunkSeconds: number): void; +} + +export interface DecodeResult { + text: string; + /** Segments with chunk-relative timestamps. */ + chunks: TranscriptChunk[]; + /** Tokens the decoder generated (prompt excluded). */ + generatedTokens: number; + /** Mean log-probability of the generated tokens; null when not measured. */ + avgLogProb: number | null; +} + +export interface LongFormDeps { + /** Silero VAD speech probability per 512-sample window of `audio`. */ + speechProbabilities(audio: Float32Array): Promise; + decode(audio: Float32Array, request: DecodeRequest): Promise; + /** Language code for `audio`; absent for English-only models. */ + detectLanguage?(audio: Float32Array): Promise; +} + +/** Decoder-side repetition guard (D-035's proposal; D-WS3a measured it). */ +export interface RepetitionGuard { + noRepeatNgramSize: number; + repetitionPenalty: number; +} + +export interface LongFormOptions { + /** Whisper language code; undefined/"auto" detects once, then pins it. */ + language?: string; + task?: WhisperTask; + /** Passed to every decode; a no-op when undefined. */ + prompt?: string; + vad?: Partial; + guard?: RepetitionGuard; + /** Temperature fallback; `false` disables it, `{}` uses the defaults. */ + fallback?: Partial | false; + onPartial?(text: string): void; + onProgress?(processedSeconds: number, totalSeconds: number): void; +} + +export interface LongFormResult { + text: string; + /** Segments with timestamps on the original file's timeline. */ + chunks: TranscriptChunk[]; + /** The language every chunk was decoded in (detected or given). */ + language?: string; + /** VAD found no speech at all — nothing was decoded. */ + noSpeech: boolean; + /** Seconds of audio actually decoded (speech + padding). */ + speechSeconds: number; + /** Decoder calls, fallback retries included. */ + decodeCalls: number; + /** Generated tokens, fallback retries included. */ + generatedTokens: number; +} diff --git a/tests/longform-pipeline.test.ts b/tests/longform-pipeline.test.ts new file mode 100644 index 0000000..c3a69ac --- /dev/null +++ b/tests/longform-pipeline.test.ts @@ -0,0 +1,230 @@ +import { describe, expect, test } from "bun:test"; +import { + transcribeLongForm, + type DecodeRequest, + type DecodeResult, + type LongFormDeps, +} from "../src/worker/longform"; +import { SAMPLE_RATE, VAD_WINDOW_SAMPLES } from "../src/worker/longform/vad"; + +const MAX_CHUNK_SAMPLES = 30 * SAMPLE_RATE; + +/** Speech probabilities for audio of `seconds` with speech in [start, end) ranges. */ +function maskProbs(seconds: number, speech: [number, number][]): Float32Array { + const windows = Math.ceil((seconds * SAMPLE_RATE) / VAD_WINDOW_SAMPLES); + const probs = new Float32Array(windows).fill(0.02); + for (const [start, end] of speech) { + const from = Math.floor((start * SAMPLE_RATE) / VAD_WINDOW_SAMPLES); + const to = Math.min(windows, Math.floor((end * SAMPLE_RATE) / VAD_WINDOW_SAMPLES)); + for (let i = from; i < to; i++) probs[i] = 0.95; + } + return probs; +} + +interface Recorder { + deps: LongFormDeps; + decodes: { audio: Float32Array; request: DecodeRequest }[]; + vadInputs: Float32Array[]; + detectCalls: Float32Array[]; +} + +function fakeDeps( + probs: Float32Array, + decode: (audio: Float32Array, request: DecodeRequest, call: number) => DecodeResult = ( + audio, + ) => ({ + text: ` words ${audio.length}`, + chunks: [{ text: ` words ${audio.length}`, timestamp: [0, audio.length / SAMPLE_RATE] }], + generatedTokens: 5, + avgLogProb: -0.2, + }), + detect: string | null = null, +): Recorder { + const recorder: Recorder = { decodes: [], vadInputs: [], detectCalls: [], deps: null as never }; + recorder.deps = { + speechProbabilities: async (audio) => { + recorder.vadInputs.push(audio); + return probs; + }, + decode: async (audio, request) => { + recorder.decodes.push({ audio, request }); + request.onPartial?.("x"); + request.onTimestamp?.(1); + return decode(audio, request, recorder.decodes.length); + }, + ...(detect !== null && { + detectLanguage: async (audio: Float32Array) => { + recorder.detectCalls.push(audio); + return detect; + }, + }), + }; + return recorder; +} + +describe("transcribeLongForm", () => { + test("no speech → empty result, decode never called", async () => { + const audio = new Float32Array(60 * SAMPLE_RATE); + const { deps, decodes } = fakeDeps(maskProbs(60, [])); + const progress: number[] = []; + const result = await transcribeLongForm(audio, deps, { + onProgress: (processed) => progress.push(processed), + }); + expect(decodes).toHaveLength(0); + expect(result.text).toBe(""); + expect(result.chunks).toEqual([]); + expect(result.noSpeech).toBe(true); + expect(progress.at(-1)).toBe(60); + }); + + test("speech regions are decoded per chunk and timestamps mapped to the file", async () => { + // Two 5 s regions 15 s apart → one ≤ 30 s chunk holding both spans. + const audio = new Float32Array(40 * SAMPLE_RATE); + const { deps, decodes } = fakeDeps(maskProbs(40, [[5, 10], [25, 30]]), (chunk) => ({ + text: " hello there", + chunks: [ + { text: " hello", timestamp: [0.5, 2] }, + { text: " there", timestamp: [6.5, 8] }, + ], + generatedTokens: 4, + avgLogProb: -0.1, + })); + const result = await transcribeLongForm(audio, deps, {}); + expect(decodes).toHaveLength(1); + // Window-aligned spans padded by 400 ms: [73 472, 166 144) and + // [393 472, 486 144) → 2 × 92 672 samples, silence between them dropped. + expect(decodes[0]!.audio.length).toBe(2 * 92_672); + expect(result.text).toBe("hello there"); + expect(result.noSpeech).toBe(false); + // The chunk starts at 4.592 s; 6.5 s into the chunk is past the first + // span (5.792 s long) → 0.708 s into the second, which starts at 24.592 s. + expect(result.chunks[0]!.timestamp[0]).toBeCloseTo(4.592 + 0.5, 3); + expect(result.chunks[1]!.timestamp[0]).toBeCloseTo(24.592 + 0.708, 3); + expect(result.chunks[1]!.timestamp[0]).toBeGreaterThan(24); + expect(result.chunks[1]!.timestamp[1]!).toBeLessThan(30.5); + }); + + test("auto language: detected once on the first speech chunk, then pinned", async () => { + const audio = new Float32Array(100 * SAMPLE_RATE); + const recorder = fakeDeps(maskProbs(100, [[0, 25], [40, 65], [75, 99]]), undefined, "da"); + const result = await transcribeLongForm(audio, recorder.deps, { language: "auto" }); + expect(recorder.detectCalls).toHaveLength(1); + expect(recorder.detectCalls[0]).toBe(recorder.decodes[0]!.audio); + expect(recorder.decodes.length).toBeGreaterThanOrEqual(3); + for (const { request } of recorder.decodes) expect(request.language).toBe("da"); + expect(result.language).toBe("da"); + }); + + test("an explicit language skips detection; English-only models send none", async () => { + const audio = new Float32Array(20 * SAMPLE_RATE); + const pinned = fakeDeps(maskProbs(20, [[2, 10]]), undefined, "de"); + await transcribeLongForm(audio, pinned.deps, { language: "en" }); + expect(pinned.detectCalls).toHaveLength(0); + expect(pinned.decodes[0]!.request.language).toBe("en"); + + const englishOnly = fakeDeps(maskProbs(20, [[2, 10]])); + await transcribeLongForm(audio, englishOnly.deps, {}); + expect(englishOnly.decodes[0]!.request.language).toBeUndefined(); + }); + + test("task, prompt and the repetition guard reach every decode", async () => { + const audio = new Float32Array(100 * SAMPLE_RATE); + const recorder = fakeDeps(maskProbs(100, [[0, 25], [40, 65], [75, 99]])); + await transcribeLongForm(audio, recorder.deps, { + task: "translate", + prompt: "Voices of the Void", + guard: { noRepeatNgramSize: 5, repetitionPenalty: 1.1 }, + }); + expect(recorder.decodes.length).toBeGreaterThanOrEqual(3); + for (const { request } of recorder.decodes) { + expect(request.task).toBe("translate"); + expect(request.prompt).toBe("Voices of the Void"); + expect(request.noRepeatNgramSize).toBe(5); + expect(request.repetitionPenalty).toBe(1.1); + expect(request.temperature).toBe(0); + } + }); + + test("fallback: a looping decode is retried at the next temperature", async () => { + const audio = new Float32Array(20 * SAMPLE_RATE); + const loop = " the main clar,".repeat(40); + const recorder = fakeDeps(maskProbs(20, [[2, 10]]), (_audio, request) => ({ + text: request.temperature === 0 ? loop : " a fine sentence about festivals", + chunks: [], + generatedTokens: request.temperature === 0 ? 400 : 8, + avgLogProb: -0.2, + })); + const result = await transcribeLongForm(audio, recorder.deps, { fallback: {} }); + expect(recorder.decodes.map((d) => d.request.temperature)).toEqual([0, 0.2]); + expect(result.text).toBe("a fine sentence about festivals"); + expect(result.generatedTokens).toBe(408); + }); + + test("fallback: when every temperature fails, the best attempt is kept", async () => { + const audio = new Float32Array(20 * SAMPLE_RATE); + const recorder = fakeDeps(maskProbs(20, [[2, 10]]), (_audio, request) => ({ + text: request.temperature === 0.6 ? " ok ok ok ok ok ok ok ok ok ok ok ok ok" : " la".repeat(80), + chunks: [], + generatedTokens: 10, + avgLogProb: null, + })); + const result = await transcribeLongForm(audio, recorder.deps, { fallback: {} }); + expect(recorder.decodes).toHaveLength(6); + expect(result.text).toBe("ok ok ok ok ok ok ok ok ok ok ok ok ok"); + }); + + test("fallback off → exactly one decode per chunk", async () => { + const audio = new Float32Array(20 * SAMPLE_RATE); + const recorder = fakeDeps(maskProbs(20, [[2, 10]]), () => ({ + text: " la".repeat(80), + chunks: [], + generatedTokens: 10, + avgLogProb: -3, + })); + await transcribeLongForm(audio, recorder.deps, { fallback: false }); + expect(recorder.decodes).toHaveLength(1); + }); + + test("progress is monotone and ends at the total; partials are forwarded", async () => { + const audio = new Float32Array(100 * SAMPLE_RATE); + const recorder = fakeDeps(maskProbs(100, [[0, 25], [40, 65], [75, 90]])); + const progress: number[] = []; + const partials: string[] = []; + await transcribeLongForm(audio, recorder.deps, { + onProgress: (processed, total) => { + expect(total).toBe(100); + progress.push(processed); + }, + onPartial: (text) => partials.push(text), + }); + expect(progress[0]).toBe(0); + expect(progress.at(-1)).toBe(100); + for (let i = 1; i < progress.length; i++) { + expect(progress[i]!).toBeGreaterThan(progress[i - 1]!); + } + expect(partials.length).toBe(recorder.decodes.length); + }); + + test("90-minute input: VAD sees the caller's buffer, decode sees ≤ 30 s chunks", async () => { + const seconds = 90 * 60; + const audio = new Float32Array(seconds * SAMPLE_RATE); + const speech: [number, number][] = []; + for (let t = 0; t < seconds; t += 25) speech.push([t, t + 20]); + const recorder = fakeDeps(maskProbs(seconds, speech)); + const started = performance.now(); + const result = await transcribeLongForm(audio, recorder.deps, {}); + expect(performance.now() - started).toBeLessThan(5_000); + expect(recorder.vadInputs).toHaveLength(1); + expect(recorder.vadInputs[0]).toBe(audio); // no copy before VAD + let decoded = 0; + for (const { audio: chunk } of recorder.decodes) { + expect(chunk.length).toBeLessThanOrEqual(MAX_CHUNK_SAMPLES); + expect(chunk.buffer).not.toBe(audio.buffer); + decoded += chunk.length; + } + // Every speech second is decoded once (padding included), silence never. + expect(decoded).toBeLessThan(audio.length); + expect(result.speechSeconds).toBeCloseTo(decoded / SAMPLE_RATE, 3); + expect(result.decodeCalls).toBe(recorder.decodes.length); + }); +});