From e0a4b2796f86b26b42a091b58253f369c9663aa1 Mon Sep 17 00:00:00 2001 From: Lucas Winther Date: Thu, 29 Oct 2026 21:43:18 +0100 Subject: [PATCH] Add pure VAD speech-timestamp and chunking functions for the long-form pipeline Port of faster-whisper 1.2.1 vad.py (checked identical on a random 20 000-window probability trace for both min-silence settings). Co-Authored-By: Claude Opus 5.5 --- src/worker/longform/vad.ts | 193 +++++++++++++++++++++++++++++++++++++ tests/longform-vad.test.ts | 110 +++++++++++++++++++++ 2 files changed, 303 insertions(+) create mode 100644 src/worker/longform/vad.ts create mode 100644 tests/longform-vad.test.ts diff --git a/src/worker/longform/vad.ts b/src/worker/longform/vad.ts new file mode 100644 index 0000000..ef18447 --- /dev/null +++ b/src/worker/longform/vad.ts @@ -0,0 +1,193 @@ +/** + * Speech segmentation for the long-form pipeline (D-WS3a). Pure: turns + * per-window Silero VAD speech probabilities into speech spans, and packs the + * spans into ≤ 30 s chunks cut on speech boundaries. A port of faster-whisper + * 1.2.1 `vad.py` (`get_speech_timestamps` after the model call, and the + * batched pipeline's `collect_chunks(…, max_duration=chunk_length)`), so the + * same thresholds mean the same thing on both engines. + * + * Nothing here copies audio except `chunkAudio`, which builds exactly one + * ≤ 30 s chunk when it is about to be decoded (a 90-minute file is never + * duplicated per stage — Review Focus 1 of plan 001-0). + */ + +/** Audio sample rate the whole app standardizes on (D-012). */ +export const SAMPLE_RATE = 16_000; +/** Silero VAD v5 window at 16 kHz. */ +export const VAD_WINDOW_SAMPLES = 512; + +export interface VadOptions { + /** Speech when the probability is at or above this. */ + threshold: number; + /** Silence when below this; defaults to max(threshold − 0.15, 0.01). */ + negThreshold?: number; + /** Speech spans shorter than this are dropped. */ + minSpeechDurationMs: number; + /** Spans longer than this are split (at a ≥ 98 ms pause when there is one). */ + maxSpeechDurationS: number; + /** A span ends only after this much silence. */ + minSilenceDurationMs: number; + /** Padding added on both sides of each span. */ + speechPadMs: number; +} + +/** + * faster-whisper's `VadOptions` defaults (the spec's starting point), with + * `max_speech_duration_s` at Whisper's 30 s window as in its batched pipeline. + */ +export const DEFAULT_VAD_OPTIONS: VadOptions = { + threshold: 0.5, + minSpeechDurationMs: 0, + maxSpeechDurationS: 30, + minSilenceDurationMs: 2000, + speechPadMs: 400, +}; + +/** A run of samples, `end` exclusive. */ +export interface SampleSpan { + start: number; + end: number; +} + +/** + * Speech spans (in samples) from per-window speech probabilities — + * `probs[i]` covers samples [i·512, (i+1)·512). + */ +export function getSpeechTimestamps( + probs: ArrayLike, + audioLength: number, + options: Partial = {}, +): SampleSpan[] { + const o = { ...DEFAULT_VAD_OPTIONS, ...options }; + const w = VAD_WINDOW_SAMPLES; + const minSpeechSamples = (SAMPLE_RATE * o.minSpeechDurationMs) / 1000; + const speechPadSamples = (SAMPLE_RATE * o.speechPadMs) / 1000; + const maxSpeechSamples = SAMPLE_RATE * o.maxSpeechDurationS - w - 2 * speechPadSamples; + const minSilenceSamples = (SAMPLE_RATE * o.minSilenceDurationMs) / 1000; + const minSilenceSamplesAtMaxSpeech = (SAMPLE_RATE * 98) / 1000; + const negThreshold = o.negThreshold ?? Math.max(o.threshold - 0.15, 0.01); + + let triggered = false; + const speeches: SampleSpan[] = []; + let current: Partial = {}; + let tempEnd = 0; + let prevEnd = 0; + let nextStart = 0; + + for (let i = 0; i < probs.length; i++) { + const prob = probs[i]!; + const at = w * i; + if (prob >= o.threshold && tempEnd) { + tempEnd = 0; + if (nextStart < prevEnd) nextStart = at; + } + if (prob >= o.threshold && !triggered) { + triggered = true; + current.start = at; + continue; + } + if (triggered && at - current.start! > maxSpeechSamples) { + if (prevEnd) { + speeches.push({ start: current.start!, end: prevEnd }); + current = {}; + if (nextStart < prevEnd) triggered = false; + else current.start = nextStart; + prevEnd = nextStart = tempEnd = 0; + } else { + speeches.push({ start: current.start!, end: at }); + current = {}; + prevEnd = nextStart = tempEnd = 0; + triggered = false; + continue; + } + } + if (prob < negThreshold && triggered) { + if (!tempEnd) tempEnd = at; + if (at - tempEnd > minSilenceSamplesAtMaxSpeech) prevEnd = tempEnd; + if (at - tempEnd < minSilenceSamples) continue; + if (tempEnd - current.start! > minSpeechSamples) { + speeches.push({ start: current.start!, end: tempEnd }); + } + current = {}; + prevEnd = nextStart = tempEnd = 0; + triggered = false; + continue; + } + } + if (current.start !== undefined && audioLength - current.start > minSpeechSamples) { + speeches.push({ start: current.start, end: audioLength }); + } + + for (let i = 0; i < speeches.length; i++) { + const speech = speeches[i]!; + if (i === 0) speech.start = Math.trunc(Math.max(0, speech.start - speechPadSamples)); + const next = speeches[i + 1]; + if (next) { + const silence = next.start - speech.end; + if (silence < 2 * speechPadSamples) { + speech.end += Math.floor(silence / 2); + next.start = Math.trunc(Math.max(0, next.start - Math.floor(silence / 2))); + } else { + speech.end = Math.trunc(Math.min(audioLength, speech.end + speechPadSamples)); + next.start = Math.trunc(Math.max(0, next.start - speechPadSamples)); + } + } else { + speech.end = Math.trunc(Math.min(audioLength, speech.end + speechPadSamples)); + } + } + return speeches; +} + +/** Speech spans decoded together as one Whisper window. */ +export interface SpeechChunk { + spans: SampleSpan[]; + /** Total samples across `spans` (the chunk's audio length). */ + samples: number; +} + +/** + * Greedily pack consecutive spans into chunks of at most `maxSamples` + * (faster-whisper `collect_chunks`). Metadata only — no audio is copied. + */ +export function collectChunks(spans: readonly SampleSpan[], maxSamples: number): SpeechChunk[] { + const chunks: SpeechChunk[] = []; + let current: SpeechChunk = { spans: [], samples: 0 }; + for (const span of spans) { + const length = span.end - span.start; + if (current.spans.length > 0 && current.samples + length > maxSamples) { + chunks.push(current); + current = { spans: [], samples: 0 }; + } + current.spans.push(span); + current.samples += length; + } + if (current.spans.length > 0) chunks.push(current); + return chunks; +} + +/** The chunk's audio: its spans concatenated (one ≤ 30 s copy). */ +export function chunkAudio(audio: Float32Array, chunk: SpeechChunk): Float32Array { + const out = new Float32Array(chunk.samples); + let at = 0; + for (const span of chunk.spans) { + out.set(audio.subarray(span.start, span.end), at); + at += span.end - span.start; + } + return out; +} + +/** + * Map a chunk-relative time (seconds into the concatenated chunk audio) to + * seconds on the original timeline. A time exactly on a splice maps to the + * earlier span's end; times past the chunk clamp to its last span's end. + */ +export function chunkTimeToOriginal(chunk: SpeechChunk, seconds: number): number { + let remaining = Math.max(0, seconds) * SAMPLE_RATE; + for (const span of chunk.spans) { + const length = span.end - span.start; + if (remaining <= length) return (span.start + remaining) / SAMPLE_RATE; + remaining -= length; + } + const last = chunk.spans[chunk.spans.length - 1]; + return last ? last.end / SAMPLE_RATE : 0; +} diff --git a/tests/longform-vad.test.ts b/tests/longform-vad.test.ts new file mode 100644 index 0000000..200ca12 --- /dev/null +++ b/tests/longform-vad.test.ts @@ -0,0 +1,110 @@ +import { describe, expect, test } from "bun:test"; +import { + chunkAudio, + chunkTimeToOriginal, + collectChunks, + getSpeechTimestamps, + SAMPLE_RATE, + VAD_WINDOW_SAMPLES, +} from "../src/worker/longform/vad"; + +const W = VAD_WINDOW_SAMPLES; +const PAD = (400 / 1000) * SAMPLE_RATE; // speech_pad_ms 400 → 6400 samples + +/** Probabilities for `seconds` of audio with speech in the given [start, end) second ranges. */ +function probsFor(seconds: number, speech: [number, number][]): Float32Array { + const windows = Math.ceil((seconds * SAMPLE_RATE) / W); + const probs = new Float32Array(windows).fill(0.05); + for (const [start, end] of speech) { + const from = Math.floor((start * SAMPLE_RATE) / W); + const to = Math.floor((end * SAMPLE_RATE) / W); + for (let i = from; i < to; i++) probs[i] = 0.9; + } + return probs; +} + +describe("getSpeechTimestamps (port of faster-whisper vad.py)", () => { + test("no window above the threshold → no speech", () => { + const probs = new Float32Array(500).fill(0.3); + expect(getSpeechTimestamps(probs, 500 * W)).toEqual([]); + }); + + test("one burst is padded by speech_pad_ms on both sides", () => { + const probs = new Float32Array(200).fill(0.05); + for (let i = 50; i < 80; i++) probs[i] = 0.9; + expect(getSpeechTimestamps(probs, 200 * W)).toEqual([ + { start: 50 * W - PAD, end: 80 * W + PAD }, + ]); + }); + + test("padding is clamped to the audio", () => { + const probs = new Float32Array(40).fill(0.9); + expect(getSpeechTimestamps(probs, 40 * W)).toEqual([{ start: 0, end: 40 * W }]); + }); + + test("a pause shorter than min_silence_duration_ms does not split", () => { + const spans = getSpeechTimestamps(probsFor(20, [[2, 6], [7, 11]]), 20 * SAMPLE_RATE); + expect(spans).toHaveLength(1); + }); + + test("a pause longer than min_silence_duration_ms splits; close spans share the gap", () => { + const spans = getSpeechTimestamps(probsFor(30, [[2, 6], [9, 12]]), 30 * SAMPLE_RATE); + expect(spans).toHaveLength(2); + // 3 s gap ≥ 2·pad → each side padded by the full 400 ms. + expect(spans[0]!.end - Math.floor((6 * SAMPLE_RATE) / W) * W).toBe(PAD); + const narrow = getSpeechTimestamps(probsFor(30, [[2, 6], [8.5, 12]]), 30 * SAMPLE_RATE, { + minSilenceDurationMs: 2000, + speechPadMs: 1500, + }); + // 2.5 s gap < 2·1.5 s pad → the gap is split in half instead. + expect(narrow).toHaveLength(2); + expect(narrow[0]!.end).toBe(narrow[1]!.start); + }); + + test("continuous speech is cut at max_speech_duration_s", () => { + const spans = getSpeechTimestamps(probsFor(70, [[0, 70]]), 70 * SAMPLE_RATE, { + maxSpeechDurationS: 30, + }); + expect(spans.length).toBeGreaterThanOrEqual(3); + for (const span of spans) { + expect(span.end - span.start).toBeLessThanOrEqual(30 * SAMPLE_RATE); + } + }); +}); + +describe("collectChunks / chunk time mapping", () => { + const s = (sec: number) => sec * SAMPLE_RATE; + const spans = [ + { start: s(0), end: s(12) }, + { start: s(20), end: s(32) }, + { start: s(40), end: s(52) }, + ]; + + test("packs spans into chunks no longer than the maximum", () => { + const chunks = collectChunks(spans, s(30)); + expect(chunks.map((chunk) => chunk.samples)).toEqual([s(24), s(12)]); + expect(chunks[0]!.spans).toEqual([spans[0]!, spans[1]!]); + }); + + test("no spans → no chunks", () => { + expect(collectChunks([], s(30))).toEqual([]); + }); + + test("chunk-relative seconds map back to the original timeline", () => { + const [chunk] = collectChunks(spans, s(30)); + expect(chunkTimeToOriginal(chunk!, 0)).toBe(0); + expect(chunkTimeToOriginal(chunk!, 5)).toBe(5); + expect(chunkTimeToOriginal(chunk!, 12)).toBe(12); // splice → earlier span's end + expect(chunkTimeToOriginal(chunk!, 15)).toBe(23); + expect(chunkTimeToOriginal(chunk!, 99)).toBe(32); // clamps to the last span + }); + + test("chunkAudio copies exactly the chunk's spans", () => { + const audio = new Float32Array(s(60)).map((_, i) => i); + const [chunk] = collectChunks(spans, s(30)); + const out = chunkAudio(audio, chunk!); + expect(out.length).toBe(chunk!.samples); + expect(out[0]).toBe(0); + expect(out[s(12)]).toBe(s(20)); + }); +});