From 8ac81baf540ae380083c7cefcebe6ad56450e16f Mon Sep 17 00:00:00 2001 From: Lucas Winther Date: Thu, 29 Oct 2026 23:34:13 +0100 Subject: [PATCH] Tick plan 001-3 steps already done before the pause Co-Authored-By: Claude Opus 5.5 --- docs/plans/001-3-longform-pipeline.md | 38 +++++++++++++-------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/docs/plans/001-3-longform-pipeline.md b/docs/plans/001-3-longform-pipeline.md index 108f87d..7439de2 100644 --- a/docs/plans/001-3-longform-pipeline.md +++ b/docs/plans/001-3-longform-pipeline.md @@ -5,7 +5,7 @@ status: in-progress # 001-3 — Browser long-form pipeline Implementation Plan > **For agentic workers:** REQUIRED SUB-SKILL: superpowers:executing-plans (inline, no subagents — -> coordination plan `001-0`). Steps use checkbox (`- [ ]`) syntax for tracking. +> coordination plan `001-0`). Steps use checkbox (`- [x]`) syntax for tracking. **Goal:** a model-agnostic long-form file-transcription pipeline in the inference worker (Silero VAD → speech chunks ≤ 30 s → repetition guard → optional temperature fallback → language @@ -88,14 +88,14 @@ browser pipeline", "A file-mode accuracy gate", "Measurement fallback". Coordina - `chunkTimeToOriginal(chunk: SpeechChunk, seconds: number): number` — chunk-relative → absolute seconds (piecewise over spans; a time exactly on a splice maps to the earlier span's end). -- [ ] Tests: probs all < 0.35 → `[]`; one burst (windows 10–40 at 0.9) → one span padded by 6400 +- [x] Tests: probs all < 0.35 → `[]`; one burst (windows 10–40 at 0.9) → one span padded by 6400 samples each side, clamped to [0, len]; two bursts 1 s apart (< 2 s min silence) → one span; two bursts 3 s apart → two spans, pad split when gap < 2·pad; continuous speech of 70 s with `maxSpeechDurationS 30` → spans ≤ 30 s; `collectChunks` packs 12 s+12 s+12 s into [24 s, 12 s]; `chunkTimeToOriginal` maps 0 → first span start and a time past span 1's length into span 2; `chunkAudio` length equals `chunk.samples`. -- [ ] Run `bun test tests/longform-vad.test.ts` → FAIL (module missing); implement; → PASS. -- [ ] Commit "Add pure VAD speech-timestamp and chunking functions for the long-form pipeline". +- [x] Run `bun test tests/longform-vad.test.ts` → FAIL (module missing); implement; → PASS. +- [x] Commit "Add pure VAD speech-timestamp and chunking functions for the long-form pipeline". ### Task 2: Progress tracker for variable windows @@ -108,10 +108,10 @@ current window's `toAbsolute`; `finalizeWindow()` advances to that window's `nex (skipped silence counts as processed; last window → total). Existing fixed-stride constructor and its tests unchanged. -- [ ] Tests: windows `[0–20 s]`, `[35–60 s]` (with a gap): `timestamp(5)` → 5; `finalizeWindow()` → +- [x] Tests: windows `[0–20 s]`, `[35–60 s]` (with a gap): `timestamp(5)` → 5; `finalizeWindow()` → 35; `timestamp(10)` in window 2 → 45; regressions swallowed; final `finalizeWindow()` → total; timestamps beyond a window clamp to its `endSeconds`. -- [ ] FAIL → implement → PASS → commit "Let the transcribe progress tracker follow variable-length windows". +- [x] FAIL → implement → PASS → commit "Let the transcribe progress tracker follow variable-length windows". ### Task 3: Quality checks (`quality.ts`) @@ -123,9 +123,9 @@ its tests unchanged. logProbThreshold: -1.0 }`; `needsFallback(result: { compressionRatio: number; avgLogProb: number | null }, thresholds): boolean` (null log-prob never triggers). -- [ ] Tests: a 40× repeated phrase → ratio > 2.4 and `needsFallback` true; a normal English +- [x] Tests: a 40× repeated phrase → ratio > 2.4 and `needsFallback` true; a normal English sentence → ratio < 2.4; avgLogProb −1.5 → true; −0.3 and null → false. -- [ ] FAIL → implement → PASS → commit "Add compression-ratio and log-prob quality checks for decode fallback". +- [x] FAIL → implement → PASS → commit "Add compression-ratio and log-prob quality checks for decode fallback". ### Task 4: `transcribeLongForm` (pure orchestration) @@ -153,7 +153,7 @@ interface LongFormResult { text: string; chunks: TranscriptChunk[] /* absolute * `DEFAULT_GUARD` and the default `fallback` are set from Task 6's measurements (start: guard `{ noRepeatNgramSize: 0, repetitionPenalty: 1 }`, fallback off). -- [ ] Tests (fake deps; probs built from a speech mask): +- [x] Tests (fake deps; probs built from a speech mask): - `no speech → empty result, decode never called` (`noSpeech: true`, text "", progress reaches total). - two speech regions 10 s apart → two decodes, chunk timestamps mapped to absolute seconds. - `language "auto"/undefined + detectLanguage → called once with the first chunk, its result @@ -167,9 +167,9 @@ interface LongFormResult { text: string; chunks: TranscriptChunk[] /* absolute * Float32Array, speech mask alternating 20 s speech / 5 s silence; assert `speechProbabilities` got the identical object, every decode input ≤ 480 000 samples, decode calls ≈ speech/≤30 s, finishes < 5 s. -- [ ] FAIL → implement (`pipeline.ts`: VAD → spans → `collectChunks(…, 30 s)` → per chunk: +- [x] FAIL → implement (`pipeline.ts`: VAD → spans → `collectChunks(…, 30 s)` → per chunk: `chunkAudio`, decode with fallback loop, map chunks, progress via `forWindows`) → PASS. -- [ ] Commit "Add the long-form transcription pipeline behind transcribeLongForm". +- [x] Commit "Add the long-form transcription pipeline behind transcribeLongForm". ### Task 5: Transformers.js adapters (`transformers.ts`) @@ -186,22 +186,22 @@ interface LongFormResult { text: string; chunks: TranscriptChunk[] /* absolute * do_sample: temperature > 0, temperature, streamer, logits_processor})` with a fresh logits list whose last processor records log-probs and token count; `detectLanguage` runs one decoder step from `<|startoftranscript|>` with all non-language logits masked. -- [ ] Tests: `localOnly load with an empty cache rejects without calling fetch` (temp `env.cacheDir`, +- [x] Tests: `localOnly load with an empty cache rejects without calling fetch` (temp `env.cacheDir`, `globalThis.fetch` spy). Real-VAD tests (skip if neither cached nor reachable, download under the lock): 10 s of zeros, a 440 Hz tone and white noise → `getSpeechTimestamps` returns `[]`; `en_aiproblem` → spans covering ≥ 80 % of its speech. -- [ ] FAIL → implement → PASS → commit "Run Silero VAD and Whisper decodes for the long-form pipeline on Transformers.js". +- [x] FAIL → implement → PASS → commit "Run Silero VAD and Whisper decodes for the long-form pipeline on Transformers.js". ### Task 6: Meeting fixture, measurement, and tuning **Files:** Create `tests/meetingFixture.ts`, `tests/clips.ts` (ffmpeg decode to 16 kHz + WER normalize), `scripts/measure-longform.ts`. -- [ ] `buildMeetingFixture()`: 2 s lead-in, en_aiproblem, then three en_alien excerpts cut at +- [x] `buildMeetingFixture()`: 2 s lead-in, en_aiproblem, then three en_alien excerpts cut at reference-line boundaries (times read once from a timestamped turbo transcript, hard-coded with their line ranges), pauses `[3, 8, 2, 5]` s filled with deterministic −60 dBFS noise. Test (fast suite, skips without ffmpeg): length = sum of parts, reference non-empty, deterministic. -- [ ] `scripts/measure-longform.ts [--clips …] [--config name]` prints per clip: wall-clock, WER, +- [x] `scripts/measure-longform.ts [--clips …] [--config name]` prints per clip: wall-clock, WER, generated tokens, decode calls — for `old` (worker's current options) and long-form configs. Clips: en_alien, da_festuge with `da`, da_festuge auto, en_aiproblem, meeting (synthetic). - [ ] Run under the lock, old/new interleaved: baseline; long-form guard off; guard variants @@ -220,14 +220,14 @@ cache only); `TranscribeMessage.longForm?: boolean`; pure `choosePipeline({ long "longform" | "chunked"`; `EngineTranscribeOptions.longForm?: boolean`; `emptyTranscriptNotice(result): string | null` → "No speech was found in this recording." for an empty finished transcript. -- [ ] Tests: guards accept/reject `vad`/`longForm` types; `choosePipeline` → chunked when VAD not +- [x] Tests: guards accept/reject `vad`/`longForm` types; `choosePipeline` → chunked when VAD not ready or not requested; BrowserEngine forwards `longForm`; `emptyTranscriptNotice`. -- [ ] Worker: after the Whisper pipeline is ready, if `vad` load Silero (`localOnly: +- [x] Worker: after the Whisper pipeline is ready, if `vad` load Silero (`localOnly: navigator.onLine === false`), failures logged and leave `vadReady` false; transcribe dispatches per `choosePipeline`; long-form posts the same `partial` / `transcribe-progress` / `result`. -- [ ] UI per Task 6's rollout outcome: default-on, or an "Advanced" disclosure with a plain-language +- [x] UI per Task 6's rollout outcome: default-on, or an "Advanced" disclosure with a plain-language checkbox (off by default, remembered in localStorage) that also sets `vad` on loads. -- [ ] typecheck, fast suite, build → commit "Run browser file transcription through the long-form pipeline when the VAD model is available". +- [x] typecheck, fast suite, build → commit "Run browser file transcription through the long-form pipeline when the VAD model is available". ### Task 8: File-mode accuracy gate