diff --git a/src/worker/longform/quality.ts b/src/worker/longform/quality.ts new file mode 100644 index 0000000..11aa2b8 --- /dev/null +++ b/src/worker/longform/quality.ts @@ -0,0 +1,42 @@ +/** + * Per-chunk decode quality checks for the long-form pipeline's temperature + * fallback (D-WS3a) — faster-whisper's `generate_with_fallback` criteria. + */ + +export interface FallbackOptions { + /** Decode temperatures in order; the first is the normal (greedy) pass. */ + temperatures: number[]; + /** Reject a decode whose text compresses better than this (a loop). */ + compressionRatioThreshold: number; + /** Reject a decode whose average token log-probability is below this. */ + logProbThreshold: number; +} + +/** faster-whisper / OpenAI Whisper defaults (spec Goal 3). */ +export const DEFAULT_FALLBACK: FallbackOptions = { + temperatures: [0, 0.2, 0.4, 0.6, 0.8, 1.0], + compressionRatioThreshold: 2.4, + logProbThreshold: -1.0, +}; + +/** + * UTF-8 length over zlib-deflated length — `get_compression_ratio` in + * faster-whisper (`zlib.compress`). `CompressionStream("deflate")` is the + * same zlib format and exists in workers and Bun alike. + */ +export async function compressionRatio(text: string): Promise { + const bytes = new TextEncoder().encode(text); + if (bytes.length === 0) return 0; + const stream = new Blob([bytes]).stream().pipeThrough(new CompressionStream("deflate")); + const compressed = await new Response(stream).arrayBuffer(); + return bytes.length / compressed.byteLength; +} + +/** Whether a decode should be retried at the next temperature. */ +export function needsFallback( + result: { compressionRatio: number; avgLogProb: number | null }, + thresholds: Pick = DEFAULT_FALLBACK, +): boolean { + if (result.compressionRatio > thresholds.compressionRatioThreshold) return true; + return result.avgLogProb !== null && result.avgLogProb < thresholds.logProbThreshold; +} diff --git a/tests/longform-quality.test.ts b/tests/longform-quality.test.ts new file mode 100644 index 0000000..7dad2d1 --- /dev/null +++ b/tests/longform-quality.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, test } from "bun:test"; +import { deflateSync } from "node:zlib"; +import { + compressionRatio, + DEFAULT_FALLBACK, + needsFallback, +} from "../src/worker/longform/quality"; + +describe("compressionRatio (faster-whisper get_compression_ratio)", () => { + test("matches zlib.compress on the same bytes", async () => { + const text = "The problem I find with AI is that it is probably a bit overhyped."; + const bytes = new TextEncoder().encode(text); + expect(await compressionRatio(text)).toBeCloseTo( + bytes.length / deflateSync(bytes).length, + 5, + ); + }); + + test("a repetition loop compresses far above 2.4, normal speech below", async () => { + expect(await compressionRatio("the main clar, ".repeat(40))).toBeGreaterThan(2.4); + expect( + await compressionRatio( + "På Aarhus Festuges sekretariat har vi løbet rigtig stærkt de sidste måneder.", + ), + ).toBeLessThan(2.4); + }); + + test("empty text is not a loop", async () => { + expect(await compressionRatio("")).toBe(0); + }); +}); + +describe("needsFallback", () => { + test("uses the spec thresholds by default", () => { + expect(DEFAULT_FALLBACK.compressionRatioThreshold).toBe(2.4); + expect(DEFAULT_FALLBACK.logProbThreshold).toBe(-1.0); + expect(DEFAULT_FALLBACK.temperatures).toEqual([0, 0.2, 0.4, 0.6, 0.8, 1.0]); + }); + + test("rejects a high compression ratio or a low average log-prob", () => { + expect(needsFallback({ compressionRatio: 3.1, avgLogProb: -0.2 })).toBe(true); + expect(needsFallback({ compressionRatio: 1.5, avgLogProb: -1.5 })).toBe(true); + expect(needsFallback({ compressionRatio: 1.5, avgLogProb: -0.3 })).toBe(false); + expect(needsFallback({ compressionRatio: 1.5, avgLogProb: null })).toBe(false); + }); +});