Add compression-ratio and log-prob quality checks for decode fallback

Co-Authored-By: Claude Opus 5.5 <[email protected]>

src/worker/longform/quality.ts

@@ -0,0 +1,42 @@
+/**
+ * Per-chunk decode quality checks for the long-form pipeline's temperature
+ * fallback (D-WS3a) — faster-whisper's `generate_with_fallback` criteria.
+ */
+
+export interface FallbackOptions {
+  /** Decode temperatures in order; the first is the normal (greedy) pass. */
+  temperatures: number[];
+  /** Reject a decode whose text compresses better than this (a loop). */
+  compressionRatioThreshold: number;
+  /** Reject a decode whose average token log-probability is below this. */
+  logProbThreshold: number;
+}
+
+/** faster-whisper / OpenAI Whisper defaults (spec Goal 3). */
+export const DEFAULT_FALLBACK: FallbackOptions = {
+  temperatures: [0, 0.2, 0.4, 0.6, 0.8, 1.0],
+  compressionRatioThreshold: 2.4,
+  logProbThreshold: -1.0,
+};
+
+/**
+ * UTF-8 length over zlib-deflated length — `get_compression_ratio` in
+ * faster-whisper (`zlib.compress`). `CompressionStream("deflate")` is the
+ * same zlib format and exists in workers and Bun alike.
+ */
+export async function compressionRatio(text: string): Promise<number> {
+  const bytes = new TextEncoder().encode(text);
+  if (bytes.length === 0) return 0;
+  const stream = new Blob([bytes]).stream().pipeThrough(new CompressionStream("deflate"));
+  const compressed = await new Response(stream).arrayBuffer();
+  return bytes.length / compressed.byteLength;
+}
+
+/** Whether a decode should be retried at the next temperature. */
+export function needsFallback(
+  result: { compressionRatio: number; avgLogProb: number | null },
+  thresholds: Pick<FallbackOptions, "compressionRatioThreshold" | "logProbThreshold"> = DEFAULT_FALLBACK,
+): boolean {
+  if (result.compressionRatio > thresholds.compressionRatioThreshold) return true;
+  return result.avgLogProb !== null && result.avgLogProb < thresholds.logProbThreshold;
+}

tests/longform-quality.test.ts

@@ -0,0 +1,46 @@
+import { describe, expect, test } from "bun:test";
+import { deflateSync } from "node:zlib";
+import {
+  compressionRatio,
+  DEFAULT_FALLBACK,
+  needsFallback,
+} from "../src/worker/longform/quality";
+
+describe("compressionRatio (faster-whisper get_compression_ratio)", () => {
+  test("matches zlib.compress on the same bytes", async () => {
+    const text = "The problem I find with AI is that it is probably a bit overhyped.";
+    const bytes = new TextEncoder().encode(text);
+    expect(await compressionRatio(text)).toBeCloseTo(
+      bytes.length / deflateSync(bytes).length,
+      5,
+    );
+  });
+
+  test("a repetition loop compresses far above 2.4, normal speech below", async () => {
+    expect(await compressionRatio("the main clar, ".repeat(40))).toBeGreaterThan(2.4);
+    expect(
+      await compressionRatio(
+        "På Aarhus Festuges sekretariat har vi løbet rigtig stærkt de sidste måneder.",
+      ),
+    ).toBeLessThan(2.4);
+  });
+
+  test("empty text is not a loop", async () => {
+    expect(await compressionRatio("")).toBe(0);
+  });
+});
+
+describe("needsFallback", () => {
+  test("uses the spec thresholds by default", () => {
+    expect(DEFAULT_FALLBACK.compressionRatioThreshold).toBe(2.4);
+    expect(DEFAULT_FALLBACK.logProbThreshold).toBe(-1.0);
+    expect(DEFAULT_FALLBACK.temperatures).toEqual([0, 0.2, 0.4, 0.6, 0.8, 1.0]);
+  });
+
+  test("rejects a high compression ratio or a low average log-prob", () => {
+    expect(needsFallback({ compressionRatio: 3.1, avgLogProb: -0.2 })).toBe(true);
+    expect(needsFallback({ compressionRatio: 1.5, avgLogProb: -1.5 })).toBe(true);
+    expect(needsFallback({ compressionRatio: 1.5, avgLogProb: -0.3 })).toBe(false);
+    expect(needsFallback({ compressionRatio: 1.5, avgLogProb: null })).toBe(false);
+  });
+});