Add compression-ratio and log-prob quality checks for decode fallback
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
221ac594eb
commit
ad2eb10a28
2 files changed
+88
No files matched your search
@@ -0,0 +1,46 @@
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import { deflateSync } from "node:zlib";
|
||||
import {
|
||||
compressionRatio,
|
||||
DEFAULT_FALLBACK,
|
||||
needsFallback,
|
||||
} from "../src/worker/longform/quality";
|
||||
|
||||
describe("compressionRatio (faster-whisper get_compression_ratio)", () => {
|
||||
test("matches zlib.compress on the same bytes", async () => {
|
||||
const text = "The problem I find with AI is that it is probably a bit overhyped.";
|
||||
const bytes = new TextEncoder().encode(text);
|
||||
expect(await compressionRatio(text)).toBeCloseTo(
|
||||
bytes.length / deflateSync(bytes).length,
|
||||
5,
|
||||
);
|
||||
});
|
||||
|
||||
test("a repetition loop compresses far above 2.4, normal speech below", async () => {
|
||||
expect(await compressionRatio("the main clar, ".repeat(40))).toBeGreaterThan(2.4);
|
||||
expect(
|
||||
await compressionRatio(
|
||||
"På Aarhus Festuges sekretariat har vi løbet rigtig stærkt de sidste måneder.",
|
||||
),
|
||||
).toBeLessThan(2.4);
|
||||
});
|
||||
|
||||
test("empty text is not a loop", async () => {
|
||||
expect(await compressionRatio("")).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe("needsFallback", () => {
|
||||
test("uses the spec thresholds by default", () => {
|
||||
expect(DEFAULT_FALLBACK.compressionRatioThreshold).toBe(2.4);
|
||||
expect(DEFAULT_FALLBACK.logProbThreshold).toBe(-1.0);
|
||||
expect(DEFAULT_FALLBACK.temperatures).toEqual([0, 0.2, 0.4, 0.6, 0.8, 1.0]);
|
||||
});
|
||||
|
||||
test("rejects a high compression ratio or a low average log-prob", () => {
|
||||
expect(needsFallback({ compressionRatio: 3.1, avgLogProb: -0.2 })).toBe(true);
|
||||
expect(needsFallback({ compressionRatio: 1.5, avgLogProb: -1.5 })).toBe(true);
|
||||
expect(needsFallback({ compressionRatio: 1.5, avgLogProb: -0.3 })).toBe(false);
|
||||
expect(needsFallback({ compressionRatio: 1.5, avgLogProb: null })).toBe(false);
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user