65 lines
2.6 KiB
TypeScript
65 lines
2.6 KiB
TypeScript
import { describe, expect, test } from "bun:test";
|
|
import { join } from "node:path";
|
|
import { fitHintToBudget, normalizeHint, PROMPT_TOKEN_BUDGET } from "../src/lib/hint";
|
|
|
|
// D-WS2a: the server engine truncates the hint in Python (server-py/sidecar.py,
|
|
// with faster-whisper's tokenizer) and the browser engine in TypeScript
|
|
// (src/lib/hint.ts, with the Transformers.js tokenizer). Both must apply the
|
|
// same rule; with the same deterministic fake tokenizer they must agree
|
|
// exactly. sidecar.py imports faster-whisper only inside main(), so the
|
|
// helpers import without it.
|
|
|
|
const SIDECAR = join(import.meta.dir, "../server-py/sidecar.py");
|
|
|
|
const CASES: { hint: string; budget: number }[] = [
|
|
{ hint: "", budget: 223 },
|
|
{ hint: " \n\t ", budget: 223 },
|
|
{ hint: "Maxwell\nVoices of the Void", budget: 223 },
|
|
{ hint: " Århus\r\n\r\nÆrø, Festuge \n", budget: 223 },
|
|
{ hint: "Max\u0000well\u0007", budget: 223 },
|
|
{ hint: "alpha, beta gamma", budget: 10 },
|
|
{ hint: "alpha; beta; gamma; delta", budget: 14 },
|
|
{ hint: "x".repeat(50), budget: 10 },
|
|
{ hint: "Maxwell, ".repeat(300), budget: 223 },
|
|
{ hint: Array.from({ length: 400 }, (_, i) => `term${i}`).join("\n"), budget: 223 },
|
|
];
|
|
|
|
/** One token per UTF-16 code unit — the same function on both sides. */
|
|
const charEncode = (text: string) => [...text].map((c) => c.charCodeAt(0));
|
|
|
|
const PYTHON = `
|
|
import importlib.util, json, sys
|
|
spec = importlib.util.spec_from_file_location("sidecar", sys.argv[1])
|
|
sidecar = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(sidecar)
|
|
encode = lambda text: [ord(c) for c in text]
|
|
cases = json.load(sys.stdin)
|
|
print(json.dumps({"budget": sidecar.PROMPT_TOKEN_BUDGET, "results": [
|
|
{"normalized": sidecar.normalize_hint(c["hint"]),
|
|
"ids": sidecar.fit_prompt_ids(c["hint"], encode, c["budget"])}
|
|
for c in cases
|
|
]}))
|
|
`;
|
|
|
|
describe("hint truncation parity: TypeScript vs sidecar.py", () => {
|
|
test("normalize_hint / fit_prompt_ids match normalizeHint / fitHintToBudget", () => {
|
|
const proc = Bun.spawnSync(["python3", "-I", "-c", PYTHON, SIDECAR], {
|
|
stdin: new TextEncoder().encode(JSON.stringify(CASES)),
|
|
stdout: "pipe",
|
|
stderr: "pipe",
|
|
});
|
|
expect(proc.stderr.toString()).toBe("");
|
|
const python = JSON.parse(proc.stdout.toString()) as {
|
|
budget: number;
|
|
results: { normalized: string; ids: number[] }[];
|
|
};
|
|
expect(python.budget).toBe(PROMPT_TOKEN_BUDGET);
|
|
expect(python.results).toEqual(
|
|
CASES.map(({ hint, budget }) => ({
|
|
normalized: normalizeHint(hint),
|
|
ids: fitHintToBudget(hint, charEncode, budget),
|
|
})),
|
|
);
|
|
});
|
|
});
|