Lucas WintherandClaude Opus 5
5b0f73ae0e
Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
65 lines
2.2 KiB
TypeScript
65 lines
2.2 KiB
TypeScript
import { describe, expect, test } from "bun:test";
|
|
import {
|
|
normalizeText,
|
|
tokenize,
|
|
tokenizeQuery,
|
|
tokenizeWithOffsets,
|
|
} from "../../src/search/tokenize.ts";
|
|
|
|
describe("normalizeText", () => {
|
|
test("lowercases and strips diacritics", () => {
|
|
expect(normalizeText("Müller")).toBe("muller");
|
|
expect(normalizeText("Café")).toBe("cafe");
|
|
expect(normalizeText("RÉSUMÉ")).toBe("resume");
|
|
});
|
|
|
|
test("leaves plain ascii untouched apart from case", () => {
|
|
expect(normalizeText("Budget Approval")).toBe("budget approval");
|
|
});
|
|
});
|
|
|
|
describe("tokenize", () => {
|
|
test("splits on non-alphanumeric boundaries", () => {
|
|
expect(tokenize("Approval of Q3 budget!")).toEqual(["approval", "of", "q3", "budget"]);
|
|
});
|
|
|
|
test("returns an empty array for empty or punctuation-only input", () => {
|
|
expect(tokenize("")).toEqual([]);
|
|
expect(tokenize("--- ...")).toEqual([]);
|
|
});
|
|
|
|
test("preserves non-latin scripts", () => {
|
|
expect(tokenize("Привет мир")).toEqual(["привет", "мир"]);
|
|
});
|
|
});
|
|
|
|
describe("tokenizeQuery", () => {
|
|
test("strips English stopwords from a natural-language query", () => {
|
|
expect(tokenizeQuery("what did we decide about hiring")).toEqual(["decide", "hiring"]);
|
|
expect(tokenizeQuery("approval of the budget")).toEqual(["approval", "budget"]);
|
|
});
|
|
|
|
test("keeps stopwords when nothing else would survive", () => {
|
|
expect(tokenizeQuery("the who")).toEqual(["the", "who"]);
|
|
});
|
|
|
|
test("leaves a stopword-free query untouched", () => {
|
|
expect(tokenizeQuery("budget approval")).toEqual(["budget", "approval"]);
|
|
});
|
|
});
|
|
|
|
describe("tokenizeWithOffsets", () => {
|
|
test("reports offsets into the original text, not the normalized text", () => {
|
|
const tokens = tokenizeWithOffsets("Der Müller Bericht");
|
|
expect(tokens.map((t) => t.value)).toEqual(["der", "muller", "bericht"]);
|
|
expect(tokens.map((t) => t.offset)).toEqual([0, 4, 11]);
|
|
});
|
|
|
|
test("offsets index back into the source string correctly", () => {
|
|
const source = "Approval of Q3 budget";
|
|
const tokens = tokenizeWithOffsets(source);
|
|
const budget = tokens.find((t) => t.value === "budget")!;
|
|
expect(source.slice(budget.offset, budget.offset + 6)).toBe("budget");
|
|
});
|
|
});
|