diff --git a/src/search/tokenize.ts b/src/search/tokenize.ts new file mode 100644 index 0000000..ca351b7 --- /dev/null +++ b/src/search/tokenize.ts @@ -0,0 +1,88 @@ +/** + * Text normalization and tokenization for fuzzy search. + * + * Query text and field text must travel through an identical path; any + * divergence between the two sides silently breaks matching, so there is + * exactly one implementation of each operation here. + */ + +export interface Token { + /** Normalized token value. */ + value: string; + /** Character offset of the token in the ORIGINAL (un-normalized) text. */ + offset: number; +} + +const COMBINING_MARKS = /[̀-ͯ]/g; +const TOKEN_PATTERN = /[\p{L}\p{N}]+/gu; + +/** + * Lowercases text and strips diacritics via NFKD decomposition, so that + * "Müller" matches "muller" and "café" matches "cafe". + */ +export function normalizeText(text: string): string { + return text.normalize("NFKD").replace(COMBINING_MARKS, "").toLowerCase(); +} + +/** + * Splits text into normalized tokens, each carrying its offset in the + * original string. + * + * Token boundaries are found on the ORIGINAL text and normalization is + * applied per token, because NFKD can change string length and would + * otherwise invalidate the offsets. + */ +export function tokenizeWithOffsets(text: string): Token[] { + const tokens: Token[] = []; + if (!text) { + return tokens; + } + + TOKEN_PATTERN.lastIndex = 0; + let match: RegExpExecArray | null; + while ((match = TOKEN_PATTERN.exec(text)) !== null) { + const value = normalizeText(match[0]); + if (value.length > 0) { + tokens.push({ value, offset: match.index }); + } + } + + return tokens; +} + +/** + * Splits text into normalized tokens, discarding offsets. + */ +export function tokenize(text: string): string[] { + return tokenizeWithOffsets(text).map((token) => token.value); +} + +/** + * English function words carrying no retrieval signal. + * + * LLM queries are phrased as questions ("what did we decide about hiring"), + * and the coverage multiplier in scoreField divides by the query token count. + * Left in, these words drive coverage — and therefore the score — toward zero + * for exactly the natural-language queries this feature exists to serve. + */ +const STOPWORDS = new Set([ + "a", "about", "an", "and", "are", "as", "at", "be", "but", "by", "can", + "could", "did", "do", "does", "for", "from", "had", "has", "have", "he", + "her", "his", "how", "i", "if", "in", "is", "it", "its", "me", "my", "of", + "on", "or", "our", "she", "should", "so", "than", "that", "the", "their", + "them", "then", "there", "these", "they", "this", "to", "us", "was", "we", + "were", "what", "when", "where", "which", "who", "why", "will", "with", + "would", "you", "your", +]); + +/** + * Tokenizes a QUERY, stripping stopwords. + * + * Stopwords are kept when removing them would leave nothing, so that a query + * such as "the who" still searches for something. + */ +export function tokenizeQuery(query: string): string[] { + const tokens = tokenize(query); + const meaningful = tokens.filter((token) => !STOPWORDS.has(token)); + return meaningful.length > 0 ? meaningful : tokens; +} diff --git a/tests/search/tokenize.test.ts b/tests/search/tokenize.test.ts new file mode 100644 index 0000000..728ea96 --- /dev/null +++ b/tests/search/tokenize.test.ts @@ -0,0 +1,64 @@ +import { describe, expect, test } from "bun:test"; +import { + normalizeText, + tokenize, + tokenizeQuery, + tokenizeWithOffsets, +} from "../../src/search/tokenize.ts"; + +describe("normalizeText", () => { + test("lowercases and strips diacritics", () => { + expect(normalizeText("Müller")).toBe("muller"); + expect(normalizeText("Café")).toBe("cafe"); + expect(normalizeText("RÉSUMÉ")).toBe("resume"); + }); + + test("leaves plain ascii untouched apart from case", () => { + expect(normalizeText("Budget Approval")).toBe("budget approval"); + }); +}); + +describe("tokenize", () => { + test("splits on non-alphanumeric boundaries", () => { + expect(tokenize("Approval of Q3 budget!")).toEqual(["approval", "of", "q3", "budget"]); + }); + + test("returns an empty array for empty or punctuation-only input", () => { + expect(tokenize("")).toEqual([]); + expect(tokenize("--- ...")).toEqual([]); + }); + + test("preserves non-latin scripts", () => { + expect(tokenize("Привет мир")).toEqual(["привет", "мир"]); + }); +}); + +describe("tokenizeQuery", () => { + test("strips English stopwords from a natural-language query", () => { + expect(tokenizeQuery("what did we decide about hiring")).toEqual(["decide", "hiring"]); + expect(tokenizeQuery("approval of the budget")).toEqual(["approval", "budget"]); + }); + + test("keeps stopwords when nothing else would survive", () => { + expect(tokenizeQuery("the who")).toEqual(["the", "who"]); + }); + + test("leaves a stopword-free query untouched", () => { + expect(tokenizeQuery("budget approval")).toEqual(["budget", "approval"]); + }); +}); + +describe("tokenizeWithOffsets", () => { + test("reports offsets into the original text, not the normalized text", () => { + const tokens = tokenizeWithOffsets("Der Müller Bericht"); + expect(tokens.map((t) => t.value)).toEqual(["der", "muller", "bericht"]); + expect(tokens.map((t) => t.offset)).toEqual([0, 4, 11]); + }); + + test("offsets index back into the source string correctly", () => { + const source = "Approval of Q3 budget"; + const tokens = tokenizeWithOffsets(source); + const budget = tokens.find((t) => t.value === "budget")!; + expect(source.slice(budget.offset, budget.offset + 6)).toBe("budget"); + }); +});