feat(search): add unicode-aware tokenizer with original-text offsets

Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>

src/search/tokenize.ts

@@ -0,0 +1,88 @@
+/**
+ * Text normalization and tokenization for fuzzy search.
+ *
+ * Query text and field text must travel through an identical path; any
+ * divergence between the two sides silently breaks matching, so there is
+ * exactly one implementation of each operation here.
+ */
+
+export interface Token {
+  /** Normalized token value. */
+  value: string;
+  /** Character offset of the token in the ORIGINAL (un-normalized) text. */
+  offset: number;
+}
+
+const COMBINING_MARKS = /[̀-ͯ]/g;
+const TOKEN_PATTERN = /[\p{L}\p{N}]+/gu;
+
+/**
+ * Lowercases text and strips diacritics via NFKD decomposition, so that
+ * "Müller" matches "muller" and "café" matches "cafe".
+ */
+export function normalizeText(text: string): string {
+  return text.normalize("NFKD").replace(COMBINING_MARKS, "").toLowerCase();
+}
+
+/**
+ * Splits text into normalized tokens, each carrying its offset in the
+ * original string.
+ *
+ * Token boundaries are found on the ORIGINAL text and normalization is
+ * applied per token, because NFKD can change string length and would
+ * otherwise invalidate the offsets.
+ */
+export function tokenizeWithOffsets(text: string): Token[] {
+  const tokens: Token[] = [];
+  if (!text) {
+    return tokens;
+  }
+
+  TOKEN_PATTERN.lastIndex = 0;
+  let match: RegExpExecArray | null;
+  while ((match = TOKEN_PATTERN.exec(text)) !== null) {
+    const value = normalizeText(match[0]);
+    if (value.length > 0) {
+      tokens.push({ value, offset: match.index });
+    }
+  }
+
+  return tokens;
+}
+
+/**
+ * Splits text into normalized tokens, discarding offsets.
+ */
+export function tokenize(text: string): string[] {
+  return tokenizeWithOffsets(text).map((token) => token.value);
+}
+
+/**
+ * English function words carrying no retrieval signal.
+ *
+ * LLM queries are phrased as questions ("what did we decide about hiring"),
+ * and the coverage multiplier in scoreField divides by the query token count.
+ * Left in, these words drive coverage — and therefore the score — toward zero
+ * for exactly the natural-language queries this feature exists to serve.
+ */
+const STOPWORDS = new Set([
+  "a", "about", "an", "and", "are", "as", "at", "be", "but", "by", "can",
+  "could", "did", "do", "does", "for", "from", "had", "has", "have", "he",
+  "her", "his", "how", "i", "if", "in", "is", "it", "its", "me", "my", "of",
+  "on", "or", "our", "she", "should", "so", "than", "that", "the", "their",
+  "them", "then", "there", "these", "they", "this", "to", "us", "was", "we",
+  "were", "what", "when", "where", "which", "who", "why", "will", "with",
+  "would", "you", "your",
+]);
+
+/**
+ * Tokenizes a QUERY, stripping stopwords.
+ *
+ * Stopwords are kept when removing them would leave nothing, so that a query
+ * such as "the who" still searches for something.
+ */
+export function tokenizeQuery(query: string): string[] {
+  const tokens = tokenize(query);
+  const meaningful = tokens.filter((token) => !STOPWORDS.has(token));
+  return meaningful.length > 0 ? meaningful : tokens;
+}

tests/search/tokenize.test.ts

@@ -0,0 +1,64 @@
+import { describe, expect, test } from "bun:test";
+import {
+  normalizeText,
+  tokenize,
+  tokenizeQuery,
+  tokenizeWithOffsets,
+} from "../../src/search/tokenize.ts";
+
+describe("normalizeText", () => {
+  test("lowercases and strips diacritics", () => {
+    expect(normalizeText("Müller")).toBe("muller");
+    expect(normalizeText("Café")).toBe("cafe");
+    expect(normalizeText("RÉSUMÉ")).toBe("resume");
+  });
+
+  test("leaves plain ascii untouched apart from case", () => {
+    expect(normalizeText("Budget Approval")).toBe("budget approval");
+  });
+});
+
+describe("tokenize", () => {
+  test("splits on non-alphanumeric boundaries", () => {
+    expect(tokenize("Approval of Q3 budget!")).toEqual(["approval", "of", "q3", "budget"]);
+  });
+
+  test("returns an empty array for empty or punctuation-only input", () => {
+    expect(tokenize("")).toEqual([]);
+    expect(tokenize("---  ...")).toEqual([]);
+  });
+
+  test("preserves non-latin scripts", () => {
+    expect(tokenize("Привет мир")).toEqual(["привет", "мир"]);
+  });
+});
+
+describe("tokenizeQuery", () => {
+  test("strips English stopwords from a natural-language query", () => {
+    expect(tokenizeQuery("what did we decide about hiring")).toEqual(["decide", "hiring"]);
+    expect(tokenizeQuery("approval of the budget")).toEqual(["approval", "budget"]);
+  });
+
+  test("keeps stopwords when nothing else would survive", () => {
+    expect(tokenizeQuery("the who")).toEqual(["the", "who"]);
+  });
+
+  test("leaves a stopword-free query untouched", () => {
+    expect(tokenizeQuery("budget approval")).toEqual(["budget", "approval"]);
+  });
+});
+
+describe("tokenizeWithOffsets", () => {
+  test("reports offsets into the original text, not the normalized text", () => {
+    const tokens = tokenizeWithOffsets("Der Müller Bericht");
+    expect(tokens.map((t) => t.value)).toEqual(["der", "muller", "bericht"]);
+    expect(tokens.map((t) => t.offset)).toEqual([0, 4, 11]);
+  });
+
+  test("offsets index back into the source string correctly", () => {
+    const source = "Approval of Q3 budget";
+    const tokens = tokenizeWithOffsets(source);
+    const budget = tokens.find((t) => t.value === "budget")!;
+    expect(source.slice(budget.offset, budget.offset + 6)).toBe("budget");
+  });
+});