feat(search): add unicode-aware tokenizer with original-text offsets

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

tests/search/tokenize.test.ts

@@ -0,0 +1,64 @@
+import { describe, expect, test } from "bun:test";
+import {
+  normalizeText,
+  tokenize,
+  tokenizeQuery,
+  tokenizeWithOffsets,
+} from "../../src/search/tokenize.ts";
+
+describe("normalizeText", () => {
+  test("lowercases and strips diacritics", () => {
+    expect(normalizeText("Müller")).toBe("muller");
+    expect(normalizeText("Café")).toBe("cafe");
+    expect(normalizeText("RÉSUMÉ")).toBe("resume");
+  });
+
+  test("leaves plain ascii untouched apart from case", () => {
+    expect(normalizeText("Budget Approval")).toBe("budget approval");
+  });
+});
+
+describe("tokenize", () => {
+  test("splits on non-alphanumeric boundaries", () => {
+    expect(tokenize("Approval of Q3 budget!")).toEqual(["approval", "of", "q3", "budget"]);
+  });
+
+  test("returns an empty array for empty or punctuation-only input", () => {
+    expect(tokenize("")).toEqual([]);
+    expect(tokenize("---  ...")).toEqual([]);
+  });
+
+  test("preserves non-latin scripts", () => {
+    expect(tokenize("Привет мир")).toEqual(["привет", "мир"]);
+  });
+});
+
+describe("tokenizeQuery", () => {
+  test("strips English stopwords from a natural-language query", () => {
+    expect(tokenizeQuery("what did we decide about hiring")).toEqual(["decide", "hiring"]);
+    expect(tokenizeQuery("approval of the budget")).toEqual(["approval", "budget"]);
+  });
+
+  test("keeps stopwords when nothing else would survive", () => {
+    expect(tokenizeQuery("the who")).toEqual(["the", "who"]);
+  });
+
+  test("leaves a stopword-free query untouched", () => {
+    expect(tokenizeQuery("budget approval")).toEqual(["budget", "approval"]);
+  });
+});
+
+describe("tokenizeWithOffsets", () => {
+  test("reports offsets into the original text, not the normalized text", () => {
+    const tokens = tokenizeWithOffsets("Der Müller Bericht");
+    expect(tokens.map((t) => t.value)).toEqual(["der", "muller", "bericht"]);
+    expect(tokens.map((t) => t.offset)).toEqual([0, 4, 11]);
+  });
+
+  test("offsets index back into the source string correctly", () => {
+    const source = "Approval of Q3 budget";
+    const tokens = tokenizeWithOffsets(source);
+    const budget = tokens.find((t) => t.value === "budget")!;
+    expect(source.slice(budget.offset, budget.offset + 6)).toBe("budget");
+  });
+});