diff --git a/src/search/rank.ts b/src/search/rank.ts new file mode 100644 index 0000000..2868aa3 --- /dev/null +++ b/src/search/rank.ts @@ -0,0 +1,170 @@ +/** + * Generic relevance ranking over record collections. + * + * Services declare WHAT to search via FieldSpec declarations; this module + * decides HOW WELL each record matches. + */ + +import { normalizeText, tokenizeQuery } from "./tokenize.ts"; +import { scoreField, type FieldScore } from "./score.ts"; +import { extractSnippet } from "./snippet.ts"; + +export type MatchMode = "fuzzy" | "exact"; + +export interface FieldSpec { + /** Field name reported back in FieldMatch.field. */ + name: string; + /** Relative importance; scores are normalized by the highest declared weight. */ + weight: number; + /** Pulls the searchable text out of a record. */ + extract: (record: T) => string | string[] | undefined; +} + +export interface FieldMatch { + field: string; + score: number; + snippet?: string; +} + +export interface Ranked { + record: T; + /** 0..1, the best weighted field score. */ + score: number; + /** Only fields that scored above zero, highest first. */ + matches: FieldMatch[]; +} + +export interface RankOptions { + minScore?: number; + limit?: number; + matchMode?: MatchMode; + /** Used for deterministic tie-breaking. Defaults to reading `record.id`. */ + idOf?: (record: T) => number; +} + +export const DEFAULT_MIN_SCORE = 0.35; + +/** Below this query length, edit distance is meaningless (spec section 5). */ +const MIN_FUZZY_QUERY_LENGTH = 3; + +function defaultIdOf(record: unknown): number { + return (record as { id?: number })?.id ?? 0; +} + +function fieldTexts(record: T, spec: FieldSpec): string[] { + const raw = spec.extract(record); + if (raw === undefined || raw === null) { + return []; + } + if (Array.isArray(raw)) { + return raw.filter((value): value is string => typeof value === "string" && value.length > 0); + } + return raw.length > 0 ? [raw] : []; +} + +/** + * Case-insensitive substring containment, reproducing the pre-fuzzy behaviour + * byte for byte. + */ +function scoreExact(query: string, text: string): FieldScore { + const haystack = normalizeText(text); + const needle = normalizeText(query); + if (needle.length === 0) { + return { score: 0, matchedOffsets: [] }; + } + const index = haystack.indexOf(needle); + return index === -1 ? { score: 0, matchedOffsets: [] } : { score: 1, matchedOffsets: [index] }; +} + +/** + * Ranks records against a query using the supplied field declarations. + * + * Record score is the best weighted field score: + * recordScore = max over fields of (fieldScore * weight) / maxDeclaredWeight + * + * Normalizing by the maximum DECLARED weight (not the best matching weight) + * means a hit on a low-weight field yields a proportionally lower record + * score rather than being rescaled up to 1. + * + * The minScore gate is applied to the RAW field score, never to this + * weighted value. + */ +export function rankRecords( + records: T[], + query: string, + fields: FieldSpec[], + opts?: RankOptions +): Ranked[] { + const matchMode = opts?.matchMode ?? "fuzzy"; + const minScore = opts?.minScore ?? DEFAULT_MIN_SCORE; + const idOf = opts?.idOf ?? defaultIdOf; + + const trimmedQuery = query.trim(); + if (trimmedQuery.length === 0) { + return records.map((record) => ({ record, score: 0, matches: [] })); + } + + const queryTokens = tokenizeQuery(trimmedQuery); + const useExact = + matchMode === "exact" || + trimmedQuery.length < MIN_FUZZY_QUERY_LENGTH || + queryTokens.length === 0; + + // Exact mode must not apply minScore: a containment hit on a low-weight + // field legitimately scores below it, and dropping those would break + // backward compatibility. + const threshold = useExact ? Number.EPSILON : minScore; + + const maxWeight = fields.reduce((highest, field) => Math.max(highest, field.weight), 0) || 1; + + const ranked: Ranked[] = []; + + for (const record of records) { + const matches: FieldMatch[] = []; + let bestWeighted = 0; + let bestRaw = 0; + + for (const spec of fields) { + let bestScore: FieldScore = { score: 0, matchedOffsets: [] }; + let bestText = ""; + + for (const text of fieldTexts(record, spec)) { + const result = useExact ? scoreExact(trimmedQuery, text) : scoreField(queryTokens, text); + if (result.score > bestScore.score) { + bestScore = result; + bestText = text; + } + } + + if (bestScore.score > 0) { + matches.push({ + field: spec.name, + score: Number(bestScore.score.toFixed(2)), + snippet: extractSnippet(bestText, bestScore.matchedOffsets) || undefined, + }); + + if (bestScore.score > bestRaw) { + bestRaw = bestScore.score; + } + + const weighted = (bestScore.score * spec.weight) / maxWeight; + if (weighted > bestWeighted) { + bestWeighted = weighted; + } + } + } + + // Gate on the RAW score (is this a real match?), order by the weighted + // score (how important is where it matched?). Gating on the weighted + // score would make every low-weight field unreachable — see the note in + // the plan for this task. + if (bestRaw >= threshold) { + matches.sort((a, b) => b.score - a.score); + ranked.push({ record, score: Number(bestWeighted.toFixed(2)), matches }); + } + } + + ranked.sort((a, b) => b.score - a.score || idOf(a.record) - idOf(b.record)); + + return opts?.limit !== undefined && opts.limit > 0 ? ranked.slice(0, opts.limit) : ranked; +} diff --git a/tests/search/rank.test.ts b/tests/search/rank.test.ts new file mode 100644 index 0000000..1120b49 --- /dev/null +++ b/tests/search/rank.test.ts @@ -0,0 +1,114 @@ +import { describe, expect, test } from "bun:test"; +import { rankRecords, type FieldSpec } from "../../src/search/rank.ts"; + +interface Doc { + id: number; + title: string; + body?: string; + tags?: string[]; +} + +const FIELDS: FieldSpec[] = [ + { name: "title", weight: 3, extract: (d) => d.title }, + { name: "body", weight: 1, extract: (d) => d.body }, + { name: "tags", weight: 0.5, extract: (d) => d.tags }, +]; + +const DOCS: Doc[] = [ + { id: 1, title: "Budget approval", body: "Nothing relevant here" }, + { id: 2, title: "Weekly sync", body: "We discussed the budget approval process at length" }, + { id: 3, title: "Retrospective", body: "No overlap", tags: ["budget"] }, + { id: 4, title: "Unrelated", body: "Completely different subject matter" }, +]; + +describe("rankRecords", () => { + test("ranks a title match above a body match", () => { + const results = rankRecords(DOCS, "budget approval", FIELDS); + expect(results[0]!.record.id).toBe(1); + expect(results[1]!.record.id).toBe(2); + }); + + test("excludes records below the minimum score", () => { + const results = rankRecords(DOCS, "budget approval", FIELDS); + expect(results.some((r) => r.record.id === 4)).toBe(false); + }); + + test("tolerates typos", () => { + const results = rankRecords(DOCS, "budgt aproval", FIELDS); + expect(results[0]!.record.id).toBe(1); + }); + + test("tolerates reordered words", () => { + const results = rankRecords(DOCS, "approval budget", FIELDS); + expect(results[0]!.record.id).toBe(1); + }); + + test("reports which fields matched, with snippets", () => { + const results = rankRecords(DOCS, "budget approval", FIELDS); + const second = results.find((r) => r.record.id === 2)!; + expect(second.matches.map((m) => m.field)).toContain("body"); + expect(second.matches[0]!.snippet).toBeDefined(); + }); + + test("searches array-valued fields", () => { + const results = rankRecords(DOCS, "budget", FIELDS); + expect(results.some((r) => r.record.id === 3)).toBe(true); + }); + + test("breaks ties by ascending record id for determinism", () => { + const tied: Doc[] = [ + { id: 9, title: "Budget" }, + { id: 4, title: "Budget" }, + { id: 7, title: "Budget" }, + ]; + const results = rankRecords(tied, "budget", FIELDS); + expect(results.map((r) => r.record.id)).toEqual([4, 7, 9]); + }); + + test("applies the limit after sorting", () => { + const results = rankRecords(DOCS, "budget", FIELDS, { limit: 1 }); + expect(results).toHaveLength(1); + expect(results[0]!.record.id).toBe(1); + }); + + test("exact mode matches only literal substrings", () => { + const results = rankRecords(DOCS, "budget approval", FIELDS, { matchMode: "exact" }); + expect(results.map((r) => r.record.id).sort()).toEqual([1, 2]); + }); + + test("exact mode finds no results for a typo", () => { + const results = rankRecords(DOCS, "budgt aproval", FIELDS, { matchMode: "exact" }); + expect(results).toHaveLength(0); + }); + + test("exact mode keeps low-weight field hits that fall under minScore", () => { + const results = rankRecords(DOCS, "budget", FIELDS, { matchMode: "exact" }); + expect(results.some((r) => r.record.id === 3)).toBe(true); + }); + + test("a blank query returns every record unranked", () => { + const results = rankRecords(DOCS, " ", FIELDS); + expect(results).toHaveLength(DOCS.length); + expect(results.every((r) => r.score === 0)).toBe(true); + }); + + test("a query under three characters falls back to substring containment", () => { + const results = rankRecords(DOCS, "bu", FIELDS); + expect(results.length).toBeGreaterThan(0); + expect( + results.every((r) => { + const haystack = [r.record.title, r.record.body ?? "", ...(r.record.tags ?? [])] + .join(" ") + .toLowerCase(); + return haystack.includes("bu"); + }) + ).toBe(true); + }); + + test("a body-only match is returned despite its low field weight", () => { + const bodyOnly: Doc[] = [{ id: 1, title: "Unrelated", body: "the hiring freeze was agreed" }]; + const results = rankRecords(bodyOnly, "hiring freeze", FIELDS); + expect(results).toHaveLength(1); + expect(results[0]!.matches[0]!.field).toBe("body"); + }); +});