diff --git a/src/search/score.ts b/src/search/score.ts new file mode 100644 index 0000000..82102ca --- /dev/null +++ b/src/search/score.ts @@ -0,0 +1,213 @@ +/** + * Fuzzy scoring primitives. + */ + +import { tokenizeWithOffsets } from "./tokenize.ts"; + +export interface FieldScore { + /** Aggregate field score in the range 0..1. */ + score: number; + /** Offsets of matching tokens in the ORIGINAL field text. */ + matchedOffsets: number[]; +} + +/** Multiplier applied to the normalized edit distance when scoring a typo. */ +const EDIT_PENALTY = 0.6; + +/** Added when the full query appears verbatim in the field. */ +const PHRASE_BONUS = 0.15; + +/** + * The base score is scaled by this before the phrase bonus is added. + * + * Load-bearing: without it a document matching every query token already + * scores 1.0, the phrase bonus clamps to nothing, and a verbatim phrase + * cannot outrank a scattered one — the bonus becomes dead code in exactly + * the case it exists for. Scaling reserves the top 0.15 for verbatim hits. + */ +const BASE_SCALE = 0.85; + +/** Minimum shared prefix length for the morphological-variant rule. */ +const MIN_SHARED_PREFIX = 4; + +/** Below this length a token only ever matches exactly. */ +const MIN_FUZZY_TOKEN_LENGTH = 3; + +/** Below this length a token is not eligible for containment matching. */ +const MIN_CONTAINS_TOKEN_LENGTH = 4; + +/** + * Levenshtein distance with an early exit. + * + * Returns the true distance when it is <= max, otherwise max + 1. The + * overwhelmingly common case is a non-match, so the row-minimum early exit + * is what keeps scoring cheap over long documents. + */ +export function boundedLevenshtein(a: string, b: string, max: number): number { + if (a === b) { + return 0; + } + if (Math.abs(a.length - b.length) > max) { + return max + 1; + } + if (a.length === 0) { + return b.length <= max ? b.length : max + 1; + } + if (b.length === 0) { + return a.length <= max ? a.length : max + 1; + } + + let previous: number[] = Array.from({ length: b.length + 1 }, (_, index) => index); + let current: number[] = new Array(b.length + 1).fill(0); + + for (let i = 1; i <= a.length; i++) { + current[0] = i; + let rowMinimum = i; + + for (let j = 1; j <= b.length; j++) { + const substitutionCost = a.charCodeAt(i - 1) === b.charCodeAt(j - 1) ? 0 : 1; + const value = Math.min( + previous[j]! + 1, + current[j - 1]! + 1, + previous[j - 1]! + substitutionCost + ); + current[j] = value; + if (value < rowMinimum) { + rowMinimum = value; + } + } + + if (rowMinimum > max) { + return max + 1; + } + + const swap = previous; + previous = current; + current = swap; + } + + const distance = previous[b.length]!; + return distance <= max ? distance : max + 1; +} + +/** + * Length of the common prefix shared by two tokens. + */ +function sharedPrefixLength(a: string, b: string): number { + const limit = Math.min(a.length, b.length); + let index = 0; + while (index < limit && a.charCodeAt(index) === b.charCodeAt(index)) { + index++; + } + return index; +} + +/** + * Scores a single query token against a single field token. + * + * Rules are applied in order and the first match wins. The length guards are + * load-bearing: without them "cat" is edit distance 3 from "dog" and every + * short token fuzzily matches every other short token. + */ +export function scoreToken(queryToken: string, fieldToken: string): number { + if (queryToken === fieldToken) { + return 1; + } + if (queryToken.length < MIN_FUZZY_TOKEN_LENGTH) { + return 0; + } + if (fieldToken.startsWith(queryToken)) { + return 0.9; + } + if (queryToken.length >= MIN_CONTAINS_TOKEN_LENGTH && fieldToken.includes(queryToken)) { + return 0.75; + } + + // Morphological variants: "decide"/"decision", "deploy"/"deployment". + // These are 3-4 edits apart, and no edit budget wide enough to match them + // is narrow enough to reject unrelated words. A shared prefix is. + const sharedPrefix = sharedPrefixLength(queryToken, fieldToken); + if (sharedPrefix >= MIN_SHARED_PREFIX) { + const longest = Math.max(queryToken.length, fieldToken.length); + return 0.55 + 0.25 * (sharedPrefix / longest); + } + + const maxDistance = queryToken.length < 6 ? 1 : 2; + const distance = boundedLevenshtein(queryToken, fieldToken, maxDistance); + if (distance > maxDistance) { + return 0; + } + + return 1 - (distance / queryToken.length) * EDIT_PENALTY; +} + +/** + * Scores a set of query tokens against one field's text. + * + * Aggregation is mean(best score per query token) * coverage * BASE_SCALE, + * plus PHRASE_BONUS when the full query appears verbatim. Coverage is the + * fraction of query tokens that matched anything, and is what stops a + * two-word query from ranking a document that matched only the common word. + * + * A field score reaches 1.0 only via the phrase bonus. + */ +export function scoreField(queryTokens: string[], fieldText: string): FieldScore { + const empty: FieldScore = { score: 0, matchedOffsets: [] }; + + if (queryTokens.length === 0 || !fieldText) { + return empty; + } + + const fieldTokens = tokenizeWithOffsets(fieldText); + if (fieldTokens.length === 0) { + return empty; + } + + let total = 0; + let matchedCount = 0; + const matchedOffsets: number[] = []; + + for (const queryToken of queryTokens) { + let best = 0; + let bestOffset = -1; + + for (const fieldToken of fieldTokens) { + const score = scoreToken(queryToken, fieldToken.value); + if (score > best) { + best = score; + bestOffset = fieldToken.offset; + if (best === 1) { + break; + } + } + } + + total += best; + if (best > 0) { + matchedCount++; + if (bestOffset >= 0) { + matchedOffsets.push(bestOffset); + } + } + } + + if (matchedCount === 0) { + return empty; + } + + const coverage = matchedCount / queryTokens.length; + let score = (total / queryTokens.length) * coverage * BASE_SCALE; + + // Phrase bonus, compared over joined token values so that punctuation + // between words does not defeat the check. + const phrase = queryTokens.join(" "); + const joinedField = fieldTokens.map((token) => token.value).join(" "); + if (joinedField.includes(phrase)) { + score += PHRASE_BONUS; + } + + return { + score: Math.min(1, score), + matchedOffsets: matchedOffsets.sort((a, b) => a - b), + }; +} diff --git a/tests/search/score.test.ts b/tests/search/score.test.ts new file mode 100644 index 0000000..a948bd0 --- /dev/null +++ b/tests/search/score.test.ts @@ -0,0 +1,124 @@ +import { describe, expect, test } from "bun:test"; +import { boundedLevenshtein, scoreField, scoreToken } from "../../src/search/score.ts"; +import { tokenize } from "../../src/search/tokenize.ts"; + +describe("boundedLevenshtein", () => { + test("returns 0 for identical strings", () => { + expect(boundedLevenshtein("budget", "budget", 2)).toBe(0); + }); + + test("computes small distances within budget", () => { + expect(boundedLevenshtein("budget", "budgt", 2)).toBe(1); + expect(boundedLevenshtein("approval", "aproval", 2)).toBe(1); + }); + + test("returns max + 1 when the distance exceeds the budget", () => { + expect(boundedLevenshtein("cat", "dog", 1)).toBe(2); + expect(boundedLevenshtein("meeting", "xyz", 2)).toBe(3); + }); + + test("short-circuits on length difference alone", () => { + expect(boundedLevenshtein("a", "abcdefgh", 2)).toBe(3); + }); +}); + +describe("scoreToken", () => { + test("scores an exact match at 1", () => { + expect(scoreToken("budget", "budget")).toBe(1); + }); + + test("scores a prefix match at 0.9 when the query token is at least 3 chars", () => { + expect(scoreToken("bud", "budget")).toBe(0.9); + }); + + test("does not prefix-match tokens shorter than 3 characters", () => { + expect(scoreToken("bu", "budget")).toBe(0); + }); + + test("scores a containment match at 0.75 when the query token is at least 4 chars", () => { + expect(scoreToken("prov", "approval")).toBe(0.75); + }); + + test("scores a single-typo match below an exact match but above zero", () => { + const score = scoreToken("budgt", "budget"); + expect(score).toBeGreaterThan(0); + expect(score).toBeLessThan(1); + }); + + test("does NOT match unrelated short tokens", () => { + expect(scoreToken("cat", "dog")).toBe(0); + expect(scoreToken("red", "car")).toBe(0); + }); + + test("does not fuzzy-match tokens shorter than 3 characters", () => { + expect(scoreToken("ab", "ac")).toBe(0); + }); + + test("matches morphological variants via a shared prefix", () => { + // Neither token is a true prefix of the other, so these exercise the + // shared-prefix rule rather than being intercepted by the startsWith rule. + expect(scoreToken("decide", "decision")).toBeGreaterThan(0.5); + expect(scoreToken("deployed", "deployment")).toBeGreaterThan(0.5); + }); + + test("does not shared-prefix-match on fewer than 4 common characters", () => { + // "cars"/"carton" share only "car" (3 < MIN_SHARED_PREFIX) and neither is + // a true prefix of the other, so every rule falls through to zero. + expect(scoreToken("cars", "carton")).toBe(0); + expect(scoreToken("bees", "beetle")).toBe(0); + }); + + test("ranks a shared-prefix match below a true prefix match", () => { + expect(scoreToken("decide", "decision")).toBeLessThan(scoreToken("deci", "decision")); + }); +}); + +describe("scoreField", () => { + test("scores a full phrase match highest", () => { + const query = tokenize("budget approval"); + const both = scoreField(query, "Approval of the budget"); + const one = scoreField(query, "Approval of the agenda"); + expect(both.score).toBeGreaterThan(one.score); + }); + + test("coverage penalises documents matching only one query token", () => { + const query = tokenize("hiring decision"); + const partial = scoreField(query, "Decision log"); + expect(partial.score).toBeLessThan(0.6); + }); + + test("gives a verbatim phrase a bonus over scattered tokens", () => { + const query = tokenize("budget approval"); + const verbatim = scoreField(query, "budget approval meeting"); + const scattered = scoreField(query, "approval of the yearly budget"); + expect(verbatim.score).toBeGreaterThan(scattered.score); + }); + + test("tolerates a typo in the query", () => { + const query = tokenize("budgt aproval"); + const result = scoreField(query, "Approval of Q3 budget"); + expect(result.score).toBeGreaterThan(0.5); + }); + + test("reports matched offsets that index into the original field text", () => { + const field = "Approval of Q3 budget"; + const result = scoreField(tokenize("budget"), field); + expect(result.matchedOffsets.length).toBeGreaterThan(0); + expect(field.slice(result.matchedOffsets[0]!, result.matchedOffsets[0]! + 6)).toBe("budget"); + }); + + test("returns zero for empty input", () => { + expect(scoreField([], "anything").score).toBe(0); + expect(scoreField(tokenize("budget"), "").score).toBe(0); + }); + + test("an all-tokens match without the phrase tops out below 1.0", () => { + const result = scoreField(tokenize("budget approval"), "approval of the budget"); + expect(result.score).toBeCloseTo(0.85, 5); + }); + + test("only a verbatim phrase reaches 1.0", () => { + const result = scoreField(tokenize("budget approval"), "the budget approval doc"); + expect(result.score).toBeCloseTo(1, 5); + }); +});