feat(search): add bounded levenshtein and coverage-weighted field scoring
Adds boundedLevenshtein with row-minimum early exit, token scoring (exact / prefix / contains / shared-prefix / edit distance), and coverage-weighted field aggregation scaled by BASE_SCALE so that only a verbatim phrase match reaches 1.0. Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
This commit is contained in:
Lucas Winther
co-authored by
Claude Opus 5
Claude Opus 5
parent
c4e303c8b7
commit
926e241c7b
@@ -0,0 +1,213 @@
|
||||
/**
|
||||
* Fuzzy scoring primitives.
|
||||
*/
|
||||
|
||||
import { tokenizeWithOffsets } from "./tokenize.ts";
|
||||
|
||||
export interface FieldScore {
|
||||
/** Aggregate field score in the range 0..1. */
|
||||
score: number;
|
||||
/** Offsets of matching tokens in the ORIGINAL field text. */
|
||||
matchedOffsets: number[];
|
||||
}
|
||||
|
||||
/** Multiplier applied to the normalized edit distance when scoring a typo. */
|
||||
const EDIT_PENALTY = 0.6;
|
||||
|
||||
/** Added when the full query appears verbatim in the field. */
|
||||
const PHRASE_BONUS = 0.15;
|
||||
|
||||
/**
|
||||
* The base score is scaled by this before the phrase bonus is added.
|
||||
*
|
||||
* Load-bearing: without it a document matching every query token already
|
||||
* scores 1.0, the phrase bonus clamps to nothing, and a verbatim phrase
|
||||
* cannot outrank a scattered one — the bonus becomes dead code in exactly
|
||||
* the case it exists for. Scaling reserves the top 0.15 for verbatim hits.
|
||||
*/
|
||||
const BASE_SCALE = 0.85;
|
||||
|
||||
/** Minimum shared prefix length for the morphological-variant rule. */
|
||||
const MIN_SHARED_PREFIX = 4;
|
||||
|
||||
/** Below this length a token only ever matches exactly. */
|
||||
const MIN_FUZZY_TOKEN_LENGTH = 3;
|
||||
|
||||
/** Below this length a token is not eligible for containment matching. */
|
||||
const MIN_CONTAINS_TOKEN_LENGTH = 4;
|
||||
|
||||
/**
|
||||
* Levenshtein distance with an early exit.
|
||||
*
|
||||
* Returns the true distance when it is <= max, otherwise max + 1. The
|
||||
* overwhelmingly common case is a non-match, so the row-minimum early exit
|
||||
* is what keeps scoring cheap over long documents.
|
||||
*/
|
||||
export function boundedLevenshtein(a: string, b: string, max: number): number {
|
||||
if (a === b) {
|
||||
return 0;
|
||||
}
|
||||
if (Math.abs(a.length - b.length) > max) {
|
||||
return max + 1;
|
||||
}
|
||||
if (a.length === 0) {
|
||||
return b.length <= max ? b.length : max + 1;
|
||||
}
|
||||
if (b.length === 0) {
|
||||
return a.length <= max ? a.length : max + 1;
|
||||
}
|
||||
|
||||
let previous: number[] = Array.from({ length: b.length + 1 }, (_, index) => index);
|
||||
let current: number[] = new Array<number>(b.length + 1).fill(0);
|
||||
|
||||
for (let i = 1; i <= a.length; i++) {
|
||||
current[0] = i;
|
||||
let rowMinimum = i;
|
||||
|
||||
for (let j = 1; j <= b.length; j++) {
|
||||
const substitutionCost = a.charCodeAt(i - 1) === b.charCodeAt(j - 1) ? 0 : 1;
|
||||
const value = Math.min(
|
||||
previous[j]! + 1,
|
||||
current[j - 1]! + 1,
|
||||
previous[j - 1]! + substitutionCost
|
||||
);
|
||||
current[j] = value;
|
||||
if (value < rowMinimum) {
|
||||
rowMinimum = value;
|
||||
}
|
||||
}
|
||||
|
||||
if (rowMinimum > max) {
|
||||
return max + 1;
|
||||
}
|
||||
|
||||
const swap = previous;
|
||||
previous = current;
|
||||
current = swap;
|
||||
}
|
||||
|
||||
const distance = previous[b.length]!;
|
||||
return distance <= max ? distance : max + 1;
|
||||
}
|
||||
|
||||
/**
|
||||
* Length of the common prefix shared by two tokens.
|
||||
*/
|
||||
function sharedPrefixLength(a: string, b: string): number {
|
||||
const limit = Math.min(a.length, b.length);
|
||||
let index = 0;
|
||||
while (index < limit && a.charCodeAt(index) === b.charCodeAt(index)) {
|
||||
index++;
|
||||
}
|
||||
return index;
|
||||
}
|
||||
|
||||
/**
|
||||
* Scores a single query token against a single field token.
|
||||
*
|
||||
* Rules are applied in order and the first match wins. The length guards are
|
||||
* load-bearing: without them "cat" is edit distance 3 from "dog" and every
|
||||
* short token fuzzily matches every other short token.
|
||||
*/
|
||||
export function scoreToken(queryToken: string, fieldToken: string): number {
|
||||
if (queryToken === fieldToken) {
|
||||
return 1;
|
||||
}
|
||||
if (queryToken.length < MIN_FUZZY_TOKEN_LENGTH) {
|
||||
return 0;
|
||||
}
|
||||
if (fieldToken.startsWith(queryToken)) {
|
||||
return 0.9;
|
||||
}
|
||||
if (queryToken.length >= MIN_CONTAINS_TOKEN_LENGTH && fieldToken.includes(queryToken)) {
|
||||
return 0.75;
|
||||
}
|
||||
|
||||
// Morphological variants: "decide"/"decision", "deploy"/"deployment".
|
||||
// These are 3-4 edits apart, and no edit budget wide enough to match them
|
||||
// is narrow enough to reject unrelated words. A shared prefix is.
|
||||
const sharedPrefix = sharedPrefixLength(queryToken, fieldToken);
|
||||
if (sharedPrefix >= MIN_SHARED_PREFIX) {
|
||||
const longest = Math.max(queryToken.length, fieldToken.length);
|
||||
return 0.55 + 0.25 * (sharedPrefix / longest);
|
||||
}
|
||||
|
||||
const maxDistance = queryToken.length < 6 ? 1 : 2;
|
||||
const distance = boundedLevenshtein(queryToken, fieldToken, maxDistance);
|
||||
if (distance > maxDistance) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
return 1 - (distance / queryToken.length) * EDIT_PENALTY;
|
||||
}
|
||||
|
||||
/**
|
||||
* Scores a set of query tokens against one field's text.
|
||||
*
|
||||
* Aggregation is mean(best score per query token) * coverage * BASE_SCALE,
|
||||
* plus PHRASE_BONUS when the full query appears verbatim. Coverage is the
|
||||
* fraction of query tokens that matched anything, and is what stops a
|
||||
* two-word query from ranking a document that matched only the common word.
|
||||
*
|
||||
* A field score reaches 1.0 only via the phrase bonus.
|
||||
*/
|
||||
export function scoreField(queryTokens: string[], fieldText: string): FieldScore {
|
||||
const empty: FieldScore = { score: 0, matchedOffsets: [] };
|
||||
|
||||
if (queryTokens.length === 0 || !fieldText) {
|
||||
return empty;
|
||||
}
|
||||
|
||||
const fieldTokens = tokenizeWithOffsets(fieldText);
|
||||
if (fieldTokens.length === 0) {
|
||||
return empty;
|
||||
}
|
||||
|
||||
let total = 0;
|
||||
let matchedCount = 0;
|
||||
const matchedOffsets: number[] = [];
|
||||
|
||||
for (const queryToken of queryTokens) {
|
||||
let best = 0;
|
||||
let bestOffset = -1;
|
||||
|
||||
for (const fieldToken of fieldTokens) {
|
||||
const score = scoreToken(queryToken, fieldToken.value);
|
||||
if (score > best) {
|
||||
best = score;
|
||||
bestOffset = fieldToken.offset;
|
||||
if (best === 1) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
total += best;
|
||||
if (best > 0) {
|
||||
matchedCount++;
|
||||
if (bestOffset >= 0) {
|
||||
matchedOffsets.push(bestOffset);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (matchedCount === 0) {
|
||||
return empty;
|
||||
}
|
||||
|
||||
const coverage = matchedCount / queryTokens.length;
|
||||
let score = (total / queryTokens.length) * coverage * BASE_SCALE;
|
||||
|
||||
// Phrase bonus, compared over joined token values so that punctuation
|
||||
// between words does not defeat the check.
|
||||
const phrase = queryTokens.join(" ");
|
||||
const joinedField = fieldTokens.map((token) => token.value).join(" ");
|
||||
if (joinedField.includes(phrase)) {
|
||||
score += PHRASE_BONUS;
|
||||
}
|
||||
|
||||
return {
|
||||
score: Math.min(1, score),
|
||||
matchedOffsets: matchedOffsets.sort((a, b) => a - b),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,124 @@
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import { boundedLevenshtein, scoreField, scoreToken } from "../../src/search/score.ts";
|
||||
import { tokenize } from "../../src/search/tokenize.ts";
|
||||
|
||||
describe("boundedLevenshtein", () => {
|
||||
test("returns 0 for identical strings", () => {
|
||||
expect(boundedLevenshtein("budget", "budget", 2)).toBe(0);
|
||||
});
|
||||
|
||||
test("computes small distances within budget", () => {
|
||||
expect(boundedLevenshtein("budget", "budgt", 2)).toBe(1);
|
||||
expect(boundedLevenshtein("approval", "aproval", 2)).toBe(1);
|
||||
});
|
||||
|
||||
test("returns max + 1 when the distance exceeds the budget", () => {
|
||||
expect(boundedLevenshtein("cat", "dog", 1)).toBe(2);
|
||||
expect(boundedLevenshtein("meeting", "xyz", 2)).toBe(3);
|
||||
});
|
||||
|
||||
test("short-circuits on length difference alone", () => {
|
||||
expect(boundedLevenshtein("a", "abcdefgh", 2)).toBe(3);
|
||||
});
|
||||
});
|
||||
|
||||
describe("scoreToken", () => {
|
||||
test("scores an exact match at 1", () => {
|
||||
expect(scoreToken("budget", "budget")).toBe(1);
|
||||
});
|
||||
|
||||
test("scores a prefix match at 0.9 when the query token is at least 3 chars", () => {
|
||||
expect(scoreToken("bud", "budget")).toBe(0.9);
|
||||
});
|
||||
|
||||
test("does not prefix-match tokens shorter than 3 characters", () => {
|
||||
expect(scoreToken("bu", "budget")).toBe(0);
|
||||
});
|
||||
|
||||
test("scores a containment match at 0.75 when the query token is at least 4 chars", () => {
|
||||
expect(scoreToken("prov", "approval")).toBe(0.75);
|
||||
});
|
||||
|
||||
test("scores a single-typo match below an exact match but above zero", () => {
|
||||
const score = scoreToken("budgt", "budget");
|
||||
expect(score).toBeGreaterThan(0);
|
||||
expect(score).toBeLessThan(1);
|
||||
});
|
||||
|
||||
test("does NOT match unrelated short tokens", () => {
|
||||
expect(scoreToken("cat", "dog")).toBe(0);
|
||||
expect(scoreToken("red", "car")).toBe(0);
|
||||
});
|
||||
|
||||
test("does not fuzzy-match tokens shorter than 3 characters", () => {
|
||||
expect(scoreToken("ab", "ac")).toBe(0);
|
||||
});
|
||||
|
||||
test("matches morphological variants via a shared prefix", () => {
|
||||
// Neither token is a true prefix of the other, so these exercise the
|
||||
// shared-prefix rule rather than being intercepted by the startsWith rule.
|
||||
expect(scoreToken("decide", "decision")).toBeGreaterThan(0.5);
|
||||
expect(scoreToken("deployed", "deployment")).toBeGreaterThan(0.5);
|
||||
});
|
||||
|
||||
test("does not shared-prefix-match on fewer than 4 common characters", () => {
|
||||
// "cars"/"carton" share only "car" (3 < MIN_SHARED_PREFIX) and neither is
|
||||
// a true prefix of the other, so every rule falls through to zero.
|
||||
expect(scoreToken("cars", "carton")).toBe(0);
|
||||
expect(scoreToken("bees", "beetle")).toBe(0);
|
||||
});
|
||||
|
||||
test("ranks a shared-prefix match below a true prefix match", () => {
|
||||
expect(scoreToken("decide", "decision")).toBeLessThan(scoreToken("deci", "decision"));
|
||||
});
|
||||
});
|
||||
|
||||
describe("scoreField", () => {
|
||||
test("scores a full phrase match highest", () => {
|
||||
const query = tokenize("budget approval");
|
||||
const both = scoreField(query, "Approval of the budget");
|
||||
const one = scoreField(query, "Approval of the agenda");
|
||||
expect(both.score).toBeGreaterThan(one.score);
|
||||
});
|
||||
|
||||
test("coverage penalises documents matching only one query token", () => {
|
||||
const query = tokenize("hiring decision");
|
||||
const partial = scoreField(query, "Decision log");
|
||||
expect(partial.score).toBeLessThan(0.6);
|
||||
});
|
||||
|
||||
test("gives a verbatim phrase a bonus over scattered tokens", () => {
|
||||
const query = tokenize("budget approval");
|
||||
const verbatim = scoreField(query, "budget approval meeting");
|
||||
const scattered = scoreField(query, "approval of the yearly budget");
|
||||
expect(verbatim.score).toBeGreaterThan(scattered.score);
|
||||
});
|
||||
|
||||
test("tolerates a typo in the query", () => {
|
||||
const query = tokenize("budgt aproval");
|
||||
const result = scoreField(query, "Approval of Q3 budget");
|
||||
expect(result.score).toBeGreaterThan(0.5);
|
||||
});
|
||||
|
||||
test("reports matched offsets that index into the original field text", () => {
|
||||
const field = "Approval of Q3 budget";
|
||||
const result = scoreField(tokenize("budget"), field);
|
||||
expect(result.matchedOffsets.length).toBeGreaterThan(0);
|
||||
expect(field.slice(result.matchedOffsets[0]!, result.matchedOffsets[0]! + 6)).toBe("budget");
|
||||
});
|
||||
|
||||
test("returns zero for empty input", () => {
|
||||
expect(scoreField([], "anything").score).toBe(0);
|
||||
expect(scoreField(tokenize("budget"), "").score).toBe(0);
|
||||
});
|
||||
|
||||
test("an all-tokens match without the phrase tops out below 1.0", () => {
|
||||
const result = scoreField(tokenize("budget approval"), "approval of the budget");
|
||||
expect(result.score).toBeCloseTo(0.85, 5);
|
||||
});
|
||||
|
||||
test("only a verbatim phrase reaches 1.0", () => {
|
||||
const result = scoreField(tokenize("budget approval"), "the budget approval doc");
|
||||
expect(result.score).toBeCloseTo(1, 5);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user