diff --git a/src/ingest/merge.ts b/src/ingest/merge.ts new file mode 100644 index 0000000..69a29a6 --- /dev/null +++ b/src/ingest/merge.ts @@ -0,0 +1,174 @@ +import type { GachaEvent } from "../shared/schema.ts"; + +/** + * Combine events for one game from several sources. + * + * Two sources covering the same game will disagree: different titles for the + * same event, dates that differ by a day, one listing something the other + * misses. This decides what the feed shows. + * + * The rules, in order: + * 1. Same event ID → same event. Keep the higher-confidence copy. + * 2. Near match → same event under different titles. Keep the + * higher-confidence copy and record corroboration. + * 3. Otherwise → distinct events; keep both. + * + * Corroboration is the point of running multiple sources: two independent + * sources agreeing on a date is much stronger evidence than one asserting it, + * and that shows up as a confidence bump. Two sources *disagreeing* on an end + * date is flagged rather than silently resolved — see `conflicts`. + */ + +export interface MergeResult { + events: GachaEvent[]; + /** + * Pairs that look like the same event but disagree on an end date by more + * than the tolerance. These are the cases a human should look at; the + * pipeline routes them to quarantine. + */ + conflicts: Array<{ + kept: GachaEvent; + rejected: GachaEvent; + field: "endsAt" | "startsAt"; + deltaHours: number; + }>; +} + +export interface MergeOptions { + /** How far two boundaries may differ and still count as agreement. */ + toleranceHours?: number; + /** Title similarity above which two events are considered the same. */ + titleThreshold?: number; + /** Confidence added when an independent source agrees. */ + corroborationBonus?: number; +} + +const DEFAULTS = { + toleranceHours: 24, + /** + * 0.8, not higher: a two-word title with one extra decorative token + * ("Stygian Onslaught" vs "Stygian Onslaught Event") scores exactly 0.8, and + * failing to merge those puts duplicate rows in front of the user. The real + * guard against false positives is start-date proximity, not this number — a + * rerun reuses the name but starts months later. + */ + titleThreshold: 0.8, + corroborationBonus: 0.1, +} as const; + +export function mergeEvents( + groups: GachaEvent[][], + options: MergeOptions = {}, +): MergeResult { + const toleranceHours = options.toleranceHours ?? DEFAULTS.toleranceHours; + const titleThreshold = options.titleThreshold ?? DEFAULTS.titleThreshold; + const bonus = options.corroborationBonus ?? DEFAULTS.corroborationBonus; + + const kept: GachaEvent[] = []; + const conflicts: MergeResult["conflicts"] = []; + + for (const incoming of groups.flat()) { + const matchIndex = kept.findIndex( + (existing) => + existing.game === incoming.game && + isSameEvent(existing, incoming, titleThreshold, toleranceHours), + ); + + if (matchIndex === -1) { + kept.push(incoming); + continue; + } + + const existing = kept[matchIndex]; + if (existing === undefined) continue; + + const conflict = findConflict(existing, incoming, toleranceHours); + const [winner, loser] = + incoming.confidence > existing.confidence + ? ([incoming, existing] as const) + : ([existing, incoming] as const); + + if (conflict !== null) { + // Independent sources disagree on when this ends. Do not average, do not + // silently prefer one — surface it. A wrong end date is the failure this + // product exists to prevent. + conflicts.push({ kept: winner, rejected: loser, ...conflict }); + kept[matchIndex] = winner; + continue; + } + + // Agreement from a different source is real evidence; from the same source + // it is just the same row seen twice. + const corroborated = + winner.sourceId !== loser.sourceId + ? { ...winner, confidence: Math.min(1, winner.confidence + bonus) } + : winner; + + kept[matchIndex] = corroborated; + } + + kept.sort((a, b) => + a.startsAt === b.startsAt + ? a.id.localeCompare(b.id) + : a.startsAt.localeCompare(b.startsAt), + ); + + return { events: kept, conflicts }; +} + +function isSameEvent( + a: GachaEvent, + b: GachaEvent, + titleThreshold: number, + toleranceHours: number, +): boolean { + if (a.id === b.id) return true; + if (titleSimilarity(a.title, b.title) < titleThreshold) return false; + // Similar titles are not enough — reruns reuse names. Require the start dates + // to be close before treating two entries as one event. + return hoursBetween(a.startsAt, b.startsAt) <= toleranceHours; +} + +function findConflict( + a: GachaEvent, + b: GachaEvent, + toleranceHours: number, +): { field: "endsAt" | "startsAt"; deltaHours: number } | null { + if (a.endsAt !== null && b.endsAt !== null) { + const delta = hoursBetween(a.endsAt, b.endsAt); + if (delta > toleranceHours) return { field: "endsAt", deltaHours: delta }; + } + const startDelta = hoursBetween(a.startsAt, b.startsAt); + if (startDelta > toleranceHours) { + return { field: "startsAt", deltaHours: startDelta }; + } + return null; +} + +function hoursBetween(a: string, b: string): number { + return Math.abs(Date.parse(a) - Date.parse(b)) / 3_600_000; +} + +/** + * Token-overlap (Dice) similarity on normalised titles. Deliberately simple: + * it only needs to catch "Stygian Onslaught" vs "Stygian Onslaught (Event)", + * not to do fuzzy natural-language matching. + */ +export function titleSimilarity(a: string, b: string): number { + const ta = tokens(a); + const tb = tokens(b); + if (ta.size === 0 || tb.size === 0) return 0; + let shared = 0; + for (const t of ta) if (tb.has(t)) shared += 1; + return (2 * shared) / (ta.size + tb.size); +} + +function tokens(title: string): Set { + return new Set( + title + .toLowerCase() + .replace(/[^a-z0-9\s]/g, " ") + .split(/\s+/) + .filter((w) => w.length > 0), + ); +} diff --git a/test/merge.test.ts b/test/merge.test.ts new file mode 100644 index 0000000..0fd3c0a --- /dev/null +++ b/test/merge.test.ts @@ -0,0 +1,136 @@ +import { describe, expect, test } from "bun:test"; +import { mergeEvents, titleSimilarity } from "../src/ingest/merge.ts"; +import type { GachaEvent } from "../src/shared/schema.ts"; + +const NOW = "2026-08-14T00:00:00.000Z"; + +function event(overrides: Partial = {}): GachaEvent { + return { + id: "genshin:test-event:2026-08-12", + game: "genshin", + title: "Test Event", + type: "other", + summary: null, + startsAt: "2026-08-12T00:00:00.000Z", + startPrecision: "day", + endsAt: "2026-08-24T00:00:00.000Z", + endPrecision: "day", + regionScoped: false, + regionEnds: null, + sourceUrl: "https://example.test/a", + sourceId: "source-a", + status: "published", + confidence: 0.9, + extractionMethod: "parser", + version: 1, + firstSeenAt: NOW, + updatedAt: NOW, + ...overrides, + }; +} + +describe("mergeEvents", () => { + test("keeps distinct events from different sources", () => { + const a = event({ id: "genshin:a:2026-08-12", title: "Alpha" }); + const b = event({ + id: "genshin:b:2026-09-01", + title: "Beta", + startsAt: "2026-09-01T00:00:00.000Z", + endsAt: "2026-09-10T00:00:00.000Z", + sourceId: "source-b", + }); + const { events, conflicts } = mergeEvents([[a], [b]]); + expect(events).toHaveLength(2); + expect(conflicts).toHaveLength(0); + }); + + test("collapses the same event seen by two sources", () => { + const a = event({ sourceId: "source-a", confidence: 0.85 }); + const b = event({ sourceId: "source-b", confidence: 0.9 }); + const { events } = mergeEvents([[a], [b]]); + expect(events).toHaveLength(1); + }); + + test("raises confidence when an independent source corroborates", () => { + const a = event({ sourceId: "source-a", confidence: 0.85 }); + const b = event({ sourceId: "source-b", confidence: 0.85 }); + const { events } = mergeEvents([[a], [b]]); + // Two sources independently agreeing is stronger evidence than one. + expect(events[0]?.confidence).toBeCloseTo(0.95, 5); + }); + + test("does not corroborate a duplicate from the same source", () => { + const a = event({ sourceId: "source-a", confidence: 0.85 }); + const { events } = mergeEvents([[a, { ...a }]]); + expect(events).toHaveLength(1); + expect(events[0]?.confidence).toBeCloseTo(0.85, 5); + }); + + test("matches the same event under slightly different titles", () => { + const a = event({ title: "Stygian Onslaught", sourceId: "source-a" }); + const b = event({ + id: "genshin:stygian-onslaught-event:2026-08-12", + title: "Stygian Onslaught Event", + sourceId: "source-b", + }); + const { events } = mergeEvents([[a], [b]]); + expect(events).toHaveLength(1); + }); + + test("flags an end-date disagreement instead of picking silently", () => { + const a = event({ sourceId: "source-a", confidence: 0.9 }); + const b = event({ + sourceId: "source-b", + confidence: 0.8, + endsAt: "2026-08-28T00:00:00.000Z", // 4 days later + }); + const { events, conflicts } = mergeEvents([[a], [b]]); + expect(events).toHaveLength(1); + expect(conflicts).toHaveLength(1); + expect(conflicts[0]?.field).toBe("endsAt"); + expect(conflicts[0]?.deltaHours).toBe(96); + // The conflict is surfaced, not averaged away, and confidence is NOT + // bumped — disagreement is the opposite of corroboration. + expect(events[0]?.confidence).toBeCloseTo(0.9, 5); + }); + + test("treats a same-name rerun months later as a separate event", () => { + const a = event({ title: "Windblume Festival" }); + const b = event({ + id: "genshin:windblume-festival:2027-03-01", + title: "Windblume Festival", + startsAt: "2027-03-01T00:00:00.000Z", + endsAt: "2027-03-20T00:00:00.000Z", + sourceId: "source-b", + }); + const { events } = mergeEvents([[a], [b]]); + expect(events).toHaveLength(2); + }); + + test("returns events sorted by start date", () => { + const late = event({ + id: "genshin:late:2026-09-01", + title: "Late", + startsAt: "2026-09-01T00:00:00.000Z", + endsAt: "2026-09-05T00:00:00.000Z", + }); + const early = event({ id: "genshin:early:2026-08-01", title: "Early", + startsAt: "2026-08-01T00:00:00.000Z" }); + const { events } = mergeEvents([[late], [early]]); + expect(events.map((e) => e.title)).toEqual(["Early", "Late"]); + }); +}); + +describe("titleSimilarity", () => { + test("identical titles score 1", () => { + expect(titleSimilarity("Stygian Onslaught", "Stygian Onslaught")).toBe(1); + }); + + test("ignores case and punctuation", () => { + expect(titleSimilarity("Gold Clash!", "gold clash")).toBe(1); + }); + + test("unrelated titles score low", () => { + expect(titleSimilarity("Gold Clash", "Fishing Frenzy")).toBeLessThan(0.3); + }); +});