From 6b00d9a9c0dbf0cc3e12ab0f360e60be8cdd1b41 Mon Sep 17 00:00:00 2001 From: Lucas Winther Date: Wed, 19 Aug 2026 03:52:15 +0200 Subject: [PATCH] fix(merge): fuse near matches only across sources MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Near matching exists to reconcile two sources describing one event under different titles. Within one source it has no such job: the page has already told us these are two rows, the parser has already dropped repeats of the same id, and fusing them overrules a distinction the publisher made on purpose. The loose 0.80 threshold that makes the rule useful across sources makes it actively wrong within one. Umamusume's banner list runs "3 Star Guaranteed 1.5 Anniversary Scout (Character)" and "(Support)" concurrently — titles differing by a single parenthetical, starting the same day — and fusing them dropped a live banner off the calendar with nothing anywhere reporting it. A silent drop is the failure this codebase ranks as the dangerous one. Same-id fusion is untouched, so nothing duplicates. Co-Authored-By: Claude Opus 5 (1M context) --- AGENTS.md | 2 +- docs/INGESTION.md | 24 ++++++++++++++++++++--- src/ingest/merge.ts | 12 ++++++++++++ test/merge.test.ts | 47 +++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 81 insertions(+), 4 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 8d7e686..197e73f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -119,7 +119,7 @@ src/client/ React app, service worker, manifest theme.ts — dark or light, and what a game hue reads as on each scripts/ build-feed.ts, build-static.ts, parse-fixture.ts (offline), refresh-sources.ts (fetches) serve.ts static server + /api/health -test/ 683 tests +test/ 686 tests fixtures// raw HTML + .expected.json per source — pinned, kept forever snapshots/ current page per source, rewritten by refresh — see its README ``` diff --git a/docs/INGESTION.md b/docs/INGESTION.md index 4e9f9b5..92cabaf 100644 --- a/docs/INGESTION.md +++ b/docs/INGESTION.md @@ -338,9 +338,10 @@ Only meaningful when a game has more than one source; a single-source game passe `mergeEvents(groups)` compares events across sources: -1. **Same ID** → same event; keep the higher-confidence copy. -2. **Near match** — same game, title similarity ≥ 0.80, starts within 24h — → same event under - different titles; keep the higher-confidence copy. +1. **Same ID** → same event; keep the higher-confidence copy. Applies within a source as well as + across sources — an identical id is the same row seen twice, whatever it is called. +2. **Near match** — **different sources**, same game, title similarity ≥ 0.80, starts within 24h — + → same event under different titles; keep the higher-confidence copy. 3. **Otherwise** → distinct events; keep both. Title similarity alone would merge a rerun with its original, since reruns reuse the name. The @@ -348,6 +349,23 @@ start-date proximity check is the actual guard; the title threshold is deliberat that "Stygian Onslaught" and "Stygian Onslaught Event" collapse into one row rather than showing the user a duplicate. +**Near matching is cross-source only, and that restriction is load-bearing** (2026-08-19). Fusing +two rows from *one* page overrules a distinction the publisher made on purpose, and the loose +threshold that makes rule 2 useful across sources makes it actively wrong within one: Game8's +Umamusume banner list runs `3 Star Guaranteed 1.5 Anniversary Scout (Character)` and `(Support)` +concurrently, titles differing by a single parenthetical and starting the same day. That scores far +above 0.80, and fusing them dropped a live banner off the calendar with nothing anywhere reporting +it — a silent drop, which § Silent drops ranks as the dangerous failure. Rule 1 still fuses repeats +within a source, so nothing is duplicated; the parsers dedupe by id before merge is reached anyway. + +**Agreement raises confidence (+0.10) only across different `sourceId`s.** The same row seen twice +in one document is not corroboration. + +**Disagreement is surfaced, never averaged.** Two sources whose `endsAt` differ by more than 24 +hours produce a `conflicts` entry; the pipeline routes those to quarantine. Splitting the difference +between two dates would produce a value neither source asserts — the worst possible answer for a +product whose promise is date accuracy. + ## Stage 4 — validate Zod parse against `GachaEvent`, then calendar sanity rules. Anything failing a hard rule goes to diff --git a/src/ingest/merge.ts b/src/ingest/merge.ts index 6792153..5ca823a 100644 --- a/src/ingest/merge.ts +++ b/src/ingest/merge.ts @@ -124,6 +124,18 @@ function isSameEvent( ): boolean { if (a.id === b.id) return true; + // Near-match fusion reconciles two *sources* describing one event under + // different titles. Within one source it has no such job to do: the page has + // already told us these are two rows, the parser has already dropped repeats + // of the same id, and fusing them here overrules a distinction the publisher + // made on purpose. Game8's Umamusume page is the case that surfaced it — its + // `3 Star Guaranteed 1.5 Anniversary Scout (Character)` and `(Support)` are + // two concurrent banners whose titles differ by one parenthetical, which + // scores far above any workable threshold and starts on the same day. Fusing + // them dropped a real banner off the calendar with nothing reporting it, + // which is the silent drop this codebase treats as the dangerous failure. + if (a.sourceId === b.sourceId) return false; + // Overlap alone misses a source that appends a qualifier: "Bedazzling // Dawnstar" vs "Bedazzling Dawnstar Sign-In" scores 0.67, well under any // safe threshold, yet is plainly one event. diff --git a/test/merge.test.ts b/test/merge.test.ts index 31fd70e..b8d10e9 100644 --- a/test/merge.test.ts +++ b/test/merge.test.ts @@ -191,3 +191,50 @@ describe("merging two sources for one game", () => { expect(conflicts[0]?.field).toBe("endsAt"); }); }); + +describe("near matches within one source", () => { + test("two rows from one page stay two events", () => { + // The page has already told us these are two things, and the parser has + // already dropped repeats of the same id — so fusing them here overrules a + // distinction the publisher made on purpose. Game8's Umamusume banner list + // is the real case: two concurrent banners whose titles differ by one + // parenthetical and which start on the same day. Fusing them dropped a live + // banner off the calendar with nothing reporting it. + const character = event({ + id: "uma:3-star-guaranteed-1-5-anniversary-scout-character:2026-07-22", + game: "uma", + title: "3 Star Guaranteed 1.5 Anniversary Scout (Character)", + sourceId: "uma-game8-events", + }); + const support = event({ + id: "uma:3-star-guaranteed-1-5-anniversary-scout-support:2026-07-22", + game: "uma", + title: "3 Star Guaranteed 1.5 Anniversary Scout (Support)", + sourceId: "uma-game8-events", + }); + + const merged = mergeEvents([[character, support]]); + expect(merged.events).toHaveLength(2); + expect(merged.conflicts).toHaveLength(0); + }); + + test("the same id twice from one source is still one event", () => { + // The exception that keeps the rule safe: an identical id is the same row + // seen twice, whatever it is called. + const once = event({ sourceId: "source-a" }); + const twice = event({ sourceId: "source-a", title: "Test Event (again)" }); + expect(mergeEvents([[once, twice]]).events).toHaveLength(1); + }); + + test("near matches across two sources still fuse", () => { + // The behaviour this rule narrows, not removes: reconciling two sources + // that describe one event under different titles is the whole job. + const a = event({ title: "Bedazzling Dawnstar", sourceId: "source-a" }); + const b = event({ + id: "genshin:bedazzling-dawnstar-sign-in:2026-08-12", + title: "Bedazzling Dawnstar Sign-In", + sourceId: "source-b", + }); + expect(mergeEvents([[a], [b]]).events).toHaveLength(1); + }); +});