diff --git a/AGENTS.md b/AGENTS.md index 8d7e686..197e73f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -119,7 +119,7 @@ src/client/ React app, service worker, manifest theme.ts — dark or light, and what a game hue reads as on each scripts/ build-feed.ts, build-static.ts, parse-fixture.ts (offline), refresh-sources.ts (fetches) serve.ts static server + /api/health -test/ 683 tests +test/ 686 tests fixtures// raw HTML + .expected.json per source — pinned, kept forever snapshots/ current page per source, rewritten by refresh — see its README ``` diff --git a/docs/INGESTION.md b/docs/INGESTION.md index 4e9f9b5..92cabaf 100644 --- a/docs/INGESTION.md +++ b/docs/INGESTION.md @@ -338,9 +338,10 @@ Only meaningful when a game has more than one source; a single-source game passe `mergeEvents(groups)` compares events across sources: -1. **Same ID** → same event; keep the higher-confidence copy. -2. **Near match** — same game, title similarity ≥ 0.80, starts within 24h — → same event under - different titles; keep the higher-confidence copy. +1. **Same ID** → same event; keep the higher-confidence copy. Applies within a source as well as + across sources — an identical id is the same row seen twice, whatever it is called. +2. **Near match** — **different sources**, same game, title similarity ≥ 0.80, starts within 24h — + → same event under different titles; keep the higher-confidence copy. 3. **Otherwise** → distinct events; keep both. Title similarity alone would merge a rerun with its original, since reruns reuse the name. The @@ -348,6 +349,23 @@ start-date proximity check is the actual guard; the title threshold is deliberat that "Stygian Onslaught" and "Stygian Onslaught Event" collapse into one row rather than showing the user a duplicate. +**Near matching is cross-source only, and that restriction is load-bearing** (2026-08-19). Fusing +two rows from *one* page overrules a distinction the publisher made on purpose, and the loose +threshold that makes rule 2 useful across sources makes it actively wrong within one: Game8's +Umamusume banner list runs `3 Star Guaranteed 1.5 Anniversary Scout (Character)` and `(Support)` +concurrently, titles differing by a single parenthetical and starting the same day. That scores far +above 0.80, and fusing them dropped a live banner off the calendar with nothing anywhere reporting +it — a silent drop, which § Silent drops ranks as the dangerous failure. Rule 1 still fuses repeats +within a source, so nothing is duplicated; the parsers dedupe by id before merge is reached anyway. + +**Agreement raises confidence (+0.10) only across different `sourceId`s.** The same row seen twice +in one document is not corroboration. + +**Disagreement is surfaced, never averaged.** Two sources whose `endsAt` differ by more than 24 +hours produce a `conflicts` entry; the pipeline routes those to quarantine. Splitting the difference +between two dates would produce a value neither source asserts — the worst possible answer for a +product whose promise is date accuracy. + ## Stage 4 — validate Zod parse against `GachaEvent`, then calendar sanity rules. Anything failing a hard rule goes to diff --git a/src/ingest/merge.ts b/src/ingest/merge.ts index 6792153..5ca823a 100644 --- a/src/ingest/merge.ts +++ b/src/ingest/merge.ts @@ -124,6 +124,18 @@ function isSameEvent( ): boolean { if (a.id === b.id) return true; + // Near-match fusion reconciles two *sources* describing one event under + // different titles. Within one source it has no such job to do: the page has + // already told us these are two rows, the parser has already dropped repeats + // of the same id, and fusing them here overrules a distinction the publisher + // made on purpose. Game8's Umamusume page is the case that surfaced it — its + // `3 Star Guaranteed 1.5 Anniversary Scout (Character)` and `(Support)` are + // two concurrent banners whose titles differ by one parenthetical, which + // scores far above any workable threshold and starts on the same day. Fusing + // them dropped a real banner off the calendar with nothing reporting it, + // which is the silent drop this codebase treats as the dangerous failure. + if (a.sourceId === b.sourceId) return false; + // Overlap alone misses a source that appends a qualifier: "Bedazzling // Dawnstar" vs "Bedazzling Dawnstar Sign-In" scores 0.67, well under any // safe threshold, yet is plainly one event. diff --git a/test/merge.test.ts b/test/merge.test.ts index 31fd70e..b8d10e9 100644 --- a/test/merge.test.ts +++ b/test/merge.test.ts @@ -191,3 +191,50 @@ describe("merging two sources for one game", () => { expect(conflicts[0]?.field).toBe("endsAt"); }); }); + +describe("near matches within one source", () => { + test("two rows from one page stay two events", () => { + // The page has already told us these are two things, and the parser has + // already dropped repeats of the same id — so fusing them here overrules a + // distinction the publisher made on purpose. Game8's Umamusume banner list + // is the real case: two concurrent banners whose titles differ by one + // parenthetical and which start on the same day. Fusing them dropped a live + // banner off the calendar with nothing reporting it. + const character = event({ + id: "uma:3-star-guaranteed-1-5-anniversary-scout-character:2026-07-22", + game: "uma", + title: "3 Star Guaranteed 1.5 Anniversary Scout (Character)", + sourceId: "uma-game8-events", + }); + const support = event({ + id: "uma:3-star-guaranteed-1-5-anniversary-scout-support:2026-07-22", + game: "uma", + title: "3 Star Guaranteed 1.5 Anniversary Scout (Support)", + sourceId: "uma-game8-events", + }); + + const merged = mergeEvents([[character, support]]); + expect(merged.events).toHaveLength(2); + expect(merged.conflicts).toHaveLength(0); + }); + + test("the same id twice from one source is still one event", () => { + // The exception that keeps the rule safe: an identical id is the same row + // seen twice, whatever it is called. + const once = event({ sourceId: "source-a" }); + const twice = event({ sourceId: "source-a", title: "Test Event (again)" }); + expect(mergeEvents([[once, twice]]).events).toHaveLength(1); + }); + + test("near matches across two sources still fuse", () => { + // The behaviour this rule narrows, not removes: reconciling two sources + // that describe one event under different titles is the whole job. + const a = event({ title: "Bedazzling Dawnstar", sourceId: "source-a" }); + const b = event({ + id: "genshin:bedazzling-dawnstar-sign-in:2026-08-12", + title: "Bedazzling Dawnstar Sign-In", + sourceId: "source-b", + }); + expect(mergeEvents([[a], [b]]).events).toHaveLength(1); + }); +});