From 5576e5ad5c8265c519d74d280a080ef7883e62b4 Mon Sep 17 00:00:00 2001 From: Lucas Winther Date: Sun, 1 Nov 2026 04:08:36 +0100 Subject: [PATCH] feat(ingest): preserve source content classifications Keep the sources' Miliastra, battle-pass and TCG distinctions so rotating names do not need a new title table every patch. Classification survives cross-source merges while event IDs, dates and source identities stay fixed. Conflicting classifications are reported instead of selecting a destination. --- docs/INGESTION.md | 23 +++++++ .../fandom-events-2026-09-12.expected.json | 25 +++++++ .../game8-events-2026-08-14.expected.json | 4 ++ ...om-nameless-honor-2026-10-16.expected.json | 4 ++ ...ndom-mira-journey-2026-10-16.expected.json | 4 ++ .../fandom-city-fund-2026-10-16.expected.json | 4 ++ scripts/build-feed.ts | 2 + src/ingest/merge.ts | 20 +++++- src/ingest/parsers/fandom.ts | 2 + src/ingest/parsers/fandomTable.ts | 1 + src/ingest/parsers/game8.ts | 1 + src/ingest/parsers/versionTable.ts | 1 + src/shared/classification.ts | 67 +++++++++++++++++++ src/shared/games.ts | 1 + src/shared/schema.ts | 11 +++ test/classification.test.ts | 40 +++++++++++ 16 files changed, 207 insertions(+), 3 deletions(-) create mode 100644 src/shared/classification.ts create mode 100644 test/classification.test.ts diff --git a/docs/INGESTION.md b/docs/INGESTION.md index ce148f5..ace6c02 100644 --- a/docs/INGESTION.md +++ b/docs/INGESTION.md @@ -704,3 +704,26 @@ which sets `confirmedDead` on the matching code (by its `["game","CODE"]` id) an that matched nothing; the build logs those so they can be pruned. Nothing else about the code changes, and the check is never used to add a code. The field is optional, so `schemaVersion` stays 1. + +## Preserving source classifications (2026-11-01) + +Genshin's Fandom `Type(s)` column now survives parsing as optional structured +`classification`: Miliastra Wonderland supplies a destination lane, Battle Pass +and Miliastra Pass supply a pass family, and the established TCG category families +supply a `tcg` tag. Pass-only table/version sources also identify their family; +Game8 retains its explicit Battle Pass prefix as classification while its title +normalization remains unchanged. Classification never changes an ID or a date. + +`mergeEvents` retains metadata from a matched secondary source even when the +identity-priority winner lacks it. Conflicting explicit lane/activity/family +claims are reported in `classificationConflicts` and the survivor is marked +`unresolved`; the build prints those conflicts rather than choosing a destination. +Current title/date matching and `mergedIds` behavior are unchanged. Classification +is not evidence that two otherwise distinct events are duplicates. + +The wire game's value remains the original source game. Client classification +resolves Miliastra's logical lane after merging, so Genshin's two sources are still +fetched once each and old cached bundles can continue validating the feed. A +small, evidenced article-family resolver supports old feeds/saved snapshots; +unknown content stays in its original category. See DATA-MODEL's source identity +section and the 2026-11-01 content-classification design. diff --git a/fixtures/genshin/fandom-events-2026-09-12.expected.json b/fixtures/genshin/fandom-events-2026-09-12.expected.json index 6292e7d..05e29a3 100644 --- a/fixtures/genshin/fandom-events-2026-09-12.expected.json +++ b/fixtures/genshin/fandom-events-2026-09-12.expected.json @@ -88,6 +88,10 @@ "game": "genshin", "title": "Frostfarer", "type": "other", + "classification": { + "activity": "battle-pass", + "family": "battle-pass" + }, "summary": null, "startsAt": "2026-08-12T00:00:00.000Z", "startPrecision": "day", @@ -109,6 +113,11 @@ "game": "genshin", "title": "Phantasmagoric Chronicle", "type": "other", + "classification": { + "lane": "miliastra", + "activity": "battle-pass", + "family": "miliastra-pass" + }, "summary": null, "startsAt": "2026-08-12T00:00:00.000Z", "startPrecision": "day", @@ -130,6 +139,9 @@ "game": "genshin", "title": "Raiment Collection: Gentle Warmth", "type": "other", + "classification": { + "lane": "miliastra" + }, "summary": null, "startsAt": "2026-08-12T00:00:00.000Z", "startPrecision": "day", @@ -151,6 +163,11 @@ "game": "genshin", "title": "The Forge Realm's Temper: Game of Wits", "type": "other", + "classification": { + "tags": [ + "tcg" + ] + }, "summary": null, "startsAt": "2026-08-12T00:00:00.000Z", "startPrecision": "day", @@ -235,6 +252,11 @@ "game": "genshin", "title": "Heated Battle Mode: Tactical Formation", "type": "other", + "classification": { + "tags": [ + "tcg" + ] + }, "summary": null, "startsAt": "2026-08-29T00:00:00.000Z", "startPrecision": "day", @@ -277,6 +299,9 @@ "game": "genshin", "title": "Starlight Voyage: Joyous Moment", "type": "other", + "classification": { + "lane": "miliastra" + }, "summary": null, "startsAt": "2026-09-07T00:00:00.000Z", "startPrecision": "day", diff --git a/fixtures/genshin/game8-events-2026-08-14.expected.json b/fixtures/genshin/game8-events-2026-08-14.expected.json index 56ff6b4..c243dd2 100644 --- a/fixtures/genshin/game8-events-2026-08-14.expected.json +++ b/fixtures/genshin/game8-events-2026-08-14.expected.json @@ -45,6 +45,10 @@ "id": "genshin:frostfarer:2026-08-12", "game": "genshin", "title": "Frostfarer", + "classification": { + "activity": "battle-pass", + "family": "battle-pass" + }, "type": "other", "summary": "Participate in the free Sojourner's Battle Pass and the for-purchase Gnostic Hymn to receive prizes as you increase your BP Level.", "startsAt": "2026-08-12T00:00:00.000Z", diff --git a/fixtures/hsr/fandom-nameless-honor-2026-10-16.expected.json b/fixtures/hsr/fandom-nameless-honor-2026-10-16.expected.json index 62bccde..2a550c4 100644 --- a/fixtures/hsr/fandom-nameless-honor-2026-10-16.expected.json +++ b/fixtures/hsr/fandom-nameless-honor-2026-10-16.expected.json @@ -4,6 +4,10 @@ "game": "hsr", "title": "Nameless Honor", "type": "other", + "classification": { + "activity": "battle-pass", + "family": "nameless-honor" + }, "summary": null, "startsAt": "2026-09-28T00:00:00.000Z", "startPrecision": "day", diff --git a/fixtures/nikki/fandom-mira-journey-2026-10-16.expected.json b/fixtures/nikki/fandom-mira-journey-2026-10-16.expected.json index 206cf46..2c1fb31 100644 --- a/fixtures/nikki/fandom-mira-journey-2026-10-16.expected.json +++ b/fixtures/nikki/fandom-mira-journey-2026-10-16.expected.json @@ -4,6 +4,10 @@ "game": "nikki", "title": "Mira Journey", "type": "other", + "classification": { + "activity": "battle-pass", + "family": "mira-journey" + }, "summary": "Follows version 2.10 (The Feather of Justice). That the pass ends with the version was confirmed in game, 2026-10-16.", "startsAt": "2026-09-23T00:00:00.000Z", "startPrecision": "day", diff --git a/fixtures/zzz/fandom-city-fund-2026-10-16.expected.json b/fixtures/zzz/fandom-city-fund-2026-10-16.expected.json index 9303005..761e37d 100644 --- a/fixtures/zzz/fandom-city-fund-2026-10-16.expected.json +++ b/fixtures/zzz/fandom-city-fund-2026-10-16.expected.json @@ -4,6 +4,10 @@ "game": "zzz", "title": "New Eridu City Fund", "type": "other", + "classification": { + "activity": "battle-pass", + "family": "city-fund" + }, "summary": null, "startsAt": "2026-09-09T00:00:00.000Z", "startPrecision": "day", diff --git a/scripts/build-feed.ts b/scripts/build-feed.ts index d216542..41b90bd 100644 --- a/scripts/build-feed.ts +++ b/scripts/build-feed.ts @@ -124,6 +124,8 @@ for (const [, groups] of byGame) { const merged = mergeEvents(sorted); events.push(...merged.events); conflictCount += merged.conflicts.length; + conflictCount += merged.classificationConflicts.length; + for (const c of merged.classificationConflicts) console.warn(` ! classification conflict: "${c.kept.title}" between ${c.kept.sourceId} and ${c.rejected.sourceId}; routing left unresolved`); for (const c of merged.conflicts) { console.warn( ` ! conflict: "${c.kept.title}" ${c.field} differs by ${c.deltaHours}h between sources`, diff --git a/src/ingest/merge.ts b/src/ingest/merge.ts index 9a899eb..a354ab3 100644 --- a/src/ingest/merge.ts +++ b/src/ingest/merge.ts @@ -22,6 +22,7 @@ import { recordedSame } from "./aliases.ts"; export interface MergeResult { events: GachaEvent[]; + classificationConflicts: Array<{ kept: GachaEvent; rejected: GachaEvent }>; /** * Pairs that look like the same event but disagree on an end date by more * than the tolerance. These are the cases a human should look at; the @@ -67,6 +68,7 @@ export function mergeEvents( const kept: GachaEvent[] = []; const conflicts: MergeResult["conflicts"] = []; + const classificationConflicts: MergeResult["classificationConflicts"] = []; const all = groups.flat(); for (const incoming of all) { @@ -89,7 +91,15 @@ export function mergeEvents( incoming.confidence > existing.confidence ? ([incoming, existing] as const) : ([existing, incoming] as const); - const winner = absorb(ranked, loser); + let winner = absorb(ranked, loser); + if (["lane", "activity", "family"].some(key => { + const a = existing.classification?.[key as "lane" | "activity" | "family"]; + const b = incoming.classification?.[key as "lane" | "activity" | "family"]; + return a !== undefined && b !== undefined && a !== b; + }) || existing.classification?.unresolved || incoming.classification?.unresolved) { + winner = { ...winner, classification: { unresolved: true } }; + classificationConflicts.push({ kept: winner, rejected: loser }); + } if (conflict !== null) { // Independent sources disagree on when this ends. Do not average, do not @@ -116,7 +126,7 @@ export function mergeEvents( : a.startsAt.localeCompare(b.startsAt), ); - return { events: kept, conflicts }; + return { events: kept, conflicts, classificationConflicts }; } /** @@ -129,12 +139,16 @@ export function mergeEvents( * new fusion never needs a hand-edited table to keep a reader's ticks. */ function absorb(winner: GachaEvent, loser: GachaEvent): GachaEvent { + // Identity priority must not discard a second source's explicit categories. + const classification = { ...loser.classification, ...winner.classification }; + if (winner.classification?.tags || loser.classification?.tags) + classification.tags = [...new Set([...(winner.classification?.tags ?? []), ...(loser.classification?.tags ?? [])])]; const ids = [ ...(winner.mergedIds ?? []), ...(loser.id === winner.id ? [] : [loser.id]), ...(loser.mergedIds ?? []), ].filter((id, i, all) => id !== winner.id && all.indexOf(id) === i); - return ids.length === 0 ? winner : { ...winner, mergedIds: ids }; + return { ...winner, ...(Object.keys(classification).length ? { classification } : {}), ...(ids.length ? { mergedIds: ids } : {}) }; } function isSameEvent( diff --git a/src/ingest/parsers/fandom.ts b/src/ingest/parsers/fandom.ts index f6928ca..08cdc5c 100644 --- a/src/ingest/parsers/fandom.ts +++ b/src/ingest/parsers/fandom.ts @@ -9,6 +9,7 @@ import { import { decodeEntities, text } from "../html.ts"; import type { ParseContext } from "../adapters/types.ts"; import { inferType } from "./game8.ts"; +import { classifySourceLabels } from "../../shared/classification.ts"; import type { SourceParser } from "./types.ts"; /** @@ -841,6 +842,7 @@ function parseGenshinEvents( // "Login" — is a far better signal than a Genshin event title, which // is usually a piece of poetry. type: inferType(`${title} ${stated}`), + ...(Object.keys(classifySourceLabels(stated)).length ? { classification: classifySourceLabels(stated) } : {}), // No description column, and the prose lives on each event's own // article rather than further down this page — so there is nothing to // summarise, and `Type(s)` is a classification rather than a blurb. diff --git a/src/ingest/parsers/fandomTable.ts b/src/ingest/parsers/fandomTable.ts index e1cf608..c9ff4dc 100644 --- a/src/ingest/parsers/fandomTable.ts +++ b/src/ingest/parsers/fandomTable.ts @@ -191,6 +191,7 @@ export function parseFandomHistoryTable( game: ctx.game, title, type: "other", + classification: { activity: "battle-pass", family: ctx.sourceId === "hsr-fandom-nameless-honor" ? "nameless-honor" : "city-fund" }, summary: null, startsAt: startsAt.iso, startPrecision: startsAt.precision, diff --git a/src/ingest/parsers/game8.ts b/src/ingest/parsers/game8.ts index f77615f..b61790f 100644 --- a/src/ingest/parsers/game8.ts +++ b/src/ingest/parsers/game8.ts @@ -479,6 +479,7 @@ function toEvent(c: Candidate, ctx: ParseContext): GachaEvent { id: eventId(ctx.game, title, c.start.iso), game: ctx.game, title, + ...(/^Battle Pass\s*-/i.test(c.title) ? { classification: { activity: "battle-pass" as const, family: "battle-pass" } } : {}), type: inferType(title), summary: c.summary, startsAt: c.start.iso, diff --git a/src/ingest/parsers/versionTable.ts b/src/ingest/parsers/versionTable.ts index 267e5b6..b825719 100644 --- a/src/ingest/parsers/versionTable.ts +++ b/src/ingest/parsers/versionTable.ts @@ -191,6 +191,7 @@ export function parseFandomVersionTable( game: ctx.game, title: pass.title, type: "other", + classification: { activity: "battle-pass", family: "mira-journey" }, summary: `Follows version ${row.number} (${row.name}). ${pass.link}`, startsAt: row.start.iso, startPrecision: row.start.precision, diff --git a/src/shared/classification.ts b/src/shared/classification.ts new file mode 100644 index 0000000..61825e3 --- /dev/null +++ b/src/shared/classification.ts @@ -0,0 +1,67 @@ +import type { DisplayEvent } from "./custom.ts"; +import type { EventClassification } from "./schema.ts"; + +/** The source's category vocabulary, rather than similarity between titles. */ +export function classifySourceLabels(labels: string): EventClassification { + const result: EventClassification = {}; + if (/\bMiliastra Wonderland\b/i.test(labels)) result.lane = "miliastra"; + if (/\bMiliastra Pass\b/i.test(labels)) { + result.lane = "miliastra"; + result.activity = "battle-pass"; + result.family = "miliastra-pass"; + } else if (/\bBattle Pass\b/i.test(labels)) { + result.activity = "battle-pass"; + result.family = "battle-pass"; + } + // Named TCG families in Genshin Event's type glossary. Forge Realm was also + // confirmed in game by the owner, 2026-11-01 (docs event audit). + if (/Genius Invokation TCG|The Forge Realm['’]s Temper|Heated Battle Mode/i.test(labels)) result.tags = ["tcg"]; + return result; +} + +/** Compatibility for source-backed snapshots/feeds predating the metadata. */ +export function classificationFor(event: DisplayEvent): EventClassification { + if (event.id.startsWith("myevent:") || event.sourceUrl === null) return {}; + if (event.classification?.unresolved) return {}; + let known: EventClassification = {}; + if (event.game === "genshin" || event.game === "miliastra") { + let path = ""; + try { + const url = new URL(event.sourceUrl); + if (url.hostname === "genshin-impact.fandom.com") path = decodeURIComponent(url.pathname).replace(/_/g, " "); + } catch { /* Unknown links provide no evidence. */ } + if (/^\/wiki\/(Miliastra Pass|Starlit Gala|Starlight Voyage|Raiment Collection)(?:[/:]|$)/i.test(path)) known.lane = "miliastra"; + if (/^\/wiki\/Miliastra Pass\//i.test(path)) known = { ...known, activity: "battle-pass", family: "miliastra-pass" }; + if (/^\/wiki\/Battle Pass\//i.test(path)) known = { activity: "battle-pass", family: "battle-pass" }; + if (/^\/wiki\/(The Forge Realm['’]s Temper|Heated Battle Mode)(?:[/:]|$)/i.test(path)) known.tags = ["tcg"]; + // Game8's overview link does not identify the article. These two already + // published pass names are pinned fixtures; future passes use source labels. + if (event.sourceId === "genshin-game8-events" && /^(?:Battle Pass - )?(Moontrace|Frostfarer)$/.test(event.title)) known = { activity: "battle-pass", family: "battle-pass" }; + } + const families: Record = { + "hsr-fandom-nameless-honor": "nameless-honor", + "zzz-fandom-city-fund": "city-fund", + "nikki-fandom-mira-journey": "mira-journey", + }; + const family = families[event.sourceId]; + if (family) known = { activity: "battle-pass", family }; + return { ...known, ...event.classification }; +} + +export const displayGameFor = (event: DisplayEvent): string => classificationFor(event).lane ?? event.game; +export const passFamilyFor = (event: DisplayEvent): string | null => { + const c = classificationFor(event); + return c.activity === "battle-pass" ? c.family ?? "battle-pass" : null; +}; +export const passPlacementId = (event: DisplayEvent): string | null => { + const family = passFamilyFor(event); + return family === null ? null : `feed-cycle:${displayGameFor(event)}:${family}`; +}; +export const isTcgEvent = (event: DisplayEvent): boolean => classificationFor(event).tags?.includes("tcg") === true; +export const tracksEventCategory = (event: DisplayEvent, prefs: Record): boolean => !(displayGameFor(event) === "genshin" && prefs.genshinTcg === false && isTcgEvent(event)); +export function classifiedEvent(event: DisplayEvent): DisplayEvent { + const game = displayGameFor(event); + return game === event.game ? event : { ...event, game, classification: classificationFor(event) }; +} +/** Miliastra consumes the same source bytes, freshness and attribution. */ +export const sourceGameFor = (game: string): string => game === "miliastra" ? "genshin" : game; diff --git a/src/shared/games.ts b/src/shared/games.ts index 32478b8..c281e8f 100644 --- a/src/shared/games.ts +++ b/src/shared/games.ts @@ -62,6 +62,7 @@ export interface GameMeta { } export const GAMES: Record = { + miliastra: { id: "miliastra", name: "Miliastra Wonderland", short: "Miliastra", hue: "#9C83D6", studio: "HoYoverse", dailyTasks: "" }, // EU reset at 04:00 UTC+1 confirmed in game by the owner on 2026-10-16: the // Trounce Domains menu reads "every Monday 4AM GMT+1". genshin: { id: "genshin", name: "Genshin Impact", short: "Genshin", hue: "#4EA8DE" , studio: "HoYoverse", dailyTasks: "Commissions, resin" }, diff --git a/src/shared/schema.ts b/src/shared/schema.ts index 2927e69..fc43add 100644 --- a/src/shared/schema.ts +++ b/src/shared/schema.ts @@ -23,6 +23,7 @@ export const GameId = z.enum([ "nikke", // Goddess of Victory: Nikke "hi3", // Honkai Impact 3rd "pgr", // Punishing: Gray Raven + "miliastra", // Logical lane; source identities remain Genshin. ]); export type GameId = z.infer; @@ -50,6 +51,15 @@ export type Region = z.infer; export const Precision = z.enum(["exact", "day", "unknown"]); export type Precision = z.infer; +export const EventClassification = z.object({ + unresolved: z.boolean().optional(), + lane: z.literal("miliastra").optional(), + activity: z.literal("battle-pass").optional(), + family: z.string().min(1).optional(), + tags: z.array(z.string()).optional(), +}); +export type EventClassification = z.infer; + export const GachaEvent = z .object({ id: z.string(), @@ -85,6 +95,7 @@ export const GachaEvent = z * client's non-strict parse drops it unread. */ mergedIds: z.array(z.string()).optional(), + classification: EventClassification.optional(), }) // These invariants are cheap here and prevent a whole class of wrong-date bugs // reaching the feed. See docs/INGESTION.md § Stage 4. diff --git a/test/classification.test.ts b/test/classification.test.ts new file mode 100644 index 0000000..531a8a0 --- /dev/null +++ b/test/classification.test.ts @@ -0,0 +1,40 @@ +import { expect, test } from "bun:test"; +import { classificationFor, classifySourceLabels, displayGameFor, passFamilyFor, isTcgEvent } from "../src/shared/classification.ts"; +import { GachaEvent } from "../src/shared/schema.ts"; +import { mergeEvents } from "../src/ingest/merge.ts"; +import { sampleEvent } from "./helpers/reader.ts"; +import { ADAPTERS } from "../src/ingest/adapters/index.ts"; + +test("source labels retain Miliastra and pass identity independently", () => { + expect(classifySourceLabels("In-Game, Miliastra Pass, Miliastra Wonderland")).toEqual({ lane: "miliastra", activity: "battle-pass", family: "miliastra-pass" }); + expect(classifySourceLabels("In-Game, Battle Pass")).toEqual({ activity: "battle-pass", family: "battle-pass" }); + expect(classifySourceLabels("In-Game, The Forge Realm's Temper")).toEqual({ tags: ["tcg"] }); + expect(classifySourceLabels("Challenge")).toEqual({}); +}); +test("cached article paths route poetic names without changing IDs", () => { + const e = { ...sampleEvent, sourceUrl: "https://genshin-impact.fandom.com/wiki/Miliastra_Pass/2026-09-23" }; + expect(displayGameFor(e)).toBe("miliastra"); + expect(passFamilyFor(e)).toBe("miliastra-pass"); + expect(classificationFor({ ...sampleEvent, sourceUrl: "https://genshin-impact.fandom.com/wiki/Starlight_Voyage/2026-10-19" }).lane).toBe("miliastra"); + expect(classificationFor(sampleEvent)).toEqual({}); + expect(classificationFor({ ...e, id: "myevent:abcdef", sourceUrl: null })).toEqual({}); + expect(isTcgEvent({ ...sampleEvent, sourceUrl: "https://genshin-impact.fandom.com/wiki/The_Forge_Realm%27s_Temper/2026-09-23" })).toBe(true); +}); +test("optional classification survives schema and the losing source's merge", () => { + const labeled = GachaEvent.parse({ ...sampleEvent, sourceId: "second", confidence: 0.5, classification: { activity: "battle-pass", family: "battle-pass" } }); + expect(mergeEvents([[sampleEvent], [labeled]]).events[0]?.classification).toEqual({ activity: "battle-pass", family: "battle-pass" }); + expect(GachaEvent.parse(sampleEvent).id).toBe(sampleEvent.id); +}); +test("the Genshin parser preserves its real type column", async () => { + const a = ADAPTERS.find(a => a.id === "genshin-fandom-events")!; + const events = a.parse(await Bun.file("fixtures/genshin/fandom-events-2026-09-12.html").text(), { now: "2026-09-12T12:00:00.000Z", sourceId: a.id, sourceUrl: a.url, game: a.game }); + expect(events.find(e => e.sourceUrl.includes("Miliastra_Pass"))?.classification).toHaveProperty("lane", "miliastra"); + expect(events.find(e => e.sourceUrl.includes("Battle_Pass"))?.classification).toHaveProperty("activity", "battle-pass"); +}); +test("conflicting explicit pass families are reported without choosing a routing", () => { + const a = { ...sampleEvent, classification: { activity: "battle-pass" as const, family: "first" } }; + const b = { ...sampleEvent, sourceId: "second", classification: { activity: "battle-pass" as const, family: "second" } }; + const result = mergeEvents([[a], [b]]); + expect(result.classificationConflicts).toHaveLength(1); + expect(result.events[0]?.classification).toEqual({ unresolved: true }); +});