feat(ingest): preserve source content classifications

Keep the sources' Miliastra, battle-pass and TCG distinctions so rotating
names do not need a new title table every patch. Classification survives
cross-source merges while event IDs, dates and source identities stay fixed.
Conflicting classifications are reported instead of selecting a destination.
This commit is contained in:
Lucas Winther
2026-11-01 04:08:36 +01:00
parent 4faf3c9ceb
commit 5576e5ad5c
16 changed files with 207 additions and 3 deletions
+23
View File
@@ -704,3 +704,26 @@ which sets `confirmedDead` on the matching code (by its `["game","CODE"]` id) an
that matched nothing; the build logs those so they can be pruned. Nothing else about the code
changes, and the check is never used to add a code. The field is optional, so `schemaVersion`
stays 1.
## Preserving source classifications (2026-11-01)
Genshin's Fandom `Type(s)` column now survives parsing as optional structured
`classification`: Miliastra Wonderland supplies a destination lane, Battle Pass
and Miliastra Pass supply a pass family, and the established TCG category families
supply a `tcg` tag. Pass-only table/version sources also identify their family;
Game8 retains its explicit Battle Pass prefix as classification while its title
normalization remains unchanged. Classification never changes an ID or a date.
`mergeEvents` retains metadata from a matched secondary source even when the
identity-priority winner lacks it. Conflicting explicit lane/activity/family
claims are reported in `classificationConflicts` and the survivor is marked
`unresolved`; the build prints those conflicts rather than choosing a destination.
Current title/date matching and `mergedIds` behavior are unchanged. Classification
is not evidence that two otherwise distinct events are duplicates.
The wire game's value remains the original source game. Client classification
resolves Miliastra's logical lane after merging, so Genshin's two sources are still
fetched once each and old cached bundles can continue validating the feed. A
small, evidenced article-family resolver supports old feeds/saved snapshots;
unknown content stays in its original category. See DATA-MODEL's source identity
section and the 2026-11-01 content-classification design.
@@ -88,6 +88,10 @@
"game": "genshin",
"title": "Frostfarer",
"type": "other",
"classification": {
"activity": "battle-pass",
"family": "battle-pass"
},
"summary": null,
"startsAt": "2026-08-12T00:00:00.000Z",
"startPrecision": "day",
@@ -109,6 +113,11 @@
"game": "genshin",
"title": "Phantasmagoric Chronicle",
"type": "other",
"classification": {
"lane": "miliastra",
"activity": "battle-pass",
"family": "miliastra-pass"
},
"summary": null,
"startsAt": "2026-08-12T00:00:00.000Z",
"startPrecision": "day",
@@ -130,6 +139,9 @@
"game": "genshin",
"title": "Raiment Collection: Gentle Warmth",
"type": "other",
"classification": {
"lane": "miliastra"
},
"summary": null,
"startsAt": "2026-08-12T00:00:00.000Z",
"startPrecision": "day",
@@ -151,6 +163,11 @@
"game": "genshin",
"title": "The Forge Realm's Temper: Game of Wits",
"type": "other",
"classification": {
"tags": [
"tcg"
]
},
"summary": null,
"startsAt": "2026-08-12T00:00:00.000Z",
"startPrecision": "day",
@@ -235,6 +252,11 @@
"game": "genshin",
"title": "Heated Battle Mode: Tactical Formation",
"type": "other",
"classification": {
"tags": [
"tcg"
]
},
"summary": null,
"startsAt": "2026-08-29T00:00:00.000Z",
"startPrecision": "day",
@@ -277,6 +299,9 @@
"game": "genshin",
"title": "Starlight Voyage: Joyous Moment",
"type": "other",
"classification": {
"lane": "miliastra"
},
"summary": null,
"startsAt": "2026-09-07T00:00:00.000Z",
"startPrecision": "day",
@@ -45,6 +45,10 @@
"id": "genshin:frostfarer:2026-08-12",
"game": "genshin",
"title": "Frostfarer",
"classification": {
"activity": "battle-pass",
"family": "battle-pass"
},
"type": "other",
"summary": "Participate in the free Sojourner's Battle Pass and the for-purchase Gnostic Hymn to receive prizes as you increase your BP Level.",
"startsAt": "2026-08-12T00:00:00.000Z",
@@ -4,6 +4,10 @@
"game": "hsr",
"title": "Nameless Honor",
"type": "other",
"classification": {
"activity": "battle-pass",
"family": "nameless-honor"
},
"summary": null,
"startsAt": "2026-09-28T00:00:00.000Z",
"startPrecision": "day",
@@ -4,6 +4,10 @@
"game": "nikki",
"title": "Mira Journey",
"type": "other",
"classification": {
"activity": "battle-pass",
"family": "mira-journey"
},
"summary": "Follows version 2.10 (The Feather of Justice). That the pass ends with the version was confirmed in game, 2026-10-16.",
"startsAt": "2026-09-23T00:00:00.000Z",
"startPrecision": "day",
@@ -4,6 +4,10 @@
"game": "zzz",
"title": "New Eridu City Fund",
"type": "other",
"classification": {
"activity": "battle-pass",
"family": "city-fund"
},
"summary": null,
"startsAt": "2026-09-09T00:00:00.000Z",
"startPrecision": "day",
+2
View File
@@ -124,6 +124,8 @@ for (const [, groups] of byGame) {
const merged = mergeEvents(sorted);
events.push(...merged.events);
conflictCount += merged.conflicts.length;
conflictCount += merged.classificationConflicts.length;
for (const c of merged.classificationConflicts) console.warn(` ! classification conflict: "${c.kept.title}" between ${c.kept.sourceId} and ${c.rejected.sourceId}; routing left unresolved`);
for (const c of merged.conflicts) {
console.warn(
` ! conflict: "${c.kept.title}" ${c.field} differs by ${c.deltaHours}h between sources`,
+17 -3
View File
@@ -22,6 +22,7 @@ import { recordedSame } from "./aliases.ts";
export interface MergeResult {
events: GachaEvent[];
classificationConflicts: Array<{ kept: GachaEvent; rejected: GachaEvent }>;
/**
* Pairs that look like the same event but disagree on an end date by more
* than the tolerance. These are the cases a human should look at; the
@@ -67,6 +68,7 @@ export function mergeEvents(
const kept: GachaEvent[] = [];
const conflicts: MergeResult["conflicts"] = [];
const classificationConflicts: MergeResult["classificationConflicts"] = [];
const all = groups.flat();
for (const incoming of all) {
@@ -89,7 +91,15 @@ export function mergeEvents(
incoming.confidence > existing.confidence
? ([incoming, existing] as const)
: ([existing, incoming] as const);
const winner = absorb(ranked, loser);
let winner = absorb(ranked, loser);
if (["lane", "activity", "family"].some(key => {
const a = existing.classification?.[key as "lane" | "activity" | "family"];
const b = incoming.classification?.[key as "lane" | "activity" | "family"];
return a !== undefined && b !== undefined && a !== b;
}) || existing.classification?.unresolved || incoming.classification?.unresolved) {
winner = { ...winner, classification: { unresolved: true } };
classificationConflicts.push({ kept: winner, rejected: loser });
}
if (conflict !== null) {
// Independent sources disagree on when this ends. Do not average, do not
@@ -116,7 +126,7 @@ export function mergeEvents(
: a.startsAt.localeCompare(b.startsAt),
);
return { events: kept, conflicts };
return { events: kept, conflicts, classificationConflicts };
}
/**
@@ -129,12 +139,16 @@ export function mergeEvents(
* new fusion never needs a hand-edited table to keep a reader's ticks.
*/
function absorb(winner: GachaEvent, loser: GachaEvent): GachaEvent {
// Identity priority must not discard a second source's explicit categories.
const classification = { ...loser.classification, ...winner.classification };
if (winner.classification?.tags || loser.classification?.tags)
classification.tags = [...new Set([...(winner.classification?.tags ?? []), ...(loser.classification?.tags ?? [])])];
const ids = [
...(winner.mergedIds ?? []),
...(loser.id === winner.id ? [] : [loser.id]),
...(loser.mergedIds ?? []),
].filter((id, i, all) => id !== winner.id && all.indexOf(id) === i);
return ids.length === 0 ? winner : { ...winner, mergedIds: ids };
return { ...winner, ...(Object.keys(classification).length ? { classification } : {}), ...(ids.length ? { mergedIds: ids } : {}) };
}
function isSameEvent(
+2
View File
@@ -9,6 +9,7 @@ import {
import { decodeEntities, text } from "../html.ts";
import type { ParseContext } from "../adapters/types.ts";
import { inferType } from "./game8.ts";
import { classifySourceLabels } from "../../shared/classification.ts";
import type { SourceParser } from "./types.ts";
/**
@@ -841,6 +842,7 @@ function parseGenshinEvents(
// "Login" — is a far better signal than a Genshin event title, which
// is usually a piece of poetry.
type: inferType(`${title} ${stated}`),
...(Object.keys(classifySourceLabels(stated)).length ? { classification: classifySourceLabels(stated) } : {}),
// No description column, and the prose lives on each event's own
// article rather than further down this page — so there is nothing to
// summarise, and `Type(s)` is a classification rather than a blurb.
+1
View File
@@ -191,6 +191,7 @@ export function parseFandomHistoryTable(
game: ctx.game,
title,
type: "other",
classification: { activity: "battle-pass", family: ctx.sourceId === "hsr-fandom-nameless-honor" ? "nameless-honor" : "city-fund" },
summary: null,
startsAt: startsAt.iso,
startPrecision: startsAt.precision,
+1
View File
@@ -479,6 +479,7 @@ function toEvent(c: Candidate, ctx: ParseContext): GachaEvent {
id: eventId(ctx.game, title, c.start.iso),
game: ctx.game,
title,
...(/^Battle Pass\s*-/i.test(c.title) ? { classification: { activity: "battle-pass" as const, family: "battle-pass" } } : {}),
type: inferType(title),
summary: c.summary,
startsAt: c.start.iso,
+1
View File
@@ -191,6 +191,7 @@ export function parseFandomVersionTable(
game: ctx.game,
title: pass.title,
type: "other",
classification: { activity: "battle-pass", family: "mira-journey" },
summary: `Follows version ${row.number} (${row.name}). ${pass.link}`,
startsAt: row.start.iso,
startPrecision: row.start.precision,
+67
View File
@@ -0,0 +1,67 @@
import type { DisplayEvent } from "./custom.ts";
import type { EventClassification } from "./schema.ts";
/** The source's category vocabulary, rather than similarity between titles. */
export function classifySourceLabels(labels: string): EventClassification {
const result: EventClassification = {};
if (/\bMiliastra Wonderland\b/i.test(labels)) result.lane = "miliastra";
if (/\bMiliastra Pass\b/i.test(labels)) {
result.lane = "miliastra";
result.activity = "battle-pass";
result.family = "miliastra-pass";
} else if (/\bBattle Pass\b/i.test(labels)) {
result.activity = "battle-pass";
result.family = "battle-pass";
}
// Named TCG families in Genshin Event's type glossary. Forge Realm was also
// confirmed in game by the owner, 2026-11-01 (docs event audit).
if (/Genius Invokation TCG|The Forge Realm['’]s Temper|Heated Battle Mode/i.test(labels)) result.tags = ["tcg"];
return result;
}
/** Compatibility for source-backed snapshots/feeds predating the metadata. */
export function classificationFor(event: DisplayEvent): EventClassification {
if (event.id.startsWith("myevent:") || event.sourceUrl === null) return {};
if (event.classification?.unresolved) return {};
let known: EventClassification = {};
if (event.game === "genshin" || event.game === "miliastra") {
let path = "";
try {
const url = new URL(event.sourceUrl);
if (url.hostname === "genshin-impact.fandom.com") path = decodeURIComponent(url.pathname).replace(/_/g, " ");
} catch { /* Unknown links provide no evidence. */ }
if (/^\/wiki\/(Miliastra Pass|Starlit Gala|Starlight Voyage|Raiment Collection)(?:[/:]|$)/i.test(path)) known.lane = "miliastra";
if (/^\/wiki\/Miliastra Pass\//i.test(path)) known = { ...known, activity: "battle-pass", family: "miliastra-pass" };
if (/^\/wiki\/Battle Pass\//i.test(path)) known = { activity: "battle-pass", family: "battle-pass" };
if (/^\/wiki\/(The Forge Realm['’]s Temper|Heated Battle Mode)(?:[/:]|$)/i.test(path)) known.tags = ["tcg"];
// Game8's overview link does not identify the article. These two already
// published pass names are pinned fixtures; future passes use source labels.
if (event.sourceId === "genshin-game8-events" && /^(?:Battle Pass - )?(Moontrace|Frostfarer)$/.test(event.title)) known = { activity: "battle-pass", family: "battle-pass" };
}
const families: Record<string, string> = {
"hsr-fandom-nameless-honor": "nameless-honor",
"zzz-fandom-city-fund": "city-fund",
"nikki-fandom-mira-journey": "mira-journey",
};
const family = families[event.sourceId];
if (family) known = { activity: "battle-pass", family };
return { ...known, ...event.classification };
}
export const displayGameFor = (event: DisplayEvent): string => classificationFor(event).lane ?? event.game;
export const passFamilyFor = (event: DisplayEvent): string | null => {
const c = classificationFor(event);
return c.activity === "battle-pass" ? c.family ?? "battle-pass" : null;
};
export const passPlacementId = (event: DisplayEvent): string | null => {
const family = passFamilyFor(event);
return family === null ? null : `feed-cycle:${displayGameFor(event)}:${family}`;
};
export const isTcgEvent = (event: DisplayEvent): boolean => classificationFor(event).tags?.includes("tcg") === true;
export const tracksEventCategory = (event: DisplayEvent, prefs: Record<string, unknown>): boolean => !(displayGameFor(event) === "genshin" && prefs.genshinTcg === false && isTcgEvent(event));
export function classifiedEvent(event: DisplayEvent): DisplayEvent {
const game = displayGameFor(event);
return game === event.game ? event : { ...event, game, classification: classificationFor(event) };
}
/** Miliastra consumes the same source bytes, freshness and attribution. */
export const sourceGameFor = (game: string): string => game === "miliastra" ? "genshin" : game;
+1
View File
@@ -62,6 +62,7 @@ export interface GameMeta {
}
export const GAMES: Record<GameId, GameMeta> = {
miliastra: { id: "miliastra", name: "Miliastra Wonderland", short: "Miliastra", hue: "#9C83D6", studio: "HoYoverse", dailyTasks: "" },
// EU reset at 04:00 UTC+1 confirmed in game by the owner on 2026-10-16: the
// Trounce Domains menu reads "every Monday 4AM GMT+1".
genshin: { id: "genshin", name: "Genshin Impact", short: "Genshin", hue: "#4EA8DE" , studio: "HoYoverse", dailyTasks: "Commissions, resin" },
+11
View File
@@ -23,6 +23,7 @@ export const GameId = z.enum([
"nikke", // Goddess of Victory: Nikke
"hi3", // Honkai Impact 3rd
"pgr", // Punishing: Gray Raven
"miliastra", // Logical lane; source identities remain Genshin.
]);
export type GameId = z.infer<typeof GameId>;
@@ -50,6 +51,15 @@ export type Region = z.infer<typeof Region>;
export const Precision = z.enum(["exact", "day", "unknown"]);
export type Precision = z.infer<typeof Precision>;
export const EventClassification = z.object({
unresolved: z.boolean().optional(),
lane: z.literal("miliastra").optional(),
activity: z.literal("battle-pass").optional(),
family: z.string().min(1).optional(),
tags: z.array(z.string()).optional(),
});
export type EventClassification = z.infer<typeof EventClassification>;
export const GachaEvent = z
.object({
id: z.string(),
@@ -85,6 +95,7 @@ export const GachaEvent = z
* client's non-strict parse drops it unread.
*/
mergedIds: z.array(z.string()).optional(),
classification: EventClassification.optional(),
})
// These invariants are cheap here and prevent a whole class of wrong-date bugs
// reaching the feed. See docs/INGESTION.md § Stage 4.
+40
View File
@@ -0,0 +1,40 @@
import { expect, test } from "bun:test";
import { classificationFor, classifySourceLabels, displayGameFor, passFamilyFor, isTcgEvent } from "../src/shared/classification.ts";
import { GachaEvent } from "../src/shared/schema.ts";
import { mergeEvents } from "../src/ingest/merge.ts";
import { sampleEvent } from "./helpers/reader.ts";
import { ADAPTERS } from "../src/ingest/adapters/index.ts";
test("source labels retain Miliastra and pass identity independently", () => {
expect(classifySourceLabels("In-Game, Miliastra Pass, Miliastra Wonderland")).toEqual({ lane: "miliastra", activity: "battle-pass", family: "miliastra-pass" });
expect(classifySourceLabels("In-Game, Battle Pass")).toEqual({ activity: "battle-pass", family: "battle-pass" });
expect(classifySourceLabels("In-Game, The Forge Realm's Temper")).toEqual({ tags: ["tcg"] });
expect(classifySourceLabels("Challenge")).toEqual({});
});
test("cached article paths route poetic names without changing IDs", () => {
const e = { ...sampleEvent, sourceUrl: "https://genshin-impact.fandom.com/wiki/Miliastra_Pass/2026-09-23" };
expect(displayGameFor(e)).toBe("miliastra");
expect(passFamilyFor(e)).toBe("miliastra-pass");
expect(classificationFor({ ...sampleEvent, sourceUrl: "https://genshin-impact.fandom.com/wiki/Starlight_Voyage/2026-10-19" }).lane).toBe("miliastra");
expect(classificationFor(sampleEvent)).toEqual({});
expect(classificationFor({ ...e, id: "myevent:abcdef", sourceUrl: null })).toEqual({});
expect(isTcgEvent({ ...sampleEvent, sourceUrl: "https://genshin-impact.fandom.com/wiki/The_Forge_Realm%27s_Temper/2026-09-23" })).toBe(true);
});
test("optional classification survives schema and the losing source's merge", () => {
const labeled = GachaEvent.parse({ ...sampleEvent, sourceId: "second", confidence: 0.5, classification: { activity: "battle-pass", family: "battle-pass" } });
expect(mergeEvents([[sampleEvent], [labeled]]).events[0]?.classification).toEqual({ activity: "battle-pass", family: "battle-pass" });
expect(GachaEvent.parse(sampleEvent).id).toBe(sampleEvent.id);
});
test("the Genshin parser preserves its real type column", async () => {
const a = ADAPTERS.find(a => a.id === "genshin-fandom-events")!;
const events = a.parse(await Bun.file("fixtures/genshin/fandom-events-2026-09-12.html").text(), { now: "2026-09-12T12:00:00.000Z", sourceId: a.id, sourceUrl: a.url, game: a.game });
expect(events.find(e => e.sourceUrl.includes("Miliastra_Pass"))?.classification).toHaveProperty("lane", "miliastra");
expect(events.find(e => e.sourceUrl.includes("Battle_Pass"))?.classification).toHaveProperty("activity", "battle-pass");
});
test("conflicting explicit pass families are reported without choosing a routing", () => {
const a = { ...sampleEvent, classification: { activity: "battle-pass" as const, family: "first" } };
const b = { ...sampleEvent, sourceId: "second", classification: { activity: "battle-pass" as const, family: "second" } };
const result = mergeEvents([[a], [b]]);
expect(result.classificationConflicts).toHaveLength(1);
expect(result.events[0]?.classification).toEqual({ unresolved: true });
});