feat(ingest): preserve source content classifications
Keep the sources' Miliastra, battle-pass and TCG distinctions so rotating names do not need a new title table every patch. Classification survives cross-source merges while event IDs, dates and source identities stay fixed. Conflicting classifications are reported instead of selecting a destination.
This commit is contained in:
@@ -704,3 +704,26 @@ which sets `confirmedDead` on the matching code (by its `["game","CODE"]` id) an
|
||||
that matched nothing; the build logs those so they can be pruned. Nothing else about the code
|
||||
changes, and the check is never used to add a code. The field is optional, so `schemaVersion`
|
||||
stays 1.
|
||||
|
||||
## Preserving source classifications (2026-11-01)
|
||||
|
||||
Genshin's Fandom `Type(s)` column now survives parsing as optional structured
|
||||
`classification`: Miliastra Wonderland supplies a destination lane, Battle Pass
|
||||
and Miliastra Pass supply a pass family, and the established TCG category families
|
||||
supply a `tcg` tag. Pass-only table/version sources also identify their family;
|
||||
Game8 retains its explicit Battle Pass prefix as classification while its title
|
||||
normalization remains unchanged. Classification never changes an ID or a date.
|
||||
|
||||
`mergeEvents` retains metadata from a matched secondary source even when the
|
||||
identity-priority winner lacks it. Conflicting explicit lane/activity/family
|
||||
claims are reported in `classificationConflicts` and the survivor is marked
|
||||
`unresolved`; the build prints those conflicts rather than choosing a destination.
|
||||
Current title/date matching and `mergedIds` behavior are unchanged. Classification
|
||||
is not evidence that two otherwise distinct events are duplicates.
|
||||
|
||||
The wire game's value remains the original source game. Client classification
|
||||
resolves Miliastra's logical lane after merging, so Genshin's two sources are still
|
||||
fetched once each and old cached bundles can continue validating the feed. A
|
||||
small, evidenced article-family resolver supports old feeds/saved snapshots;
|
||||
unknown content stays in its original category. See DATA-MODEL's source identity
|
||||
section and the 2026-11-01 content-classification design.
|
||||
|
||||
@@ -88,6 +88,10 @@
|
||||
"game": "genshin",
|
||||
"title": "Frostfarer",
|
||||
"type": "other",
|
||||
"classification": {
|
||||
"activity": "battle-pass",
|
||||
"family": "battle-pass"
|
||||
},
|
||||
"summary": null,
|
||||
"startsAt": "2026-08-12T00:00:00.000Z",
|
||||
"startPrecision": "day",
|
||||
@@ -109,6 +113,11 @@
|
||||
"game": "genshin",
|
||||
"title": "Phantasmagoric Chronicle",
|
||||
"type": "other",
|
||||
"classification": {
|
||||
"lane": "miliastra",
|
||||
"activity": "battle-pass",
|
||||
"family": "miliastra-pass"
|
||||
},
|
||||
"summary": null,
|
||||
"startsAt": "2026-08-12T00:00:00.000Z",
|
||||
"startPrecision": "day",
|
||||
@@ -130,6 +139,9 @@
|
||||
"game": "genshin",
|
||||
"title": "Raiment Collection: Gentle Warmth",
|
||||
"type": "other",
|
||||
"classification": {
|
||||
"lane": "miliastra"
|
||||
},
|
||||
"summary": null,
|
||||
"startsAt": "2026-08-12T00:00:00.000Z",
|
||||
"startPrecision": "day",
|
||||
@@ -151,6 +163,11 @@
|
||||
"game": "genshin",
|
||||
"title": "The Forge Realm's Temper: Game of Wits",
|
||||
"type": "other",
|
||||
"classification": {
|
||||
"tags": [
|
||||
"tcg"
|
||||
]
|
||||
},
|
||||
"summary": null,
|
||||
"startsAt": "2026-08-12T00:00:00.000Z",
|
||||
"startPrecision": "day",
|
||||
@@ -235,6 +252,11 @@
|
||||
"game": "genshin",
|
||||
"title": "Heated Battle Mode: Tactical Formation",
|
||||
"type": "other",
|
||||
"classification": {
|
||||
"tags": [
|
||||
"tcg"
|
||||
]
|
||||
},
|
||||
"summary": null,
|
||||
"startsAt": "2026-08-29T00:00:00.000Z",
|
||||
"startPrecision": "day",
|
||||
@@ -277,6 +299,9 @@
|
||||
"game": "genshin",
|
||||
"title": "Starlight Voyage: Joyous Moment",
|
||||
"type": "other",
|
||||
"classification": {
|
||||
"lane": "miliastra"
|
||||
},
|
||||
"summary": null,
|
||||
"startsAt": "2026-09-07T00:00:00.000Z",
|
||||
"startPrecision": "day",
|
||||
|
||||
@@ -45,6 +45,10 @@
|
||||
"id": "genshin:frostfarer:2026-08-12",
|
||||
"game": "genshin",
|
||||
"title": "Frostfarer",
|
||||
"classification": {
|
||||
"activity": "battle-pass",
|
||||
"family": "battle-pass"
|
||||
},
|
||||
"type": "other",
|
||||
"summary": "Participate in the free Sojourner's Battle Pass and the for-purchase Gnostic Hymn to receive prizes as you increase your BP Level.",
|
||||
"startsAt": "2026-08-12T00:00:00.000Z",
|
||||
|
||||
@@ -4,6 +4,10 @@
|
||||
"game": "hsr",
|
||||
"title": "Nameless Honor",
|
||||
"type": "other",
|
||||
"classification": {
|
||||
"activity": "battle-pass",
|
||||
"family": "nameless-honor"
|
||||
},
|
||||
"summary": null,
|
||||
"startsAt": "2026-09-28T00:00:00.000Z",
|
||||
"startPrecision": "day",
|
||||
|
||||
@@ -4,6 +4,10 @@
|
||||
"game": "nikki",
|
||||
"title": "Mira Journey",
|
||||
"type": "other",
|
||||
"classification": {
|
||||
"activity": "battle-pass",
|
||||
"family": "mira-journey"
|
||||
},
|
||||
"summary": "Follows version 2.10 (The Feather of Justice). That the pass ends with the version was confirmed in game, 2026-10-16.",
|
||||
"startsAt": "2026-09-23T00:00:00.000Z",
|
||||
"startPrecision": "day",
|
||||
|
||||
@@ -4,6 +4,10 @@
|
||||
"game": "zzz",
|
||||
"title": "New Eridu City Fund",
|
||||
"type": "other",
|
||||
"classification": {
|
||||
"activity": "battle-pass",
|
||||
"family": "city-fund"
|
||||
},
|
||||
"summary": null,
|
||||
"startsAt": "2026-09-09T00:00:00.000Z",
|
||||
"startPrecision": "day",
|
||||
|
||||
@@ -124,6 +124,8 @@ for (const [, groups] of byGame) {
|
||||
const merged = mergeEvents(sorted);
|
||||
events.push(...merged.events);
|
||||
conflictCount += merged.conflicts.length;
|
||||
conflictCount += merged.classificationConflicts.length;
|
||||
for (const c of merged.classificationConflicts) console.warn(` ! classification conflict: "${c.kept.title}" between ${c.kept.sourceId} and ${c.rejected.sourceId}; routing left unresolved`);
|
||||
for (const c of merged.conflicts) {
|
||||
console.warn(
|
||||
` ! conflict: "${c.kept.title}" ${c.field} differs by ${c.deltaHours}h between sources`,
|
||||
|
||||
+17
-3
@@ -22,6 +22,7 @@ import { recordedSame } from "./aliases.ts";
|
||||
|
||||
export interface MergeResult {
|
||||
events: GachaEvent[];
|
||||
classificationConflicts: Array<{ kept: GachaEvent; rejected: GachaEvent }>;
|
||||
/**
|
||||
* Pairs that look like the same event but disagree on an end date by more
|
||||
* than the tolerance. These are the cases a human should look at; the
|
||||
@@ -67,6 +68,7 @@ export function mergeEvents(
|
||||
|
||||
const kept: GachaEvent[] = [];
|
||||
const conflicts: MergeResult["conflicts"] = [];
|
||||
const classificationConflicts: MergeResult["classificationConflicts"] = [];
|
||||
|
||||
const all = groups.flat();
|
||||
for (const incoming of all) {
|
||||
@@ -89,7 +91,15 @@ export function mergeEvents(
|
||||
incoming.confidence > existing.confidence
|
||||
? ([incoming, existing] as const)
|
||||
: ([existing, incoming] as const);
|
||||
const winner = absorb(ranked, loser);
|
||||
let winner = absorb(ranked, loser);
|
||||
if (["lane", "activity", "family"].some(key => {
|
||||
const a = existing.classification?.[key as "lane" | "activity" | "family"];
|
||||
const b = incoming.classification?.[key as "lane" | "activity" | "family"];
|
||||
return a !== undefined && b !== undefined && a !== b;
|
||||
}) || existing.classification?.unresolved || incoming.classification?.unresolved) {
|
||||
winner = { ...winner, classification: { unresolved: true } };
|
||||
classificationConflicts.push({ kept: winner, rejected: loser });
|
||||
}
|
||||
|
||||
if (conflict !== null) {
|
||||
// Independent sources disagree on when this ends. Do not average, do not
|
||||
@@ -116,7 +126,7 @@ export function mergeEvents(
|
||||
: a.startsAt.localeCompare(b.startsAt),
|
||||
);
|
||||
|
||||
return { events: kept, conflicts };
|
||||
return { events: kept, conflicts, classificationConflicts };
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -129,12 +139,16 @@ export function mergeEvents(
|
||||
* new fusion never needs a hand-edited table to keep a reader's ticks.
|
||||
*/
|
||||
function absorb(winner: GachaEvent, loser: GachaEvent): GachaEvent {
|
||||
// Identity priority must not discard a second source's explicit categories.
|
||||
const classification = { ...loser.classification, ...winner.classification };
|
||||
if (winner.classification?.tags || loser.classification?.tags)
|
||||
classification.tags = [...new Set([...(winner.classification?.tags ?? []), ...(loser.classification?.tags ?? [])])];
|
||||
const ids = [
|
||||
...(winner.mergedIds ?? []),
|
||||
...(loser.id === winner.id ? [] : [loser.id]),
|
||||
...(loser.mergedIds ?? []),
|
||||
].filter((id, i, all) => id !== winner.id && all.indexOf(id) === i);
|
||||
return ids.length === 0 ? winner : { ...winner, mergedIds: ids };
|
||||
return { ...winner, ...(Object.keys(classification).length ? { classification } : {}), ...(ids.length ? { mergedIds: ids } : {}) };
|
||||
}
|
||||
|
||||
function isSameEvent(
|
||||
|
||||
@@ -9,6 +9,7 @@ import {
|
||||
import { decodeEntities, text } from "../html.ts";
|
||||
import type { ParseContext } from "../adapters/types.ts";
|
||||
import { inferType } from "./game8.ts";
|
||||
import { classifySourceLabels } from "../../shared/classification.ts";
|
||||
import type { SourceParser } from "./types.ts";
|
||||
|
||||
/**
|
||||
@@ -841,6 +842,7 @@ function parseGenshinEvents(
|
||||
// "Login" — is a far better signal than a Genshin event title, which
|
||||
// is usually a piece of poetry.
|
||||
type: inferType(`${title} ${stated}`),
|
||||
...(Object.keys(classifySourceLabels(stated)).length ? { classification: classifySourceLabels(stated) } : {}),
|
||||
// No description column, and the prose lives on each event's own
|
||||
// article rather than further down this page — so there is nothing to
|
||||
// summarise, and `Type(s)` is a classification rather than a blurb.
|
||||
|
||||
@@ -191,6 +191,7 @@ export function parseFandomHistoryTable(
|
||||
game: ctx.game,
|
||||
title,
|
||||
type: "other",
|
||||
classification: { activity: "battle-pass", family: ctx.sourceId === "hsr-fandom-nameless-honor" ? "nameless-honor" : "city-fund" },
|
||||
summary: null,
|
||||
startsAt: startsAt.iso,
|
||||
startPrecision: startsAt.precision,
|
||||
|
||||
@@ -479,6 +479,7 @@ function toEvent(c: Candidate, ctx: ParseContext): GachaEvent {
|
||||
id: eventId(ctx.game, title, c.start.iso),
|
||||
game: ctx.game,
|
||||
title,
|
||||
...(/^Battle Pass\s*-/i.test(c.title) ? { classification: { activity: "battle-pass" as const, family: "battle-pass" } } : {}),
|
||||
type: inferType(title),
|
||||
summary: c.summary,
|
||||
startsAt: c.start.iso,
|
||||
|
||||
@@ -191,6 +191,7 @@ export function parseFandomVersionTable(
|
||||
game: ctx.game,
|
||||
title: pass.title,
|
||||
type: "other",
|
||||
classification: { activity: "battle-pass", family: "mira-journey" },
|
||||
summary: `Follows version ${row.number} (${row.name}). ${pass.link}`,
|
||||
startsAt: row.start.iso,
|
||||
startPrecision: row.start.precision,
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
import type { DisplayEvent } from "./custom.ts";
|
||||
import type { EventClassification } from "./schema.ts";
|
||||
|
||||
/** The source's category vocabulary, rather than similarity between titles. */
|
||||
export function classifySourceLabels(labels: string): EventClassification {
|
||||
const result: EventClassification = {};
|
||||
if (/\bMiliastra Wonderland\b/i.test(labels)) result.lane = "miliastra";
|
||||
if (/\bMiliastra Pass\b/i.test(labels)) {
|
||||
result.lane = "miliastra";
|
||||
result.activity = "battle-pass";
|
||||
result.family = "miliastra-pass";
|
||||
} else if (/\bBattle Pass\b/i.test(labels)) {
|
||||
result.activity = "battle-pass";
|
||||
result.family = "battle-pass";
|
||||
}
|
||||
// Named TCG families in Genshin Event's type glossary. Forge Realm was also
|
||||
// confirmed in game by the owner, 2026-11-01 (docs event audit).
|
||||
if (/Genius Invokation TCG|The Forge Realm['’]s Temper|Heated Battle Mode/i.test(labels)) result.tags = ["tcg"];
|
||||
return result;
|
||||
}
|
||||
|
||||
/** Compatibility for source-backed snapshots/feeds predating the metadata. */
|
||||
export function classificationFor(event: DisplayEvent): EventClassification {
|
||||
if (event.id.startsWith("myevent:") || event.sourceUrl === null) return {};
|
||||
if (event.classification?.unresolved) return {};
|
||||
let known: EventClassification = {};
|
||||
if (event.game === "genshin" || event.game === "miliastra") {
|
||||
let path = "";
|
||||
try {
|
||||
const url = new URL(event.sourceUrl);
|
||||
if (url.hostname === "genshin-impact.fandom.com") path = decodeURIComponent(url.pathname).replace(/_/g, " ");
|
||||
} catch { /* Unknown links provide no evidence. */ }
|
||||
if (/^\/wiki\/(Miliastra Pass|Starlit Gala|Starlight Voyage|Raiment Collection)(?:[/:]|$)/i.test(path)) known.lane = "miliastra";
|
||||
if (/^\/wiki\/Miliastra Pass\//i.test(path)) known = { ...known, activity: "battle-pass", family: "miliastra-pass" };
|
||||
if (/^\/wiki\/Battle Pass\//i.test(path)) known = { activity: "battle-pass", family: "battle-pass" };
|
||||
if (/^\/wiki\/(The Forge Realm['’]s Temper|Heated Battle Mode)(?:[/:]|$)/i.test(path)) known.tags = ["tcg"];
|
||||
// Game8's overview link does not identify the article. These two already
|
||||
// published pass names are pinned fixtures; future passes use source labels.
|
||||
if (event.sourceId === "genshin-game8-events" && /^(?:Battle Pass - )?(Moontrace|Frostfarer)$/.test(event.title)) known = { activity: "battle-pass", family: "battle-pass" };
|
||||
}
|
||||
const families: Record<string, string> = {
|
||||
"hsr-fandom-nameless-honor": "nameless-honor",
|
||||
"zzz-fandom-city-fund": "city-fund",
|
||||
"nikki-fandom-mira-journey": "mira-journey",
|
||||
};
|
||||
const family = families[event.sourceId];
|
||||
if (family) known = { activity: "battle-pass", family };
|
||||
return { ...known, ...event.classification };
|
||||
}
|
||||
|
||||
export const displayGameFor = (event: DisplayEvent): string => classificationFor(event).lane ?? event.game;
|
||||
export const passFamilyFor = (event: DisplayEvent): string | null => {
|
||||
const c = classificationFor(event);
|
||||
return c.activity === "battle-pass" ? c.family ?? "battle-pass" : null;
|
||||
};
|
||||
export const passPlacementId = (event: DisplayEvent): string | null => {
|
||||
const family = passFamilyFor(event);
|
||||
return family === null ? null : `feed-cycle:${displayGameFor(event)}:${family}`;
|
||||
};
|
||||
export const isTcgEvent = (event: DisplayEvent): boolean => classificationFor(event).tags?.includes("tcg") === true;
|
||||
export const tracksEventCategory = (event: DisplayEvent, prefs: Record<string, unknown>): boolean => !(displayGameFor(event) === "genshin" && prefs.genshinTcg === false && isTcgEvent(event));
|
||||
export function classifiedEvent(event: DisplayEvent): DisplayEvent {
|
||||
const game = displayGameFor(event);
|
||||
return game === event.game ? event : { ...event, game, classification: classificationFor(event) };
|
||||
}
|
||||
/** Miliastra consumes the same source bytes, freshness and attribution. */
|
||||
export const sourceGameFor = (game: string): string => game === "miliastra" ? "genshin" : game;
|
||||
@@ -62,6 +62,7 @@ export interface GameMeta {
|
||||
}
|
||||
|
||||
export const GAMES: Record<GameId, GameMeta> = {
|
||||
miliastra: { id: "miliastra", name: "Miliastra Wonderland", short: "Miliastra", hue: "#9C83D6", studio: "HoYoverse", dailyTasks: "" },
|
||||
// EU reset at 04:00 UTC+1 confirmed in game by the owner on 2026-10-16: the
|
||||
// Trounce Domains menu reads "every Monday 4AM GMT+1".
|
||||
genshin: { id: "genshin", name: "Genshin Impact", short: "Genshin", hue: "#4EA8DE" , studio: "HoYoverse", dailyTasks: "Commissions, resin" },
|
||||
|
||||
@@ -23,6 +23,7 @@ export const GameId = z.enum([
|
||||
"nikke", // Goddess of Victory: Nikke
|
||||
"hi3", // Honkai Impact 3rd
|
||||
"pgr", // Punishing: Gray Raven
|
||||
"miliastra", // Logical lane; source identities remain Genshin.
|
||||
]);
|
||||
export type GameId = z.infer<typeof GameId>;
|
||||
|
||||
@@ -50,6 +51,15 @@ export type Region = z.infer<typeof Region>;
|
||||
export const Precision = z.enum(["exact", "day", "unknown"]);
|
||||
export type Precision = z.infer<typeof Precision>;
|
||||
|
||||
export const EventClassification = z.object({
|
||||
unresolved: z.boolean().optional(),
|
||||
lane: z.literal("miliastra").optional(),
|
||||
activity: z.literal("battle-pass").optional(),
|
||||
family: z.string().min(1).optional(),
|
||||
tags: z.array(z.string()).optional(),
|
||||
});
|
||||
export type EventClassification = z.infer<typeof EventClassification>;
|
||||
|
||||
export const GachaEvent = z
|
||||
.object({
|
||||
id: z.string(),
|
||||
@@ -85,6 +95,7 @@ export const GachaEvent = z
|
||||
* client's non-strict parse drops it unread.
|
||||
*/
|
||||
mergedIds: z.array(z.string()).optional(),
|
||||
classification: EventClassification.optional(),
|
||||
})
|
||||
// These invariants are cheap here and prevent a whole class of wrong-date bugs
|
||||
// reaching the feed. See docs/INGESTION.md § Stage 4.
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
import { expect, test } from "bun:test";
|
||||
import { classificationFor, classifySourceLabels, displayGameFor, passFamilyFor, isTcgEvent } from "../src/shared/classification.ts";
|
||||
import { GachaEvent } from "../src/shared/schema.ts";
|
||||
import { mergeEvents } from "../src/ingest/merge.ts";
|
||||
import { sampleEvent } from "./helpers/reader.ts";
|
||||
import { ADAPTERS } from "../src/ingest/adapters/index.ts";
|
||||
|
||||
test("source labels retain Miliastra and pass identity independently", () => {
|
||||
expect(classifySourceLabels("In-Game, Miliastra Pass, Miliastra Wonderland")).toEqual({ lane: "miliastra", activity: "battle-pass", family: "miliastra-pass" });
|
||||
expect(classifySourceLabels("In-Game, Battle Pass")).toEqual({ activity: "battle-pass", family: "battle-pass" });
|
||||
expect(classifySourceLabels("In-Game, The Forge Realm's Temper")).toEqual({ tags: ["tcg"] });
|
||||
expect(classifySourceLabels("Challenge")).toEqual({});
|
||||
});
|
||||
test("cached article paths route poetic names without changing IDs", () => {
|
||||
const e = { ...sampleEvent, sourceUrl: "https://genshin-impact.fandom.com/wiki/Miliastra_Pass/2026-09-23" };
|
||||
expect(displayGameFor(e)).toBe("miliastra");
|
||||
expect(passFamilyFor(e)).toBe("miliastra-pass");
|
||||
expect(classificationFor({ ...sampleEvent, sourceUrl: "https://genshin-impact.fandom.com/wiki/Starlight_Voyage/2026-10-19" }).lane).toBe("miliastra");
|
||||
expect(classificationFor(sampleEvent)).toEqual({});
|
||||
expect(classificationFor({ ...e, id: "myevent:abcdef", sourceUrl: null })).toEqual({});
|
||||
expect(isTcgEvent({ ...sampleEvent, sourceUrl: "https://genshin-impact.fandom.com/wiki/The_Forge_Realm%27s_Temper/2026-09-23" })).toBe(true);
|
||||
});
|
||||
test("optional classification survives schema and the losing source's merge", () => {
|
||||
const labeled = GachaEvent.parse({ ...sampleEvent, sourceId: "second", confidence: 0.5, classification: { activity: "battle-pass", family: "battle-pass" } });
|
||||
expect(mergeEvents([[sampleEvent], [labeled]]).events[0]?.classification).toEqual({ activity: "battle-pass", family: "battle-pass" });
|
||||
expect(GachaEvent.parse(sampleEvent).id).toBe(sampleEvent.id);
|
||||
});
|
||||
test("the Genshin parser preserves its real type column", async () => {
|
||||
const a = ADAPTERS.find(a => a.id === "genshin-fandom-events")!;
|
||||
const events = a.parse(await Bun.file("fixtures/genshin/fandom-events-2026-09-12.html").text(), { now: "2026-09-12T12:00:00.000Z", sourceId: a.id, sourceUrl: a.url, game: a.game });
|
||||
expect(events.find(e => e.sourceUrl.includes("Miliastra_Pass"))?.classification).toHaveProperty("lane", "miliastra");
|
||||
expect(events.find(e => e.sourceUrl.includes("Battle_Pass"))?.classification).toHaveProperty("activity", "battle-pass");
|
||||
});
|
||||
test("conflicting explicit pass families are reported without choosing a routing", () => {
|
||||
const a = { ...sampleEvent, classification: { activity: "battle-pass" as const, family: "first" } };
|
||||
const b = { ...sampleEvent, sourceId: "second", classification: { activity: "battle-pass" as const, family: "second" } };
|
||||
const result = mergeEvents([[a], [b]]);
|
||||
expect(result.classificationConflicts).toHaveLength(1);
|
||||
expect(result.events[0]?.classification).toEqual({ unresolved: true });
|
||||
});
|
||||
Reference in New Issue
Block a user