From 24327ea99b2ea99131df239f0642b43cb2114aa4 Mon Sep 17 00:00:00 2001 From: Lucas Winther Date: Sat, 7 Nov 2026 02:16:57 +0100 Subject: [PATCH] feat(feed): show the end a Genshin article states Genshin and Miliastra rows take the end their own wiki article states, because the table's date can mean 03:59, 14:59, 17:59 or one instant worldwide. Rows with no usable time stay on the daily-reset reading. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 18 ++++- docs/2026-11-01-genshin-event-audit.md | 16 +++- docs/INGESTION.md | 40 ++++++++++ docs/PRD.md | 3 + .../2026-11-07-exact-end-times-design.md | 2 +- scripts/build-feed.ts | 21 +++++- test/article-feed.test.ts | 75 +++++++++++++++++++ 7 files changed, 169 insertions(+), 6 deletions(-) create mode 100644 test/article-feed.test.ts diff --git a/AGENTS.md b/AGENTS.md index 0a923a5..2ac8b00 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -173,6 +173,7 @@ src/shared/ schema.ts (the contract), time.ts, daily.ts, effort.ts, games. custom.ts — reader-authored games and events, and their key spaces src/ingest/ html.ts, dates.ts (sixteen formats), merge.ts, sanitize.ts, robots.ts, snapshots.ts health.ts — which of the three empties a source's zero was; pure + articleEnds.ts, articleStore.ts — the end a Genshin article states, and where it is kept parsers/ game8.ts, wikigg.ts, akwiki.ts, fandom.ts, bawiki.ts, holodori.ts, iopwiki.ts, stellasora.ts, arustats.ts, karendar.ts — keyed by SITE, not game adapters/ index.ts — SOURCES registry binding url+game+parser, and the sanitize seam @@ -185,7 +186,7 @@ src/client/ ReaderApp (shared interfaces), components/redesign/, service w zoom.ts — the timeline's scale ladder; pure lanes.ts — how the timeline stacks: a lane per game, or one deadline queue; pure theme.ts — dark or light, and what a game hue reads as on each -scripts/ build-feed.ts, build-static.ts, parse-fixture.ts (offline), refresh-sources.ts (fetches) +scripts/ build-feed.ts, build-static.ts, parse-fixture.ts (offline), refresh-sources.ts, refresh-articles.ts (fetch) serve.ts static server + /api/health test/ `bun test` — one file per module or feature; no count here, it only goes stale fixtures// raw HTML + .expected.json per source — pinned, kept forever @@ -228,6 +229,12 @@ These come from how gacha games actually schedule things, and they cause most bu deadline they were counting down to vanish on its last day. So `latestBoundaryMs` answers the same question for the *last* region, and `bawiki.ts` and three branches of `fandom.ts` ask it. Nothing stored changes: it is one comparison, not a resolved boundary written to the feed. + **Genshin and Miliastra rows do better than that reading when they can.** Fandom's table prints a + date, and on 2026-11-07 that date meant 03:59 server time on nine live rows, 14:59 or 17:59 on four + and 05:59 GMT+8 on two: up to 14 hours early and 5 late. So the refresh keeps each row's article + and the build applies the end its infobox states, after the merge (`src/ingest/articleEnds.ts`, + `docs/INGESTION.md` § Article ends). The reset reading stays as the fallback, and it is still the + only one for every other game. - **Patch cycles are ~6 weeks, and a span over a year is a parse error rather than a long event.** `test/adapters/game8.test.ts` rejects one across every fixture. Two corrections to what this rule used to say, both load-bearing: @@ -401,6 +408,10 @@ Sources are community wikis. Treat them as a guest would: the shape an edge network throttles, and what a burst looks like from the far end regardless of our intent. - Send `If-None-Match` / `If-Modified-Since`; treat `304` as "skip, unchanged". +- **Genshin event articles are the one exception to one request per source**, and they are bounded + rather than open-ended: only rows the table lists now, only the lead section, one request per + article per 6 hours at most, under the same robots gate, per-host spacing and User-Agent. `--force` + does not reach them. - Cache raw snapshots so re-parsing never re-fetches. **Iterate against fixtures, not the network.** - Record `sourceUrl` on every event and surface attribution in the UI. @@ -794,6 +805,11 @@ source CI can refresh. Four things to know about it: - **Run-date suffixes are stripped.** Recurring events often carry their run date in the subpage title (`Overflowing Abundance 2026-09-14`). The date names the run, the start date is already half the event ID, and keeping it would duplicate the date on the calendar. +- **The table's date is not the event's end time; the article's infobox is.** The `Event` table is + still this source's one registered page, but its rows' articles are read too, at most one request + each per 6 hours and normally once a week: `scripts/refresh-articles.ts`, and § Article ends in + `docs/INGESTION.md`. Read only `data-source="time_end"`. The `Ends in` lines are derived, and the body + prose gives `05:59` with no zone on a row whose infobox says `GMT+8`. - **`latestBoundaryMs` gates currency.** Like FGO and Infinity Nikki, this source prints day-precision dates (`Aug. 14, 2026 – Aug. 24, 2026`), so `parseFullRange` stores UTC midnight as a placeholder. Currency is checked against `latestBoundaryMs` so an event is not retired prematurely on its final day diff --git a/docs/2026-11-01-genshin-event-audit.md b/docs/2026-11-01-genshin-event-audit.md index eb412e4..20ada40 100644 --- a/docs/2026-11-01-genshin-event-audit.md +++ b/docs/2026-11-01-genshin-event-audit.md @@ -109,7 +109,9 @@ disagreement between type label and article path. The owner stopped the in-game walk here because it is slow to navigate, so the other Genshin rows have no in-game confirmation. -### Open items, none fixed +### Open items + +Item 1 was then built: see its note. The others are not fixed. 1. **Day-precision ends can be hours wrong, in both directions.** Fandom's event table prints dates only, so the app resolves an end to that game-day's reset. @@ -127,6 +129,18 @@ in-game confirmation. remain rounded observations and must not be used to mint timestamps. The likely fix is reading exact times from article pages, which adds requests per refresh and so needs its own design and a look at § Scraping conduct. + + **Done:** `superpowers/specs/2026-11-07-exact-end-times-design.md`. Every live row + was then checked: 9 of 15 were correct. In-game, Test Run read 17d 16h as its article + implies, where the app said 17d 2h. + + **One reading disagrees, and is accepted rather than special-cased.** Forge Realm's + Temper is not in the event tab. Its timer at The Cat's Tail read 38d 22h at about + 00:30 UTC on 2026-11-07, implying 15 Dec about 22:30-23:30 UTC. Its article says + 05:59 GMT+8 on 15 Dec, which is 14 Dec 21:59 UTC, about 25h earlier, and the app had + been about 20h earlier. The timer may cover something beyond the event, or the + article may be wrong: neither is established. The error is on the safe side. Recheck + when the next `The Forge Realm's Temper` run starts. 2. **Wonderland Treasures has no row in either lane.** In game, `Wonderland Treasures - Polyphonic Harmony` appears in Genshin's event tab, 38d 13h 47m remaining around 00:12 UTC on 2026-11-07. It is probably the 7.2 counterpart of diff --git a/docs/INGESTION.md b/docs/INGESTION.md index b27025e..a864ed3 100644 --- a/docs/INGESTION.md +++ b/docs/INGESTION.md @@ -736,3 +736,43 @@ fetched once each and old cached bundles can continue validating the feed. A small, evidenced article-family resolver supports old feeds/saved snapshots; unknown content stays in its original category. See DATA-MODEL's source identity section and the 2026-11-01 content-classification design. + +## Article ends (2026-11-07) + +Fandom's Genshin `Event` table prints a date, and the same date can mean 03:59 server time, 14:59, +17:59 or one instant worldwide. Each row's article states which, so the refresh keeps those articles +and the build uses them. Genshin and Miliastra only: `genshin-fandom-events` is the one source wired +up, and other games' articles are laid out differently. Reasoning and evidence are in +`superpowers/specs/2026-11-07-exact-end-times-design.md` and `2026-11-01-genshin-event-audit.md`. + +**Fetch.** After the source loop, `runRefresh` lists the live rows from the stored table and calls +`scripts/refresh-articles.ts`. It requests `action=parse§ion=0`, about 8 KB, and stores the +response under `snapshots/articles/genshin-fandom/`. An article is due when it was never read, when +it stated no time and 6 hours have passed, or when it stated a time and 7 days have passed since a +read that worked. The listing passes the stored ends to the parser: on a row's last day they are what +keeps it listed, and an unlisted article is deleted. + +| Result of a read | Stored article | Row shows | +|---|---|---| +| Network error, non-200 other than 404, robots refusal, or a 200 that is not the API's answer | Kept | The last good time, if any | +| 404 or `missingtitle` | Body deleted; metadata kept for the 6-hour floor | Fallback | +| A real article with no usable End | Overwritten | Fallback, logged but not warned: stubs are ordinary | + +None of these touches a source's failure streak, health or broken count. + +**Parse.** `parseArticleEnd` reads only the `` cell, in two shapes: +`December 15, 2026 14:59 (Server Time)` gives `regionEnds` at each server's clock, with `endsAt` the +earliest; `December 16, 2026 05:59 GMT+8` gives one instant. Anything else is no usable time. + +**Apply.** `applyArticleEnds` runs in `scripts/build-feed.ts` after `mergeEvents`, because the merge +keeps the higher-confidence copy, often Game8's, and would discard an end set on the Fandom row. It +matches by ID or `mergedIds`, and writes only `endsAt`, `endPrecision`, `regionScoped` and +`regionEnds`. The article's date must equal both the table's and the survivor's. If not, the row +stays as merged and the build logs `! article:`. A survivor already at `exact` is left alone. + +**Currency.** `fandom.ts` retires a row on its stated end when the article agrees on the date. Without +this, a row ending 14:59 server time would leave the feed on its last day, hours before it closes. +The parser reads `ctx.articleEnds` for this alone and never writes one onto a row. + +**Known exception.** Forge Realm's Temper: the Cat's Tail timer read about a day later than its article +on 2026-11-07. Recheck when the next run starts; see the audit doc. diff --git a/docs/PRD.md b/docs/PRD.md index ee48260..122d565 100644 --- a/docs/PRD.md +++ b/docs/PRD.md @@ -663,6 +663,9 @@ The app's entire value is that the dates are right. Therefore: reading retires an event up to nine hours before the game does and the reader is standing in the game while we say it. This is a reading of the date the source printed, not a time invented for it, and the detail sheet says as much. +- **A time the source does state is used.** Genshin and Miliastra rows take the end their own wiki + article states, because the same printed date can mean three different times. A row whose article + states none, or a different date, stays on the reading above. An empty calendar is a recoverable disappointment. A confidently wrong end date is the failure this product exists to prevent. diff --git a/docs/superpowers/specs/2026-11-07-exact-end-times-design.md b/docs/superpowers/specs/2026-11-07-exact-end-times-design.md index edff6c6..05ca034 100644 --- a/docs/superpowers/specs/2026-11-07-exact-end-times-design.md +++ b/docs/superpowers/specs/2026-11-07-exact-end-times-design.md @@ -1,6 +1,6 @@ # Exact end times for Genshin and Miliastra — design -Date: 2026-11-07. Status: **approved** by the owner 2026-11-07; plan at `../plans/2026-11-07-exact-end-times.md`. +Date: 2026-11-07. Status: **built** 2026-11-07; plan at `../plans/2026-11-07-exact-end-times.md`. Detail of the shipped behaviour is in `docs/INGESTION.md` § Article ends. ## Why diff --git a/scripts/build-feed.ts b/scripts/build-feed.ts index 41b90bd..4167500 100644 --- a/scripts/build-feed.ts +++ b/scripts/build-feed.ts @@ -10,14 +10,18 @@ * bun run build:feed */ import { ADAPTERS } from "../src/ingest/adapters/index.ts"; +import { applyArticleEnds } from "../src/ingest/articleEnds.ts"; +import { ArticleStore } from "../src/ingest/articleStore.ts"; import { sourceHealth } from "../src/ingest/health.ts"; import { mergeEvents } from "../src/ingest/merge.ts"; import { SnapshotStore, freshnessAt, pickFixture } from "../src/ingest/snapshots.ts"; import { EventFeed, SCHEMA_VERSION, type SourceHealth } from "../src/shared/feed.ts"; import type { GachaEvent, GameId } from "../src/shared/schema.ts"; -const OUT = "public/data/events.v1.json"; +const OUT = process.env["EVENT_FEED_OUT"] ?? "public/data/events.v1.json"; const snapshots = new SnapshotStore(process.env["SNAPSHOT_DIR"] ?? "snapshots"); +/** The one source whose rows have articles stating an exact end. */ +const ARTICLE_SOURCE_ID = "genshin-fandom-events"; /** * Newest fixture for one *source*, not one game. @@ -73,7 +77,10 @@ async function documentFor(adapterId: string, game: GameId) { }; } -const now = new Date().toISOString(); +// Overridable so a test against a pinned fixture does not expire with its events. +const now = process.env["EVENT_FEED_NOW"] ?? new Date().toISOString(); +const articleEnds = await new ArticleStore(snapshots.root).readEnds(); +let articleRows: GachaEvent[] = []; const byGame = new Map>(); const sources: SourceHealth[] = []; @@ -87,7 +94,9 @@ for (const adapter of ADAPTERS) { sourceUrl: adapter.url, sourceId: adapter.id, game: adapter.game, + ...(adapter.id === ARTICLE_SOURCE_ID ? { articleEnds } : {}), }); + if (adapter.id === ARTICLE_SOURCE_ID) articleRows = events; // Which of the three empties this is, decided in a module a test can reach // rather than here — see `src/ingest/health.ts` for why that matters. @@ -122,7 +131,13 @@ for (const [, groups] of byGame) { .sort((a, b) => b.priority - a.priority) .map((g) => g.events); const merged = mergeEvents(sorted); - events.push(...merged.events); + // After the merge: it keeps the higher-confidence copy, often Game8's, which + // would discard an end set on the Fandom row (spec decision 10). + events.push( + ...applyArticleEnds(merged.events, articleRows, articleEnds, (line) => + console.log(` ! article: ${line}`), + ), + ); conflictCount += merged.conflicts.length; conflictCount += merged.classificationConflicts.length; for (const c of merged.classificationConflicts) console.warn(` ! classification conflict: "${c.kept.title}" between ${c.kept.sourceId} and ${c.rejected.sourceId}; routing left unresolved`); diff --git a/test/article-feed.test.ts b/test/article-feed.test.ts new file mode 100644 index 0000000..65913cc --- /dev/null +++ b/test/article-feed.test.ts @@ -0,0 +1,75 @@ +import { afterEach, beforeEach, expect, test } from "bun:test"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { ArticleStore } from "../src/ingest/articleStore.ts"; +import { EventFeed } from "../src/shared/feed.ts"; +import { SnapshotStore } from "../src/ingest/snapshots.ts"; + +const NOW = "2026-11-07T00:00:00.000Z"; +const dir = "fixtures/genshin/articles"; +let root: string; + +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), "event-clock-feed-")); + await new SnapshotStore(root).save("genshin-fandom-events", { + url: "https://genshin-impact.fandom.com/api.php?action=parse&page=Event", + body: await Bun.file(`${dir}/event-table-2026-11-06.html`).text(), + etag: null, + lastModified: null, + at: "2026-11-06T17:27:59.858Z", + eventCount: 15, + }); +}); +afterEach(async () => { + await rm(root, { recursive: true, force: true }); +}); + +async function buildFeed() { + const out = join(root, "feed.json"); + const proc = Bun.spawn(["bun", "run", "scripts/build-feed.ts"], { + env: { ...process.env, SNAPSHOT_DIR: root, EVENT_FEED_OUT: out, EVENT_FEED_NOW: NOW }, + stdout: "pipe", + stderr: "pipe", + }); + const stderr = await new Response(proc.stderr).text(); + const log = await new Response(proc.stdout).text(); + expect(`${await proc.exited} ${stderr}`).toBe("0 "); + return { feed: EventFeed.parse(await Bun.file(out).json()), log }; +} + +const hourOfRuin = "genshin:hour-of-ruin:2026-11-04"; + +test("the feed carries the end the article states, on an unchanged id", async () => { + await new ArticleStore(root).saveRead( + "Hour_of_Ruin_(Event)", + await Bun.file(`${dir}/hour-of-ruin-2026-11-07.html`).text(), + NOW, + ); + const { feed } = await buildFeed(); + const row = feed.events.find((e) => e.id === hourOfRuin)!; + expect(row.endPrecision).toBe("exact"); + expect(row.regionScoped).toBe(true); + expect(row.regionEnds?.europe).toBe("2026-12-15T13:59:00.000Z"); + expect(row.startsAt).toBe("2026-11-04T00:00:00.000Z"); +}); + +test("no articles directory builds the same day-precision row as today", async () => { + const { feed } = await buildFeed(); + const row = feed.events.find((e) => e.id === hourOfRuin)!; + expect(row.endPrecision).toBe("day"); + expect(row.endsAt).toBe("2026-12-15T00:00:00.000Z"); + expect(row.regionEnds).toBeNull(); +}); + +test("an article that disagrees on the date is named in the build log", async () => { + const body = (await Bun.file(`${dir}/hour-of-ruin-2026-11-07.html`).text()).replace( + "December 15, 2026 14:59", + "December 20, 2026 14:59", + ); + await new ArticleStore(root).saveRead("Hour_of_Ruin_(Event)", body, NOW); + const { feed, log } = await buildFeed(); + expect(feed.events.find((e) => e.id === hourOfRuin)?.endPrecision).toBe("day"); + expect(log).toContain("article:"); + expect(log).toContain("Hour of Ruin"); +});