feat: refresh sources on a schedule

The feed was generated from checked-in fixtures and only moved when
somebody captured a page by hand. This fetches.

bun run refresh caches each page raw under snapshots/ and rebuilds the
feed from what it cached; build-feed prefers a snapshot and falls back to
the fixture, so a clean checkout and the container build stay offline and
reproducible. The workflow runs it twice a day and commits only when a
page's bytes actually changed — a 304, an identical body or a rejected
parse all leave the tree clean — then dispatches ci.yml, which already
knows how to test, build and deploy.

Scraping conduct is enforced in code rather than left to good
intentions: one request per source per cycle, a six-hour floor checked
per source, conditional requests, a User-Agent with a contact URL, and
robots.txt honoured — failing closed, because a permission we could not
read is not a permission we have. No retries; a retry is a second
request.

A body that yields zero events is rejected and the previous snapshot
kept, so a redesigned wiki shows up as a stale timestamp rather than an
emptied calendar. One source down is a warning; all of them down fails
the run, so a cycle that learned nothing is never committed.

Tested entirely offline against an injected fetch and clock — no request
has ever been made to a live wiki from this code.

Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>

scripts/build-feed.ts

@@ -1,18 +1,22 @@
 /**
- * Build the static event feed from checked-in fixtures.
+ * Build the static event feed from cached snapshots, falling back to fixtures.
  *
- * Offline: this reads fixtures, never the network. It exists so the client can
- * be developed and demoed against real parsed data before the server and
- * database land, and it emits exactly the shape `GET /api/events.json` will.
+ * Offline: this reads files on disk, never the network. Fetching is
+ * `scripts/refresh-sources.ts`'s job; this stage only parses what that left in
+ * the snapshot cache. On a clean checkout — and in the container build — no
+ * snapshot exists and the checked-in fixture is used instead, so the build
+ * stays reproducible and a wiki being down never breaks it.
  *
  *   bun run build:feed
  */
 import { ADAPTERS } from "../src/ingest/adapters/index.ts";
 import { mergeEvents } from "../src/ingest/merge.ts";
+import { SnapshotStore, freshnessAt } from "../src/ingest/snapshots.ts";
 import { EventFeed, SCHEMA_VERSION, type SourceHealth } from "../src/shared/feed.ts";
 import type { GachaEvent, GameId } from "../src/shared/schema.ts";
 
 const OUT = "public/data/events.v1.json";
+const snapshots = new SnapshotStore(process.env["SNAPSHOT_DIR"] ?? "snapshots");
 
 /**
  * Newest fixture for one *source*, not one game.
@@ -32,12 +36,32 @@ async function latestFixture(adapterId: string, game: GameId) {
   return { file, html: await Bun.file(file).text() };
 }
 
+/**
+ * The document to parse for one source: the live snapshot when the refresh
+ * runner has cached one, otherwise the newest checked-in fixture.
+ *
+ * `at` is what the UI's staleness badge reads, so it must never claim to be
+ * fresher than the bytes actually are — a fixture reports its capture date.
+ */
+async function documentFor(adapterId: string, game: GameId) {
+  const cached = await snapshots.read(adapterId);
+  if (cached !== null) {
+    return {
+      file: snapshots.bodyPath(adapterId),
+      html: cached.html,
+      at: freshnessAt(cached),
+    };
+  }
+  const { file, html } = await latestFixture(adapterId, game);
+  return { file, html, at: fixtureDate(file) };
+}
+
 const now = new Date().toISOString();
 const byGame = new Map<GameId, GachaEvent[][]>();
 const sources: SourceHealth[] = [];
 
 for (const adapter of ADAPTERS) {
-  const { file, html } = await latestFixture(adapter.id, adapter.game);
+  const { file, html, at } = await documentFor(adapter.id, adapter.game);
   const events = adapter.parse(html, {
     now,
     sourceUrl: adapter.url,
@@ -53,10 +77,9 @@ for (const adapter of ADAPTERS) {
     sourceId: adapter.id,
     game: adapter.game,
     url: adapter.url,
-    // Fixture capture date stands in for a real fetch timestamp until the
-    // scheduler exists. The UI's staleness badge reads this, so it must not
-    // claim to be fresher than the data actually is.
-    lastSuccessAt: fixtureDate(file),
+    // When the bytes were last confirmed live; a fixture's capture date when
+    // this source has never been refreshed.
+    lastSuccessAt: at,
     eventCount: events.length,
   });