feat(feed): record crawler check and content update timestamps in SourceHealth

Add lastConfirmedAt and contentChangedAt to SourceHealth and enrich freshness() to track per-source check and content update times.

docs/DATA-MODEL.md

@@ -635,3 +635,8 @@ takes the offline promise down with it. A default must also be the *strict* read
 `parsedCount` defaults to `null` — an older feed never recorded the figure, which is an absence of
 information and not evidence of health — and `statesNoEvents` to `false`, because a feed built
 before the field existed never asked the page and cannot be read as having got an answer.
+
+Two timestamp fields distinguish crawler activity from site updates: `lastConfirmedAt` records when
+the crawler last successfully contacted and verified the source (HTTP 200 or 304 Not Modified), and
+`contentChangedAt` records when the served bytes last changed. Both default to `null` so feeds built
+before their introduction continue to validate.

scripts/build-feed.ts

@@ -51,10 +51,19 @@ async function documentFor(adapterId: string, game: GameId) {
       file: snapshots.bodyPath(adapterId),
       html: cached.html,
       at: freshnessAt(cached),
+      lastConfirmedAt: cached.state.lastConfirmedAt,
+      contentChangedAt: cached.meta.contentChangedAt,
     };
   }
   const { file, html } = await latestFixture(adapterId, game);
-  return { file, html, at: fixtureDate(file) };
+  const date = fixtureDate(file);
+  return {
+    file,
+    html,
+    at: date,
+    lastConfirmedAt: null,
+    contentChangedAt: date,
+  };
 }
 
 const now = new Date().toISOString();
@@ -62,7 +71,10 @@ const byGame = new Map<GameId, Array<{ priority: number; events: GachaEvent[] }>
 const sources: SourceHealth[] = [];
 
 for (const adapter of ADAPTERS) {
-  const { file, html, at } = await documentFor(adapter.id, adapter.game);
+  const { file, html, at, lastConfirmedAt, contentChangedAt } = await documentFor(
+    adapter.id,
+    adapter.game,
+  );
   const events = adapter.parse(html, {
     now,
     sourceUrl: adapter.url,
@@ -72,7 +84,10 @@ for (const adapter of ADAPTERS) {
 
   // Which of the three empties this is, decided in a module a test can reach
   // rather than here — see `src/ingest/health.ts` for why that matters.
-  const health = sourceHealth(adapter, html, at, events.length);
+  const health = sourceHealth(adapter, html, at, events.length, {
+    lastConfirmedAt,
+    contentChangedAt,
+  });
   const { parsedCount } = health;
 
   const groups = byGame.get(adapter.game) ?? [];

src/ingest/health.ts

@@ -44,6 +44,10 @@ export function sourceHealth(
   html: string,
   at: string | null,
   eventCount: number,
+  timestamps?: {
+    lastConfirmedAt?: string | null;
+    contentChangedAt?: string | null;
+  },
 ): SourceHealth {
   // Parsed a second time as of the document's own capture date, when nothing in
   // it had expired yet. That figure is what separates "this parser has stopped
@@ -82,6 +86,8 @@ export function sourceHealth(
     // When the bytes were last confirmed live; a fixture's capture date when
     // this source has never been refreshed.
     lastSuccessAt: at,
+    lastConfirmedAt: timestamps?.lastConfirmedAt ?? null,
+    contentChangedAt: timestamps?.contentChangedAt ?? at,
     eventCount,
     parsedCount,
     statesNoEvents,

src/shared/feed.ts

@@ -15,6 +15,16 @@ export const SourceHealth = z.object({
   game: GameId,
   url: z.string().url(),
   lastSuccessAt: z.string().datetime().nullable(),
+  /**
+   * When the crawler last successfully contacted and confirmed this source
+   * (HTTP 200 or 304 Not Modified). Null when never checked (e.g. clean checkout
+   * using fixtures, or older feed).
+   */
+  lastConfirmedAt: z.string().datetime().nullable().default(null),
+  /**
+   * When the served bytes last changed.
+   */
+  contentChangedAt: z.string().datetime().nullable().default(null),
   /** Events this source contributed to the feed — after expired ones are dropped. */
   eventCount: z.number().int().nonnegative(),
   /**
@@ -69,6 +79,22 @@ export type EventFeed = z.infer<typeof EventFeed>;
 /** A game's data is stale past this age (PRD F7). */
 export const STALE_AFTER_MS = 48 * 60 * 60 * 1000;
 
+export interface StaleSourceInfo {
+  sourceId: string;
+  url: string;
+  lastSuccessAt: string | null;
+  lastConfirmedAt: string | null;
+  contentChangedAt: string | null;
+}
+
+export interface StaleGame {
+  game: GameId;
+  lastSuccessAt: string | null;
+  lastConfirmedAt: string | null;
+  contentChangedAt: string | null;
+  sources: StaleSourceInfo[];
+}
+
 export interface Freshness {
   /**
    * When any source last had its bytes confirmed — the newest `lastSuccessAt`.
@@ -84,7 +110,7 @@ export interface Freshness {
    */
   refreshedAt: string | null;
   /** Per game, oldest first: what has not refreshed inside `STALE_AFTER_MS`. */
-  stale: Array<{ game: GameId; lastSuccessAt: string | null }>;
+  stale: StaleGame[];
 }
 
 /**
@@ -101,6 +127,7 @@ export function freshness(
   now: number,
 ): Freshness {
   const oldestPerGame = new Map<GameId, string | null>();
+  const sourcesByGame = new Map<GameId, SourceHealth[]>();
   let refreshedAt: string | null = null;
 
   for (const source of sources) {
@@ -117,11 +144,56 @@ export function freshness(
     if (known === undefined || (known !== null && (at === null || at < known))) {
       oldestPerGame.set(source.game, at);
     }
+
+    const list = sourcesByGame.get(source.game) ?? [];
+    list.push(source);
+    sourcesByGame.set(source.game, list);
   }
 
-  const stale = [...oldestPerGame.entries()]
+  const stale: StaleGame[] = [...oldestPerGame.entries()]
     .filter(([, at]) => at === null || now - Date.parse(at) > STALE_AFTER_MS)
-    .map(([game, lastSuccessAt]) => ({ game, lastSuccessAt }))
+    .map(([game, lastSuccessAt]) => {
+      const gameSources = sourcesByGame.get(game) ?? [];
+      const sourcesInfo: StaleSourceInfo[] = gameSources
+        .map((s) => ({
+          sourceId: s.sourceId,
+          url: s.url,
+          lastSuccessAt: s.lastSuccessAt,
+          lastConfirmedAt: s.lastConfirmedAt ?? null,
+          contentChangedAt: s.contentChangedAt ?? s.lastSuccessAt,
+        }))
+        .sort((a, b) => a.sourceId.localeCompare(b.sourceId));
+
+      let lastConfirmedAt: string | null = null;
+      let hasConfirmed = false;
+      for (const s of sourcesInfo) {
+        if (!hasConfirmed) {
+          lastConfirmedAt = s.lastConfirmedAt;
+          hasConfirmed = true;
+        } else if (lastConfirmedAt !== null && (s.lastConfirmedAt === null || s.lastConfirmedAt < lastConfirmedAt)) {
+          lastConfirmedAt = s.lastConfirmedAt;
+        }
+      }
+
+      let contentChangedAt: string | null = null;
+      let hasContent = false;
+      for (const s of sourcesInfo) {
+        if (!hasContent) {
+          contentChangedAt = s.contentChangedAt;
+          hasContent = true;
+        } else if (contentChangedAt !== null && (s.contentChangedAt === null || s.contentChangedAt < contentChangedAt)) {
+          contentChangedAt = s.contentChangedAt;
+        }
+      }
+
+      return {
+        game,
+        lastSuccessAt,
+        lastConfirmedAt,
+        contentChangedAt,
+        sources: sourcesInfo,
+      };
+    })
     .sort((a, b) => (a.lastSuccessAt ?? "").localeCompare(b.lastSuccessAt ?? ""));
 
   return { refreshedAt, stale };

test/feed.test.ts

@@ -27,12 +27,16 @@ function source(
   game: GameId,
   lastSuccessAt: string | null,
   sourceId = `${game}-src`,
+  lastConfirmedAt: string | null = null,
+  contentChangedAt: string | null = lastSuccessAt,
 ): SourceHealth {
   return {
     sourceId,
     game,
     url: "https://example.test/events",
     lastSuccessAt,
+    lastConfirmedAt,
+    contentChangedAt,
     eventCount: 3,
 
     parsedCount: 3,
@@ -67,7 +71,28 @@ describe("freshness", () => {
     );
     expect(result.refreshedAt).toBe("2026-08-17T11:00:00.000Z");
     expect(result.stale).toEqual([
-      { game: "endfield", lastSuccessAt: "2026-08-10T11:00:00.000Z" },
+      {
+        game: "endfield",
+        lastSuccessAt: "2026-08-10T11:00:00.000Z",
+        lastConfirmedAt: null,
+        contentChangedAt: "2026-08-10T11:00:00.000Z",
+        sources: [
+          {
+            sourceId: "endfield-game8-events",
+            url: "https://example.test/events",
+            lastSuccessAt: "2026-08-10T11:00:00.000Z",
+            lastConfirmedAt: null,
+            contentChangedAt: "2026-08-10T11:00:00.000Z",
+          },
+          {
+            sourceId: "endfield-wikigg-events",
+            url: "https://example.test/events",
+            lastSuccessAt: "2026-08-17T11:00:00.000Z",
+            lastConfirmedAt: null,
+            contentChangedAt: "2026-08-17T11:00:00.000Z",
+          },
+        ],
+      },
     ]);
   });
 
@@ -79,7 +104,30 @@ describe("freshness", () => {
       ],
       NOW,
     );
-    expect(result.stale).toEqual([{ game: "endfield", lastSuccessAt: null }]);
+    expect(result.stale).toEqual([
+      {
+        game: "endfield",
+        lastSuccessAt: null,
+        lastConfirmedAt: null,
+        contentChangedAt: null,
+        sources: [
+          {
+            sourceId: "endfield-game8-events",
+            url: "https://example.test/events",
+            lastSuccessAt: null,
+            lastConfirmedAt: null,
+            contentChangedAt: null,
+          },
+          {
+            sourceId: "endfield-wikigg-events",
+            url: "https://example.test/events",
+            lastSuccessAt: "2026-08-17T11:00:00.000Z",
+            lastConfirmedAt: null,
+            contentChangedAt: "2026-08-17T11:00:00.000Z",
+          },
+        ],
+      },
+    ]);
   });
 
   test("order of sources does not change the answer", () => {
@@ -114,7 +162,44 @@ describe("freshness", () => {
     // footer must say something honest rather than format a null.
     const result = freshness([source("genshin", null)], NOW);
     expect(result.refreshedAt).toBeNull();
-    expect(result.stale).toEqual([{ game: "genshin", lastSuccessAt: null }]);
+    expect(result.stale).toEqual([
+      {
+        game: "genshin",
+        lastSuccessAt: null,
+        lastConfirmedAt: null,
+        contentChangedAt: null,
+        sources: [
+          {
+            sourceId: "genshin-src",
+            url: "https://example.test/events",
+            lastSuccessAt: null,
+            lastConfirmedAt: null,
+            contentChangedAt: null,
+          },
+        ],
+      },
+    ]);
+  });
+
+  test("differentiates blame when data was pulled recently but site has not changed", () => {
+    const PULL_TIME = new Date(NOW - 3 * HOUR).toISOString();
+    const OLD_SITE = new Date(NOW - 60 * HOUR).toISOString();
+    const result = freshness(
+      [
+        {
+          ...source("endfield", OLD_SITE, "endfield-wikigg-events"),
+          lastConfirmedAt: PULL_TIME,
+          contentChangedAt: OLD_SITE,
+        },
+      ],
+      NOW,
+    );
+    expect(result.stale).toHaveLength(1);
+    const staleEndfield = result.stale[0]!;
+    expect(staleEndfield.lastConfirmedAt).toBe(PULL_TIME);
+    expect(staleEndfield.contentChangedAt).toBe(OLD_SITE);
+    expect(staleEndfield.sources[0]?.lastConfirmedAt).toBe(PULL_TIME);
+    expect(staleEndfield.sources[0]?.contentChangedAt).toBe(OLD_SITE);
   });
 
   test("an empty feed reports nothing rather than throwing", () => {
@@ -133,6 +218,8 @@ describe("telling a broken source from a stale one", () => {
     game: "nikki" as GameId,
     url: "https://example.test/nikki",
     lastSuccessAt: "2026-08-19T00:00:00.000Z",
+    lastConfirmedAt: null,
+    contentChangedAt: "2026-08-19T00:00:00.000Z",
     eventCount: 0,
     parsedCount: 7,
     statesNoEvents: false,