From 6c09accc204495f91867de1ed3055180eeb0c2ca Mon Sep 17 00:00:00 2001 From: Lucas Winther Date: Fri, 18 Sep 2026 08:53:44 +0200 Subject: [PATCH] feat(feed): record crawler check and content update timestamps in SourceHealth Add lastConfirmedAt and contentChangedAt to SourceHealth and enrich freshness() to track per-source check and content update times. --- docs/DATA-MODEL.md | 5 +++ scripts/build-feed.ts | 21 ++++++++-- src/ingest/health.ts | 6 +++ src/shared/feed.ts | 78 ++++++++++++++++++++++++++++++++++-- test/feed.test.ts | 93 +++++++++++++++++++++++++++++++++++++++++-- 5 files changed, 194 insertions(+), 9 deletions(-) diff --git a/docs/DATA-MODEL.md b/docs/DATA-MODEL.md index cd215d0..c101253 100644 --- a/docs/DATA-MODEL.md +++ b/docs/DATA-MODEL.md @@ -635,3 +635,8 @@ takes the offline promise down with it. A default must also be the *strict* read `parsedCount` defaults to `null` — an older feed never recorded the figure, which is an absence of information and not evidence of health — and `statesNoEvents` to `false`, because a feed built before the field existed never asked the page and cannot be read as having got an answer. + +Two timestamp fields distinguish crawler activity from site updates: `lastConfirmedAt` records when +the crawler last successfully contacted and verified the source (HTTP 200 or 304 Not Modified), and +`contentChangedAt` records when the served bytes last changed. Both default to `null` so feeds built +before their introduction continue to validate. diff --git a/scripts/build-feed.ts b/scripts/build-feed.ts index af2e621..c10ec88 100644 --- a/scripts/build-feed.ts +++ b/scripts/build-feed.ts @@ -51,10 +51,19 @@ async function documentFor(adapterId: string, game: GameId) { file: snapshots.bodyPath(adapterId), html: cached.html, at: freshnessAt(cached), + lastConfirmedAt: cached.state.lastConfirmedAt, + contentChangedAt: cached.meta.contentChangedAt, }; } const { file, html } = await latestFixture(adapterId, game); - return { file, html, at: fixtureDate(file) }; + const date = fixtureDate(file); + return { + file, + html, + at: date, + lastConfirmedAt: null, + contentChangedAt: date, + }; } const now = new Date().toISOString(); @@ -62,7 +71,10 @@ const byGame = new Map const sources: SourceHealth[] = []; for (const adapter of ADAPTERS) { - const { file, html, at } = await documentFor(adapter.id, adapter.game); + const { file, html, at, lastConfirmedAt, contentChangedAt } = await documentFor( + adapter.id, + adapter.game, + ); const events = adapter.parse(html, { now, sourceUrl: adapter.url, @@ -72,7 +84,10 @@ for (const adapter of ADAPTERS) { // Which of the three empties this is, decided in a module a test can reach // rather than here — see `src/ingest/health.ts` for why that matters. - const health = sourceHealth(adapter, html, at, events.length); + const health = sourceHealth(adapter, html, at, events.length, { + lastConfirmedAt, + contentChangedAt, + }); const { parsedCount } = health; const groups = byGame.get(adapter.game) ?? []; diff --git a/src/ingest/health.ts b/src/ingest/health.ts index a0c47f4..566af16 100644 --- a/src/ingest/health.ts +++ b/src/ingest/health.ts @@ -44,6 +44,10 @@ export function sourceHealth( html: string, at: string | null, eventCount: number, + timestamps?: { + lastConfirmedAt?: string | null; + contentChangedAt?: string | null; + }, ): SourceHealth { // Parsed a second time as of the document's own capture date, when nothing in // it had expired yet. That figure is what separates "this parser has stopped @@ -82,6 +86,8 @@ export function sourceHealth( // When the bytes were last confirmed live; a fixture's capture date when // this source has never been refreshed. lastSuccessAt: at, + lastConfirmedAt: timestamps?.lastConfirmedAt ?? null, + contentChangedAt: timestamps?.contentChangedAt ?? at, eventCount, parsedCount, statesNoEvents, diff --git a/src/shared/feed.ts b/src/shared/feed.ts index c1c7089..1d5dc01 100644 --- a/src/shared/feed.ts +++ b/src/shared/feed.ts @@ -15,6 +15,16 @@ export const SourceHealth = z.object({ game: GameId, url: z.string().url(), lastSuccessAt: z.string().datetime().nullable(), + /** + * When the crawler last successfully contacted and confirmed this source + * (HTTP 200 or 304 Not Modified). Null when never checked (e.g. clean checkout + * using fixtures, or older feed). + */ + lastConfirmedAt: z.string().datetime().nullable().default(null), + /** + * When the served bytes last changed. + */ + contentChangedAt: z.string().datetime().nullable().default(null), /** Events this source contributed to the feed — after expired ones are dropped. */ eventCount: z.number().int().nonnegative(), /** @@ -69,6 +79,22 @@ export type EventFeed = z.infer; /** A game's data is stale past this age (PRD F7). */ export const STALE_AFTER_MS = 48 * 60 * 60 * 1000; +export interface StaleSourceInfo { + sourceId: string; + url: string; + lastSuccessAt: string | null; + lastConfirmedAt: string | null; + contentChangedAt: string | null; +} + +export interface StaleGame { + game: GameId; + lastSuccessAt: string | null; + lastConfirmedAt: string | null; + contentChangedAt: string | null; + sources: StaleSourceInfo[]; +} + export interface Freshness { /** * When any source last had its bytes confirmed — the newest `lastSuccessAt`. @@ -84,7 +110,7 @@ export interface Freshness { */ refreshedAt: string | null; /** Per game, oldest first: what has not refreshed inside `STALE_AFTER_MS`. */ - stale: Array<{ game: GameId; lastSuccessAt: string | null }>; + stale: StaleGame[]; } /** @@ -101,6 +127,7 @@ export function freshness( now: number, ): Freshness { const oldestPerGame = new Map(); + const sourcesByGame = new Map(); let refreshedAt: string | null = null; for (const source of sources) { @@ -117,11 +144,56 @@ export function freshness( if (known === undefined || (known !== null && (at === null || at < known))) { oldestPerGame.set(source.game, at); } + + const list = sourcesByGame.get(source.game) ?? []; + list.push(source); + sourcesByGame.set(source.game, list); } - const stale = [...oldestPerGame.entries()] + const stale: StaleGame[] = [...oldestPerGame.entries()] .filter(([, at]) => at === null || now - Date.parse(at) > STALE_AFTER_MS) - .map(([game, lastSuccessAt]) => ({ game, lastSuccessAt })) + .map(([game, lastSuccessAt]) => { + const gameSources = sourcesByGame.get(game) ?? []; + const sourcesInfo: StaleSourceInfo[] = gameSources + .map((s) => ({ + sourceId: s.sourceId, + url: s.url, + lastSuccessAt: s.lastSuccessAt, + lastConfirmedAt: s.lastConfirmedAt ?? null, + contentChangedAt: s.contentChangedAt ?? s.lastSuccessAt, + })) + .sort((a, b) => a.sourceId.localeCompare(b.sourceId)); + + let lastConfirmedAt: string | null = null; + let hasConfirmed = false; + for (const s of sourcesInfo) { + if (!hasConfirmed) { + lastConfirmedAt = s.lastConfirmedAt; + hasConfirmed = true; + } else if (lastConfirmedAt !== null && (s.lastConfirmedAt === null || s.lastConfirmedAt < lastConfirmedAt)) { + lastConfirmedAt = s.lastConfirmedAt; + } + } + + let contentChangedAt: string | null = null; + let hasContent = false; + for (const s of sourcesInfo) { + if (!hasContent) { + contentChangedAt = s.contentChangedAt; + hasContent = true; + } else if (contentChangedAt !== null && (s.contentChangedAt === null || s.contentChangedAt < contentChangedAt)) { + contentChangedAt = s.contentChangedAt; + } + } + + return { + game, + lastSuccessAt, + lastConfirmedAt, + contentChangedAt, + sources: sourcesInfo, + }; + }) .sort((a, b) => (a.lastSuccessAt ?? "").localeCompare(b.lastSuccessAt ?? "")); return { refreshedAt, stale }; diff --git a/test/feed.test.ts b/test/feed.test.ts index c7c61f6..798d5b1 100644 --- a/test/feed.test.ts +++ b/test/feed.test.ts @@ -27,12 +27,16 @@ function source( game: GameId, lastSuccessAt: string | null, sourceId = `${game}-src`, + lastConfirmedAt: string | null = null, + contentChangedAt: string | null = lastSuccessAt, ): SourceHealth { return { sourceId, game, url: "https://example.test/events", lastSuccessAt, + lastConfirmedAt, + contentChangedAt, eventCount: 3, parsedCount: 3, @@ -67,7 +71,28 @@ describe("freshness", () => { ); expect(result.refreshedAt).toBe("2026-08-17T11:00:00.000Z"); expect(result.stale).toEqual([ - { game: "endfield", lastSuccessAt: "2026-08-10T11:00:00.000Z" }, + { + game: "endfield", + lastSuccessAt: "2026-08-10T11:00:00.000Z", + lastConfirmedAt: null, + contentChangedAt: "2026-08-10T11:00:00.000Z", + sources: [ + { + sourceId: "endfield-game8-events", + url: "https://example.test/events", + lastSuccessAt: "2026-08-10T11:00:00.000Z", + lastConfirmedAt: null, + contentChangedAt: "2026-08-10T11:00:00.000Z", + }, + { + sourceId: "endfield-wikigg-events", + url: "https://example.test/events", + lastSuccessAt: "2026-08-17T11:00:00.000Z", + lastConfirmedAt: null, + contentChangedAt: "2026-08-17T11:00:00.000Z", + }, + ], + }, ]); }); @@ -79,7 +104,30 @@ describe("freshness", () => { ], NOW, ); - expect(result.stale).toEqual([{ game: "endfield", lastSuccessAt: null }]); + expect(result.stale).toEqual([ + { + game: "endfield", + lastSuccessAt: null, + lastConfirmedAt: null, + contentChangedAt: null, + sources: [ + { + sourceId: "endfield-game8-events", + url: "https://example.test/events", + lastSuccessAt: null, + lastConfirmedAt: null, + contentChangedAt: null, + }, + { + sourceId: "endfield-wikigg-events", + url: "https://example.test/events", + lastSuccessAt: "2026-08-17T11:00:00.000Z", + lastConfirmedAt: null, + contentChangedAt: "2026-08-17T11:00:00.000Z", + }, + ], + }, + ]); }); test("order of sources does not change the answer", () => { @@ -114,7 +162,44 @@ describe("freshness", () => { // footer must say something honest rather than format a null. const result = freshness([source("genshin", null)], NOW); expect(result.refreshedAt).toBeNull(); - expect(result.stale).toEqual([{ game: "genshin", lastSuccessAt: null }]); + expect(result.stale).toEqual([ + { + game: "genshin", + lastSuccessAt: null, + lastConfirmedAt: null, + contentChangedAt: null, + sources: [ + { + sourceId: "genshin-src", + url: "https://example.test/events", + lastSuccessAt: null, + lastConfirmedAt: null, + contentChangedAt: null, + }, + ], + }, + ]); + }); + + test("differentiates blame when data was pulled recently but site has not changed", () => { + const PULL_TIME = new Date(NOW - 3 * HOUR).toISOString(); + const OLD_SITE = new Date(NOW - 60 * HOUR).toISOString(); + const result = freshness( + [ + { + ...source("endfield", OLD_SITE, "endfield-wikigg-events"), + lastConfirmedAt: PULL_TIME, + contentChangedAt: OLD_SITE, + }, + ], + NOW, + ); + expect(result.stale).toHaveLength(1); + const staleEndfield = result.stale[0]!; + expect(staleEndfield.lastConfirmedAt).toBe(PULL_TIME); + expect(staleEndfield.contentChangedAt).toBe(OLD_SITE); + expect(staleEndfield.sources[0]?.lastConfirmedAt).toBe(PULL_TIME); + expect(staleEndfield.sources[0]?.contentChangedAt).toBe(OLD_SITE); }); test("an empty feed reports nothing rather than throwing", () => { @@ -133,6 +218,8 @@ describe("telling a broken source from a stale one", () => { game: "nikki" as GameId, url: "https://example.test/nikki", lastSuccessAt: "2026-08-19T00:00:00.000Z", + lastConfirmedAt: null, + contentChangedAt: "2026-08-19T00:00:00.000Z", eventCount: 0, parsedCount: 7, statesNoEvents: false,