From 84441b6d6d67b7eeae9e1eb5245fbc47ee60f43c Mon Sep 17 00:00:00 2001 From: Lucas Winther Date: Fri, 18 Sep 2026 10:37:37 +0200 Subject: [PATCH] fix(snapshots): persist lastConfirmedAt into snapshot metadata upon pull confirmation Record lastConfirmedAt in snapshot metadata when a source is confirmed (200 or 304). This ensures git-committed snapshot metadata reflects when sources like wiki.gg were successfully pulled even when page HTML did not change, avoiding false pull overdue notices on downstream builds like GitHub Pages. --- scripts/build-feed.ts | 2 +- snapshots/README.md | 16 +++++++--------- snapshots/arknights-akwiki-events.meta.json | 1 + snapshots/endfield-wikigg-events.meta.json | 1 + src/ingest/snapshots.ts | 16 ++++++++++++++-- test/refresh.test.ts | 1 + test/snapshots.test.ts | 18 +++++++++++++++--- 7 files changed, 40 insertions(+), 15 deletions(-) diff --git a/scripts/build-feed.ts b/scripts/build-feed.ts index 7f19d13..efc3f2b 100644 --- a/scripts/build-feed.ts +++ b/scripts/build-feed.ts @@ -47,7 +47,7 @@ async function latestFixture(adapterId: string, game: GameId) { async function documentFor(adapterId: string, game: GameId) { const cached = await snapshots.read(adapterId); if (cached !== null) { - const rawConfirmed = cached.state.lastConfirmedAt; + const rawConfirmed = cached.meta.lastConfirmedAt ?? cached.state.lastConfirmedAt; const contentChangedAt = cached.meta.contentChangedAt; const lastConfirmedAt = rawConfirmed !== null && diff --git a/snapshots/README.md b/snapshots/README.md index 1c78473..4c677e9 100644 --- a/snapshots/README.md +++ b/snapshots/README.md @@ -4,8 +4,8 @@ Raw pages, exactly as fetched. `scripts/refresh-sources.ts` writes them; nothing ``` .html the body verbatim — tracked -.meta.json hash, size, charset, ETag, Last-Modified, when the bytes last changed — tracked -.state.json when we last checked, and failure streak — gitignored +.meta.json hash, size, charset, ETag, Last-Modified, contentChangedAt, lastConfirmedAt — tracked +.state.json transient run bookkeeping: last attempt and failure streak — gitignored ``` "Verbatim" means the bytes as served, not text we re-encoded. `charset` in the metadata records @@ -29,14 +29,12 @@ Three reasons this is committed rather than cached: - **The build stays offline.** `bun run build:feed` parses whichever of these exists and falls back to `fixtures/` otherwise, so a clean checkout and the container build work with no network. -The `.state.json` files are the exception: they change every cycle whether or not a page did, and -committing them would mean a commit per run saying nothing happened. CI keeps them in the actions -cache instead: `refresh.yml` saves that cache, and `ci.yml` restores it read-only before building -the feed. Both halves matter — `lastConfirmedAt` lives only there, and without the restore the feed -falls back to `contentChangedAt` and the UI calls every source stale two days after its bytes last -moved. +The `.state.json` files track transient check attempts and failure streaks. When a source is +confirmed (200 or 304), `lastConfirmedAt` is recorded in `.meta.json` so downstream builds +(including GitHub Pages, which deploys from git without access to the Gitea Actions cache) know the +exact time the source was confirmed current. -The metadata is rewritten on an unchanged page in one case: the server rotating an `ETag` or +The metadata is also rewritten on an unchanged page when the server rotates an `ETag` or `Last-Modified` while serving the same bytes. Keeping the old validator would mean sending a stale `If-None-Match` forever and being served the whole page every cycle, so that diff is worth the commit — `contentChangedAt` and the body stay put, so it is still visibly not a content change. diff --git a/snapshots/arknights-akwiki-events.meta.json b/snapshots/arknights-akwiki-events.meta.json index 54cf12e..7c7eba3 100644 --- a/snapshots/arknights-akwiki-events.meta.json +++ b/snapshots/arknights-akwiki-events.meta.json @@ -6,6 +6,7 @@ "etag": null, "lastModified": "Tue, 08 Sep 2026 04:50:40 GMT", "contentChangedAt": "2026-09-08T09:51:10.314Z", + "lastConfirmedAt": "2026-09-18T08:04:45.000Z", "eventCount": 3, "charset": "utf-8" } diff --git a/snapshots/endfield-wikigg-events.meta.json b/snapshots/endfield-wikigg-events.meta.json index 5d3fdb6..daa1942 100644 --- a/snapshots/endfield-wikigg-events.meta.json +++ b/snapshots/endfield-wikigg-events.meta.json @@ -6,6 +6,7 @@ "etag": null, "lastModified": "Tue, 15 Sep 2026 05:34:53 GMT", "contentChangedAt": "2026-09-15T10:11:51.059Z", + "lastConfirmedAt": "2026-09-18T08:04:45.000Z", "eventCount": 6, "charset": "utf-8" } diff --git a/src/ingest/snapshots.ts b/src/ingest/snapshots.ts index dd3e6fe..8ef4371 100644 --- a/src/ingest/snapshots.ts +++ b/src/ingest/snapshots.ts @@ -32,6 +32,8 @@ export interface SnapshotMeta { lastModified: string | null; /** ISO timestamp of the fetch that last produced *different* bytes. */ contentChangedAt: string; + /** ISO timestamp of the last fetch that confirmed the body is still current. */ + lastConfirmedAt?: string | null; /** Events the adapter yielded from this body, for drop detection. */ eventCount: number | null; /** @@ -320,6 +322,7 @@ export class SnapshotStore { etag: input.etag, lastModified: input.lastModified, contentChangedAt: changed ? input.at : (previous?.contentChangedAt ?? input.at), + lastConfirmedAt: changed ? input.at : (previous?.lastConfirmedAt ?? input.at), // New bytes mean the count they yielded, even when that is unknown; the // same bytes keep the count we already recorded for them. eventCount: changed @@ -347,7 +350,7 @@ export class SnapshotStore { return { changed: true, meta }; } - /** Record an attempt: success, 304 or failure. Writes only state. */ + /** Record an attempt: success, 304 or failure. Writes state, and updates lastConfirmedAt in meta if confirmed. */ async recordCheck( sourceId: string, check: { at: string; status: number | null; ok: boolean }, @@ -363,6 +366,15 @@ export class SnapshotStore { await mkdir(this.root, { recursive: true }); await writeAtomic(this.statePath(sourceId), `${JSON.stringify(state, null, 2)}\n`); + + if (check.ok) { + const meta = await this.readMeta(sourceId); + if (meta !== null && meta.lastConfirmedAt !== check.at) { + meta.lastConfirmedAt = check.at; + await writeAtomic(this.metaPath(sourceId), `${JSON.stringify(meta, null, 2)}\n`); + } + } + return state; } @@ -386,7 +398,7 @@ export class SnapshotStore { * that overstates is worse than one that lags. */ export function freshnessAt(snapshot: Snapshot): string { - const confirmed = snapshot.state.lastConfirmedAt; + const confirmed = snapshot.meta.lastConfirmedAt ?? snapshot.state.lastConfirmedAt; if (confirmed === null) return snapshot.meta.contentChangedAt; return Date.parse(confirmed) > Date.parse(snapshot.meta.contentChangedAt) ? confirmed diff --git a/test/refresh.test.ts b/test/refresh.test.ts index b93b67e..7577996 100644 --- a/test/refresh.test.ts +++ b/test/refresh.test.ts @@ -173,6 +173,7 @@ describe("a normal cycle", () => { const snapshot = await store.read("genshin-game8-events"); expect(snapshot?.html).toBe(""); expect(snapshot?.meta.contentChangedAt).toBe("2026-08-01T00:00:00.000Z"); + expect(snapshot?.meta.lastConfirmedAt).toBe(NOW.toISOString()); expect(snapshot?.state.lastConfirmedAt).toBe(NOW.toISOString()); }); diff --git a/test/snapshots.test.ts b/test/snapshots.test.ts index c8e8d8d..3ce770f 100644 --- a/test/snapshots.test.ts +++ b/test/snapshots.test.ts @@ -385,16 +385,28 @@ describe("the six-hour floor", () => { expect(state.lastConfirmedAt).toBe(T1); }); - test("check bookkeeping lives outside the committed metadata", async () => { + test("failed check bookkeeping lives in state without modifying meta", async () => { await save("one", T0); const before = await Bun.file(store.metaPath("genshin-game8-events")).text(); + await store.recordCheck("genshin-game8-events", { + at: T1, + status: 503, + ok: false, + }); + const after = await Bun.file(store.metaPath("genshin-game8-events")).text(); + expect(after).toBe(before); + }); + + test("successful confirmation updates lastConfirmedAt in metadata", async () => { + await save("one", T0); await store.recordCheck("genshin-game8-events", { at: T1, status: 304, ok: true, }); - const after = await Bun.file(store.metaPath("genshin-game8-events")).text(); - expect(after).toBe(before); + const meta = await store.readMeta("genshin-game8-events"); + expect(meta?.lastConfirmedAt).toBe(T1); + expect(meta?.contentChangedAt).toBe(T0); }); });