fix(snapshots): persist lastConfirmedAt into snapshot metadata upon pull confirmation
CI / Typecheck, test, feed (push) Successful in 20s
CI / Build site (push) Successful in 11s
CI / Container image (push) Successful in 41s

Record lastConfirmedAt in snapshot metadata when a source is confirmed (200 or 304). This ensures git-committed snapshot metadata reflects when sources like wiki.gg were successfully pulled even when page HTML did not change, avoiding false pull overdue notices on downstream builds like GitHub Pages.
This commit is contained in:
Lucas Winther
2026-09-18 10:37:37 +02:00
parent f07607911b
commit 84441b6d6d
7 changed files with 40 additions and 15 deletions
+1 -1
View File
@@ -47,7 +47,7 @@ async function latestFixture(adapterId: string, game: GameId) {
async function documentFor(adapterId: string, game: GameId) {
const cached = await snapshots.read(adapterId);
if (cached !== null) {
const rawConfirmed = cached.state.lastConfirmedAt;
const rawConfirmed = cached.meta.lastConfirmedAt ?? cached.state.lastConfirmedAt;
const contentChangedAt = cached.meta.contentChangedAt;
const lastConfirmedAt =
rawConfirmed !== null &&
+7 -9
View File
@@ -4,8 +4,8 @@ Raw pages, exactly as fetched. `scripts/refresh-sources.ts` writes them; nothing
```
<source-id>.html the body verbatim — tracked
<source-id>.meta.json hash, size, charset, ETag, Last-Modified, when the bytes last changed — tracked
<source-id>.state.json when we last checked, and failure streak — gitignored
<source-id>.meta.json hash, size, charset, ETag, Last-Modified, contentChangedAt, lastConfirmedAt — tracked
<source-id>.state.json transient run bookkeeping: last attempt and failure streak — gitignored
```
"Verbatim" means the bytes as served, not text we re-encoded. `charset` in the metadata records
@@ -29,14 +29,12 @@ Three reasons this is committed rather than cached:
- **The build stays offline.** `bun run build:feed` parses whichever of these exists and falls back
to `fixtures/` otherwise, so a clean checkout and the container build work with no network.
The `.state.json` files are the exception: they change every cycle whether or not a page did, and
committing them would mean a commit per run saying nothing happened. CI keeps them in the actions
cache instead: `refresh.yml` saves that cache, and `ci.yml` restores it read-only before building
the feed. Both halves matter — `lastConfirmedAt` lives only there, and without the restore the feed
falls back to `contentChangedAt` and the UI calls every source stale two days after its bytes last
moved.
The `.state.json` files track transient check attempts and failure streaks. When a source is
confirmed (200 or 304), `lastConfirmedAt` is recorded in `<source-id>.meta.json` so downstream builds
(including GitHub Pages, which deploys from git without access to the Gitea Actions cache) know the
exact time the source was confirmed current.
The metadata is rewritten on an unchanged page in one case: the server rotating an `ETag` or
The metadata is also rewritten on an unchanged page when the server rotates an `ETag` or
`Last-Modified` while serving the same bytes. Keeping the old validator would mean sending a stale
`If-None-Match` forever and being served the whole page every cycle, so that diff is worth the
commit — `contentChangedAt` and the body stay put, so it is still visibly not a content change.
@@ -6,6 +6,7 @@
"etag": null,
"lastModified": "Tue, 08 Sep 2026 04:50:40 GMT",
"contentChangedAt": "2026-09-08T09:51:10.314Z",
"lastConfirmedAt": "2026-09-18T08:04:45.000Z",
"eventCount": 3,
"charset": "utf-8"
}
@@ -6,6 +6,7 @@
"etag": null,
"lastModified": "Tue, 15 Sep 2026 05:34:53 GMT",
"contentChangedAt": "2026-09-15T10:11:51.059Z",
"lastConfirmedAt": "2026-09-18T08:04:45.000Z",
"eventCount": 6,
"charset": "utf-8"
}
+14 -2
View File
@@ -32,6 +32,8 @@ export interface SnapshotMeta {
lastModified: string | null;
/** ISO timestamp of the fetch that last produced *different* bytes. */
contentChangedAt: string;
/** ISO timestamp of the last fetch that confirmed the body is still current. */
lastConfirmedAt?: string | null;
/** Events the adapter yielded from this body, for drop detection. */
eventCount: number | null;
/**
@@ -320,6 +322,7 @@ export class SnapshotStore {
etag: input.etag,
lastModified: input.lastModified,
contentChangedAt: changed ? input.at : (previous?.contentChangedAt ?? input.at),
lastConfirmedAt: changed ? input.at : (previous?.lastConfirmedAt ?? input.at),
// New bytes mean the count they yielded, even when that is unknown; the
// same bytes keep the count we already recorded for them.
eventCount: changed
@@ -347,7 +350,7 @@ export class SnapshotStore {
return { changed: true, meta };
}
/** Record an attempt: success, 304 or failure. Writes only state. */
/** Record an attempt: success, 304 or failure. Writes state, and updates lastConfirmedAt in meta if confirmed. */
async recordCheck(
sourceId: string,
check: { at: string; status: number | null; ok: boolean },
@@ -363,6 +366,15 @@ export class SnapshotStore {
await mkdir(this.root, { recursive: true });
await writeAtomic(this.statePath(sourceId), `${JSON.stringify(state, null, 2)}\n`);
if (check.ok) {
const meta = await this.readMeta(sourceId);
if (meta !== null && meta.lastConfirmedAt !== check.at) {
meta.lastConfirmedAt = check.at;
await writeAtomic(this.metaPath(sourceId), `${JSON.stringify(meta, null, 2)}\n`);
}
}
return state;
}
@@ -386,7 +398,7 @@ export class SnapshotStore {
* that overstates is worse than one that lags.
*/
export function freshnessAt(snapshot: Snapshot): string {
const confirmed = snapshot.state.lastConfirmedAt;
const confirmed = snapshot.meta.lastConfirmedAt ?? snapshot.state.lastConfirmedAt;
if (confirmed === null) return snapshot.meta.contentChangedAt;
return Date.parse(confirmed) > Date.parse(snapshot.meta.contentChangedAt)
? confirmed
+1
View File
@@ -173,6 +173,7 @@ describe("a normal cycle", () => {
const snapshot = await store.read("genshin-game8-events");
expect(snapshot?.html).toBe("<html><event></event></html>");
expect(snapshot?.meta.contentChangedAt).toBe("2026-08-01T00:00:00.000Z");
expect(snapshot?.meta.lastConfirmedAt).toBe(NOW.toISOString());
expect(snapshot?.state.lastConfirmedAt).toBe(NOW.toISOString());
});
+15 -3
View File
@@ -385,16 +385,28 @@ describe("the six-hour floor", () => {
expect(state.lastConfirmedAt).toBe(T1);
});
test("check bookkeeping lives outside the committed metadata", async () => {
test("failed check bookkeeping lives in state without modifying meta", async () => {
await save("<html>one</html>", T0);
const before = await Bun.file(store.metaPath("genshin-game8-events")).text();
await store.recordCheck("genshin-game8-events", {
at: T1,
status: 503,
ok: false,
});
const after = await Bun.file(store.metaPath("genshin-game8-events")).text();
expect(after).toBe(before);
});
test("successful confirmation updates lastConfirmedAt in metadata", async () => {
await save("<html>one</html>", T0);
await store.recordCheck("genshin-game8-events", {
at: T1,
status: 304,
ok: true,
});
const after = await Bun.file(store.metaPath("genshin-game8-events")).text();
expect(after).toBe(before);
const meta = await store.readMeta("genshin-game8-events");
expect(meta?.lastConfirmedAt).toBe(T1);
expect(meta?.contentChangedAt).toBe(T0);
});
});