feat: refresh sources on a schedule

The feed was generated from checked-in fixtures and only moved when somebody captured a page by hand. This fetches. bun run refresh caches each page raw under snapshots/ and rebuilds the feed from what it cached; build-feed prefers a snapshot and falls back to the fixture, so a clean checkout and the container build stay offline and reproducible. The workflow runs it twice a day and commits only when a page's bytes actually changed — a 304, an identical body or a rejected parse all leave the tree clean — then dispatches ci.yml, which already knows how to test, build and deploy. Scraping conduct is enforced in code rather than left to good intentions: one request per source per cycle, a six-hour floor checked per source, conditional requests, a User-Agent with a contact URL, and robots.txt honoured — failing closed, because a permission we could not read is not a permission we have. No retries; a retry is a second request. A body that yields zero events is rejected and the previous snapshot kept, so a redesigned wiki shows up as a stale timestamp rather than an emptied calendar. One source down is a warning; all of them down fails the run, so a cycle that learned nothing is never committed. Tested entirely offline against an injected fetch and clock — no request has ever been made to a live wiki from this code. Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
2026-08-15 21:15:25 +02:00
parent 18b9652aed
commit b085087b05
12 changed files with 2251 additions and 12 deletions
@@ -0,0 +1,439 @@
+import { afterEach, beforeEach, describe, expect, test } from "bun:test";
+import { mkdtemp, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import {
+  parseArgs,
+  runRefresh,
+  type RefreshOptions,
+  type RobotsGate,
+} from "../scripts/refresh-sources.ts";
+import type { Adapter, ParseContext } from "../src/ingest/adapters/types.ts";
+import { SnapshotStore } from "../src/ingest/snapshots.ts";
+import type { GachaEvent } from "../src/shared/schema.ts";
+
+const SIX_HOURS_MS = 6 * 60 * 60 * 1000;
+const NOW = new Date("2026-08-15T12:00:00.000Z");
+const UA = "gacha-event-tracker/1.0 (+https://example.test)";
+
+let root: string;
+let store: SnapshotStore;
+
+beforeEach(async () => {
+  root = await mkdtemp(join(tmpdir(), "event-clock-refresh-"));
+  store = new SnapshotStore(root);
+});
+
+afterEach(async () => {
+  await rm(root, { recursive: true, force: true });
+});
+
+/**
+ * A stand-in adapter whose "parser" counts `<event>` tags, so a test can make a
+ * body parse well, badly, or not at all without touching a real parser.
+ */
+function adapter(overrides: Partial<Adapter> = {}): Adapter {
+  return {
+    id: "genshin-game8-events",
+    game: "genshin",
+    url: "https://game8.co/games/Genshin-Impact/archives/301601",
+    parserId: "game8",
+    minIntervalMs: SIX_HOURS_MS,
+    priority: 0,
+    parse(html: string, _ctx: ParseContext): GachaEvent[] {
+      if (html.includes("<broken>")) throw new Error("template not recognised");
+      const count = html.match(/<event>/g)?.length ?? 0;
+      return Array.from({ length: count }) as GachaEvent[];
+    },
+    ...overrides,
+  };
+}
+
+const ALLOW_ALL: RobotsGate = {
+  allows: async () => ({ allowed: true, reason: "robots.txt ok" }),
+};
+
+interface Call {
+  url: string;
+  headers: Record<string, string>;
+}
+
+function options(
+  over: Partial<RefreshOptions> & { responder?: (call: Call) => Response },
+): { opts: RefreshOptions; calls: Call[]; rebuilds: { count: number } } {
+  const calls: Call[] = [];
+  const rebuilds = { count: 0 };
+  const responder =
+    over.responder ?? (() => new Response("<html><event></event></html>"));
+
+  const opts: RefreshOptions = {
+    adapters: [adapter()],
+    store,
+    robots: ALLOW_ALL,
+    fetchImpl: async (url, init) => {
+      const headers: Record<string, string> = {};
+      for (const [k, v] of Object.entries(
+        (init?.headers ?? {}) as Record<string, string>,
+      )) {
+        headers[k] = v;
+      }
+      const call = { url, headers };
+      calls.push(call);
+      return responder(call);
+    },
+    userAgent: UA,
+    now: () => NOW,
+    dryRun: false,
+    only: null,
+    timeoutMs: 1000,
+    log: () => {},
+    rebuildFeed: async () => {
+      rebuilds.count += 1;
+    },
+    ...over,
+  };
+
+  return { opts, calls, rebuilds };
+}
+
+async function seed(html: string, at: string, eventCount: number | null) {
+  await store.save("genshin-game8-events", {
+    url: "https://game8.co/games/Genshin-Impact/archives/301601",
+    html,
+    etag: 'W/"v1"',
+    lastModified: "Fri, 14 Aug 2026 09:00:00 GMT",
+    at,
+    eventCount,
+  });
+}
+
+describe("a normal cycle", () => {
+  test("fetches once, stores the body, rebuilds the feed", async () => {
+    const { opts, calls, rebuilds } = options({});
+    const summary = await runRefresh(opts);
+
+    expect(calls).toHaveLength(1);
+    expect(summary.outcomes[0]?.result).toBe("fetched");
+    expect(summary.changed).toBe(1);
+    expect(summary.hardFailure).toBeNull();
+    expect(rebuilds.count).toBe(1);
+
+    const snapshot = await store.read("genshin-game8-events");
+    expect(snapshot?.html).toBe("<html><event></event></html>");
+    expect(snapshot?.meta.eventCount).toBe(1);
+    expect(snapshot?.state.lastConfirmedAt).toBe(NOW.toISOString());
+  });
+
+  test("identifies itself with a contact URL", async () => {
+    const { opts, calls } = options({});
+    await runRefresh(opts);
+    expect(calls[0]?.headers["User-Agent"]).toBe(UA);
+    expect(calls[0]?.headers["User-Agent"]).toContain("+https://");
+  });
+
+  test("sends the validators it was given last time", async () => {
+    await seed("<html><event></event></html>", "2026-08-01T00:00:00.000Z", 1);
+    const { opts, calls } = options({});
+    await runRefresh(opts);
+
+    expect(calls[0]?.headers["If-None-Match"]).toBe('W/"v1"');
+    expect(calls[0]?.headers["If-Modified-Since"]).toBe(
+      "Fri, 14 Aug 2026 09:00:00 GMT",
+    );
+  });
+
+  test("304 reuses the cached snapshot and changes nothing", async () => {
+    await seed("<html><event></event></html>", "2026-08-01T00:00:00.000Z", 1);
+    const { opts, rebuilds } = options({
+      responder: () => new Response(null, { status: 304 }),
+    });
+    const summary = await runRefresh(opts);
+
+    expect(summary.outcomes[0]?.result).toBe("unchanged");
+    expect(summary.changed).toBe(0);
+    expect(rebuilds.count).toBe(0);
+
+    const snapshot = await store.read("genshin-game8-events");
+    expect(snapshot?.html).toBe("<html><event></event></html>");
+    expect(snapshot?.meta.contentChangedAt).toBe("2026-08-01T00:00:00.000Z");
+    expect(snapshot?.state.lastConfirmedAt).toBe(NOW.toISOString());
+  });
+
+  test("a 200 with identical bytes is not a change either", async () => {
+    await seed("<html><event></event></html>", "2026-08-01T00:00:00.000Z", 1);
+    const { opts, rebuilds } = options({});
+    const summary = await runRefresh(opts);
+
+    expect(summary.outcomes[0]?.result).toBe("unchanged");
+    expect(summary.changed).toBe(0);
+    expect(rebuilds.count).toBe(0);
+  });
+});
+
+describe("one request per source per six hours", () => {
+  test("skips a source checked less than six hours ago", async () => {
+    await store.recordCheck("genshin-game8-events", {
+      at: new Date(NOW.getTime() - SIX_HOURS_MS + 1000).toISOString(),
+      status: 200,
+      ok: true,
+    });
+
+    const { opts, calls } = options({});
+    const summary = await runRefresh(opts);
+
+    expect(calls).toHaveLength(0);
+    expect(summary.outcomes[0]?.result).toBe("skipped_interval");
+    expect(summary.attempted).toBe(0);
+    expect(summary.hardFailure).toBeNull();
+  });
+
+  test("fetches again once the interval has elapsed", async () => {
+    await store.recordCheck("genshin-game8-events", {
+      at: new Date(NOW.getTime() - SIX_HOURS_MS).toISOString(),
+      status: 200,
+      ok: true,
+    });
+
+    const { opts, calls } = options({});
+    await runRefresh(opts);
+    expect(calls).toHaveLength(1);
+  });
+
+  test("never retries a failure inside the same cycle", async () => {
+    const { opts, calls } = options({
+      responder: () => new Response("nope", { status: 500 }),
+    });
+    const summary = await runRefresh(opts);
+
+    expect(calls).toHaveLength(1);
+    expect(summary.outcomes[0]?.result).toBe("failed");
+  });
+});
+
+describe("robots", () => {
+  test("does not fetch a source robots.txt disallows", async () => {
+    const { opts, calls } = options({
+      robots: {
+        allows: async () => ({ allowed: false, reason: "disallowed by robots" }),
+      },
+    });
+    const summary = await runRefresh(opts);
+
+    expect(calls).toHaveLength(0);
+    expect(summary.outcomes[0]?.result).toBe("skipped_robots");
+    expect(summary.warnings).toHaveLength(1);
+  });
+
+  test("one source blocked is a warning; all of them is a failure", async () => {
+    const blocked = {
+      allows: async (url: string) => ({
+        allowed: !url.includes("Genshin"),
+        reason: "disallowed by robots",
+      }),
+    };
+    const two = [
+      adapter(),
+      adapter({
+        id: "nte-game8-events",
+        game: "nte",
+        url: "https://game8.co/games/Neverness-to-Everness/archives/592073",
+      }),
+    ];
+
+    const partial = await runRefresh(options({ adapters: two, robots: blocked }).opts);
+    expect(partial.hardFailure).toBeNull();
+    expect(partial.warnings).toHaveLength(1);
+
+    // Fresh ids: the partial run above already checked one of the pair, and a
+    // source checked minutes ago is skipped for the interval, not for robots.
+    const all = await runRefresh(
+      options({
+        adapters: [
+          adapter({ id: "hsr-game8-events", game: "hsr", url: "https://game8.co/a" }),
+          adapter({ id: "zzz-game8-events", game: "zzz", url: "https://game8.co/b" }),
+        ],
+        robots: {
+          allows: async () => ({ allowed: false, reason: "disallowed by robots" }),
+        },
+      }).opts,
+    );
+    expect(all.hardFailure).toContain("blocked all 2 sources");
+  });
+
+  test("robots is consulted before the page is requested", async () => {
+    const order: string[] = [];
+    const { opts } = options({
+      robots: {
+        allows: async () => {
+          order.push("robots");
+          return { allowed: true, reason: "ok" };
+        },
+      },
+      responder: () => {
+        order.push("page");
+        return new Response("<html><event></event></html>");
+      },
+    });
+    await runRefresh(opts);
+    expect(order).toEqual(["robots", "page"]);
+  });
+});
+
+describe("a source being down never blanks the feed", () => {
+  test("an unreachable source is a warning, not a failure", async () => {
+    await seed("<html><event></event></html>", "2026-08-01T00:00:00.000Z", 1);
+    const { opts } = options({
+      adapters: [
+        adapter(),
+        adapter({
+          id: "nte-game8-events",
+          game: "nte",
+          url: "https://game8.co/games/Neverness-to-Everness/archives/592073",
+        }),
+      ],
+      responder: (call) => {
+        if (call.url.includes("Genshin")) throw new Error("ETIMEDOUT");
+        return new Response("<html><event></event><event></event></html>");
+      },
+    });
+
+    const summary = await runRefresh(opts);
+
+    expect(summary.outcomes[0]?.result).toBe("failed");
+    expect(summary.outcomes[1]?.result).toBe("fetched");
+    expect(summary.warnings).toHaveLength(1);
+    expect(summary.hardFailure).toBeNull();
+    // The old snapshot is untouched, so the feed keeps this game's events.
+    expect((await store.read("genshin-game8-events"))?.html).toBe(
+      "<html><event></event></html>",
+    );
+  });
+
+  test("every source failing is a hard failure", async () => {
+    const { opts, rebuilds } = options({
+      adapters: [adapter(), adapter({ id: "nte-game8-events", game: "nte" })],
+      responder: () => new Response("", { status: 503 }),
+    });
+    const summary = await runRefresh(opts);
+
+    expect(summary.hardFailure).toContain("all 2 attempted sources failed");
+    expect(rebuilds.count).toBe(0);
+  });
+
+  test("a body that no longer parses keeps the previous snapshot", async () => {
+    await seed("<html><event></event></html>", "2026-08-01T00:00:00.000Z", 1);
+    const { opts } = options({
+      responder: () => new Response("<broken>"),
+    });
+    const summary = await runRefresh(opts);
+
+    expect(summary.outcomes[0]?.result).toBe("rejected");
+    expect(summary.warnings[0]).toContain("did not parse");
+    expect((await store.read("genshin-game8-events"))?.html).toBe(
+      "<html><event></event></html>",
+    );
+  });
+
+  test("a body that suddenly yields no events keeps the previous snapshot", async () => {
+    await seed("<html><event></event></html>", "2026-08-01T00:00:00.000Z", 1);
+    const { opts } = options({
+      responder: () => new Response("<html>redesigned</html>"),
+    });
+    const summary = await runRefresh(opts);
+
+    expect(summary.outcomes[0]?.result).toBe("rejected");
+    expect(summary.warnings[0]).toContain("0 events");
+    expect((await store.read("genshin-game8-events"))?.meta.eventCount).toBe(1);
+  });
+
+  test("a first fetch that yields nothing is not stored either", async () => {
+    // With no snapshot yet, storing an empty parse would make build-feed prefer
+    // it over the checked-in fixture and quietly empty that game's calendar.
+    const { opts } = options({
+      responder: () => new Response("<html>redesigned</html>"),
+    });
+    const summary = await runRefresh(opts);
+
+    expect(summary.outcomes[0]?.result).toBe("rejected");
+    expect(summary.changed).toBe(0);
+    expect(await store.read("genshin-game8-events")).toBeNull();
+  });
+
+  test("a steep drop is stored but flagged", async () => {
+    await seed("<html>" + "<event></event>".repeat(10) + "</html>", "2026-08-01T00:00:00.000Z", 10);
+    const { opts } = options({
+      responder: () => new Response("<html><event></event></html>"),
+    });
+    const summary = await runRefresh(opts);
+
+    expect(summary.outcomes[0]?.result).toBe("fetched");
+    expect(summary.outcomes[0]?.note).toContain("down from 10");
+  });
+
+  test("a feed that will not rebuild fails the run", async () => {
+    const { opts } = options({
+      rebuildFeed: async () => {
+        throw new Error("build-feed exited 1");
+      },
+    });
+    const summary = await runRefresh(opts);
+    expect(summary.hardFailure).toContain("feed rebuild failed");
+  });
+});
+
+describe("flags", () => {
+  test("--dry-run makes no requests and writes nothing", async () => {
+    const { opts, calls, rebuilds } = options({ dryRun: true });
+    const summary = await runRefresh(opts);
+
+    expect(calls).toHaveLength(0);
+    expect(rebuilds.count).toBe(0);
+    expect(summary.outcomes[0]?.result).toBe("planned");
+    expect(summary.outcomes[0]?.note).toContain("would GET");
+    expect(await store.read("genshin-game8-events")).toBeNull();
+    expect(await store.readState("genshin-game8-events")).toMatchObject({
+      lastCheckedAt: null,
+    });
+  });
+
+  test("--only refreshes one source", async () => {
+    const { opts, calls } = options({
+      adapters: [adapter(), adapter({ id: "nte-game8-events", game: "nte" })],
+      only: "nte-game8-events",
+    });
+    const summary = await runRefresh(opts);
+
+    expect(calls).toHaveLength(1);
+    expect(summary.outcomes).toHaveLength(1);
+    expect(summary.outcomes[0]?.sourceId).toBe("nte-game8-events");
+  });
+
+  test("--only with an unknown id is a hard failure", async () => {
+    const { opts, calls } = options({ only: "does-not-exist" });
+    const summary = await runRefresh(opts);
+
+    expect(calls).toHaveLength(0);
+    expect(summary.hardFailure).toContain("unknown source");
+  });
+
+  test("parseArgs reads the flags", () => {
+    const args = parseArgs([
+      "--dry-run",
+      "--only",
+      "nte-game8-events",
+      "--snapshots",
+      "/tmp/x",
+      "--no-feed",
+    ]);
+    expect(args).toMatchObject({
+      dryRun: true,
+      only: "nte-game8-events",
+      root: "/tmp/x",
+      rebuild: false,
+    });
+  });
+
+  test("parseArgs rejects an unknown flag rather than ignoring it", () => {
+    expect(() => parseArgs(["--force"])).toThrow("unknown flag");
+  });
+});
@@ -0,0 +1,260 @@
+import { describe, expect, test } from "bun:test";
+import {
+  agentToken,
+  crawlDelayMs,
+  groupFor,
+  isAllowed,
+  parseRobots,
+  patternMatches,
+  requestTarget,
+  RobotsCache,
+} from "../src/ingest/robots.ts";
+
+const UA = "gacha-event-tracker/1.0 (+https://example.test/contact)";
+
+describe("parseRobots", () => {
+  test("groups consecutive user-agent lines together", () => {
+    const robots = parseRobots(`
+User-agent: alpha
+User-agent: beta
+Disallow: /private
+
+User-agent: *
+Disallow: /
+`);
+    expect(robots.groups).toHaveLength(2);
+    expect(robots.groups[0]?.agents).toEqual(["alpha", "beta"]);
+    expect(robots.groups[1]?.agents).toEqual(["*"]);
+  });
+
+  test("ignores comments, blank lines and unknown directives", () => {
+    const robots = parseRobots(
+      "# a comment\r\nUser-agent: *  # trailing\r\nHost: example.test\r\nDisallow: /x\r\nSitemap: https://example.test/sitemap.xml\r\n",
+    );
+    expect(robots.groups[0]?.rules).toEqual([{ allow: false, pattern: "/x" }]);
+    expect(robots.sitemaps).toEqual(["https://example.test/sitemap.xml"]);
+  });
+
+  test("an empty Disallow restricts nothing", () => {
+    const robots = parseRobots("User-agent: *\nDisallow:\n");
+    expect(robots.groups[0]?.rules).toEqual([]);
+    expect(isAllowed(robots, UA, "/anything")).toBe(true);
+  });
+
+  test("reads crawl-delay", () => {
+    const robots = parseRobots("User-agent: *\nCrawl-delay: 10\nDisallow: /x\n");
+    expect(crawlDelayMs(robots, UA)).toBe(10_000);
+  });
+});
+
+describe("group selection", () => {
+  test("takes the product token out of a full User-Agent header", () => {
+    expect(agentToken(UA)).toBe("gacha-event-tracker");
+  });
+
+  test("a named group beats the wildcard group", () => {
+    const robots = parseRobots(`
+User-agent: *
+Disallow: /
+
+User-agent: gacha-event-tracker
+Disallow: /admin
+`);
+    expect(isAllowed(robots, UA, "/games/Genshin-Impact/archives/301601")).toBe(
+      true,
+    );
+    expect(isAllowed(robots, UA, "/admin/panel")).toBe(false);
+  });
+
+  test("falls back to the wildcard group when nothing names us", () => {
+    const robots = parseRobots("User-agent: *\nDisallow: /games/\n");
+    expect(isAllowed(robots, UA, "/games/x")).toBe(false);
+  });
+
+  test("no applicable group at all means allowed", () => {
+    const robots = parseRobots("User-agent: gptbot\nDisallow: /\n");
+    expect(groupFor(robots, UA)).toBeNull();
+    expect(isAllowed(robots, UA, "/games/x")).toBe(true);
+  });
+
+  test("the longest matching agent name wins", () => {
+    const robots = parseRobots(`
+User-agent: googlebot
+Disallow: /
+
+User-agent: googlebot-news
+Allow: /
+`);
+    expect(isAllowed(robots, "Googlebot-News/1.0", "/anything")).toBe(true);
+    expect(isAllowed(robots, "Googlebot/2.1", "/anything")).toBe(false);
+  });
+
+  test("merges rules from several groups naming the same agent", () => {
+    const robots = parseRobots(`
+User-agent: gacha-event-tracker
+Disallow: /a
+
+User-agent: gacha-event-tracker
+Disallow: /b
+`);
+    expect(isAllowed(robots, UA, "/a")).toBe(false);
+    expect(isAllowed(robots, UA, "/b")).toBe(false);
+    expect(isAllowed(robots, UA, "/c")).toBe(true);
+  });
+});
+
+describe("path matching", () => {
+  test("prefix match, wildcards and the end anchor", () => {
+    expect(patternMatches("/games/", "/games/Genshin")).toBe(true);
+    expect(patternMatches("/*.json", "/data/events.json")).toBe(true);
+    expect(patternMatches("/x$", "/x")).toBe(true);
+    expect(patternMatches("/x$", "/x/y")).toBe(false);
+    expect(patternMatches("", "/x")).toBe(false);
+  });
+
+  test("longest matching rule wins", () => {
+    const robots = parseRobots(`
+User-agent: *
+Disallow: /wiki/
+Allow: /wiki/Event
+`);
+    expect(isAllowed(robots, UA, "/wiki/Special:Random")).toBe(false);
+    expect(isAllowed(robots, UA, "/wiki/Event")).toBe(true);
+  });
+
+  test("allow wins a tie of equal length", () => {
+    const robots = parseRobots("User-agent: *\nDisallow: /page\nAllow: /page\n");
+    expect(isAllowed(robots, UA, "/page")).toBe(true);
+  });
+
+  test("the query string is part of the matched target", () => {
+    const robots = parseRobots("User-agent: *\nDisallow: /*?action=edit\n");
+    expect(requestTarget("https://x.test/wiki/Event?action=edit")).toBe(
+      "/wiki/Event?action=edit",
+    );
+    expect(isAllowed(robots, UA, "/wiki/Event?action=edit")).toBe(false);
+    expect(isAllowed(robots, UA, "/wiki/Event")).toBe(true);
+  });
+
+  test("a path is matched with a leading slash even if given without one", () => {
+    const robots = parseRobots("User-agent: *\nDisallow: /x\n");
+    expect(isAllowed(robots, UA, "x")).toBe(false);
+  });
+});
+
+describe("the sources we actually fetch", () => {
+  // Game8 opts out of AI-training crawlers by name and leaves everyone else
+  // alone (CLAUDE.md § Scraping conduct). If that ever changes, this is where
+  // it should be noticed.
+  const game8 = parseRobots(`
+User-agent: GPTBot
+Disallow: /
+
+User-agent: Google-Extended
+Disallow: /
+
+User-agent: *
+Disallow: /admin/
+Disallow: /*?utm_source=
+`);
+
+  test("our agent may fetch a Game8 article page", () => {
+    expect(
+      isAllowed(game8, UA, "/games/Genshin-Impact/archives/301601"),
+    ).toBe(true);
+  });
+
+  test("the AI-training opt-outs still bind those crawlers", () => {
+    expect(isAllowed(game8, "GPTBot/1.2", "/games/x")).toBe(false);
+    expect(isAllowed(game8, "Google-Extended", "/games/x")).toBe(false);
+  });
+
+  test("the wildcard rules that do exist are obeyed", () => {
+    expect(isAllowed(game8, UA, "/admin/")).toBe(false);
+    expect(isAllowed(game8, UA, "/games/x?utm_source=y")).toBe(false);
+  });
+});
+
+describe("RobotsCache", () => {
+  function cacheWith(
+    responder: (url: string) => Response | Promise<Response>,
+    calls: string[] = [],
+  ) {
+    return {
+      calls,
+      cache: new RobotsCache({
+        userAgent: UA,
+        fetchImpl: async (url) => {
+          calls.push(url);
+          return responder(url);
+        },
+      }),
+    };
+  }
+
+  test("fetches robots.txt once per host and reuses it", async () => {
+    const { cache, calls } = cacheWith(
+      () => new Response("User-agent: *\nDisallow: /admin\n", { status: 200 }),
+    );
+    expect((await cache.allows("https://game8.co/games/a")).allowed).toBe(true);
+    expect((await cache.allows("https://game8.co/games/b")).allowed).toBe(true);
+    expect((await cache.allows("https://game8.co/admin")).allowed).toBe(false);
+    expect(calls).toEqual(["https://game8.co/robots.txt"]);
+    expect(cache.fetches).toBe(1);
+  });
+
+  test("fetches once per distinct host", async () => {
+    const { cache, calls } = cacheWith(() => new Response("", { status: 200 }));
+    await cache.allows("https://game8.co/a");
+    await cache.allows("https://endfield.wiki.gg/wiki/Event");
+    expect(calls).toEqual([
+      "https://game8.co/robots.txt",
+      "https://endfield.wiki.gg/robots.txt",
+    ]);
+  });
+
+  test("a missing robots.txt means no restrictions", async () => {
+    const { cache } = cacheWith(() => new Response("nope", { status: 404 }));
+    const decision = await cache.allows("https://x.test/wiki/Event");
+    expect(decision.allowed).toBe(true);
+    expect(decision.reason).toBe("no robots.txt");
+  });
+
+  test("fails closed on a server error", async () => {
+    const { cache } = cacheWith(() => new Response("", { status: 503 }));
+    const decision = await cache.allows("https://x.test/wiki/Event");
+    expect(decision.allowed).toBe(false);
+    expect(decision.reason).toContain("503");
+  });
+
+  test("fails closed when robots.txt is unreachable", async () => {
+    const { cache } = cacheWith(() => {
+      throw new Error("ECONNREFUSED");
+    });
+    const decision = await cache.allows("https://x.test/wiki/Event");
+    expect(decision.allowed).toBe(false);
+    expect(decision.reason).toContain("unreachable");
+  });
+
+  test("expires an entry after its TTL", async () => {
+    const calls: string[] = [];
+    let clock = 0;
+    const cache = new RobotsCache({
+      userAgent: UA,
+      fetchImpl: async (url) => {
+        calls.push(url);
+        return new Response("User-agent: *\nDisallow:\n", { status: 200 });
+      },
+      ttlMs: 1000,
+      now: () => clock,
+    });
+
+    await cache.allows("https://x.test/a");
+    clock = 999;
+    await cache.allows("https://x.test/a");
+    expect(calls).toHaveLength(1);
+    clock = 1001;
+    await cache.allows("https://x.test/a");
+    expect(calls).toHaveLength(2);
+  });
+});
@@ -0,0 +1,223 @@
+import { afterEach, beforeEach, describe, expect, test } from "bun:test";
+import { mkdtemp, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import {
+  freshnessAt,
+  hashBody,
+  SnapshotStore,
+  type SnapshotMeta,
+} from "../src/ingest/snapshots.ts";
+
+const SIX_HOURS_MS = 6 * 60 * 60 * 1000;
+const T0 = "2026-08-15T00:00:00.000Z";
+const T1 = "2026-08-15T12:00:00.000Z";
+
+let root: string;
+let store: SnapshotStore;
+
+beforeEach(async () => {
+  root = await mkdtemp(join(tmpdir(), "event-clock-snapshots-"));
+  store = new SnapshotStore(root);
+});
+
+afterEach(async () => {
+  await rm(root, { recursive: true, force: true });
+});
+
+function save(html: string, at: string, extra: Partial<{ etag: string | null; lastModified: string | null; eventCount: number | null }> = {}) {
+  return store.save("genshin-game8-events", {
+    url: "https://game8.co/games/Genshin-Impact/archives/301601",
+    html,
+    etag: extra.etag ?? null,
+    lastModified: extra.lastModified ?? null,
+    at,
+    eventCount: extra.eventCount ?? null,
+  });
+}
+
+describe("SnapshotStore", () => {
+  test("an unfetched source reads as nothing, not as an error", async () => {
+    expect(await store.read("genshin-game8-events")).toBeNull();
+    expect(await store.readMeta("genshin-game8-events")).toBeNull();
+    expect(await store.list()).toEqual([]);
+  });
+
+  test("stores the body verbatim and its metadata", async () => {
+    const { changed, meta } = await save("<html>one</html>", T0, {
+      etag: 'W/"abc"',
+      lastModified: "Fri, 14 Aug 2026 09:00:00 GMT",
+      eventCount: 9,
+    });
+
+    expect(changed).toBe(true);
+    expect(meta.contentHash).toBe(hashBody("<html>one</html>"));
+    expect(meta.eventCount).toBe(9);
+
+    const snapshot = await store.read("genshin-game8-events");
+    expect(snapshot?.html).toBe("<html>one</html>");
+    expect(snapshot?.meta.etag).toBe('W/"abc"');
+    expect(await store.list()).toEqual(["genshin-game8-events"]);
+  });
+
+  test("identical bytes are not a change and do not rewrite metadata", async () => {
+    await save("<html>one</html>", T0, { etag: '"v1"', eventCount: 9 });
+    const again = await save("<html>one</html>", T1, {
+      etag: '"v2"',
+      eventCount: 9,
+    });
+
+    expect(again.changed).toBe(false);
+    // contentChangedAt still points at the fetch that produced these bytes,
+    // which is what keeps an unchanged cycle out of the commit log.
+    expect(again.meta.contentChangedAt).toBe(T0);
+    expect(again.meta.etag).toBe('"v1"');
+  });
+
+  test("different bytes are a change", async () => {
+    await save("<html>one</html>", T0, { eventCount: 9 });
+    const next = await save("<html>two</html>", T1, { eventCount: 10 });
+
+    expect(next.changed).toBe(true);
+    expect(next.meta.contentChangedAt).toBe(T1);
+    expect((await store.read("genshin-game8-events"))?.html).toBe(
+      "<html>two</html>",
+    );
+  });
+
+  test("re-saves when the metadata survived but the body did not", async () => {
+    await save("<html>one</html>", T0);
+    await rm(store.bodyPath("genshin-game8-events"));
+    expect(await store.read("genshin-game8-events")).toBeNull();
+    expect((await save("<html>one</html>", T1)).changed).toBe(true);
+  });
+
+  test("unreadable metadata is treated as no cache rather than crashing", async () => {
+    await save("<html>one</html>", T0);
+    await writeFile(store.metaPath("genshin-game8-events"), "{ truncated");
+    expect(await store.readMeta("genshin-game8-events")).toBeNull();
+  });
+});
+
+describe("conditional requests", () => {
+  test("emits both validators when both are known", async () => {
+    const { meta } = await save("<html>one</html>", T0, {
+      etag: 'W/"abc"',
+      lastModified: "Fri, 14 Aug 2026 09:00:00 GMT",
+    });
+    expect(store.conditionalHeaders(meta)).toEqual({
+      "If-None-Match": 'W/"abc"',
+      "If-Modified-Since": "Fri, 14 Aug 2026 09:00:00 GMT",
+    });
+  });
+
+  test("emits only what the server gave us", async () => {
+    const { meta } = await save("<html>one</html>", T0, { etag: 'W/"abc"' });
+    expect(store.conditionalHeaders(meta)).toEqual({ "If-None-Match": 'W/"abc"' });
+  });
+
+  test("a source never fetched sends no validators", () => {
+    expect(store.conditionalHeaders(null)).toEqual({});
+  });
+});
+
+describe("the six-hour floor", () => {
+  const at = (iso: string) => Date.parse(iso);
+
+  test("a source never checked is due", async () => {
+    const state = await store.readState("genshin-game8-events");
+    expect(store.isDue(state, at(T0), SIX_HOURS_MS)).toBe(true);
+  });
+
+  test("holds off until six hours have passed", async () => {
+    await store.recordCheck("genshin-game8-events", {
+      at: T0,
+      status: 200,
+      ok: true,
+    });
+    const state = await store.readState("genshin-game8-events");
+
+    expect(store.isDue(state, at(T0) + SIX_HOURS_MS - 1, SIX_HOURS_MS)).toBe(
+      false,
+    );
+    expect(store.isDue(state, at(T0) + SIX_HOURS_MS, SIX_HOURS_MS)).toBe(true);
+    expect(store.dueAt(state, SIX_HOURS_MS)).toBe(at(T0) + SIX_HOURS_MS);
+  });
+
+  test("a failed attempt still counts as an attempt", async () => {
+    await store.recordCheck("genshin-game8-events", {
+      at: T0,
+      status: 503,
+      ok: false,
+    });
+    const state = await store.readState("genshin-game8-events");
+    expect(state.consecutiveFailures).toBe(1);
+    expect(state.lastConfirmedAt).toBeNull();
+    expect(store.isDue(state, at(T0) + 60_000, SIX_HOURS_MS)).toBe(false);
+  });
+
+  test("a success clears the failure streak", async () => {
+    await store.recordCheck("x", { at: T0, status: 503, ok: false });
+    await store.recordCheck("x", { at: T1, status: 304, ok: true });
+    const state = await store.readState("x");
+    expect(state.consecutiveFailures).toBe(0);
+    expect(state.lastConfirmedAt).toBe(T1);
+  });
+
+  test("check bookkeeping lives outside the committed metadata", async () => {
+    await save("<html>one</html>", T0);
+    const before = await Bun.file(store.metaPath("genshin-game8-events")).text();
+    await store.recordCheck("genshin-game8-events", {
+      at: T1,
+      status: 304,
+      ok: true,
+    });
+    const after = await Bun.file(store.metaPath("genshin-game8-events")).text();
+    expect(after).toBe(before);
+  });
+});
+
+describe("freshnessAt", () => {
+  const meta: SnapshotMeta = {
+    sourceId: "x",
+    url: "https://x.test",
+    contentHash: "abc",
+    bytes: 3,
+    etag: null,
+    lastModified: null,
+    contentChangedAt: T0,
+    eventCount: 1,
+  };
+
+  test("reports the last confirmation when there is one", () => {
+    expect(
+      freshnessAt({
+        meta,
+        state: {
+          sourceId: "x",
+          lastCheckedAt: T1,
+          lastConfirmedAt: T1,
+          lastStatus: 304,
+          consecutiveFailures: 0,
+        },
+        html: "",
+      }),
+    ).toBe(T1);
+  });
+
+  test("never claims to be fresher than the bytes", () => {
+    expect(
+      freshnessAt({
+        meta,
+        state: {
+          sourceId: "x",
+          lastCheckedAt: null,
+          lastConfirmedAt: null,
+          lastStatus: null,
+          consecutiveFailures: 0,
+        },
+        html: "",
+      }),
+    ).toBe(T0);
+  });
+});