fix: harden the fetch pipeline for a real server

A review of code that has never made a live request. Each of these was
reproduced before it was fixed.

robots.txt matched a group by testing whether our whole User-Agent
contained the group's name. Our contact URL carries "StereotypicalCat",
so a wiki writing `User-agent: cat` matched us — and because a named
group *replaces* the wildcard group, that silently discarded every rule
under `User-agent: *`. Match the RFC 9309 product token instead. A 200
carrying an HTML "not found" page also parsed to zero rules and read as
permission; it now fails closed, while a genuinely empty body still
means "no restrictions".

The response body was read outside the try that guarded the request, so
one truncated body aborted the whole cycle: later sources were never
fetched, and the failed source never recorded its check, meaning a
re-dispatch would ask that wiki again minutes later.

Bodies were decoded as UTF-8 unconditionally and stored re-encoded. A
page served in a legacy charset became replacement characters with the
original bytes gone — and mojibake in a title flows into slugify and
moves every localStorage key for that source. Decode by the declared
charset, keep the served bytes verbatim, and hash those.

Unchanged bytes skipped the metadata write, so once a server rotated its
ETag we sent a stale validator forever and it served full bodies instead
of 304s.

Also: snapshot writes go through a temp file and a rename, and `--only`
with no value is an argument error rather than "all sources".

Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>

scripts/refresh-sources.ts

@@ -29,7 +29,7 @@ import {
 import { SIX_HOURS_MS } from "../src/ingest/adapters/types.ts";
 import type { Adapter } from "../src/ingest/adapters/types.ts";
 import { RobotsCache, type FetchLike } from "../src/ingest/robots.ts";
-import { SnapshotStore } from "../src/ingest/snapshots.ts";
+import { decodeBody, SnapshotStore } from "../src/ingest/snapshots.ts";
 
 const DEFAULT_CONTACT =
   "https://github.com/StereotypicalCat/gacha-event-tracker";
@@ -113,7 +113,23 @@ export async function runRefresh(
   }
 
   for (const adapter of selected) {
-    const outcome = await refreshOne(adapter, options);
+    // One source can never take the cycle down with it. Everything inside
+    // refreshOne that can fail is handled there; this is the backstop that
+    // keeps an unforeseen throw from costing every source after this one its
+    // turn — the sources are independent, and a run that stops halfway leaves
+    // no summary and no record of what was already asked.
+    let outcome: SourceOutcome;
+    try {
+      outcome = await refreshOne(adapter, options);
+    } catch (error) {
+      outcome = {
+        sourceId: adapter.id,
+        result: "failed",
+        note: `unexpected error: ${String(error)}`,
+        status: null,
+        eventCount: null,
+      };
+    }
     summary.outcomes.push(outcome);
 
     if (outcome.result === "fetched") {
@@ -256,7 +272,37 @@ async function refreshOne(
     };
   }
 
-  const html = await response.text();
+  // Reading the body is a second chance to fail — a reset connection, a
+  // truncated response, or the timeout firing mid-stream. Left outside the try
+  // this rejection escapes refreshOne, aborts the whole cycle, and leaves the
+  // sources after this one unfetched and this one's `lastCheckedAt` unwritten:
+  // one bad body would both blank the run and lose the record that we had
+  // already spent this source's request.
+  let bytes: Uint8Array;
+  try {
+    bytes = new Uint8Array(await response.arrayBuffer());
+  } catch (error) {
+    await store.recordCheck(adapter.id, {
+      at: nowIso,
+      status: response.status,
+      ok: false,
+    });
+    return {
+      sourceId: adapter.id,
+      result: "failed",
+      note: `body unreadable: ${String(error)}`,
+      status: response.status,
+      eventCount: meta?.eventCount ?? null,
+    };
+  }
+
+  // Decode with the charset the server declared. Storing the raw bytes keeps
+  // the snapshot re-decodable; decoding before parsing keeps mojibake out of
+  // titles, and therefore out of the event IDs that are localStorage keys.
+  const { text: html, charset } = decodeBody(
+    bytes,
+    response.headers.get("Content-Type"),
+  );
 
   // The parse gate. A body that no longer parses, or that yields nothing where
   // it used to yield events, is a source that changed shape — publishing it
@@ -310,7 +356,8 @@ async function refreshOne(
 
   const saved = await store.save(adapter.id, {
     url: adapter.url,
-    html,
+    body: bytes,
+    charset,
     etag: response.headers.get("ETag"),
     lastModified: response.headers.get("Last-Modified"),
     at: nowIso,
@@ -377,6 +424,17 @@ export function parseArgs(argv: readonly string[]): Args {
     help: false,
   };
 
+  // A flag whose value is missing is a mistake, never a default. `--only` with
+  // nothing after it used to mean "every source", which is the opposite of
+  // what the operator typed and one request per source more than they wanted.
+  const value = (i: number, flag: string): string => {
+    const next = argv[i];
+    if (next === undefined || next.startsWith("-")) {
+      throw new Error(`${flag} requires a value`);
+    }
+    return next;
+  };
+
   for (let i = 0; i < argv.length; i += 1) {
     const arg = argv[i];
     switch (arg) {
@@ -385,15 +443,15 @@ export function parseArgs(argv: readonly string[]): Args {
         break;
       case "--only":
         i += 1;
-        args.only = argv[i] ?? null;
+        args.only = value(i, "--only");
         break;
       case "--snapshots":
         i += 1;
-        args.root = argv[i] ?? args.root;
+        args.root = value(i, "--snapshots");
         break;
       case "--user-agent":
         i += 1;
-        args.userAgent = argv[i] ?? args.userAgent;
+        args.userAgent = value(i, "--user-agent");
         break;
       case "--no-feed":
         args.rebuild = false;

snapshots/README.md

@@ -4,10 +4,21 @@ Raw pages, exactly as fetched. `scripts/refresh-sources.ts` writes them; nothing
 
 ```
 <source-id>.html        the body verbatim — tracked
-<source-id>.meta.json   hash, size, ETag, Last-Modified, when the bytes last changed — tracked
+<source-id>.meta.json   hash, size, charset, ETag, Last-Modified, when the bytes last changed — tracked
 <source-id>.state.json  when we last checked, and failure streak — gitignored
 ```
 
+"Verbatim" means the bytes as served, not text we re-encoded. `charset` in the metadata records
+what those bytes are in — the `Content-Type` header, else a `<meta charset>` in the page, else
+UTF-8 — and `bytes` is the served length. A page in Shift_JIS or Latin-1 decoded as UTF-8 would be
+stored as a field of U+FFFD with the original bytes gone; re-parsing could never recover it, and the
+mojibake would reach `slugify`, moving every event ID for that source (CLAUDE.md § Event IDs are
+localStorage keys).
+
+Every file is written to a sibling `.tmp-*` and renamed into place, body before metadata, so an
+interrupted run leaves a stray temp file rather than a truncated snapshot or metadata describing
+bytes that were never stored.
+
 Three reasons this is committed rather than cached:
 
 - **Re-parsing never re-fetches.** Iterating on a parser reads these files, not the wikis
@@ -19,7 +30,15 @@ Three reasons this is committed rather than cached:
 
 The `.state.json` files are the exception: they change every cycle whether or not a page did, and
 committing them would mean a commit per run saying nothing happened. CI keeps them in the actions
-cache instead.
+cache instead: `refresh.yml` saves that cache, and `ci.yml` restores it read-only before building
+the feed. Both halves matter — `lastConfirmedAt` lives only there, and without the restore the feed
+falls back to `contentChangedAt` and the UI calls every source stale two days after its bytes last
+moved.
+
+The metadata is rewritten on an unchanged page in one case: the server rotating an `ETag` or
+`Last-Modified` while serving the same bytes. Keeping the old validator would mean sending a stale
+`If-None-Match` forever and being served the whole page every cycle, so that diff is worth the
+commit — `contentChangedAt` and the body stay put, so it is still visibly not a content change.
 
 Fixtures are not the same thing. A fixture is pinned to a date and kept forever as the regression
 test for a page shape; a snapshot is the current page and is overwritten each time it changes.

src/ingest/robots.ts

@@ -137,6 +137,26 @@ export function agentToken(userAgent: string): string {
   return first.toLowerCase();
 }
 
+/**
+ * Does a `User-agent:` value in robots.txt name us?
+ *
+ * RFC 9309 § 2.2.1 matches the *product token* — the header up to the first
+ * `/` — not the header text. Matching anywhere in the header is actively
+ * dangerous here: our contact URL contains the string `StereotypicalCat`, so a
+ * `User-agent: cat` group elsewhere in the file would be treated as naming us,
+ * and because a named group replaces the `*` group outright, that unrelated
+ * group's rules would *discard* every rule the site actually wrote for us.
+ * Erring towards obeying more rules means never letting a coincidence take a
+ * `*` group away.
+ *
+ * A robots.txt that names us with a version (`gacha-event-tracker/1.0`) is
+ * still honoured: the group's own product token is compared too.
+ */
+function agentNames(agent: string, token: string): boolean {
+  if (agent === "*") return false;
+  return agent === token || agentToken(agent) === token;
+}
+
 /**
  * The group that applies to a user agent, with every group naming the same
  * agent merged, as RFC 9309 requires.
@@ -150,19 +170,11 @@ export function groupFor(
   userAgent: string,
 ): RobotsGroup | null {
   const token = agentToken(userAgent);
-  const full = userAgent.toLowerCase();
 
   let bestName: string | null = null;
   for (const group of robots.groups) {
     for (const agent of group.agents) {
-      if (agent === "*") continue;
-      // Match on the product token first (the spec's rule); fall back to a
-      // substring of the whole header so a group naming "gptbot" still binds a
-      // header of "Mozilla/5.0 (compatible; GPTBot/1.2)". Erring towards
-      // matching means erring towards obeying more rules, not fewer.
-      const hit =
-        token === agent || token.startsWith(agent) || full.includes(agent);
-      if (!hit) continue;
+      if (!agentNames(agent, token)) continue;
       if (bestName === null || agent.length > bestName.length) bestName = agent;
     }
   }
@@ -280,8 +292,10 @@ const DAY_MS = 24 * 60 * 60 * 1000;
  * One robots.txt fetch per host per run (cached 24h), reused by every source on
  * that host — six Game8 adapters must not mean six robots requests.
  *
- * Fails closed. A 5xx, a timeout or a network error means we do not know what
- * the site permits, and "unknown" is not permission.
+ * Fails closed. A 5xx, a timeout, a network error, a body that dies mid-read
+ * or a body that is plainly not robots.txt all mean we do not know what the
+ * site permits, and "unknown" is not permission. Only two answers open the
+ * host: a parsed robots.txt, and a 404/410 saying there is none.
  */
 export class RobotsCache {
   private readonly entries = new Map<string, CacheEntry>();
@@ -368,7 +382,50 @@ export class RobotsCache {
       };
     }
 
-    const text = await response.text();
+    // Reading the body is a second chance to fail: the connection can reset or
+    // the timeout can fire mid-stream, long after the headers arrived. Outside
+    // the try that rejection would escape as an exception rather than as "we
+    // could not read robots.txt", which is the one answer this class exists to
+    // give.
+    let text: string;
+    try {
+      text = await response.text();
+    } catch (error) {
+      return {
+        robots: ALLOW_ALL,
+        usable: false,
+        reason: `robots.txt body unreadable (${String(error)})`,
+        at,
+      };
+    }
+
+    // A soft 404 — an HTML "not found" page served with status 200 — is the
+    // commonest robots.txt misconfiguration there is, and it parses to zero
+    // groups, which is indistinguishable from "everything is permitted". We do
+    // not know what the site allows, and unknown is not permission.
+    if (looksLikeHtml(text)) {
+      return {
+        robots: ALLOW_ALL,
+        usable: false,
+        reason: "robots.txt returned HTML, not a robots.txt (soft 404?)",
+        at,
+      };
+    }
+
     return { robots: parseRobots(text), usable: true, reason: "robots.txt ok", at };
   }
 }
+
+/**
+ * Is this body markup rather than robots.txt?
+ *
+ * An empty body is *valid* robots.txt meaning "no restrictions", so emptiness
+ * is deliberately not a failure. Only markup is — no robots.txt directive can
+ * begin with `<`.
+ */
+export function looksLikeHtml(text: string): boolean {
+  const head = text.trimStart().slice(0, 512).toLowerCase();
+  if (head === "") return false;
+  if (head.startsWith("<")) return true;
+  return /<!doctype html|<html[\s>]|<head[\s>]|<body[\s>]/.test(head);
+}

src/ingest/snapshots.ts

@@ -18,14 +18,15 @@
  * the metadata, every cycle would produce a commit that says nothing, and
  * "commit only when something changed" would be unenforceable.
  */
-import { mkdir, readdir, rm, writeFile } from "node:fs/promises";
+import { mkdir, readdir, rename, rm, writeFile } from "node:fs/promises";
 import { join } from "node:path";
 
 export interface SnapshotMeta {
   sourceId: string;
   url: string;
-  /** sha256 of the body, hex. The parse stage skips work when it is unchanged. */
+  /** sha256 of the served bytes, hex. The parse stage skips work when it is unchanged. */
   contentHash: string;
+  /** The served length, in bytes — not the length of the decoded text. */
   bytes: number;
   etag: string | null;
   lastModified: string | null;
@@ -33,6 +34,11 @@ export interface SnapshotMeta {
   contentChangedAt: string;
   /** Events the adapter yielded from this body, for drop detection. */
   eventCount: number | null;
+  /**
+   * The encoding the stored bytes are in, as the server declared it. Absent in
+   * metadata written before charsets were handled, which is read as UTF-8.
+   */
+  charset?: string;
 }
 
 export interface SnapshotState {
@@ -53,7 +59,15 @@ export interface Snapshot {
 
 export interface SaveInput {
   url: string;
-  html: string;
+  /**
+   * The body as served. Bytes are the honest unit: a page in Shift_JIS or
+   * Latin-1 that we stored as re-encoded text could never be re-decoded, and
+   * `snapshots/README.md` promises "the body verbatim". A string is accepted
+   * as a convenience and is stored as UTF-8.
+   */
+  body: Uint8Array | string;
+  /** The encoding `body` is in. Defaults to UTF-8. */
+  charset?: string;
   etag: string | null;
   lastModified: string | null;
   /** ISO timestamp of this fetch. */
@@ -66,8 +80,101 @@ export interface SaveResult {
   meta: SnapshotMeta;
 }
 
-export function hashBody(html: string): string {
-  return new Bun.CryptoHasher("sha256").update(html).digest("hex");
+export function hashBody(body: Uint8Array | string): string {
+  return new Bun.CryptoHasher("sha256").update(body).digest("hex");
+}
+
+const DEFAULT_CHARSET = "utf-8";
+
+/** The charset a `Content-Type` header declares, lowercased, or null. */
+export function charsetFromContentType(contentType: string | null): string | null {
+  if (contentType === null) return null;
+  const match = /;\s*charset\s*=\s*"?([^;"\s]+)"?/i.exec(contentType);
+  return match?.[1]?.toLowerCase() ?? null;
+}
+
+/** The charset a `<meta>` tag declares in the first bytes of a document. */
+export function sniffMetaCharset(bytes: Uint8Array): string | null {
+  // Every encoding we might meet here is ASCII-compatible in its first 2 KiB,
+  // so reading the head as Latin-1 cannot lose the declaration.
+  const head = Buffer.from(
+    bytes.buffer,
+    bytes.byteOffset,
+    Math.min(bytes.byteLength, 2048),
+  ).toString("latin1");
+  const match = /<meta[^>]*?charset\s*=\s*["']?\s*([a-z0-9_\-:.]+)/i.exec(head);
+  return match?.[1]?.toLowerCase() ?? null;
+}
+
+export interface DecodedBody {
+  text: string;
+  /** The charset actually used, which is what gets recorded in the metadata. */
+  charset: string;
+}
+
+/**
+ * Decode a fetched body into text.
+ *
+ * `Response.text()` assumes UTF-8, so a page served in Shift_JIS or Latin-1
+ * comes back as a field of U+FFFD. That is not merely ugly: mojibake in a title
+ * flows through `slugify` into the event ID, which is a localStorage key, so a
+ * mis-decoded fetch silently orphans every completion mark for that source
+ * (CLAUDE.md § Event IDs are localStorage keys).
+ *
+ * Header first, then a `<meta charset>` sniff, then UTF-8. An encoding label
+ * the runtime does not know falls back to UTF-8 rather than throwing — the raw
+ * bytes are kept either way, so a wrong guess stays recoverable.
+ */
+export function decodeBody(
+  bytes: Uint8Array,
+  contentType: string | null,
+): DecodedBody {
+  const declared =
+    charsetFromContentType(contentType) ?? sniffMetaCharset(bytes) ?? DEFAULT_CHARSET;
+  try {
+    const decoder = new TextDecoder(declared);
+    return { text: decoder.decode(bytes), charset: decoder.encoding };
+  } catch {
+    return {
+      text: new TextDecoder(DEFAULT_CHARSET).decode(bytes),
+      charset: DEFAULT_CHARSET,
+    };
+  }
+}
+
+function toBytes(body: Uint8Array | string): Uint8Array {
+  return typeof body === "string" ? new TextEncoder().encode(body) : body;
+}
+
+function decodeStored(bytes: Uint8Array, charset: string | undefined): string {
+  try {
+    return new TextDecoder(charset ?? DEFAULT_CHARSET).decode(bytes);
+  } catch {
+    return new TextDecoder(DEFAULT_CHARSET).decode(bytes);
+  }
+}
+
+/**
+ * Write a file by writing a sibling temp file and renaming it into place.
+ *
+ * `rename` within a directory is atomic, so a reader — the feed build, git,
+ * the next refresh — sees either the whole old file or the whole new one, and
+ * a crash mid-write leaves a stray temp file rather than a truncated snapshot.
+ */
+async function writeAtomic(path: string, data: Uint8Array | string): Promise<void> {
+  const temp = `${path}.tmp-${process.pid.toString(36)}-${Math.random().toString(36).slice(2, 10)}`;
+  try {
+    await writeFile(temp, data);
+    await rename(temp, path);
+  } catch (error) {
+    await rm(temp, { force: true });
+    throw error;
+  }
+}
+
+/** Would writing this metadata change the file on disk? */
+function sameMeta(a: SnapshotMeta, b: SnapshotMeta): boolean {
+  return JSON.stringify(a) === JSON.stringify(b);
 }
 
 function emptyState(sourceId: string): SnapshotState {
@@ -125,7 +232,14 @@ export class SnapshotStore {
     const body = Bun.file(this.bodyPath(sourceId));
     if (!(await body.exists())) return null;
 
-    return { meta, state: await this.readState(sourceId), html: await body.text() };
+    // Decoded with the charset the bytes were stored in, so a Shift_JIS page
+    // reads back as the text the wiki published rather than as mojibake.
+    const bytes = new Uint8Array(await body.arrayBuffer());
+    return {
+      meta,
+      state: await this.readState(sourceId),
+      html: decodeStored(bytes, meta.charset),
+    };
   }
 
   /** Sources with a stored snapshot, by id. */
@@ -181,35 +295,54 @@ export class SnapshotStore {
   /**
    * Store a fetched body.
    *
-   * Identical bytes are a no-op on disk: the metadata keeps its original
-   * `contentChangedAt` and validators, so an unchanged source produces no diff
-   * for the workflow to commit.
+   * Identical bytes are a no-op on the body and keep the original
+   * `contentChangedAt`, so an unchanged source does not look like a changed
+   * one. The validators are the exception: a server is free to rotate an ETag
+   * while serving the very same bytes, and keeping the old one would mean
+   * sending a stale `If-None-Match` forever — every cycle costing the wiki a
+   * full body where a 304 was the whole point (CLAUDE.md § Scraping conduct).
+   * So the metadata is refreshed, and `changed` stays false.
    */
   async save(sourceId: string, input: SaveInput): Promise<SaveResult> {
-    const contentHash = hashBody(input.html);
+    const bytes = toBytes(input.body);
+    const charset = input.charset ?? DEFAULT_CHARSET;
+    const contentHash = hashBody(bytes);
     const previous = await this.readMeta(sourceId);
     const bodyExists = await Bun.file(this.bodyPath(sourceId)).exists();
     const changed =
       previous === null || previous.contentHash !== contentHash || !bodyExists;
 
-    if (!changed && previous !== null) {
-      return { changed: false, meta: previous };
-    }
-
     const meta: SnapshotMeta = {
       sourceId,
       url: input.url,
       contentHash,
-      bytes: Buffer.byteLength(input.html),
+      bytes: bytes.byteLength,
       etag: input.etag,
       lastModified: input.lastModified,
-      contentChangedAt: input.at,
-      eventCount: input.eventCount,
+      contentChangedAt: changed ? input.at : (previous?.contentChangedAt ?? input.at),
+      // New bytes mean the count they yielded, even when that is unknown; the
+      // same bytes keep the count we already recorded for them.
+      eventCount: changed
+        ? input.eventCount
+        : (input.eventCount ?? previous?.eventCount ?? null),
+      charset,
     };
 
     await mkdir(this.root, { recursive: true });
-    await writeFile(this.bodyPath(sourceId), input.html);
-    await writeFile(this.metaPath(sourceId), `${JSON.stringify(meta, null, 2)}\n`);
+
+    if (!changed && previous !== null) {
+      if (sameMeta(previous, meta)) return { changed: false, meta: previous };
+      await writeAtomic(this.metaPath(sourceId), `${JSON.stringify(meta, null, 2)}\n`);
+      return { changed: false, meta };
+    }
+
+    // Body first, metadata second, each renamed into place. A crash between
+    // them leaves new bytes beside older metadata, whose stale hash makes the
+    // next save rewrite both — the other order would leave metadata claiming a
+    // hash for bytes that were never written, and the next save would believe
+    // it and skip them.
+    await writeAtomic(this.bodyPath(sourceId), bytes);
+    await writeAtomic(this.metaPath(sourceId), `${JSON.stringify(meta, null, 2)}\n`);
 
     return { changed: true, meta };
   }
@@ -229,7 +362,7 @@ export class SnapshotStore {
     };
 
     await mkdir(this.root, { recursive: true });
-    await writeFile(this.statePath(sourceId), `${JSON.stringify(state, null, 2)}\n`);
+    await writeAtomic(this.statePath(sourceId), `${JSON.stringify(state, null, 2)}\n`);
     return state;
   }
 

test/refresh.test.ts

@@ -99,7 +99,7 @@ function options(
 async function seed(html: string, at: string, eventCount: number | null) {
   await store.save("genshin-game8-events", {
     url: "https://game8.co/games/Genshin-Impact/archives/301601",
-    html,
+    body: html,
     etag: 'W/"v1"',
     lastModified: "Fri, 14 Aug 2026 09:00:00 GMT",
     at,
@@ -370,6 +370,79 @@ describe("a source being down never blanks the feed", () => {
     expect(summary.outcomes[0]?.note).toContain("down from 10");
   });
 
+  test("a body that dies mid-read is one source's failure, not the cycle's", async () => {
+    // The headers arrive, then the connection resets. Read outside the try,
+    // that rejection escaped refreshOne and took the whole run with it: the
+    // sources after this one were never asked, no summary was printed, and
+    // this source's lastCheckedAt was never written — so the six-hour floor
+    // did not register a request we had already spent.
+    const { opts, calls } = options({
+      adapters: [
+        adapter(),
+        adapter({
+          id: "nte-game8-events",
+          game: "nte",
+          url: "https://game8.co/games/Neverness-to-Everness/archives/592073",
+        }),
+      ],
+      responder: (call) => {
+        if (!call.url.includes("Genshin")) {
+          return new Response("<html><event></event></html>");
+        }
+        return new Response(
+          new ReadableStream({
+            start(controller) {
+              controller.enqueue(new TextEncoder().encode("<html><event>"));
+              controller.error(new Error("ECONNRESET"));
+            },
+          }),
+          { status: 200 },
+        );
+      },
+    });
+
+    const summary = await runRefresh(opts);
+
+    expect(calls).toHaveLength(2);
+    expect(summary.outcomes[0]?.result).toBe("failed");
+    expect(summary.outcomes[0]?.note).toContain("body unreadable");
+    expect(summary.outcomes[1]?.result).toBe("fetched");
+    expect(summary.hardFailure).toBeNull();
+
+    // The request was spent, so it must be on the record.
+    const state = await store.readState("genshin-game8-events");
+    expect(state.lastCheckedAt).toBe(NOW.toISOString());
+    expect(state.consecutiveFailures).toBe(1);
+  });
+
+  test("an unforeseen error in one source does not abort the others", async () => {
+    // The robots gate itself blowing up is not something refreshOne guards;
+    // the loop's backstop is what keeps the remaining sources alive.
+    const { opts } = options({
+      adapters: [
+        adapter(),
+        adapter({
+          id: "nte-game8-events",
+          game: "nte",
+          url: "https://game8.co/games/Neverness-to-Everness/archives/592073",
+        }),
+      ],
+      robots: {
+        allows: async (url) => {
+          if (url.includes("Genshin")) throw new Error("robots cache exploded");
+          return { allowed: true, reason: "ok" };
+        },
+      },
+    });
+    const summary = await runRefresh(opts);
+
+    expect(summary.outcomes).toHaveLength(2);
+    expect(summary.outcomes[0]?.result).toBe("failed");
+    expect(summary.outcomes[0]?.note).toContain("unexpected error");
+    expect(summary.outcomes[1]?.result).toBe("fetched");
+    expect(summary.hardFailure).toBeNull();
+  });
+
   test("a feed that will not rebuild fails the run", async () => {
     const { opts } = options({
       rebuildFeed: async () => {
@@ -381,6 +454,121 @@ describe("a source being down never blanks the feed", () => {
   });
 });
 
+describe("a page that is not UTF-8", () => {
+  // "<html><event>イベント</event></html>" with the title in Shift_JIS.
+  const SJIS_TITLE = [0x83, 0x43, 0x83, 0x78, 0x83, 0x93, 0x83, 0x67];
+  const body = new Uint8Array([
+    ...new TextEncoder().encode("<html><event>"),
+    ...SJIS_TITLE,
+    ...new TextEncoder().encode("</event></html>"),
+  ]);
+
+  test("is decoded with the charset the server declared", async () => {
+    const { opts } = options({
+      responder: () =>
+        new Response(body, {
+          status: 200,
+          headers: { "Content-Type": "text/html; charset=shift_jis" },
+        }),
+    });
+    const summary = await runRefresh(opts);
+    expect(summary.outcomes[0]?.result).toBe("fetched");
+
+    const snapshot = await store.read("genshin-game8-events");
+    // Read as UTF-8 this is U+FFFD soup, and mojibake in a title flows through
+    // slugify into the event ID, which is a localStorage key.
+    expect(snapshot?.html).toBe("<html><event>イベント</event></html>");
+    expect(snapshot?.html).not.toContain("�");
+  });
+
+  test("stores the bytes as served, so a re-decode is still possible", async () => {
+    const { opts } = options({
+      responder: () =>
+        new Response(body, {
+          status: 200,
+          headers: { "Content-Type": "text/html; charset=shift_jis" },
+        }),
+    });
+    await runRefresh(opts);
+
+    const onDisk = new Uint8Array(
+      await Bun.file(store.bodyPath("genshin-game8-events")).arrayBuffer(),
+    );
+    expect([...onDisk]).toEqual([...body]);
+    // `bytes` is the served length, which is not the length of the decoded text.
+    expect((await store.readMeta("genshin-game8-events"))?.bytes).toBe(
+      body.byteLength,
+    );
+  });
+
+  test("falls back to the document's own meta charset", async () => {
+    const withMeta = new Uint8Array([
+      ...new TextEncoder().encode('<html><head><meta charset="shift_jis"></head><event>'),
+      ...SJIS_TITLE,
+      ...new TextEncoder().encode("</event></html>"),
+    ]);
+    const { opts } = options({
+      responder: () =>
+        new Response(withMeta, {
+          status: 200,
+          headers: { "Content-Type": "text/html" },
+        }),
+    });
+    await runRefresh(opts);
+    expect((await store.read("genshin-game8-events"))?.html).toContain(
+      "イベント",
+    );
+  });
+});
+
+describe("the workflows that drive the refresh", () => {
+  // These three defects live in YAML, and each one is silent: nothing fails,
+  // the site just quietly carries wrong or stale data. Asserting on the file
+  // is the only offline way to keep them fixed.
+  const read = (name: string) =>
+    Bun.file(new URL(`../.github/workflows/${name}`, import.meta.url)).text();
+
+  test("ci.yml restores the refresh bookkeeping before it builds the feed", async () => {
+    // lastConfirmedAt lives only in the gitignored snapshots/*.state.json. If
+    // the job that builds the deployed feed never restores that cache,
+    // freshnessAt falls back to contentChangedAt and the UI calls a source
+    // stale two days after its bytes last moved — which for a wiki page is
+    // most of the time.
+    const ci = await read("ci.yml");
+    const restores = ci.split("actions/cache/restore@").length - 1;
+    expect(restores).toBeGreaterThanOrEqual(2); // the check job and the build job
+    expect(ci).toContain("snapshots/*.state.json");
+    // Restore only: refresh.yml owns writing it.
+    expect(ci).not.toContain("actions/cache/save@");
+
+    const buildJob = ci.slice(ci.indexOf("  build:"), ci.indexOf("  image:"));
+    expect(buildJob).toContain("actions/cache/restore@");
+  });
+
+  test("refresh.yml saves its cache under a key that changes per attempt", async () => {
+    // github.run_id is stable across re-runs, so a re-run's save is skipped and
+    // the next run restores bookkeeping from before it — losing the record of
+    // requests we did make.
+    const refresh = await read("refresh.yml");
+    const saveKey = /key: (refresh-state-[^\n]*)\n/g;
+    const keys = [...refresh.matchAll(saveKey)].map((m) => m[1] ?? "");
+    const savedKey = keys.find((k) => k.includes("run_id"));
+    expect(savedKey).toBeDefined();
+    expect(savedKey).toContain("github.run_attempt");
+  });
+
+  test("refresh.yml survives losing a push race without ever forcing", async () => {
+    // A human push landing mid-job made the push non-fast-forward: the fetched
+    // pages were thrown away while the bookkeeping had already spent their
+    // six-hour budget.
+    const refresh = await read("refresh.yml");
+    expect(refresh).toContain("git rebase");
+    expect(refresh).toMatch(/for attempt in/);
+    expect(refresh).not.toContain("--force");
+    expect(refresh).not.toContain("-f origin");
+  });
+});
+
 describe("flags", () => {
   test("--dry-run makes no requests and writes nothing", async () => {
     const { opts, calls, rebuilds } = options({ dryRun: true });
@@ -436,4 +624,17 @@ describe("flags", () => {
   test("parseArgs rejects an unknown flag rather than ignoring it", () => {
     expect(() => parseArgs(["--force"])).toThrow("unknown flag");
   });
+
+  test("parseArgs rejects a flag whose value is missing", () => {
+    // `--only` with nothing after it used to mean "every source": the operator
+    // asked for one request and would have got seven.
+    expect(() => parseArgs(["--only"])).toThrow("--only requires a value");
+    expect(() => parseArgs(["--only", "--dry-run"])).toThrow(
+      "--only requires a value",
+    );
+    expect(() => parseArgs(["--snapshots"])).toThrow("--snapshots requires a value");
+    expect(() => parseArgs(["--user-agent"])).toThrow(
+      "--user-agent requires a value",
+    );
+  });
 });

test/robots.test.ts

@@ -77,6 +77,48 @@ Disallow: /admin
     expect(isAllowed(robots, UA, "/games/x")).toBe(true);
   });
 
+  test("an unrelated named group cannot steal the wildcard group", () => {
+    // Our contact URL contains "StereotypicalCat". Matching a group name
+    // anywhere in the header made `User-agent: cat` look like our group, and
+    // because a named group replaces `*`, that coincidence *discarded* the
+    // rules the site actually wrote for everyone.
+    const ua =
+      "gacha-event-tracker/1.0 (+https://github.com/StereotypicalCat/gacha-event-tracker)";
+    const robots = parseRobots(`
+User-agent: *
+Disallow: /games/
+
+User-agent: cat
+Disallow: /litter
+`);
+    expect(groupFor(robots, ua)?.agents).toEqual(["*"]);
+    expect(isAllowed(robots, ua, "/games/x")).toBe(false);
+    expect(isAllowed(robots, ua, "/litter")).toBe(true);
+  });
+
+  test("a substring of the product token does not name us either", () => {
+    const robots = parseRobots(`
+User-agent: *
+Disallow: /games/
+
+User-agent: gacha
+Allow: /
+`);
+    expect(isAllowed(robots, UA, "/games/x")).toBe(false);
+  });
+
+  test("a group naming us with a version still binds", () => {
+    const robots = parseRobots(`
+User-agent: *
+Disallow: /
+
+User-agent: gacha-event-tracker/1.0
+Disallow: /admin
+`);
+    expect(isAllowed(robots, UA, "/games/x")).toBe(true);
+    expect(isAllowed(robots, UA, "/admin/panel")).toBe(false);
+  });
+
   test("the longest matching agent name wins", () => {
     const robots = parseRobots(`
 User-agent: googlebot
@@ -227,6 +269,47 @@ describe("RobotsCache", () => {
     expect(decision.reason).toContain("503");
   });
 
+  test("an empty 200 body is a valid robots.txt that restricts nothing", async () => {
+    const { cache } = cacheWith(() => new Response("   \n", { status: 200 }));
+    const decision = await cache.allows("https://x.test/wiki/Event");
+    expect(decision.allowed).toBe(true);
+    expect(decision.reason).toBe("robots.txt ok");
+  });
+
+  test("fails closed on a soft 404 — an HTML page served as 200", async () => {
+    // The commonest robots.txt misconfiguration there is. It parses to zero
+    // groups, which is indistinguishable from "nothing is restricted", so
+    // reading it as permission is exactly the fail-open this class forbids.
+    const { cache } = cacheWith(
+      () =>
+        new Response(
+          "<!DOCTYPE html>\n<html><head><title>404 Not Found</title></head><body>Not found</body></html>",
+          { status: 200, headers: { "Content-Type": "text/html" } },
+        ),
+    );
+    const decision = await cache.allows("https://x.test/wiki/Event");
+    expect(decision.allowed).toBe(false);
+    expect(decision.reason).toContain("HTML");
+  });
+
+  test("fails closed when the body dies mid-read", async () => {
+    const { cache } = cacheWith(
+      () =>
+        new Response(
+          new ReadableStream({
+            start(controller) {
+              controller.enqueue(new TextEncoder().encode("User-agent: *\n"));
+              controller.error(new Error("ECONNRESET"));
+            },
+          }),
+          { status: 200 },
+        ),
+    );
+    const decision = await cache.allows("https://x.test/wiki/Event");
+    expect(decision.allowed).toBe(false);
+    expect(decision.reason).toContain("unreadable");
+  });
+
   test("fails closed when robots.txt is unreachable", async () => {
     const { cache } = cacheWith(() => {
       throw new Error("ECONNREFUSED");

test/snapshots.test.ts

@@ -1,10 +1,14 @@
 import { afterEach, beforeEach, describe, expect, test } from "bun:test";
-import { mkdtemp, rm, writeFile } from "node:fs/promises";
+import { statSync } from "node:fs";
+import { mkdtemp, readdir, rm, writeFile } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 import {
+  charsetFromContentType,
+  decodeBody,
   freshnessAt,
   hashBody,
+  sniffMetaCharset,
   SnapshotStore,
   type SnapshotMeta,
 } from "../src/ingest/snapshots.ts";
@@ -25,10 +29,20 @@ afterEach(async () => {
   await rm(root, { recursive: true, force: true });
 });
 
-function save(html: string, at: string, extra: Partial<{ etag: string | null; lastModified: string | null; eventCount: number | null }> = {}) {
+function save(
+  body: Uint8Array | string,
+  at: string,
+  extra: Partial<{
+    etag: string | null;
+    lastModified: string | null;
+    eventCount: number | null;
+    charset: string;
+  }> = {},
+) {
   return store.save("genshin-game8-events", {
     url: "https://game8.co/games/Genshin-Impact/archives/301601",
-    html,
+    body,
+    ...(extra.charset === undefined ? {} : { charset: extra.charset }),
     etag: extra.etag ?? null,
     lastModified: extra.lastModified ?? null,
     at,
@@ -60,10 +74,10 @@ describe("SnapshotStore", () => {
     expect(await store.list()).toEqual(["genshin-game8-events"]);
   });
 
-  test("identical bytes are not a change and do not rewrite metadata", async () => {
+  test("identical bytes are not a change", async () => {
     await save("<html>one</html>", T0, { etag: '"v1"', eventCount: 9 });
     const again = await save("<html>one</html>", T1, {
-      etag: '"v2"',
+      etag: '"v1"',
       eventCount: 9,
     });
 
@@ -71,7 +85,39 @@ describe("SnapshotStore", () => {
     // contentChangedAt still points at the fetch that produced these bytes,
     // which is what keeps an unchanged cycle out of the commit log.
     expect(again.meta.contentChangedAt).toBe(T0);
-    expect(again.meta.etag).toBe('"v1"');
+  });
+
+  test("identical bytes and identical validators rewrite nothing at all", async () => {
+    await save("<html>one</html>", T0, { etag: '"v1"', eventCount: 9 });
+    const before = await Bun.file(store.metaPath("genshin-game8-events")).text();
+    await save("<html>one</html>", T1, { etag: '"v1"', eventCount: 9 });
+    expect(await Bun.file(store.metaPath("genshin-game8-events")).text()).toBe(
+      before,
+    );
+  });
+
+  test("a rotated ETag is stored even though the bytes did not change", async () => {
+    // Servers rotate validators on identical bytes. Keeping the old one meant
+    // sending a stale If-None-Match forever, so the wiki would serve the whole
+    // page every cycle — the exact cost conditional requests exist to avoid.
+    await save("<html>one</html>", T0, { etag: '"v1"', eventCount: 9 });
+    const again = await save("<html>one</html>", T1, {
+      etag: '"v2"',
+      lastModified: "Sat, 15 Aug 2026 12:00:00 GMT",
+      eventCount: 9,
+    });
+
+    expect(again.changed).toBe(false);
+    expect(again.meta.contentChangedAt).toBe(T0);
+    expect(again.meta.etag).toBe('"v2"');
+
+    const persisted = await store.readMeta("genshin-game8-events");
+    expect(persisted?.etag).toBe('"v2"');
+    expect(persisted?.contentChangedAt).toBe(T0);
+    expect(store.conditionalHeaders(persisted)).toEqual({
+      "If-None-Match": '"v2"',
+      "If-Modified-Since": "Sat, 15 Aug 2026 12:00:00 GMT",
+    });
   });
 
   test("different bytes are a change", async () => {
@@ -99,6 +145,181 @@ describe("SnapshotStore", () => {
   });
 });
 
+describe("charset", () => {
+  // "イベント" as Shift_JIS, and "Café" as Latin-1: both are mojibake if the
+  // bytes are read as UTF-8.
+  const SJIS = new Uint8Array([0x83, 0x43, 0x83, 0x78, 0x83, 0x93, 0x83, 0x67]);
+  const LATIN1 = new Uint8Array([0x43, 0x61, 0x66, 0xe9]);
+
+  test("reads the charset out of a Content-Type header", () => {
+    expect(charsetFromContentType('text/html; charset="Shift_JIS"')).toBe(
+      "shift_jis",
+    );
+    expect(charsetFromContentType("text/html;charset=iso-8859-1")).toBe(
+      "iso-8859-1",
+    );
+    expect(charsetFromContentType("text/html")).toBeNull();
+    expect(charsetFromContentType(null)).toBeNull();
+  });
+
+  test("falls back to the meta charset in the document head", () => {
+    const html = new TextEncoder().encode(
+      '<!DOCTYPE html><html><head><meta charset="shift_jis"><title>x</title></head>',
+    );
+    expect(sniffMetaCharset(html)).toBe("shift_jis");
+    expect(
+      sniffMetaCharset(
+        new TextEncoder().encode(
+          '<meta http-equiv="Content-Type" content="text/html; charset=EUC-JP">',
+        ),
+      ),
+    ).toBe("euc-jp");
+    expect(sniffMetaCharset(new TextEncoder().encode("<html><body>"))).toBeNull();
+  });
+
+  test("decodes with the declared charset, not with UTF-8", () => {
+    expect(decodeBody(SJIS, "text/html; charset=shift_jis").text).toBe(
+      "イベント",
+    );
+    expect(decodeBody(LATIN1, "text/html; charset=iso-8859-1").text).toBe("Café");
+    expect(decodeBody(new TextEncoder().encode("ok"), null).text).toBe("ok");
+  });
+
+  test("an encoding label we do not know falls back to UTF-8 rather than throwing", () => {
+    const decoded = decodeBody(
+      new TextEncoder().encode("ok"),
+      "text/html; charset=x-nonesuch",
+    );
+    expect(decoded.text).toBe("ok");
+    expect(decoded.charset).toBe("utf-8");
+  });
+
+  test("stores the served bytes verbatim and reads them back as text", async () => {
+    const { meta } = await save(SJIS, T0, { charset: "shift_jis", eventCount: 2 });
+
+    // The bytes on disk are the bytes the server sent; the hash and the size
+    // describe those bytes, so a later re-decode is still possible.
+    const onDisk = new Uint8Array(
+      await Bun.file(store.bodyPath("genshin-game8-events")).arrayBuffer(),
+    );
+    expect([...onDisk]).toEqual([...SJIS]);
+    expect(meta.bytes).toBe(SJIS.byteLength);
+    expect(meta.contentHash).toBe(hashBody(SJIS));
+
+    const snapshot = await store.read("genshin-game8-events");
+    expect(snapshot?.html).toBe("イベント");
+    expect(snapshot?.html).not.toContain("�");
+  });
+
+  test("metadata written before charsets were recorded reads as UTF-8", async () => {
+    await save("<html>é</html>", T0);
+    const meta = await store.readMeta("genshin-game8-events");
+    const { charset: _dropped, ...legacy } = meta as SnapshotMeta;
+    await writeFile(
+      store.metaPath("genshin-game8-events"),
+      `${JSON.stringify(legacy, null, 2)}\n`,
+    );
+    expect((await store.read("genshin-game8-events"))?.html).toBe("<html>é</html>");
+  });
+});
+
+describe("writes land whole or not at all", () => {
+  test("leaves no temp files behind", async () => {
+    await save("<html>one</html>", T0);
+    await store.recordCheck("genshin-game8-events", {
+      at: T0,
+      status: 200,
+      ok: true,
+    });
+    const names = await readdir(root);
+    expect(names.sort()).toEqual([
+      "genshin-game8-events.html",
+      "genshin-game8-events.meta.json",
+      "genshin-game8-events.state.json",
+    ]);
+  });
+
+  test("a temp file left by a crashed run is not mistaken for a snapshot", async () => {
+    await writeFile(
+      join(root, "genshin-game8-events.meta.json.tmp-abc"),
+      "{ half",
+    );
+    expect(await store.list()).toEqual([]);
+    await save("<html>one</html>", T0);
+    expect(await store.list()).toEqual(["genshin-game8-events"]);
+  });
+
+  test("replaces each file rather than overwriting it in place", async () => {
+    // Written in place, a snapshot exists in a truncated state for as long as
+    // the write takes, and a crash there leaves it that way. Renaming a
+    // complete temp file over it cannot: the replacement is one atomic step,
+    // which shows up as a new inode.
+    await save("<html>one</html>", T0, { etag: '"v1"' });
+    const before = {
+      body: statSync(store.bodyPath("genshin-game8-events")).ino,
+      meta: statSync(store.metaPath("genshin-game8-events")).ino,
+    };
+
+    await save("<html>two</html>", T1, { etag: '"v2"' });
+    expect(statSync(store.bodyPath("genshin-game8-events")).ino).not.toBe(
+      before.body,
+    );
+    expect(statSync(store.metaPath("genshin-game8-events")).ino).not.toBe(
+      before.meta,
+    );
+
+    // Including the metadata-only rewrite that a rotated validator triggers.
+    const metaIno = statSync(store.metaPath("genshin-game8-events")).ino;
+    await save("<html>two</html>", T1, { etag: '"v3"' });
+    expect(statSync(store.metaPath("genshin-game8-events")).ino).not.toBe(
+      metaIno,
+    );
+
+    // And the state file, which is written on every single cycle.
+    await store.recordCheck("genshin-game8-events", {
+      at: T0,
+      status: 200,
+      ok: true,
+    });
+    const stateIno = statSync(store.statePath("genshin-game8-events")).ino;
+    await store.recordCheck("genshin-game8-events", {
+      at: T1,
+      status: 200,
+      ok: true,
+    });
+    expect(statSync(store.statePath("genshin-game8-events")).ino).not.toBe(
+      stateIno,
+    );
+  });
+
+  test("a reader never sees a half-written body", async () => {
+    const first = `<html>${"a".repeat(1_000_000)}</html>`;
+    const second = `<html>${"b".repeat(1_000_000)}</html>`;
+    await save(first, T0);
+
+    let stop = false;
+    const seen = new Set<string>();
+    const reader = (async () => {
+      while (!stop) {
+        const text = await Bun.file(store.bodyPath("genshin-game8-events")).text();
+        seen.add(`${text.length}:${text.slice(-8)}`);
+        await Bun.sleep(0);
+      }
+    })();
+
+    await save(second, T1);
+    stop = true;
+    await reader;
+
+    for (const observed of seen) {
+      expect([
+        `${first.length}:${first.slice(-8)}`,
+        `${second.length}:${second.slice(-8)}`,
+      ]).toContain(observed);
+    }
+  });
+});
+
 describe("conditional requests", () => {
   test("emits both validators when both are known", async () => {
     const { meta } = await save("<html>one</html>", T0, {