diff --git a/scripts/refresh-sources.ts b/scripts/refresh-sources.ts index ca7201d..03e9080 100644 --- a/scripts/refresh-sources.ts +++ b/scripts/refresh-sources.ts @@ -29,7 +29,7 @@ import { import { SIX_HOURS_MS } from "../src/ingest/adapters/types.ts"; import type { Adapter } from "../src/ingest/adapters/types.ts"; import { RobotsCache, type FetchLike } from "../src/ingest/robots.ts"; -import { SnapshotStore } from "../src/ingest/snapshots.ts"; +import { decodeBody, SnapshotStore } from "../src/ingest/snapshots.ts"; const DEFAULT_CONTACT = "https://github.com/StereotypicalCat/gacha-event-tracker"; @@ -113,7 +113,23 @@ export async function runRefresh( } for (const adapter of selected) { - const outcome = await refreshOne(adapter, options); + // One source can never take the cycle down with it. Everything inside + // refreshOne that can fail is handled there; this is the backstop that + // keeps an unforeseen throw from costing every source after this one its + // turn — the sources are independent, and a run that stops halfway leaves + // no summary and no record of what was already asked. + let outcome: SourceOutcome; + try { + outcome = await refreshOne(adapter, options); + } catch (error) { + outcome = { + sourceId: adapter.id, + result: "failed", + note: `unexpected error: ${String(error)}`, + status: null, + eventCount: null, + }; + } summary.outcomes.push(outcome); if (outcome.result === "fetched") { @@ -256,7 +272,37 @@ async function refreshOne( }; } - const html = await response.text(); + // Reading the body is a second chance to fail — a reset connection, a + // truncated response, or the timeout firing mid-stream. Left outside the try + // this rejection escapes refreshOne, aborts the whole cycle, and leaves the + // sources after this one unfetched and this one's `lastCheckedAt` unwritten: + // one bad body would both blank the run and lose the record that we had + // already spent this source's request. + let bytes: Uint8Array; + try { + bytes = new Uint8Array(await response.arrayBuffer()); + } catch (error) { + await store.recordCheck(adapter.id, { + at: nowIso, + status: response.status, + ok: false, + }); + return { + sourceId: adapter.id, + result: "failed", + note: `body unreadable: ${String(error)}`, + status: response.status, + eventCount: meta?.eventCount ?? null, + }; + } + + // Decode with the charset the server declared. Storing the raw bytes keeps + // the snapshot re-decodable; decoding before parsing keeps mojibake out of + // titles, and therefore out of the event IDs that are localStorage keys. + const { text: html, charset } = decodeBody( + bytes, + response.headers.get("Content-Type"), + ); // The parse gate. A body that no longer parses, or that yields nothing where // it used to yield events, is a source that changed shape — publishing it @@ -310,7 +356,8 @@ async function refreshOne( const saved = await store.save(adapter.id, { url: adapter.url, - html, + body: bytes, + charset, etag: response.headers.get("ETag"), lastModified: response.headers.get("Last-Modified"), at: nowIso, @@ -377,6 +424,17 @@ export function parseArgs(argv: readonly string[]): Args { help: false, }; + // A flag whose value is missing is a mistake, never a default. `--only` with + // nothing after it used to mean "every source", which is the opposite of + // what the operator typed and one request per source more than they wanted. + const value = (i: number, flag: string): string => { + const next = argv[i]; + if (next === undefined || next.startsWith("-")) { + throw new Error(`${flag} requires a value`); + } + return next; + }; + for (let i = 0; i < argv.length; i += 1) { const arg = argv[i]; switch (arg) { @@ -385,15 +443,15 @@ export function parseArgs(argv: readonly string[]): Args { break; case "--only": i += 1; - args.only = argv[i] ?? null; + args.only = value(i, "--only"); break; case "--snapshots": i += 1; - args.root = argv[i] ?? args.root; + args.root = value(i, "--snapshots"); break; case "--user-agent": i += 1; - args.userAgent = argv[i] ?? args.userAgent; + args.userAgent = value(i, "--user-agent"); break; case "--no-feed": args.rebuild = false; diff --git a/snapshots/README.md b/snapshots/README.md index 3f9b9f3..f0dbea4 100644 --- a/snapshots/README.md +++ b/snapshots/README.md @@ -4,10 +4,21 @@ Raw pages, exactly as fetched. `scripts/refresh-sources.ts` writes them; nothing ``` .html the body verbatim — tracked -.meta.json hash, size, ETag, Last-Modified, when the bytes last changed — tracked +.meta.json hash, size, charset, ETag, Last-Modified, when the bytes last changed — tracked .state.json when we last checked, and failure streak — gitignored ``` +"Verbatim" means the bytes as served, not text we re-encoded. `charset` in the metadata records +what those bytes are in — the `Content-Type` header, else a `` in the page, else +UTF-8 — and `bytes` is the served length. A page in Shift_JIS or Latin-1 decoded as UTF-8 would be +stored as a field of U+FFFD with the original bytes gone; re-parsing could never recover it, and the +mojibake would reach `slugify`, moving every event ID for that source (CLAUDE.md § Event IDs are +localStorage keys). + +Every file is written to a sibling `.tmp-*` and renamed into place, body before metadata, so an +interrupted run leaves a stray temp file rather than a truncated snapshot or metadata describing +bytes that were never stored. + Three reasons this is committed rather than cached: - **Re-parsing never re-fetches.** Iterating on a parser reads these files, not the wikis @@ -19,7 +30,15 @@ Three reasons this is committed rather than cached: The `.state.json` files are the exception: they change every cycle whether or not a page did, and committing them would mean a commit per run saying nothing happened. CI keeps them in the actions -cache instead. +cache instead: `refresh.yml` saves that cache, and `ci.yml` restores it read-only before building +the feed. Both halves matter — `lastConfirmedAt` lives only there, and without the restore the feed +falls back to `contentChangedAt` and the UI calls every source stale two days after its bytes last +moved. + +The metadata is rewritten on an unchanged page in one case: the server rotating an `ETag` or +`Last-Modified` while serving the same bytes. Keeping the old validator would mean sending a stale +`If-None-Match` forever and being served the whole page every cycle, so that diff is worth the +commit — `contentChangedAt` and the body stay put, so it is still visibly not a content change. Fixtures are not the same thing. A fixture is pinned to a date and kept forever as the regression test for a page shape; a snapshot is the current page and is overwritten each time it changes. diff --git a/src/ingest/robots.ts b/src/ingest/robots.ts index 0e7cd71..dd0ae2e 100644 --- a/src/ingest/robots.ts +++ b/src/ingest/robots.ts @@ -137,6 +137,26 @@ export function agentToken(userAgent: string): string { return first.toLowerCase(); } +/** + * Does a `User-agent:` value in robots.txt name us? + * + * RFC 9309 § 2.2.1 matches the *product token* — the header up to the first + * `/` — not the header text. Matching anywhere in the header is actively + * dangerous here: our contact URL contains the string `StereotypicalCat`, so a + * `User-agent: cat` group elsewhere in the file would be treated as naming us, + * and because a named group replaces the `*` group outright, that unrelated + * group's rules would *discard* every rule the site actually wrote for us. + * Erring towards obeying more rules means never letting a coincidence take a + * `*` group away. + * + * A robots.txt that names us with a version (`gacha-event-tracker/1.0`) is + * still honoured: the group's own product token is compared too. + */ +function agentNames(agent: string, token: string): boolean { + if (agent === "*") return false; + return agent === token || agentToken(agent) === token; +} + /** * The group that applies to a user agent, with every group naming the same * agent merged, as RFC 9309 requires. @@ -150,19 +170,11 @@ export function groupFor( userAgent: string, ): RobotsGroup | null { const token = agentToken(userAgent); - const full = userAgent.toLowerCase(); let bestName: string | null = null; for (const group of robots.groups) { for (const agent of group.agents) { - if (agent === "*") continue; - // Match on the product token first (the spec's rule); fall back to a - // substring of the whole header so a group naming "gptbot" still binds a - // header of "Mozilla/5.0 (compatible; GPTBot/1.2)". Erring towards - // matching means erring towards obeying more rules, not fewer. - const hit = - token === agent || token.startsWith(agent) || full.includes(agent); - if (!hit) continue; + if (!agentNames(agent, token)) continue; if (bestName === null || agent.length > bestName.length) bestName = agent; } } @@ -280,8 +292,10 @@ const DAY_MS = 24 * 60 * 60 * 1000; * One robots.txt fetch per host per run (cached 24h), reused by every source on * that host — six Game8 adapters must not mean six robots requests. * - * Fails closed. A 5xx, a timeout or a network error means we do not know what - * the site permits, and "unknown" is not permission. + * Fails closed. A 5xx, a timeout, a network error, a body that dies mid-read + * or a body that is plainly not robots.txt all mean we do not know what the + * site permits, and "unknown" is not permission. Only two answers open the + * host: a parsed robots.txt, and a 404/410 saying there is none. */ export class RobotsCache { private readonly entries = new Map(); @@ -368,7 +382,50 @@ export class RobotsCache { }; } - const text = await response.text(); + // Reading the body is a second chance to fail: the connection can reset or + // the timeout can fire mid-stream, long after the headers arrived. Outside + // the try that rejection would escape as an exception rather than as "we + // could not read robots.txt", which is the one answer this class exists to + // give. + let text: string; + try { + text = await response.text(); + } catch (error) { + return { + robots: ALLOW_ALL, + usable: false, + reason: `robots.txt body unreadable (${String(error)})`, + at, + }; + } + + // A soft 404 — an HTML "not found" page served with status 200 — is the + // commonest robots.txt misconfiguration there is, and it parses to zero + // groups, which is indistinguishable from "everything is permitted". We do + // not know what the site allows, and unknown is not permission. + if (looksLikeHtml(text)) { + return { + robots: ALLOW_ALL, + usable: false, + reason: "robots.txt returned HTML, not a robots.txt (soft 404?)", + at, + }; + } + return { robots: parseRobots(text), usable: true, reason: "robots.txt ok", at }; } } + +/** + * Is this body markup rather than robots.txt? + * + * An empty body is *valid* robots.txt meaning "no restrictions", so emptiness + * is deliberately not a failure. Only markup is — no robots.txt directive can + * begin with `<`. + */ +export function looksLikeHtml(text: string): boolean { + const head = text.trimStart().slice(0, 512).toLowerCase(); + if (head === "") return false; + if (head.startsWith("<")) return true; + return /]|]|]/.test(head); +} diff --git a/src/ingest/snapshots.ts b/src/ingest/snapshots.ts index 2b209a2..d2a4252 100644 --- a/src/ingest/snapshots.ts +++ b/src/ingest/snapshots.ts @@ -18,14 +18,15 @@ * the metadata, every cycle would produce a commit that says nothing, and * "commit only when something changed" would be unenforceable. */ -import { mkdir, readdir, rm, writeFile } from "node:fs/promises"; +import { mkdir, readdir, rename, rm, writeFile } from "node:fs/promises"; import { join } from "node:path"; export interface SnapshotMeta { sourceId: string; url: string; - /** sha256 of the body, hex. The parse stage skips work when it is unchanged. */ + /** sha256 of the served bytes, hex. The parse stage skips work when it is unchanged. */ contentHash: string; + /** The served length, in bytes — not the length of the decoded text. */ bytes: number; etag: string | null; lastModified: string | null; @@ -33,6 +34,11 @@ export interface SnapshotMeta { contentChangedAt: string; /** Events the adapter yielded from this body, for drop detection. */ eventCount: number | null; + /** + * The encoding the stored bytes are in, as the server declared it. Absent in + * metadata written before charsets were handled, which is read as UTF-8. + */ + charset?: string; } export interface SnapshotState { @@ -53,7 +59,15 @@ export interface Snapshot { export interface SaveInput { url: string; - html: string; + /** + * The body as served. Bytes are the honest unit: a page in Shift_JIS or + * Latin-1 that we stored as re-encoded text could never be re-decoded, and + * `snapshots/README.md` promises "the body verbatim". A string is accepted + * as a convenience and is stored as UTF-8. + */ + body: Uint8Array | string; + /** The encoding `body` is in. Defaults to UTF-8. */ + charset?: string; etag: string | null; lastModified: string | null; /** ISO timestamp of this fetch. */ @@ -66,8 +80,101 @@ export interface SaveResult { meta: SnapshotMeta; } -export function hashBody(html: string): string { - return new Bun.CryptoHasher("sha256").update(html).digest("hex"); +export function hashBody(body: Uint8Array | string): string { + return new Bun.CryptoHasher("sha256").update(body).digest("hex"); +} + +const DEFAULT_CHARSET = "utf-8"; + +/** The charset a `Content-Type` header declares, lowercased, or null. */ +export function charsetFromContentType(contentType: string | null): string | null { + if (contentType === null) return null; + const match = /;\s*charset\s*=\s*"?([^;"\s]+)"?/i.exec(contentType); + return match?.[1]?.toLowerCase() ?? null; +} + +/** The charset a `` tag declares in the first bytes of a document. */ +export function sniffMetaCharset(bytes: Uint8Array): string | null { + // Every encoding we might meet here is ASCII-compatible in its first 2 KiB, + // so reading the head as Latin-1 cannot lose the declaration. + const head = Buffer.from( + bytes.buffer, + bytes.byteOffset, + Math.min(bytes.byteLength, 2048), + ).toString("latin1"); + const match = /]*?charset\s*=\s*["']?\s*([a-z0-9_\-:.]+)/i.exec(head); + return match?.[1]?.toLowerCase() ?? null; +} + +export interface DecodedBody { + text: string; + /** The charset actually used, which is what gets recorded in the metadata. */ + charset: string; +} + +/** + * Decode a fetched body into text. + * + * `Response.text()` assumes UTF-8, so a page served in Shift_JIS or Latin-1 + * comes back as a field of U+FFFD. That is not merely ugly: mojibake in a title + * flows through `slugify` into the event ID, which is a localStorage key, so a + * mis-decoded fetch silently orphans every completion mark for that source + * (CLAUDE.md § Event IDs are localStorage keys). + * + * Header first, then a `` sniff, then UTF-8. An encoding label + * the runtime does not know falls back to UTF-8 rather than throwing — the raw + * bytes are kept either way, so a wrong guess stays recoverable. + */ +export function decodeBody( + bytes: Uint8Array, + contentType: string | null, +): DecodedBody { + const declared = + charsetFromContentType(contentType) ?? sniffMetaCharset(bytes) ?? DEFAULT_CHARSET; + try { + const decoder = new TextDecoder(declared); + return { text: decoder.decode(bytes), charset: decoder.encoding }; + } catch { + return { + text: new TextDecoder(DEFAULT_CHARSET).decode(bytes), + charset: DEFAULT_CHARSET, + }; + } +} + +function toBytes(body: Uint8Array | string): Uint8Array { + return typeof body === "string" ? new TextEncoder().encode(body) : body; +} + +function decodeStored(bytes: Uint8Array, charset: string | undefined): string { + try { + return new TextDecoder(charset ?? DEFAULT_CHARSET).decode(bytes); + } catch { + return new TextDecoder(DEFAULT_CHARSET).decode(bytes); + } +} + +/** + * Write a file by writing a sibling temp file and renaming it into place. + * + * `rename` within a directory is atomic, so a reader — the feed build, git, + * the next refresh — sees either the whole old file or the whole new one, and + * a crash mid-write leaves a stray temp file rather than a truncated snapshot. + */ +async function writeAtomic(path: string, data: Uint8Array | string): Promise { + const temp = `${path}.tmp-${process.pid.toString(36)}-${Math.random().toString(36).slice(2, 10)}`; + try { + await writeFile(temp, data); + await rename(temp, path); + } catch (error) { + await rm(temp, { force: true }); + throw error; + } +} + +/** Would writing this metadata change the file on disk? */ +function sameMeta(a: SnapshotMeta, b: SnapshotMeta): boolean { + return JSON.stringify(a) === JSON.stringify(b); } function emptyState(sourceId: string): SnapshotState { @@ -125,7 +232,14 @@ export class SnapshotStore { const body = Bun.file(this.bodyPath(sourceId)); if (!(await body.exists())) return null; - return { meta, state: await this.readState(sourceId), html: await body.text() }; + // Decoded with the charset the bytes were stored in, so a Shift_JIS page + // reads back as the text the wiki published rather than as mojibake. + const bytes = new Uint8Array(await body.arrayBuffer()); + return { + meta, + state: await this.readState(sourceId), + html: decodeStored(bytes, meta.charset), + }; } /** Sources with a stored snapshot, by id. */ @@ -181,35 +295,54 @@ export class SnapshotStore { /** * Store a fetched body. * - * Identical bytes are a no-op on disk: the metadata keeps its original - * `contentChangedAt` and validators, so an unchanged source produces no diff - * for the workflow to commit. + * Identical bytes are a no-op on the body and keep the original + * `contentChangedAt`, so an unchanged source does not look like a changed + * one. The validators are the exception: a server is free to rotate an ETag + * while serving the very same bytes, and keeping the old one would mean + * sending a stale `If-None-Match` forever — every cycle costing the wiki a + * full body where a 304 was the whole point (CLAUDE.md § Scraping conduct). + * So the metadata is refreshed, and `changed` stays false. */ async save(sourceId: string, input: SaveInput): Promise { - const contentHash = hashBody(input.html); + const bytes = toBytes(input.body); + const charset = input.charset ?? DEFAULT_CHARSET; + const contentHash = hashBody(bytes); const previous = await this.readMeta(sourceId); const bodyExists = await Bun.file(this.bodyPath(sourceId)).exists(); const changed = previous === null || previous.contentHash !== contentHash || !bodyExists; - if (!changed && previous !== null) { - return { changed: false, meta: previous }; - } - const meta: SnapshotMeta = { sourceId, url: input.url, contentHash, - bytes: Buffer.byteLength(input.html), + bytes: bytes.byteLength, etag: input.etag, lastModified: input.lastModified, - contentChangedAt: input.at, - eventCount: input.eventCount, + contentChangedAt: changed ? input.at : (previous?.contentChangedAt ?? input.at), + // New bytes mean the count they yielded, even when that is unknown; the + // same bytes keep the count we already recorded for them. + eventCount: changed + ? input.eventCount + : (input.eventCount ?? previous?.eventCount ?? null), + charset, }; await mkdir(this.root, { recursive: true }); - await writeFile(this.bodyPath(sourceId), input.html); - await writeFile(this.metaPath(sourceId), `${JSON.stringify(meta, null, 2)}\n`); + + if (!changed && previous !== null) { + if (sameMeta(previous, meta)) return { changed: false, meta: previous }; + await writeAtomic(this.metaPath(sourceId), `${JSON.stringify(meta, null, 2)}\n`); + return { changed: false, meta }; + } + + // Body first, metadata second, each renamed into place. A crash between + // them leaves new bytes beside older metadata, whose stale hash makes the + // next save rewrite both — the other order would leave metadata claiming a + // hash for bytes that were never written, and the next save would believe + // it and skip them. + await writeAtomic(this.bodyPath(sourceId), bytes); + await writeAtomic(this.metaPath(sourceId), `${JSON.stringify(meta, null, 2)}\n`); return { changed: true, meta }; } @@ -229,7 +362,7 @@ export class SnapshotStore { }; await mkdir(this.root, { recursive: true }); - await writeFile(this.statePath(sourceId), `${JSON.stringify(state, null, 2)}\n`); + await writeAtomic(this.statePath(sourceId), `${JSON.stringify(state, null, 2)}\n`); return state; } diff --git a/test/refresh.test.ts b/test/refresh.test.ts index 3e0e86c..d2b42d2 100644 --- a/test/refresh.test.ts +++ b/test/refresh.test.ts @@ -99,7 +99,7 @@ function options( async function seed(html: string, at: string, eventCount: number | null) { await store.save("genshin-game8-events", { url: "https://game8.co/games/Genshin-Impact/archives/301601", - html, + body: html, etag: 'W/"v1"', lastModified: "Fri, 14 Aug 2026 09:00:00 GMT", at, @@ -370,6 +370,79 @@ describe("a source being down never blanks the feed", () => { expect(summary.outcomes[0]?.note).toContain("down from 10"); }); + test("a body that dies mid-read is one source's failure, not the cycle's", async () => { + // The headers arrive, then the connection resets. Read outside the try, + // that rejection escaped refreshOne and took the whole run with it: the + // sources after this one were never asked, no summary was printed, and + // this source's lastCheckedAt was never written — so the six-hour floor + // did not register a request we had already spent. + const { opts, calls } = options({ + adapters: [ + adapter(), + adapter({ + id: "nte-game8-events", + game: "nte", + url: "https://game8.co/games/Neverness-to-Everness/archives/592073", + }), + ], + responder: (call) => { + if (!call.url.includes("Genshin")) { + return new Response(""); + } + return new Response( + new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode("")); + controller.error(new Error("ECONNRESET")); + }, + }), + { status: 200 }, + ); + }, + }); + + const summary = await runRefresh(opts); + + expect(calls).toHaveLength(2); + expect(summary.outcomes[0]?.result).toBe("failed"); + expect(summary.outcomes[0]?.note).toContain("body unreadable"); + expect(summary.outcomes[1]?.result).toBe("fetched"); + expect(summary.hardFailure).toBeNull(); + + // The request was spent, so it must be on the record. + const state = await store.readState("genshin-game8-events"); + expect(state.lastCheckedAt).toBe(NOW.toISOString()); + expect(state.consecutiveFailures).toBe(1); + }); + + test("an unforeseen error in one source does not abort the others", async () => { + // The robots gate itself blowing up is not something refreshOne guards; + // the loop's backstop is what keeps the remaining sources alive. + const { opts } = options({ + adapters: [ + adapter(), + adapter({ + id: "nte-game8-events", + game: "nte", + url: "https://game8.co/games/Neverness-to-Everness/archives/592073", + }), + ], + robots: { + allows: async (url) => { + if (url.includes("Genshin")) throw new Error("robots cache exploded"); + return { allowed: true, reason: "ok" }; + }, + }, + }); + const summary = await runRefresh(opts); + + expect(summary.outcomes).toHaveLength(2); + expect(summary.outcomes[0]?.result).toBe("failed"); + expect(summary.outcomes[0]?.note).toContain("unexpected error"); + expect(summary.outcomes[1]?.result).toBe("fetched"); + expect(summary.hardFailure).toBeNull(); + }); + test("a feed that will not rebuild fails the run", async () => { const { opts } = options({ rebuildFeed: async () => { @@ -381,6 +454,121 @@ describe("a source being down never blanks the feed", () => { }); }); +describe("a page that is not UTF-8", () => { + // "イベント" with the title in Shift_JIS. + const SJIS_TITLE = [0x83, 0x43, 0x83, 0x78, 0x83, 0x93, 0x83, 0x67]; + const body = new Uint8Array([ + ...new TextEncoder().encode(""), + ...SJIS_TITLE, + ...new TextEncoder().encode(""), + ]); + + test("is decoded with the charset the server declared", async () => { + const { opts } = options({ + responder: () => + new Response(body, { + status: 200, + headers: { "Content-Type": "text/html; charset=shift_jis" }, + }), + }); + const summary = await runRefresh(opts); + expect(summary.outcomes[0]?.result).toBe("fetched"); + + const snapshot = await store.read("genshin-game8-events"); + // Read as UTF-8 this is U+FFFD soup, and mojibake in a title flows through + // slugify into the event ID, which is a localStorage key. + expect(snapshot?.html).toBe("イベント"); + expect(snapshot?.html).not.toContain("�"); + }); + + test("stores the bytes as served, so a re-decode is still possible", async () => { + const { opts } = options({ + responder: () => + new Response(body, { + status: 200, + headers: { "Content-Type": "text/html; charset=shift_jis" }, + }), + }); + await runRefresh(opts); + + const onDisk = new Uint8Array( + await Bun.file(store.bodyPath("genshin-game8-events")).arrayBuffer(), + ); + expect([...onDisk]).toEqual([...body]); + // `bytes` is the served length, which is not the length of the decoded text. + expect((await store.readMeta("genshin-game8-events"))?.bytes).toBe( + body.byteLength, + ); + }); + + test("falls back to the document's own meta charset", async () => { + const withMeta = new Uint8Array([ + ...new TextEncoder().encode(''), + ...SJIS_TITLE, + ...new TextEncoder().encode(""), + ]); + const { opts } = options({ + responder: () => + new Response(withMeta, { + status: 200, + headers: { "Content-Type": "text/html" }, + }), + }); + await runRefresh(opts); + expect((await store.read("genshin-game8-events"))?.html).toContain( + "イベント", + ); + }); +}); + +describe("the workflows that drive the refresh", () => { + // These three defects live in YAML, and each one is silent: nothing fails, + // the site just quietly carries wrong or stale data. Asserting on the file + // is the only offline way to keep them fixed. + const read = (name: string) => + Bun.file(new URL(`../.github/workflows/${name}`, import.meta.url)).text(); + + test("ci.yml restores the refresh bookkeeping before it builds the feed", async () => { + // lastConfirmedAt lives only in the gitignored snapshots/*.state.json. If + // the job that builds the deployed feed never restores that cache, + // freshnessAt falls back to contentChangedAt and the UI calls a source + // stale two days after its bytes last moved — which for a wiki page is + // most of the time. + const ci = await read("ci.yml"); + const restores = ci.split("actions/cache/restore@").length - 1; + expect(restores).toBeGreaterThanOrEqual(2); // the check job and the build job + expect(ci).toContain("snapshots/*.state.json"); + // Restore only: refresh.yml owns writing it. + expect(ci).not.toContain("actions/cache/save@"); + + const buildJob = ci.slice(ci.indexOf(" build:"), ci.indexOf(" image:")); + expect(buildJob).toContain("actions/cache/restore@"); + }); + + test("refresh.yml saves its cache under a key that changes per attempt", async () => { + // github.run_id is stable across re-runs, so a re-run's save is skipped and + // the next run restores bookkeeping from before it — losing the record of + // requests we did make. + const refresh = await read("refresh.yml"); + const saveKey = /key: (refresh-state-[^\n]*)\n/g; + const keys = [...refresh.matchAll(saveKey)].map((m) => m[1] ?? ""); + const savedKey = keys.find((k) => k.includes("run_id")); + expect(savedKey).toBeDefined(); + expect(savedKey).toContain("github.run_attempt"); + }); + + test("refresh.yml survives losing a push race without ever forcing", async () => { + // A human push landing mid-job made the push non-fast-forward: the fetched + // pages were thrown away while the bookkeeping had already spent their + // six-hour budget. + const refresh = await read("refresh.yml"); + expect(refresh).toContain("git rebase"); + expect(refresh).toMatch(/for attempt in/); + expect(refresh).not.toContain("--force"); + expect(refresh).not.toContain("-f origin"); + }); +}); + describe("flags", () => { test("--dry-run makes no requests and writes nothing", async () => { const { opts, calls, rebuilds } = options({ dryRun: true }); @@ -436,4 +624,17 @@ describe("flags", () => { test("parseArgs rejects an unknown flag rather than ignoring it", () => { expect(() => parseArgs(["--force"])).toThrow("unknown flag"); }); + + test("parseArgs rejects a flag whose value is missing", () => { + // `--only` with nothing after it used to mean "every source": the operator + // asked for one request and would have got seven. + expect(() => parseArgs(["--only"])).toThrow("--only requires a value"); + expect(() => parseArgs(["--only", "--dry-run"])).toThrow( + "--only requires a value", + ); + expect(() => parseArgs(["--snapshots"])).toThrow("--snapshots requires a value"); + expect(() => parseArgs(["--user-agent"])).toThrow( + "--user-agent requires a value", + ); + }); }); diff --git a/test/robots.test.ts b/test/robots.test.ts index 1038b89..af6b419 100644 --- a/test/robots.test.ts +++ b/test/robots.test.ts @@ -77,6 +77,48 @@ Disallow: /admin expect(isAllowed(robots, UA, "/games/x")).toBe(true); }); + test("an unrelated named group cannot steal the wildcard group", () => { + // Our contact URL contains "StereotypicalCat". Matching a group name + // anywhere in the header made `User-agent: cat` look like our group, and + // because a named group replaces `*`, that coincidence *discarded* the + // rules the site actually wrote for everyone. + const ua = + "gacha-event-tracker/1.0 (+https://github.com/StereotypicalCat/gacha-event-tracker)"; + const robots = parseRobots(` +User-agent: * +Disallow: /games/ + +User-agent: cat +Disallow: /litter +`); + expect(groupFor(robots, ua)?.agents).toEqual(["*"]); + expect(isAllowed(robots, ua, "/games/x")).toBe(false); + expect(isAllowed(robots, ua, "/litter")).toBe(true); + }); + + test("a substring of the product token does not name us either", () => { + const robots = parseRobots(` +User-agent: * +Disallow: /games/ + +User-agent: gacha +Allow: / +`); + expect(isAllowed(robots, UA, "/games/x")).toBe(false); + }); + + test("a group naming us with a version still binds", () => { + const robots = parseRobots(` +User-agent: * +Disallow: / + +User-agent: gacha-event-tracker/1.0 +Disallow: /admin +`); + expect(isAllowed(robots, UA, "/games/x")).toBe(true); + expect(isAllowed(robots, UA, "/admin/panel")).toBe(false); + }); + test("the longest matching agent name wins", () => { const robots = parseRobots(` User-agent: googlebot @@ -227,6 +269,47 @@ describe("RobotsCache", () => { expect(decision.reason).toContain("503"); }); + test("an empty 200 body is a valid robots.txt that restricts nothing", async () => { + const { cache } = cacheWith(() => new Response(" \n", { status: 200 })); + const decision = await cache.allows("https://x.test/wiki/Event"); + expect(decision.allowed).toBe(true); + expect(decision.reason).toBe("robots.txt ok"); + }); + + test("fails closed on a soft 404 — an HTML page served as 200", async () => { + // The commonest robots.txt misconfiguration there is. It parses to zero + // groups, which is indistinguishable from "nothing is restricted", so + // reading it as permission is exactly the fail-open this class forbids. + const { cache } = cacheWith( + () => + new Response( + "\n404 Not FoundNot found", + { status: 200, headers: { "Content-Type": "text/html" } }, + ), + ); + const decision = await cache.allows("https://x.test/wiki/Event"); + expect(decision.allowed).toBe(false); + expect(decision.reason).toContain("HTML"); + }); + + test("fails closed when the body dies mid-read", async () => { + const { cache } = cacheWith( + () => + new Response( + new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode("User-agent: *\n")); + controller.error(new Error("ECONNRESET")); + }, + }), + { status: 200 }, + ), + ); + const decision = await cache.allows("https://x.test/wiki/Event"); + expect(decision.allowed).toBe(false); + expect(decision.reason).toContain("unreadable"); + }); + test("fails closed when robots.txt is unreachable", async () => { const { cache } = cacheWith(() => { throw new Error("ECONNREFUSED"); diff --git a/test/snapshots.test.ts b/test/snapshots.test.ts index 453896f..c8e8d8d 100644 --- a/test/snapshots.test.ts +++ b/test/snapshots.test.ts @@ -1,10 +1,14 @@ import { afterEach, beforeEach, describe, expect, test } from "bun:test"; -import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { statSync } from "node:fs"; +import { mkdtemp, readdir, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { + charsetFromContentType, + decodeBody, freshnessAt, hashBody, + sniffMetaCharset, SnapshotStore, type SnapshotMeta, } from "../src/ingest/snapshots.ts"; @@ -25,10 +29,20 @@ afterEach(async () => { await rm(root, { recursive: true, force: true }); }); -function save(html: string, at: string, extra: Partial<{ etag: string | null; lastModified: string | null; eventCount: number | null }> = {}) { +function save( + body: Uint8Array | string, + at: string, + extra: Partial<{ + etag: string | null; + lastModified: string | null; + eventCount: number | null; + charset: string; + }> = {}, +) { return store.save("genshin-game8-events", { url: "https://game8.co/games/Genshin-Impact/archives/301601", - html, + body, + ...(extra.charset === undefined ? {} : { charset: extra.charset }), etag: extra.etag ?? null, lastModified: extra.lastModified ?? null, at, @@ -60,10 +74,10 @@ describe("SnapshotStore", () => { expect(await store.list()).toEqual(["genshin-game8-events"]); }); - test("identical bytes are not a change and do not rewrite metadata", async () => { + test("identical bytes are not a change", async () => { await save("one", T0, { etag: '"v1"', eventCount: 9 }); const again = await save("one", T1, { - etag: '"v2"', + etag: '"v1"', eventCount: 9, }); @@ -71,7 +85,39 @@ describe("SnapshotStore", () => { // contentChangedAt still points at the fetch that produced these bytes, // which is what keeps an unchanged cycle out of the commit log. expect(again.meta.contentChangedAt).toBe(T0); - expect(again.meta.etag).toBe('"v1"'); + }); + + test("identical bytes and identical validators rewrite nothing at all", async () => { + await save("one", T0, { etag: '"v1"', eventCount: 9 }); + const before = await Bun.file(store.metaPath("genshin-game8-events")).text(); + await save("one", T1, { etag: '"v1"', eventCount: 9 }); + expect(await Bun.file(store.metaPath("genshin-game8-events")).text()).toBe( + before, + ); + }); + + test("a rotated ETag is stored even though the bytes did not change", async () => { + // Servers rotate validators on identical bytes. Keeping the old one meant + // sending a stale If-None-Match forever, so the wiki would serve the whole + // page every cycle — the exact cost conditional requests exist to avoid. + await save("one", T0, { etag: '"v1"', eventCount: 9 }); + const again = await save("one", T1, { + etag: '"v2"', + lastModified: "Sat, 15 Aug 2026 12:00:00 GMT", + eventCount: 9, + }); + + expect(again.changed).toBe(false); + expect(again.meta.contentChangedAt).toBe(T0); + expect(again.meta.etag).toBe('"v2"'); + + const persisted = await store.readMeta("genshin-game8-events"); + expect(persisted?.etag).toBe('"v2"'); + expect(persisted?.contentChangedAt).toBe(T0); + expect(store.conditionalHeaders(persisted)).toEqual({ + "If-None-Match": '"v2"', + "If-Modified-Since": "Sat, 15 Aug 2026 12:00:00 GMT", + }); }); test("different bytes are a change", async () => { @@ -99,6 +145,181 @@ describe("SnapshotStore", () => { }); }); +describe("charset", () => { + // "イベント" as Shift_JIS, and "Café" as Latin-1: both are mojibake if the + // bytes are read as UTF-8. + const SJIS = new Uint8Array([0x83, 0x43, 0x83, 0x78, 0x83, 0x93, 0x83, 0x67]); + const LATIN1 = new Uint8Array([0x43, 0x61, 0x66, 0xe9]); + + test("reads the charset out of a Content-Type header", () => { + expect(charsetFromContentType('text/html; charset="Shift_JIS"')).toBe( + "shift_jis", + ); + expect(charsetFromContentType("text/html;charset=iso-8859-1")).toBe( + "iso-8859-1", + ); + expect(charsetFromContentType("text/html")).toBeNull(); + expect(charsetFromContentType(null)).toBeNull(); + }); + + test("falls back to the meta charset in the document head", () => { + const html = new TextEncoder().encode( + 'x', + ); + expect(sniffMetaCharset(html)).toBe("shift_jis"); + expect( + sniffMetaCharset( + new TextEncoder().encode( + '', + ), + ), + ).toBe("euc-jp"); + expect(sniffMetaCharset(new TextEncoder().encode(""))).toBeNull(); + }); + + test("decodes with the declared charset, not with UTF-8", () => { + expect(decodeBody(SJIS, "text/html; charset=shift_jis").text).toBe( + "イベント", + ); + expect(decodeBody(LATIN1, "text/html; charset=iso-8859-1").text).toBe("Café"); + expect(decodeBody(new TextEncoder().encode("ok"), null).text).toBe("ok"); + }); + + test("an encoding label we do not know falls back to UTF-8 rather than throwing", () => { + const decoded = decodeBody( + new TextEncoder().encode("ok"), + "text/html; charset=x-nonesuch", + ); + expect(decoded.text).toBe("ok"); + expect(decoded.charset).toBe("utf-8"); + }); + + test("stores the served bytes verbatim and reads them back as text", async () => { + const { meta } = await save(SJIS, T0, { charset: "shift_jis", eventCount: 2 }); + + // The bytes on disk are the bytes the server sent; the hash and the size + // describe those bytes, so a later re-decode is still possible. + const onDisk = new Uint8Array( + await Bun.file(store.bodyPath("genshin-game8-events")).arrayBuffer(), + ); + expect([...onDisk]).toEqual([...SJIS]); + expect(meta.bytes).toBe(SJIS.byteLength); + expect(meta.contentHash).toBe(hashBody(SJIS)); + + const snapshot = await store.read("genshin-game8-events"); + expect(snapshot?.html).toBe("イベント"); + expect(snapshot?.html).not.toContain("�"); + }); + + test("metadata written before charsets were recorded reads as UTF-8", async () => { + await save("é", T0); + const meta = await store.readMeta("genshin-game8-events"); + const { charset: _dropped, ...legacy } = meta as SnapshotMeta; + await writeFile( + store.metaPath("genshin-game8-events"), + `${JSON.stringify(legacy, null, 2)}\n`, + ); + expect((await store.read("genshin-game8-events"))?.html).toBe("é"); + }); +}); + +describe("writes land whole or not at all", () => { + test("leaves no temp files behind", async () => { + await save("one", T0); + await store.recordCheck("genshin-game8-events", { + at: T0, + status: 200, + ok: true, + }); + const names = await readdir(root); + expect(names.sort()).toEqual([ + "genshin-game8-events.html", + "genshin-game8-events.meta.json", + "genshin-game8-events.state.json", + ]); + }); + + test("a temp file left by a crashed run is not mistaken for a snapshot", async () => { + await writeFile( + join(root, "genshin-game8-events.meta.json.tmp-abc"), + "{ half", + ); + expect(await store.list()).toEqual([]); + await save("one", T0); + expect(await store.list()).toEqual(["genshin-game8-events"]); + }); + + test("replaces each file rather than overwriting it in place", async () => { + // Written in place, a snapshot exists in a truncated state for as long as + // the write takes, and a crash there leaves it that way. Renaming a + // complete temp file over it cannot: the replacement is one atomic step, + // which shows up as a new inode. + await save("one", T0, { etag: '"v1"' }); + const before = { + body: statSync(store.bodyPath("genshin-game8-events")).ino, + meta: statSync(store.metaPath("genshin-game8-events")).ino, + }; + + await save("two", T1, { etag: '"v2"' }); + expect(statSync(store.bodyPath("genshin-game8-events")).ino).not.toBe( + before.body, + ); + expect(statSync(store.metaPath("genshin-game8-events")).ino).not.toBe( + before.meta, + ); + + // Including the metadata-only rewrite that a rotated validator triggers. + const metaIno = statSync(store.metaPath("genshin-game8-events")).ino; + await save("two", T1, { etag: '"v3"' }); + expect(statSync(store.metaPath("genshin-game8-events")).ino).not.toBe( + metaIno, + ); + + // And the state file, which is written on every single cycle. + await store.recordCheck("genshin-game8-events", { + at: T0, + status: 200, + ok: true, + }); + const stateIno = statSync(store.statePath("genshin-game8-events")).ino; + await store.recordCheck("genshin-game8-events", { + at: T1, + status: 200, + ok: true, + }); + expect(statSync(store.statePath("genshin-game8-events")).ino).not.toBe( + stateIno, + ); + }); + + test("a reader never sees a half-written body", async () => { + const first = `${"a".repeat(1_000_000)}`; + const second = `${"b".repeat(1_000_000)}`; + await save(first, T0); + + let stop = false; + const seen = new Set(); + const reader = (async () => { + while (!stop) { + const text = await Bun.file(store.bodyPath("genshin-game8-events")).text(); + seen.add(`${text.length}:${text.slice(-8)}`); + await Bun.sleep(0); + } + })(); + + await save(second, T1); + stop = true; + await reader; + + for (const observed of seen) { + expect([ + `${first.length}:${first.slice(-8)}`, + `${second.length}:${second.slice(-8)}`, + ]).toContain(observed); + } + }); +}); + describe("conditional requests", () => { test("emits both validators when both are known", async () => { const { meta } = await save("one", T0, {