Lucas WintherandClaude Opus 5
5b8109b556
Full `uses:` URLs as in ci.yml, the fork gate repointed at the Gitea repo, and a commit identity that is not GitHub's bot. `gh workflow run` becomes the REST dispatch the CLI would have made, because there is no `gh` on a Gitea runner. Worth stating plainly: Gitea does not suppress workflow triggers on a push made with the Actions token, which is the whole reason this step existed on GitHub, so the snapshot commit will usually have started ci.yml by itself. The step stays anyway — it is the only guarantee of publication if that behaviour is ever configured off, and a duplicate run costs nothing because ci.yml's concurrency group is per-ref with cancel-in-progress. The crawler's fallback contact URL follows the repo. It is what wiki operators see in our User-Agent when REFRESH_CONTACT_URL is unset, so leaving it pointing at a repo we no longer publish from would make the contact half of AGENTS.md § Scraping conduct a dead end. Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
1001 lines
37 KiB
TypeScript
1001 lines
37 KiB
TypeScript
/**
|
|
* Refresh every source, then regenerate the feed.
|
|
*
|
|
* bun run refresh # the real thing
|
|
* bun run refresh --dry-run # plan only, no requests, no writes
|
|
* bun run refresh --only genshin-game8-events
|
|
* bun run refresh --assume-robots-on-403 # see § the flag, below
|
|
* bun run refresh --force --only nikki-fandom-events # ignore the 6h floor
|
|
*
|
|
* This is the scheduled half of the pipeline (docs/INGESTION.md stages 1-2).
|
|
* The rules it enforces are etiquette obligations, not preferences:
|
|
*
|
|
* - robots.txt is read once per host per run and obeyed; unreadable means
|
|
* "do not fetch", never "assume yes". `--assume-robots-on-403` is the one
|
|
* opt-in exception, for a host that will not serve us the file at all while
|
|
* serving us the page: it stands in for rules a person read in a browser
|
|
* and wrote into AGENTS.md. It covers 403 only, never overrides a
|
|
* robots.txt we could read, is refused under CI, and names every host it
|
|
* touched. Nothing else about being a guest relaxes with it.
|
|
* - at most ONE request per source per cycle, and never sooner than six hours
|
|
* after the last attempt. There is deliberately no retry: a retry is a
|
|
* second request, and the next cycle is minutes-cheap compared to being a
|
|
* bad guest. `--force` is the one way past the six hours, interactive-only
|
|
* and refused under CI, and it sets aside the interval and nothing else —
|
|
* conditional headers, per-host spacing, robots and the no-retry rule all
|
|
* still apply, and every source asked early is named in the summary.
|
|
* - requests to one host are spaced, honouring its `Crawl-delay`. Eight of the
|
|
* twelve sources are game8.co pages, so without this one cycle is eight
|
|
* back-to-back requests to a single site — inside the per-source floor and
|
|
* still the behaviour an edge network throttles.
|
|
* - conditional requests always, so an unchanged page costs the wiki a 304.
|
|
* - a descriptive User-Agent carrying a contact URL.
|
|
*
|
|
* Failure policy has two tiers, because they want opposite things from CI:
|
|
*
|
|
* - One wiki being down is a warning and exit 0. The previous snapshot stays
|
|
* in place and the feed keeps its events — a source outage must never blank
|
|
* the calendar, and the sources that did answer must still be committed.
|
|
* - A source failing `BROKEN_AFTER_FAILURES` cycles in a row is not "down",
|
|
* it is broken, and the feed has been quietly serving a stale fixture for a
|
|
* day and a half. That is reported as `broken` and annotated, and the
|
|
* workflow turns the run red *after* committing what did work. Exiting
|
|
* non-zero here instead would skip the commit and throw the good pages away.
|
|
* - Hard failures (bad arguments, every source failing, a feed that will not
|
|
* rebuild) exit non-zero so CI stops before committing anything.
|
|
*/
|
|
import { appendFile } from "node:fs/promises";
|
|
import {
|
|
ADAPTERS,
|
|
adapterById,
|
|
} from "../src/ingest/adapters/index.ts";
|
|
import { SIX_HOURS_MS } from "../src/ingest/adapters/types.ts";
|
|
import type { Adapter } from "../src/ingest/adapters/types.ts";
|
|
import { RobotsCache, type FetchLike } from "../src/ingest/robots.ts";
|
|
import { decodeBody, SnapshotStore } from "../src/ingest/snapshots.ts";
|
|
|
|
const DEFAULT_CONTACT =
|
|
"https://gitea.lucaswinther.info/lucasw89/gacha-event-tracker";
|
|
|
|
export const DEFAULT_USER_AGENT = `gacha-event-tracker/1.0 (+${process.env["REFRESH_CONTACT_URL"] ?? DEFAULT_CONTACT})`;
|
|
|
|
/** How a single source's cycle ended. */
|
|
export type RefreshResult =
|
|
| "fetched" // 200 with new bytes, parsed, stored
|
|
| "unchanged" // 304, or 200 whose bytes matched what we had
|
|
| "skipped_interval" // fetched too recently to ask again
|
|
| "skipped_robots" // robots.txt says no, or could not be read
|
|
| "rejected" // fetched, but the body parsed worse than what we hold
|
|
| "failed" // unreachable or an error status
|
|
| "planned"; // --dry-run
|
|
|
|
export interface SourceOutcome {
|
|
sourceId: string;
|
|
result: RefreshResult;
|
|
note: string;
|
|
status: number | null;
|
|
eventCount: number | null;
|
|
}
|
|
|
|
/**
|
|
* A source that has stopped answering for long enough that the feed is now
|
|
* knowingly stale for that game.
|
|
*/
|
|
export interface BrokenSource {
|
|
sourceId: string;
|
|
consecutiveFailures: number;
|
|
lastStatus: number | null;
|
|
lastConfirmedAt: string | null;
|
|
}
|
|
|
|
export interface RefreshSummary {
|
|
outcomes: SourceOutcome[];
|
|
/** Sources whose stored bytes changed — the only reason to commit. */
|
|
changed: number;
|
|
/** Sources we actually sent a request to. */
|
|
attempted: number;
|
|
/** Sources that answered (200 or 304). */
|
|
confirmed: number;
|
|
warnings: string[];
|
|
/**
|
|
* Sources failing for `BROKEN_AFTER_FAILURES` cycles running. Not a hard
|
|
* failure — the workflow reports it after the commit, so a broken source
|
|
* cannot cost a working one its snapshot.
|
|
*/
|
|
broken: BrokenSource[];
|
|
/** Set when the run should exit non-zero. */
|
|
hardFailure: string | null;
|
|
/**
|
|
* Hosts fetched under `--assume-robots-on-403`. Empty on every normal run,
|
|
* and on every CI run — the flag is refused there.
|
|
*/
|
|
assumedRobots: string[];
|
|
/**
|
|
* Sources asked before their interval was up, under `--force`. Empty on every
|
|
* normal run, and on every CI run — the flag is refused there.
|
|
*/
|
|
forced: string[];
|
|
}
|
|
|
|
export interface RobotsGate {
|
|
allows(url: string): Promise<{
|
|
allowed: boolean;
|
|
reason: string;
|
|
/** From the host's `Crawl-delay`, when it states one. */
|
|
crawlDelayMs?: number | null;
|
|
/** True when `--assume-robots-on-403` is what opened this host. */
|
|
assumedOnForbidden?: boolean;
|
|
}>;
|
|
}
|
|
|
|
export interface RefreshOptions {
|
|
adapters: readonly Adapter[];
|
|
store: SnapshotStore;
|
|
robots: RobotsGate;
|
|
fetchImpl: FetchLike;
|
|
userAgent: string;
|
|
/** Injected clock — the runner is testable, like the parsers it drives. */
|
|
now: () => Date;
|
|
/**
|
|
* Injected timer, for the same reason as the clock: the per-host gap is real
|
|
* seconds on a runner and must cost a test nothing.
|
|
*/
|
|
sleep: (ms: number) => Promise<void>;
|
|
dryRun: boolean;
|
|
only: string | null;
|
|
/**
|
|
* Ignore the per-source interval floor for this run. Interactive only — see
|
|
* `--force` in USAGE, and AGENTS.md § Scraping conduct, which the flag amends
|
|
* rather than quietly contradicts.
|
|
*/
|
|
force: boolean;
|
|
timeoutMs: number;
|
|
log: (line: string) => void;
|
|
/** Called once when something changed. Null skips the rebuild (tests). */
|
|
rebuildFeed: (() => Promise<void>) | null;
|
|
}
|
|
|
|
/** A drop this steep means the page changed shape, not that events ended. */
|
|
const DROP_WARNING_RATIO = 0.5;
|
|
|
|
/**
|
|
* Cycles of failure that separate "the wiki is down" from "this source is
|
|
* broken". At two cycles a day, three is a day and a half of a game's calendar
|
|
* silently coming from a checked-in fixture — long enough to be certain, short
|
|
* enough to still be worth hearing about.
|
|
*/
|
|
export const BROKEN_AFTER_FAILURES = 3;
|
|
|
|
/**
|
|
* Gap between two requests to the same host when its robots.txt names none.
|
|
* The per-source floor is six hours, but eight sources share game8.co, so
|
|
* without this they arrive as one burst.
|
|
*/
|
|
export const DEFAULT_HOST_GAP_MS = 2_000;
|
|
|
|
/** Per-cycle state shared across sources: which hosts we have already asked. */
|
|
interface Cycle {
|
|
requestedHosts: Set<string>;
|
|
/** Hosts fetched on `--assume-robots-on-403` rather than on a file we read. */
|
|
assumedRobots: Set<string>;
|
|
/** Sources asked before their interval was up, under `--force`. */
|
|
forced: Set<string>;
|
|
}
|
|
|
|
export async function runRefresh(
|
|
options: RefreshOptions,
|
|
): Promise<RefreshSummary> {
|
|
const summary: RefreshSummary = {
|
|
outcomes: [],
|
|
changed: 0,
|
|
attempted: 0,
|
|
confirmed: 0,
|
|
warnings: [],
|
|
broken: [],
|
|
hardFailure: null,
|
|
assumedRobots: [],
|
|
forced: [],
|
|
};
|
|
|
|
const cycle: Cycle = {
|
|
requestedHosts: new Set(),
|
|
assumedRobots: new Set(),
|
|
forced: new Set(),
|
|
};
|
|
|
|
const selected =
|
|
options.only === null
|
|
? [...options.adapters]
|
|
: options.adapters.filter((a) => a.id === options.only);
|
|
|
|
if (selected.length === 0) {
|
|
summary.hardFailure = `unknown source '${options.only ?? ""}'`;
|
|
return summary;
|
|
}
|
|
|
|
for (const adapter of selected) {
|
|
// One source can never take the cycle down with it. Everything inside
|
|
// refreshOne that can fail is handled there; this is the backstop that
|
|
// keeps an unforeseen throw from costing every source after this one its
|
|
// turn — the sources are independent, and a run that stops halfway leaves
|
|
// no summary and no record of what was already asked.
|
|
let outcome: SourceOutcome;
|
|
try {
|
|
outcome = await refreshOne(adapter, options, cycle);
|
|
} catch (error) {
|
|
outcome = {
|
|
sourceId: adapter.id,
|
|
result: "failed",
|
|
note: `unexpected error: ${String(error)}`,
|
|
status: null,
|
|
eventCount: null,
|
|
};
|
|
}
|
|
summary.outcomes.push(outcome);
|
|
|
|
if (outcome.result === "fetched") {
|
|
summary.changed += 1;
|
|
summary.attempted += 1;
|
|
summary.confirmed += 1;
|
|
} else if (outcome.result === "unchanged") {
|
|
summary.attempted += 1;
|
|
summary.confirmed += 1;
|
|
} else if (outcome.result === "failed" || outcome.result === "rejected") {
|
|
summary.attempted += 1;
|
|
summary.warnings.push(`${adapter.id}: ${outcome.note}`);
|
|
} else if (outcome.result === "skipped_robots") {
|
|
summary.warnings.push(`${adapter.id}: ${outcome.note}`);
|
|
}
|
|
|
|
options.log(
|
|
` ${adapter.id.padEnd(24)} ${outcome.result.padEnd(17)} ${outcome.note}`,
|
|
);
|
|
|
|
// Read from the store rather than from this cycle's outcome: the streak is
|
|
// the point, and a source that has failed for days and is now skipped for
|
|
// the interval is still broken. A successful fetch resets it to zero, so
|
|
// this reports a standing condition, not one bad afternoon.
|
|
if (!options.dryRun) {
|
|
const health = await options.store.readState(adapter.id);
|
|
if (health.consecutiveFailures >= BROKEN_AFTER_FAILURES) {
|
|
summary.broken.push({
|
|
sourceId: adapter.id,
|
|
consecutiveFailures: health.consecutiveFailures,
|
|
lastStatus: health.lastStatus,
|
|
lastConfirmedAt: health.lastConfirmedAt,
|
|
});
|
|
}
|
|
}
|
|
}
|
|
|
|
// Named in the summary rather than only in the per-source log, so it survives
|
|
// into the job summary and cannot be scrolled past.
|
|
summary.forced = [...cycle.forced].sort();
|
|
if (summary.forced.length > 0) {
|
|
summary.warnings.push(
|
|
`--force: asked ${summary.forced.length} source(s) before their interval ` +
|
|
`was up (${summary.forced.join(", ")})`,
|
|
);
|
|
}
|
|
|
|
summary.assumedRobots = [...cycle.assumedRobots].sort();
|
|
for (const host of summary.assumedRobots) {
|
|
summary.warnings.push(
|
|
`${host}: fetched on --assume-robots-on-403 — its robots.txt was NOT read ` +
|
|
`this run. Re-read it in a browser and confirm AGENTS.md § Scraping ` +
|
|
`conduct still describes it.`,
|
|
);
|
|
}
|
|
|
|
// Every source failing is not "a wiki is down", it is us: no network, a bad
|
|
// User-Agent, a proxy. That should stop the pipeline rather than look green.
|
|
if (summary.attempted > 0 && summary.confirmed === 0) {
|
|
summary.hardFailure = `all ${summary.attempted} attempted sources failed`;
|
|
return summary;
|
|
}
|
|
|
|
// Likewise, being turned away everywhere is news. Left as a warning it would
|
|
// read as a quiet, successful, permanently empty refresh.
|
|
if (summary.outcomes.every((o) => o.result === "skipped_robots")) {
|
|
summary.hardFailure = `robots.txt blocked all ${summary.outcomes.length} sources`;
|
|
return summary;
|
|
}
|
|
|
|
if (summary.changed > 0 && options.rebuildFeed !== null) {
|
|
try {
|
|
await options.rebuildFeed();
|
|
} catch (error) {
|
|
// New snapshots are on disk but do not produce a feed. Exiting non-zero
|
|
// keeps CI from committing them.
|
|
summary.hardFailure = `feed rebuild failed: ${String(error)}`;
|
|
}
|
|
}
|
|
|
|
return summary;
|
|
}
|
|
|
|
async function refreshOne(
|
|
adapter: Adapter,
|
|
options: RefreshOptions,
|
|
cycle: Cycle,
|
|
): Promise<SourceOutcome> {
|
|
const { store } = options;
|
|
const now = options.now();
|
|
const nowIso = now.toISOString();
|
|
const meta = await store.readMeta(adapter.id);
|
|
const state = await store.readState(adapter.id);
|
|
const headers = store.conditionalHeaders(meta);
|
|
|
|
const due = store.isDue(state, now.getTime(), adapter.minIntervalMs);
|
|
if (!due && !options.force) {
|
|
const dueAt = new Date(store.dueAt(state, adapter.minIntervalMs));
|
|
return {
|
|
sourceId: adapter.id,
|
|
result: "skipped_interval",
|
|
note: `checked ${state.lastCheckedAt ?? "?"}, next due ${dueAt.toISOString()}`,
|
|
status: null,
|
|
eventCount: meta?.eventCount ?? null,
|
|
};
|
|
}
|
|
// Asking early is the one obligation `--force` sets aside, and only for a
|
|
// source that would otherwise have been skipped — a run that was due anyway
|
|
// is an ordinary run and must not be reported as forced.
|
|
if (!due) cycle.forced.add(adapter.id);
|
|
|
|
if (options.dryRun) {
|
|
const conditional = Object.keys(headers);
|
|
return {
|
|
sourceId: adapter.id,
|
|
result: "planned",
|
|
note: `would GET ${adapter.url}${
|
|
conditional.length > 0 ? ` with ${conditional.join(", ")}` : " (no validators cached)"
|
|
}`,
|
|
status: null,
|
|
eventCount: meta?.eventCount ?? null,
|
|
};
|
|
}
|
|
|
|
const decision = await options.robots.allows(adapter.url);
|
|
if (!decision.allowed) {
|
|
return {
|
|
sourceId: adapter.id,
|
|
result: "skipped_robots",
|
|
note: decision.reason,
|
|
status: null,
|
|
eventCount: meta?.eventCount ?? null,
|
|
};
|
|
}
|
|
// Fetching on a permission nobody can re-read is a thing the run has to say
|
|
// out loud, every time and per source. An override that reports nothing is an
|
|
// override that quietly becomes the default.
|
|
if (decision.assumedOnForbidden === true) {
|
|
cycle.assumedRobots.add(new URL(adapter.url).host);
|
|
}
|
|
|
|
// Space requests to a host we have already asked this cycle. This sits after
|
|
// the interval and robots gates on purpose: waiting on behalf of a source we
|
|
// then skip would buy the host nothing and cost the run a minute.
|
|
const host = new URL(adapter.url).host;
|
|
if (cycle.requestedHosts.has(host)) {
|
|
await options.sleep(decision.crawlDelayMs ?? DEFAULT_HOST_GAP_MS);
|
|
}
|
|
cycle.requestedHosts.add(host);
|
|
|
|
let response: Response;
|
|
try {
|
|
response = await options.fetchImpl(adapter.url, {
|
|
headers: {
|
|
"User-Agent": options.userAgent,
|
|
Accept: "text/html,application/xhtml+xml",
|
|
...headers,
|
|
},
|
|
signal: AbortSignal.timeout(options.timeoutMs),
|
|
redirect: "follow",
|
|
});
|
|
} catch (error) {
|
|
await store.recordCheck(adapter.id, { at: nowIso, status: null, ok: false });
|
|
return {
|
|
sourceId: adapter.id,
|
|
result: "failed",
|
|
note: `unreachable: ${String(error)}`,
|
|
status: null,
|
|
eventCount: meta?.eventCount ?? null,
|
|
};
|
|
}
|
|
|
|
if (response.status === 304) {
|
|
await store.recordCheck(adapter.id, { at: nowIso, status: 304, ok: true });
|
|
return {
|
|
sourceId: adapter.id,
|
|
result: "unchanged",
|
|
note: "304 not modified",
|
|
status: 304,
|
|
eventCount: meta?.eventCount ?? null,
|
|
};
|
|
}
|
|
|
|
// 200 is the only status that means "here is the page". `response.ok` also
|
|
// admits the rest of the 2xx range, and that cost us the diagnosis: game8.co's
|
|
// edge answers a GitHub runner with **202 Accepted** and a bot-management
|
|
// body, which sailed through this gate as a document, reached the parser,
|
|
// yielded no events and was reported as "kept previous snapshot" — describing
|
|
// the symptom while the status that explained it went unmentioned. 202 means
|
|
// the request was accepted for processing; 204 has no body and 206 is a
|
|
// fragment. None of them is a wiki page.
|
|
if (response.status !== 200) {
|
|
await store.recordCheck(adapter.id, {
|
|
at: nowIso,
|
|
status: response.status,
|
|
ok: false,
|
|
});
|
|
return {
|
|
sourceId: adapter.id,
|
|
result: "failed",
|
|
note: `HTTP ${response.status}${describeRejection(response)}`,
|
|
status: response.status,
|
|
eventCount: meta?.eventCount ?? null,
|
|
};
|
|
}
|
|
|
|
// Reading the body is a second chance to fail — a reset connection, a
|
|
// truncated response, or the timeout firing mid-stream. Left outside the try
|
|
// this rejection escapes refreshOne, aborts the whole cycle, and leaves the
|
|
// sources after this one unfetched and this one's `lastCheckedAt` unwritten:
|
|
// one bad body would both blank the run and lose the record that we had
|
|
// already spent this source's request.
|
|
let bytes: Uint8Array;
|
|
try {
|
|
bytes = new Uint8Array(await response.arrayBuffer());
|
|
} catch (error) {
|
|
await store.recordCheck(adapter.id, {
|
|
at: nowIso,
|
|
status: response.status,
|
|
ok: false,
|
|
});
|
|
return {
|
|
sourceId: adapter.id,
|
|
result: "failed",
|
|
note: `body unreadable: ${String(error)}`,
|
|
status: response.status,
|
|
eventCount: meta?.eventCount ?? null,
|
|
};
|
|
}
|
|
|
|
// Decode with the charset the server declared. Storing the raw bytes keeps
|
|
// the snapshot re-decodable; decoding before parsing keeps mojibake out of
|
|
// titles, and therefore out of the event IDs that are localStorage keys.
|
|
const { text: html, charset } = decodeBody(
|
|
bytes,
|
|
response.headers.get("Content-Type"),
|
|
);
|
|
|
|
// The parse gate. A body that no longer parses, or that yields nothing where
|
|
// it used to yield events, is a source that changed shape — publishing it
|
|
// would empty a game's calendar silently, which is the failure this pipeline
|
|
// exists to avoid. Keep what we hold and warn.
|
|
let events: number;
|
|
try {
|
|
events = adapter.parse(html, {
|
|
now: nowIso,
|
|
sourceUrl: adapter.url,
|
|
sourceId: adapter.id,
|
|
game: adapter.game,
|
|
}).length;
|
|
} catch (error) {
|
|
await store.recordCheck(adapter.id, {
|
|
at: nowIso,
|
|
status: response.status,
|
|
ok: false,
|
|
});
|
|
return {
|
|
sourceId: adapter.id,
|
|
result: "rejected",
|
|
note: `kept previous snapshot; new body did not parse: ${String(error)}`,
|
|
status: response.status,
|
|
eventCount: meta?.eventCount ?? null,
|
|
};
|
|
}
|
|
|
|
// Zero events is almost never a useful snapshot: a source in the registry
|
|
// yields events by construction, so an empty parse normally means the page
|
|
// changed shape. Refusing it keeps the previous snapshot — or, on a first
|
|
// run, the checked-in fixture — as the thing the feed is built from.
|
|
//
|
|
// The exception is a page that says so itself. A gacha calendar goes quiet
|
|
// between versions, and Infinity Nikki's wiki spent the week after 2.7 ended
|
|
// stating "There are no Events in this category" under both headings
|
|
// (2026-09-03). Read strictly, that is a source failing every cycle until the
|
|
// next version ships, reaching the `broken` tier in a day and a half and
|
|
// failing the workflow over a game that is merely between patches — while the
|
|
// held snapshot goes on ageing. So `statesNoEvents` lets a parser distinguish
|
|
// "we read the page and there is nothing on" from "we read nothing", and only
|
|
// the first is stored.
|
|
//
|
|
// The distinction has to come from the page's own words rather than a row
|
|
// count, because a redesign that broke every selector also yields zero rows —
|
|
// and storing *that* is the silently emptied calendar this gate exists for.
|
|
const previousCount = meta?.eventCount ?? null;
|
|
const statesNoEvents = events === 0 && adapter.statesNoEvents?.(html) === true;
|
|
if (events === 0 && !statesNoEvents) {
|
|
await store.recordCheck(adapter.id, {
|
|
at: nowIso,
|
|
status: response.status,
|
|
ok: false,
|
|
});
|
|
return {
|
|
sourceId: adapter.id,
|
|
result: "rejected",
|
|
note:
|
|
previousCount === null
|
|
? "did not store; body yielded 0 events"
|
|
: `kept previous snapshot; new body yielded 0 events (had ${previousCount})`,
|
|
status: response.status,
|
|
eventCount: previousCount,
|
|
};
|
|
}
|
|
|
|
const saved = await store.save(adapter.id, {
|
|
url: adapter.url,
|
|
body: bytes,
|
|
charset,
|
|
etag: response.headers.get("ETag"),
|
|
lastModified: response.headers.get("Last-Modified"),
|
|
at: nowIso,
|
|
eventCount: events,
|
|
});
|
|
await store.recordCheck(adapter.id, {
|
|
at: nowIso,
|
|
status: response.status,
|
|
ok: true,
|
|
});
|
|
|
|
if (!saved.changed) {
|
|
return {
|
|
sourceId: adapter.id,
|
|
result: "unchanged",
|
|
note: `200 but identical bytes (${events} events)`,
|
|
status: response.status,
|
|
eventCount: events,
|
|
};
|
|
}
|
|
|
|
const dropped =
|
|
!statesNoEvents &&
|
|
previousCount !== null &&
|
|
previousCount > 0 &&
|
|
events < previousCount * DROP_WARNING_RATIO;
|
|
|
|
return {
|
|
sourceId: adapter.id,
|
|
result: "fetched",
|
|
// Say which of the two empties this is. "0 events — down from 6" reads as
|
|
// the shape change it is not, and this note is what a reader checking on a
|
|
// quiet lane actually sees.
|
|
note: statesNoEvents
|
|
? "0 events — the page states it currently lists none"
|
|
: dropped
|
|
? `${events} events — down from ${previousCount}, check the page shape`
|
|
: `${events} events`,
|
|
status: response.status,
|
|
eventCount: events,
|
|
};
|
|
}
|
|
|
|
/**
|
|
* The few header words that tell a wiki being down from an edge network turning
|
|
* us away.
|
|
*
|
|
* `HTTP 403` alone cannot be acted on: it reads the same whether the page moved
|
|
* behind a login or whether a CDN has decided the runner's address is a bot
|
|
* farm. Naming the server in the note means the answer is in the run log and
|
|
* the step summary rather than in a request someone has to reproduce by hand.
|
|
*/
|
|
function describeRejection(response: Response): string {
|
|
// A header value is a string from a host we do not control, and this one ends
|
|
// up inside a `::warning::` workflow command and a markdown table cell. HTTP
|
|
// forbids a bare newline in a value, so this is belt and braces rather than a
|
|
// live hole — but a note is not worth trusting a stranger's bytes over.
|
|
const tidy = (value: string | null): string | null => {
|
|
if (value === null) return null;
|
|
const clean = value.replace(/[^\x20-\x7e]+/g, " ").trim().slice(0, 40);
|
|
return clean === "" ? null : clean;
|
|
};
|
|
|
|
const parts: string[] = [];
|
|
const server = tidy(response.headers.get("Server"));
|
|
if (server !== null) parts.push(server);
|
|
if (response.headers.get("CF-Ray") !== null) parts.push("cf-ray");
|
|
const retryAfter = tidy(response.headers.get("Retry-After"));
|
|
if (retryAfter !== null) parts.push(`retry-after ${retryAfter}`);
|
|
return parts.length === 0 ? "" : ` (${parts.join(", ")})`;
|
|
}
|
|
|
|
/**
|
|
* Workflow-command lines for a GitHub runner.
|
|
*
|
|
* Warnings printed to stdout are invisible unless someone opens the log, which
|
|
* is how six of seven sources failed every cycle for three days under a green
|
|
* tick. An annotation shows on the run page itself. `::error::` annotates
|
|
* without failing the step, which is what lets the commit still happen.
|
|
*/
|
|
export function annotations(summary: RefreshSummary): string[] {
|
|
const lines: string[] = [];
|
|
for (const b of summary.broken) {
|
|
lines.push(
|
|
`::error title=${b.sourceId} has stopped answering::` +
|
|
`${b.consecutiveFailures} cycles failing in a row; ` +
|
|
`last status ${b.lastStatus ?? "none"}; ` +
|
|
`last confirmed ${b.lastConfirmedAt ?? "never"}. ` +
|
|
`This game's calendar is being built from a checked-in fixture.`,
|
|
);
|
|
}
|
|
for (const warning of summary.warnings) {
|
|
lines.push(`::warning title=refresh::${warning}`);
|
|
}
|
|
return lines;
|
|
}
|
|
|
|
/** `$GITHUB_STEP_SUMMARY` markdown: one row per source, statuses included. */
|
|
export function stepSummary(summary: RefreshSummary): string {
|
|
const cell = (text: string) => text.replaceAll("|", "\\|");
|
|
const rows = summary.outcomes.map(
|
|
(o) =>
|
|
`| \`${o.sourceId}\` | ${o.result} | ${o.status ?? "—"} | ` +
|
|
`${o.eventCount ?? "—"} | ${cell(o.note)} |`,
|
|
);
|
|
|
|
const head =
|
|
`### Refresh: ${summary.changed} changed, ` +
|
|
`${summary.confirmed}/${summary.attempted} confirmed, ` +
|
|
`${summary.broken.length} broken\n\n` +
|
|
`| source | result | status | events | note |\n` +
|
|
`| --- | --- | --- | --- | --- |\n`;
|
|
|
|
const tail =
|
|
summary.hardFailure === null
|
|
? ""
|
|
: `\n**Hard failure:** ${summary.hardFailure}\n`;
|
|
|
|
return `${head}${rows.join("\n")}\n${tail}`;
|
|
}
|
|
|
|
/**
|
|
* `$GITHUB_OUTPUT` values the workflow branches on.
|
|
*
|
|
* `broken` is deliberately an output rather than an exit code: the workflow has
|
|
* to commit the sources that did work before it turns the run red.
|
|
*/
|
|
export function outputs(summary: RefreshSummary): string[] {
|
|
return [
|
|
`changed=${summary.changed}`,
|
|
`attempted=${summary.attempted}`,
|
|
`confirmed=${summary.confirmed}`,
|
|
`broken=${summary.broken.length}`,
|
|
];
|
|
}
|
|
|
|
/** Append to a file named by an env var, if the runner set one. */
|
|
async function appendToEnvFile(name: string, text: string): Promise<void> {
|
|
const path = process.env[name];
|
|
if (path === undefined || path === "") return;
|
|
try {
|
|
await appendFile(path, text.endsWith("\n") ? text : `${text}\n`);
|
|
} catch (error) {
|
|
// Reporting is not the job. A read-only summary file must not turn a
|
|
// successful refresh into a failed one.
|
|
console.warn(`could not write ${name}: ${String(error)}`);
|
|
}
|
|
}
|
|
|
|
/** Regenerate public/data/events.v1.json from whatever is now cached. */
|
|
export async function rebuildFeedViaScript(): Promise<void> {
|
|
const proc = Bun.spawn(["bun", "run", "scripts/build-feed.ts"], {
|
|
stdout: "inherit",
|
|
stderr: "inherit",
|
|
});
|
|
const code = await proc.exited;
|
|
if (code !== 0) throw new Error(`build-feed exited ${code}`);
|
|
}
|
|
|
|
/** Whether a person authorised this run, and who, for the override gates. */
|
|
export type Attendance =
|
|
| { readonly attended: true; readonly by: string }
|
|
| { readonly attended: false; readonly why: string };
|
|
|
|
/**
|
|
* Did a person ask for this run, or did a schedule?
|
|
*
|
|
* The two overrides below stand in for somebody deciding something — that a
|
|
* page has moved and they want it now, or that they have read a robots.txt in a
|
|
* browser. What must never happen is a *schedule* asserting either one every
|
|
* cycle, because an override on every run is not an override, it is the new
|
|
* default with extra steps (AGENTS.md § Scraping conduct).
|
|
*
|
|
* This used to be `isCi()`, which asked the coarser question and got the wrong
|
|
* answer for the case that matters: a `workflow_dispatch` sets `CI=true`, so a
|
|
* human clicking "Run workflow" was refused exactly like the cron was. A
|
|
* dispatch is a person, and a better-evidenced one than a local shell — GitHub
|
|
* records who in `GITHUB_ACTOR`, and the run log keeps it.
|
|
*
|
|
* Erring towards "unattended" stays the safe direction: any runner event that is
|
|
* not a dispatch is treated as a schedule, so a `push`- or `repository_dispatch`-
|
|
* triggered run gets no overrides either.
|
|
*/
|
|
export function runAttendance(
|
|
env: Record<string, string | undefined> = process.env,
|
|
): Attendance {
|
|
const onRunner =
|
|
(env["CI"] !== undefined && env["CI"] !== "") ||
|
|
env["GITHUB_ACTIONS"] === "true";
|
|
if (!onRunner) return { attended: true, by: "a local shell" };
|
|
|
|
const event = env["GITHUB_EVENT_NAME"];
|
|
if (event === "workflow_dispatch") {
|
|
const actor = env["GITHUB_ACTOR"];
|
|
return {
|
|
attended: true,
|
|
by:
|
|
actor === undefined || actor === ""
|
|
? "a manual dispatch"
|
|
: `a manual dispatch by ${actor}`,
|
|
};
|
|
}
|
|
|
|
return { attended: false, why: event === undefined || event === "" ? "CI" : event };
|
|
}
|
|
|
|
interface Args {
|
|
dryRun: boolean;
|
|
only: string | null;
|
|
root: string;
|
|
userAgent: string;
|
|
rebuild: boolean;
|
|
help: boolean;
|
|
/** See `--assume-robots-on-403` in USAGE, and § Scraping conduct. */
|
|
assumeRobotsOn403: boolean;
|
|
/** See `--force` in USAGE, and § Scraping conduct. */
|
|
force: boolean;
|
|
}
|
|
|
|
export function parseArgs(argv: readonly string[]): Args {
|
|
const args: Args = {
|
|
dryRun: false,
|
|
only: null,
|
|
root: process.env["SNAPSHOT_DIR"] ?? "snapshots",
|
|
userAgent: process.env["REFRESH_USER_AGENT"] ?? DEFAULT_USER_AGENT,
|
|
rebuild: true,
|
|
help: false,
|
|
assumeRobotsOn403: false,
|
|
force: false,
|
|
};
|
|
|
|
// A flag whose value is missing is a mistake, never a default. `--only` with
|
|
// nothing after it used to mean "every source", which is the opposite of
|
|
// what the operator typed and one request per source more than they wanted.
|
|
const value = (i: number, flag: string): string => {
|
|
const next = argv[i];
|
|
if (next === undefined || next.startsWith("-")) {
|
|
throw new Error(`${flag} requires a value`);
|
|
}
|
|
return next;
|
|
};
|
|
|
|
for (let i = 0; i < argv.length; i += 1) {
|
|
const arg = argv[i];
|
|
switch (arg) {
|
|
case "--dry-run":
|
|
args.dryRun = true;
|
|
break;
|
|
case "--only":
|
|
i += 1;
|
|
args.only = value(i, "--only");
|
|
break;
|
|
case "--snapshots":
|
|
i += 1;
|
|
args.root = value(i, "--snapshots");
|
|
break;
|
|
case "--user-agent":
|
|
i += 1;
|
|
args.userAgent = value(i, "--user-agent");
|
|
break;
|
|
case "--no-feed":
|
|
args.rebuild = false;
|
|
break;
|
|
case "--assume-robots-on-403":
|
|
args.assumeRobotsOn403 = true;
|
|
break;
|
|
case "--force":
|
|
args.force = true;
|
|
break;
|
|
case "--help":
|
|
case "-h":
|
|
args.help = true;
|
|
break;
|
|
default:
|
|
if (arg !== undefined && arg.startsWith("-")) {
|
|
throw new Error(`unknown flag '${arg}'`);
|
|
}
|
|
break;
|
|
}
|
|
}
|
|
|
|
return args;
|
|
}
|
|
|
|
const USAGE = `usage: bun run refresh [--dry-run] [--only <sourceId>] [--snapshots <dir>]
|
|
[--user-agent <ua>] [--no-feed] [--assume-robots-on-403]
|
|
[--force]
|
|
|
|
--dry-run report what each source would do; no requests, no writes
|
|
--only <id> refresh a single source (${ADAPTERS.map((a) => a.id).join(", ")})
|
|
--snapshots snapshot cache directory (default: snapshots, env SNAPSHOT_DIR)
|
|
--user-agent override the User-Agent (env REFRESH_USER_AGENT)
|
|
--no-feed skip regenerating public/data/events.v1.json
|
|
--assume-robots-on-403
|
|
temporary, interactive-only. When a host answers 403 to
|
|
/robots.txt itself, proceed on the permission recorded in
|
|
AGENTS.md instead of failing closed. Refused under CI.
|
|
Does NOT override a robots.txt we could read: a file that
|
|
disallows us still says no.
|
|
--force temporary, interactive-only. Ignore the 6h per-source floor
|
|
and ask now. Refused under CI. Everything else about being a
|
|
guest still holds: one request per source, per-host spacing,
|
|
conditional headers (so an unchanged page still costs a 304),
|
|
robots, no retries. Prefer it with --only.`;
|
|
|
|
async function main(): Promise<number> {
|
|
let args: Args;
|
|
try {
|
|
args = parseArgs(Bun.argv.slice(2));
|
|
} catch (error) {
|
|
console.error(String(error));
|
|
console.error(USAGE);
|
|
return 2;
|
|
}
|
|
|
|
if (args.help) {
|
|
console.log(USAGE);
|
|
return 0;
|
|
}
|
|
|
|
if (args.only !== null && adapterById(args.only) === undefined) {
|
|
console.error(`unknown source '${args.only}'`);
|
|
console.error(USAGE);
|
|
return 2;
|
|
}
|
|
|
|
// Both overrides are a person deciding something — that a page has moved and
|
|
// they want it now, or that they have read a robots.txt in a browser and
|
|
// written it down. Neither may be asserted by a *schedule*: an override on
|
|
// every cycle is not an override, it is the new default with extra steps, and
|
|
// for the robots one it would be worse than that. From an address that gets a
|
|
// challenge we never receive the file at all, so a cron standing on a recorded
|
|
// permission has no way to notice the host withdrawing it — the recorded
|
|
// permission has no expiry, and a person re-reading it is the only thing that
|
|
// ever re-validates it. AGENTS.md § Scraping conduct is the argument.
|
|
const attendance = runAttendance();
|
|
|
|
if (args.force && !attendance.attended) {
|
|
console.error(
|
|
`--force needs a person to ask for it; this run came from ` +
|
|
`${attendance.why}.\n` +
|
|
"The 6h floor is what the scheduled runner is for; change the schedule, " +
|
|
"not the floor. Dispatch the workflow by hand to pass it.",
|
|
);
|
|
return 2;
|
|
}
|
|
|
|
// `--assume-robots-on-403` is deliberately NOT gated on attendance, unlike
|
|
// `--force` above. The four Fandom sources skip on a challenged robots.txt
|
|
// every cycle, and the repository owner has taken on re-reading those files by
|
|
// hand over time rather than have four calendars go stale between manual runs
|
|
// (decided 2026-08-20). That is a standing decision about this project's own
|
|
// conduct, so the code records it instead of re-litigating it per run.
|
|
//
|
|
// What keeps it honest is the disclosure, which is why that is not optional:
|
|
// every cycle names each host it stood on a hand-recorded permission for and
|
|
// says the file was not read this run. The risk being accepted is specific —
|
|
// from a challenged address robots.txt never arrives, so a file *edited* to
|
|
// disallow us would be invisible (a plain 403 still fails closed, and the
|
|
// challenge-or-refusal split is what guarantees that much). The per-cycle
|
|
// warning is the only thing that ever prompts the re-read, so do not demote it
|
|
// to a log line.
|
|
|
|
// Who authorised the override is part of the record, not a detail. A local
|
|
// shell leaves no trace anybody else can read; a dispatch names the actor.
|
|
if (attendance.attended && (args.force || args.assumeRobotsOn403)) {
|
|
const used = [
|
|
args.force ? "--force" : null,
|
|
args.assumeRobotsOn403 ? "--assume-robots-on-403" : null,
|
|
].filter((f): f is string => f !== null);
|
|
console.log(` override: ${used.join(", ")} — authorised by ${attendance.by}`);
|
|
}
|
|
|
|
const store = new SnapshotStore(args.root);
|
|
const robots = new RobotsCache({
|
|
userAgent: args.userAgent,
|
|
fetchImpl: (input, init) => fetch(input, init),
|
|
assumeAllowedWhenForbidden: args.assumeRobotsOn403,
|
|
});
|
|
|
|
if (args.force) {
|
|
const n = args.only === null ? ADAPTERS.length : 1;
|
|
console.warn(
|
|
` ! --force: ignoring the 6h floor for ${n} source${n === 1 ? "" : "s"}. ` +
|
|
`Conditional headers still apply, so an unchanged page costs a 304.`,
|
|
);
|
|
}
|
|
|
|
if (args.assumeRobotsOn403) {
|
|
console.warn(
|
|
" ! --assume-robots-on-403: a host answering 403 to /robots.txt will be\n" +
|
|
" fetched anyway, on the permission recorded in AGENTS.md. Temporary,\n" +
|
|
" and every host it applies to is named at the end of this run.",
|
|
);
|
|
}
|
|
|
|
console.log(
|
|
`refresh: ${args.only ?? `${ADAPTERS.length} sources`}${args.dryRun ? " (dry run)" : ""}`,
|
|
);
|
|
console.log(` user-agent: ${args.userAgent}`);
|
|
console.log(` snapshots: ${args.root}`);
|
|
console.log(` interval: ${SIX_HOURS_MS / 3_600_000}h minimum per source\n`);
|
|
|
|
const summary = await runRefresh({
|
|
adapters: ADAPTERS,
|
|
store,
|
|
robots,
|
|
fetchImpl: (input, init) => fetch(input, init),
|
|
userAgent: args.userAgent,
|
|
now: () => new Date(),
|
|
sleep: (ms) => Bun.sleep(ms),
|
|
dryRun: args.dryRun,
|
|
only: args.only,
|
|
force: args.force,
|
|
timeoutMs: 20_000,
|
|
log: (line) => console.log(line),
|
|
rebuildFeed: args.dryRun || !args.rebuild ? null : rebuildFeedViaScript,
|
|
});
|
|
|
|
console.log(
|
|
`\n${summary.changed} changed, ${summary.confirmed}/${summary.attempted} confirmed, ` +
|
|
`${summary.warnings.length} warnings, ${summary.broken.length} broken`,
|
|
);
|
|
// One line per warning, and every warning is in `summary.warnings` — printing
|
|
// a category separately here once double-reported the assumed-robots hosts
|
|
// and left the "N warnings" count above disagreeing with the lines under it.
|
|
for (const warning of summary.warnings) console.warn(` ! ${warning}`);
|
|
for (const b of summary.broken) {
|
|
console.error(
|
|
` !! ${b.sourceId} has failed ${b.consecutiveFailures} cycles running ` +
|
|
`(last confirmed ${b.lastConfirmedAt ?? "never"})`,
|
|
);
|
|
}
|
|
|
|
// Only on a runner: the `!` and `!!` lines above already said all of this to a
|
|
// human, and `::warning title=…::` in a terminal is noise that reads as a bug.
|
|
if (process.env["GITHUB_ACTIONS"] === "true") {
|
|
for (const line of annotations(summary)) console.log(line);
|
|
}
|
|
await appendToEnvFile("GITHUB_STEP_SUMMARY", stepSummary(summary));
|
|
await appendToEnvFile("GITHUB_OUTPUT", outputs(summary).join("\n"));
|
|
|
|
// Kept for a human reading the log; `git status` is what the workflow trusts
|
|
// to decide whether to commit.
|
|
console.log(`changed=${summary.changed}`);
|
|
|
|
if (summary.hardFailure !== null) {
|
|
console.error(`\nrefresh failed: ${summary.hardFailure}`);
|
|
return 1;
|
|
}
|
|
|
|
// Broken sources exit 0 on purpose. The workflow reads the `broken` output and
|
|
// fails the run *after* committing, so one dead wiki cannot stop nine live
|
|
// ones from reaching the site.
|
|
return 0;
|
|
}
|
|
|
|
if (import.meta.main) {
|
|
process.exit(await main());
|
|
}
|