feat(refresh): --force, to ask before the 6h floor is up

The interval gate has no override, so a page you know has just changed cannot
be fetched until six hours after the last attempt. The only workaround was
deleting snapshots/<id>.state.json, which also wipes consecutiveFailures and
lastConfirmedAt — resetting the broken-source streak and making the footer age
every source from when its bytes last changed rather than when we last
confirmed them. A flag that says what it means is better than a side effect
nobody documented.

It sets aside the interval and nothing else. Conditional headers still go out,
which is what makes forcing defensible at all: the host is asked, not
re-served, and an unchanged page costs it a 304. Per-host spacing, robots, one
request per source and the no-retry rule all still apply — a source that was
not due and is also disallowed stays skipped, for the reason that matters.

Refused under CI, like --assume-robots-on-403 and for the same reason: a
schedule that forces every cycle is a shorter interval with extra steps, and
the interval is the obligation, not the default. So AGENTS.md § Scraping
conduct is amended rather than left to be quietly contradicted by a flag.

Every source asked early is named in summary.forced and warned about. A run
that was due anyway is never reported as forced — a summary that cried "forced"
on an ordinary run would train the reader to ignore the word.

Also repoints the "unknown flag" test, which used --force as its example and
stopped testing anything the moment --force existed.

Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>

scripts/refresh-sources.ts

@@ -5,6 +5,7 @@
  *   bun run refresh --dry-run                # plan only, no requests, no writes
  *   bun run refresh --only genshin-game8-events
  *   bun run refresh --assume-robots-on-403   # see § the flag, below
+ *   bun run refresh --force --only nikki-fandom-events   # ignore the 6h floor
  *
  * This is the scheduled half of the pipeline (docs/INGESTION.md stages 1-2).
  * The rules it enforces are etiquette obligations, not preferences:
@@ -19,7 +20,10 @@
  *   - at most ONE request per source per cycle, and never sooner than six hours
  *     after the last attempt. There is deliberately no retry: a retry is a
  *     second request, and the next cycle is minutes-cheap compared to being a
- *     bad guest.
+ *     bad guest. `--force` is the one way past the six hours, interactive-only
+ *     and refused under CI, and it sets aside the interval and nothing else —
+ *     conditional headers, per-host spacing, robots and the no-retry rule all
+ *     still apply, and every source asked early is named in the summary.
  *   - requests to one host are spaced, honouring its `Crawl-delay`. Eight of the
  *     twelve sources are game8.co pages, so without this one cycle is eight
  *     back-to-back requests to a single site — inside the per-source floor and
@@ -106,6 +110,11 @@ export interface RefreshSummary {
    * and on every CI run — the flag is refused there.
    */
   assumedRobots: string[];
+  /**
+   * Sources asked before their interval was up, under `--force`. Empty on every
+   * normal run, and on every CI run — the flag is refused there.
+   */
+  forced: string[];
 }
 
 export interface RobotsGate {
@@ -134,6 +143,12 @@ export interface RefreshOptions {
   sleep: (ms: number) => Promise<void>;
   dryRun: boolean;
   only: string | null;
+  /**
+   * Ignore the per-source interval floor for this run. Interactive only — see
+   * `--force` in USAGE, and AGENTS.md § Scraping conduct, which the flag amends
+   * rather than quietly contradicts.
+   */
+  force: boolean;
   timeoutMs: number;
   log: (line: string) => void;
   /** Called once when something changed. Null skips the rebuild (tests). */
@@ -163,6 +178,8 @@ interface Cycle {
   requestedHosts: Set<string>;
   /** Hosts fetched on `--assume-robots-on-403` rather than on a file we read. */
   assumedRobots: Set<string>;
+  /** Sources asked before their interval was up, under `--force`. */
+  forced: Set<string>;
 }
 
 export async function runRefresh(
@@ -177,9 +194,14 @@ export async function runRefresh(
     broken: [],
     hardFailure: null,
     assumedRobots: [],
+    forced: [],
   };
 
-  const cycle: Cycle = { requestedHosts: new Set(), assumedRobots: new Set() };
+  const cycle: Cycle = {
+    requestedHosts: new Set(),
+    assumedRobots: new Set(),
+    forced: new Set(),
+  };
 
   const selected =
     options.only === null
@@ -248,6 +270,14 @@ export async function runRefresh(
 
   // Named in the summary rather than only in the per-source log, so it survives
   // into the job summary and cannot be scrolled past.
+  summary.forced = [...cycle.forced].sort();
+  if (summary.forced.length > 0) {
+    summary.warnings.push(
+      `--force: asked ${summary.forced.length} source(s) before their interval ` +
+        `was up (${summary.forced.join(", ")})`,
+    );
+  }
+
   summary.assumedRobots = [...cycle.assumedRobots].sort();
   for (const host of summary.assumedRobots) {
     summary.warnings.push(
@@ -294,7 +324,8 @@ async function refreshOne(
   const state = await store.readState(adapter.id);
   const headers = store.conditionalHeaders(meta);
 
-  if (!store.isDue(state, now.getTime(), adapter.minIntervalMs)) {
+  const due = store.isDue(state, now.getTime(), adapter.minIntervalMs);
+  if (!due && !options.force) {
     const dueAt = new Date(store.dueAt(state, adapter.minIntervalMs));
     return {
       sourceId: adapter.id,
@@ -304,6 +335,10 @@ async function refreshOne(
       eventCount: meta?.eventCount ?? null,
     };
   }
+  // Asking early is the one obligation `--force` sets aside, and only for a
+  // source that would otherwise have been skipped — a run that was due anyway
+  // is an ordinary run and must not be reported as forced.
+  if (!due) cycle.forced.add(adapter.id);
 
   if (options.dryRun) {
     const conditional = Object.keys(headers);
@@ -663,6 +698,8 @@ interface Args {
   help: boolean;
   /** See `--assume-robots-on-403` in USAGE, and § Scraping conduct. */
   assumeRobotsOn403: boolean;
+  /** See `--force` in USAGE, and § Scraping conduct. */
+  force: boolean;
 }
 
 export function parseArgs(argv: readonly string[]): Args {
@@ -674,6 +711,7 @@ export function parseArgs(argv: readonly string[]): Args {
     rebuild: true,
     help: false,
     assumeRobotsOn403: false,
+    force: false,
   };
 
   // A flag whose value is missing is a mistake, never a default. `--only` with
@@ -711,6 +749,9 @@ export function parseArgs(argv: readonly string[]): Args {
       case "--assume-robots-on-403":
         args.assumeRobotsOn403 = true;
         break;
+      case "--force":
+        args.force = true;
+        break;
       case "--help":
       case "-h":
         args.help = true;
@@ -728,6 +769,7 @@ export function parseArgs(argv: readonly string[]): Args {
 
 const USAGE = `usage: bun run refresh [--dry-run] [--only <sourceId>] [--snapshots <dir>]
                        [--user-agent <ua>] [--no-feed] [--assume-robots-on-403]
+                       [--force]
 
   --dry-run       report what each source would do; no requests, no writes
   --only <id>     refresh a single source (${ADAPTERS.map((a) => a.id).join(", ")})
@@ -739,7 +781,12 @@ const USAGE = `usage: bun run refresh [--dry-run] [--only <sourceId>] [--snapsho
                   /robots.txt itself, proceed on the permission recorded in
                   AGENTS.md instead of failing closed. Refused under CI.
                   Does NOT override a robots.txt we could read: a file that
-                  disallows us still says no.`;
+                  disallows us still says no.
+  --force         temporary, interactive-only. Ignore the 6h per-source floor
+                  and ask now. Refused under CI. Everything else about being a
+                  guest still holds: one request per source, per-host spacing,
+                  conditional headers (so an unchanged page still costs a 304),
+                  robots, no retries. Prefer it with --only.`;
 
 async function main(): Promise<number> {
   let args: Args;
@@ -767,6 +814,19 @@ async function main(): Promise<number> {
   // asserting a permission nobody re-checked is how "temporary" becomes
   // permanent — so CI is refused the option outright rather than trusted not to
   // pass it. AGENTS.md § Scraping conduct is the argument.
+  // Same reasoning as the robots override: this is a person deciding, once,
+  // that a page has moved and they want it now. A schedule deciding that every
+  // run is just a shorter interval with extra steps, and the interval is the
+  // obligation.
+  if (args.force && isCi()) {
+    console.error(
+      "--force is interactive-only and refused under CI.\n" +
+        "The 6h floor is what the scheduled runner is for; change the schedule, " +
+        "not the floor.",
+    );
+    return 2;
+  }
+
   if (args.assumeRobotsOn403 && isCi()) {
     console.error(
       "--assume-robots-on-403 is interactive-only and refused under CI.\n" +
@@ -783,6 +843,14 @@ async function main(): Promise<number> {
     assumeAllowedWhenForbidden: args.assumeRobotsOn403,
   });
 
+  if (args.force) {
+    const n = args.only === null ? ADAPTERS.length : 1;
+    console.warn(
+      `  ! --force: ignoring the 6h floor for ${n} source${n === 1 ? "" : "s"}. ` +
+        `Conditional headers still apply, so an unchanged page costs a 304.`,
+    );
+  }
+
   if (args.assumeRobotsOn403) {
     console.warn(
       "  ! --assume-robots-on-403: a host answering 403 to /robots.txt will be\n" +
@@ -808,6 +876,7 @@ async function main(): Promise<number> {
     sleep: (ms) => Bun.sleep(ms),
     dryRun: args.dryRun,
     only: args.only,
+    force: args.force,
     timeoutMs: 20_000,
     log: (line) => console.log(line),
     rebuildFeed: args.dryRun || !args.rebuild ? null : rebuildFeedViaScript,