feat(refresh): --force, to ask before the 6h floor is up

The interval gate has no override, so a page you know has just changed cannot be fetched until six hours after the last attempt. The only workaround was deleting snapshots/<id>.state.json, which also wipes consecutiveFailures and lastConfirmedAt — resetting the broken-source streak and making the footer age every source from when its bytes last changed rather than when we last confirmed them. A flag that says what it means is better than a side effect nobody documented. It sets aside the interval and nothing else. Conditional headers still go out, which is what makes forcing defensible at all: the host is asked, not re-served, and an unchanged page costs it a 304. Per-host spacing, robots, one request per source and the no-retry rule all still apply — a source that was not due and is also disallowed stays skipped, for the reason that matters. Refused under CI, like --assume-robots-on-403 and for the same reason: a schedule that forces every cycle is a shorter interval with extra steps, and the interval is the obligation, not the default. So AGENTS.md § Scraping conduct is amended rather than left to be quietly contradicted by a flag. Every source asked early is named in summary.forced and warned about. A run that was due anyway is never reported as forced — a summary that cried "forced" on an ordinary run would train the reader to ignore the word. Also repoints the "unknown flag" test, which used --force as its example and stopped testing anything the moment --force existed. Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
2026-08-19 04:59:52 +02:00
parent 240e4d59d1
commit 6552528acb
4 changed files with 187 additions and 10 deletions
@@ -100,6 +100,7 @@ function options(
    },
    dryRun: false,
    only: null,
+    force: false,
    timeoutMs: 1000,
    log: () => {},
    rebuildFeed: async () => {
@@ -202,6 +203,88 @@ describe("one request per source per six hours", () => {
    expect(summary.hardFailure).toBeNull();
  });

+  test("--force asks a source that was not due, and says which", async () => {
+    await store.recordCheck("genshin-game8-events", {
+      at: new Date(NOW.getTime() - 1000).toISOString(),
+      status: 200,
+      ok: true,
+    });
+
+    const { opts, calls } = options({ force: true });
+    const summary = await runRefresh(opts);
+
+    expect(calls).toHaveLength(1);
+    expect(summary.outcomes[0]?.result).not.toBe("skipped_interval");
+    // Named, not merely permitted. Overriding an etiquette obligation quietly
+    // is how the obligation stops being one.
+    expect(summary.forced).toEqual(["genshin-game8-events"]);
+    expect(summary.warnings.some((w) => w.includes("--force"))).toBe(true);
+  });
+
+  test("--force still sends conditional headers, so an unchanged page is a 304", async () => {
+    // The whole reason forcing is defensible: the host is asked, not re-served.
+    await seed("<html><event></event></html>", NOW.toISOString(), 1);
+    await store.recordCheck("genshin-game8-events", {
+      at: new Date(NOW.getTime() - 1000).toISOString(),
+      status: 200,
+      ok: true,
+    });
+
+    const { opts, calls } = options({
+      force: true,
+      responder: () => new Response(null, { status: 304 }),
+    });
+    const summary = await runRefresh(opts);
+
+    expect(calls[0]?.headers["If-None-Match"]).toBe('W/"v1"');
+    expect(summary.outcomes[0]?.result).toBe("unchanged");
+  });
+
+  test("--force sets aside the interval and nothing else", async () => {
+    // robots is the gate it must never touch. A source that was not due AND is
+    // disallowed stays skipped for the reason that actually matters.
+    await store.recordCheck("genshin-game8-events", {
+      at: new Date(NOW.getTime() - 1000).toISOString(),
+      status: 200,
+      ok: true,
+    });
+
+    const { opts, calls } = options({
+      force: true,
+      robots: {
+        allows: async () => ({ allowed: false, reason: "disallowed by robots" }),
+      },
+    });
+    const summary = await runRefresh(opts);
+
+    expect(calls).toHaveLength(0);
+    expect(summary.outcomes[0]?.result).toBe("skipped_robots");
+  });
+
+  test("a run that was due anyway is not reported as forced", async () => {
+    // --force is a description of what happened, not of what was passed. A
+    // summary that cried "forced" on an ordinary run would train the reader to
+    // ignore the word.
+    await store.recordCheck("genshin-game8-events", {
+      at: new Date(NOW.getTime() - SIX_HOURS_MS).toISOString(),
+      status: 200,
+      ok: true,
+    });
+
+    const { opts, calls } = options({ force: true });
+    const summary = await runRefresh(opts);
+
+    expect(calls).toHaveLength(1);
+    expect(summary.forced).toEqual([]);
+    expect(summary.warnings).toEqual([]);
+  });
+
+  test("an ordinary run reports nothing forced", async () => {
+    const { opts } = options({});
+    const summary = await runRefresh(opts);
+    expect(summary.forced).toEqual([]);
+  });
+
  test("fetches again once the interval has elapsed", async () => {
    await store.recordCheck("genshin-game8-events", {
      at: new Date(NOW.getTime() - SIX_HOURS_MS).toISOString(),
@@ -798,6 +881,7 @@ describe("what the runner reports to the runner", () => {
    ],
    hardFailure: null,
    assumedRobots: [],
+    forced: [],
  };

  test("a broken source becomes an annotation on the run page", () => {
@@ -962,8 +1046,15 @@ describe("flags", () => {
    expect(parseArgs(["--assume-robots-on-403"]).assumeRobotsOn403).toBe(true);
  });

+  test("parseArgs reads --force, and it is off by default", () => {
+    expect(parseArgs([]).force).toBe(false);
+    expect(parseArgs(["--force"]).force).toBe(true);
+  });
+
  test("parseArgs rejects an unknown flag rather than ignoring it", () => {
-    expect(() => parseArgs(["--force"])).toThrow("unknown flag");
+    // Deliberately a flag nobody would add. This case used to be spelled
+    // `--force`, which stopped testing anything the day --force was built.
+    expect(() => parseArgs(["--yolo"])).toThrow("unknown flag");
  });

  test("parseArgs rejects a flag whose value is missing", () => {