From f38fe49d30e87803eaefa8c889df79f88725b4f9 Mon Sep 17 00:00:00 2001
From: Lucas Winther <lucwin@ece.au.dk>
Date: Wed, 19 Aug 2026 05:07:49 +0200
Subject: [PATCH] fix(refresh): report an assumed-robots host once, not twice
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The run just made reported each of the four Fandom hosts twice — once from the
warning pushed in runRefresh and once from a second loop in main() that printed
summary.assumedRobots again. Eight lines for four facts, and the header above
them said "5 warnings" while nine printed under it, so the count a reader uses
to judge whether to read the list was wrong in the direction that matters.

Mine, from the commit that added the flag: I put the reminder in main() to make
it loud, having already made it loud in the summary.

The instruction folds into the single warning instead, so it still says to go
re-read the file in a browser, and it now also reaches the job summary and
annotations, which only ever read summary.warnings.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 scripts/refresh-sources.ts | 13 ++++++-------
 test/refresh.test.ts       | 10 ++++++++--
 2 files changed, 14 insertions(+), 9 deletions(-)

diff --git a/scripts/refresh-sources.ts b/scripts/refresh-sources.ts
index bb786b8..4656fb7 100644
--- a/scripts/refresh-sources.ts
+++ b/scripts/refresh-sources.ts
@@ -281,7 +281,9 @@ export async function runRefresh(
   summary.assumedRobots = [...cycle.assumedRobots].sort();
   for (const host of summary.assumedRobots) {
     summary.warnings.push(
-      `${host}: fetched on --assume-robots-on-403; its robots.txt was not read this run`,
+      `${host}: fetched on --assume-robots-on-403 — its robots.txt was NOT read ` +
+        `this run. Re-read it in a browser and confirm AGENTS.md § Scraping ` +
+        `conduct still describes it.`,
     );
   }
 
@@ -886,13 +888,10 @@ async function main(): Promise<number> {
     `\n${summary.changed} changed, ${summary.confirmed}/${summary.attempted} confirmed, ` +
       `${summary.warnings.length} warnings, ${summary.broken.length} broken`,
   );
+  // One line per warning, and every warning is in `summary.warnings` — printing
+  // a category separately here once double-reported the assumed-robots hosts
+  // and left the "N warnings" count above disagreeing with the lines under it.
   for (const warning of summary.warnings) console.warn(`  ! ${warning}`);
-  for (const host of summary.assumedRobots) {
-    console.warn(
-      `  ! ${host}: robots.txt was NOT read this run. Re-read it in a browser ` +
-        `and confirm AGENTS.md § Scraping conduct still describes it.`,
-    );
-  }
   for (const b of summary.broken) {
     console.error(
       `  !! ${b.sourceId} has failed ${b.consecutiveFailures} cycles running ` +
diff --git a/test/refresh.test.ts b/test/refresh.test.ts
index 6591919..090753e 100644
--- a/test/refresh.test.ts
+++ b/test/refresh.test.ts
@@ -363,9 +363,15 @@ describe("robots", () => {
 
     expect(summary.outcomes[0]?.result).toBe("fetched");
     expect(summary.assumedRobots).toEqual(["game8.co"]);
-    expect(summary.warnings.some((w) => w.includes("--assume-robots-on-403"))).toBe(
-      true,
+    // Exactly one warning per host, and it carries the instruction. The host
+    // was reported twice for a while — once from here and once from a second
+    // loop in main() — which also left the run's "N warnings" count disagreeing
+    // with the number of lines printed under it.
+    const assumed = summary.warnings.filter((w) =>
+      w.includes("--assume-robots-on-403"),
     );
+    expect(assumed).toHaveLength(1);
+    expect(assumed[0]).toContain("Re-read it in a browser");
   });
 
   test("an ordinary run reports no assumed hosts at all", async () => {