diff --git a/scripts/refresh-sources.ts b/scripts/refresh-sources.ts index bb786b8..4656fb7 100644 --- a/scripts/refresh-sources.ts +++ b/scripts/refresh-sources.ts @@ -281,7 +281,9 @@ export async function runRefresh( summary.assumedRobots = [...cycle.assumedRobots].sort(); for (const host of summary.assumedRobots) { summary.warnings.push( - `${host}: fetched on --assume-robots-on-403; its robots.txt was not read this run`, + `${host}: fetched on --assume-robots-on-403 — its robots.txt was NOT read ` + + `this run. Re-read it in a browser and confirm AGENTS.md § Scraping ` + + `conduct still describes it.`, ); } @@ -886,13 +888,10 @@ async function main(): Promise { `\n${summary.changed} changed, ${summary.confirmed}/${summary.attempted} confirmed, ` + `${summary.warnings.length} warnings, ${summary.broken.length} broken`, ); + // One line per warning, and every warning is in `summary.warnings` — printing + // a category separately here once double-reported the assumed-robots hosts + // and left the "N warnings" count above disagreeing with the lines under it. for (const warning of summary.warnings) console.warn(` ! ${warning}`); - for (const host of summary.assumedRobots) { - console.warn( - ` ! ${host}: robots.txt was NOT read this run. Re-read it in a browser ` + - `and confirm AGENTS.md § Scraping conduct still describes it.`, - ); - } for (const b of summary.broken) { console.error( ` !! ${b.sourceId} has failed ${b.consecutiveFailures} cycles running ` + diff --git a/test/refresh.test.ts b/test/refresh.test.ts index 6591919..090753e 100644 --- a/test/refresh.test.ts +++ b/test/refresh.test.ts @@ -363,9 +363,15 @@ describe("robots", () => { expect(summary.outcomes[0]?.result).toBe("fetched"); expect(summary.assumedRobots).toEqual(["game8.co"]); - expect(summary.warnings.some((w) => w.includes("--assume-robots-on-403"))).toBe( - true, + // Exactly one warning per host, and it carries the instruction. The host + // was reported twice for a while — once from here and once from a second + // loop in main() — which also left the run's "N warnings" count disagreeing + // with the number of lines printed under it. + const assumed = summary.warnings.filter((w) => + w.includes("--assume-robots-on-403"), ); + expect(assumed).toHaveLength(1); + expect(assumed[0]).toContain("Re-read it in a browser"); }); test("an ordinary run reports no assumed hosts at all", async () => {