From f12398a29bab3509ced2b9ef0932943ab3ce3e05 Mon Sep 17 00:00:00 2001 From: Lucas Winther Date: Thu, 20 Aug 2026 00:01:49 +0200 Subject: [PATCH] refresh.yml: offer both overrides to a dispatch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four Fandom sources have been skipping on a challenged robots.txt every cycle, and the only thing that clears it is a person passing the recorded permission. Until now there was no way to do that from the workflow, so it meant checking out the repo and running the refresh by hand. Both overrides are dispatch inputs now, guarded on the literal string "true" and empty on the cron, so the schedule still cannot reach either one. The force-push guard needed narrowing to allow this. It asserted the workflow contains no "--force" anywhere, as a proxy for never rewriting the branch, and the refresh runner's own --force flag trips that substring while having nothing to do with pushing. It now matches force on a git push specifically, including --force-with-lease, which is still one — a tighter assertion than the string it replaces, checked against both a real force push and the flag it must ignore. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/refresh.yml | 21 +++++++++++++++++++++ docs/INGESTION.md | 3 ++- test/refresh.test.ts | 34 +++++++++++++++++++++++++++++++++- 3 files changed, 56 insertions(+), 2 deletions(-) diff --git a/.github/workflows/refresh.yml b/.github/workflows/refresh.yml index f590a1d..f0c281c 100644 --- a/.github/workflows/refresh.yml +++ b/.github/workflows/refresh.yml @@ -21,6 +21,18 @@ on: description: "Refresh a single source id (blank = all)" type: string default: "" + # Both of these are refused on the scheduled run and only work here, on a + # dispatch, because a dispatch is a person and GitHub records which one. + # See AGENTS.md § Scraping conduct: an override on every cycle is just a + # new default. Prefer pairing either with `only`. + force: + description: "Ask sources before their 6h floor is up (one run only)" + type: boolean + default: false + assume_robots_on_403: + description: "Treat a challenged robots.txt 403 as the recorded permission" + type: boolean + default: false # Never two refreshes at once: they would both fetch, and the second would race # the first's commit. Queue instead of cancelling — a half-finished refresh that @@ -79,6 +91,9 @@ jobs: # run script, so a dispatch input cannot become shell. ONLY: ${{ inputs.only }} DRY_RUN: ${{ inputs.dry_run }} + # Empty on a scheduled run, so neither flag is ever passed by the cron. + FORCE: ${{ inputs.force }} + ASSUME_ROBOTS_ON_403: ${{ inputs.assume_robots_on_403 }} run: | args=() if [ "$DRY_RUN" = "true" ]; then @@ -87,6 +102,12 @@ jobs: if [ -n "$ONLY" ]; then args+=(--only "$ONLY") fi + if [ "$FORCE" = "true" ]; then + args+=(--force) + fi + if [ "$ASSUME_ROBOTS_ON_403" = "true" ]; then + args+=(--assume-robots-on-403) + fi bun run refresh "${args[@]}" # The key must differ every time this step runs. `run_id` is stable across diff --git a/docs/INGESTION.md b/docs/INGESTION.md index b780482..f60d92d 100644 --- a/docs/INGESTION.md +++ b/docs/INGESTION.md @@ -297,7 +297,8 @@ section but must never claim the event title. names the actor), while a `schedule` or any other runner event is not. The `schedule` refusal is load-bearing rather than ceremonial: from a challenged address `robots.txt` never arrives, so a cron standing on the recorded permission could never see the host withdraw it, and the recorded - permission has no expiry. + permission has no expiry. `refresh.yml` therefore exposes both overrides as + `workflow_dispatch` inputs, which are empty on the cron and so can never be passed by it. Every host it applied to is named in the run's warnings and in `summary.assumedRobots` — an override that reports nothing is one nobody withdraws. It changes no other obligation: still one diff --git a/test/refresh.test.ts b/test/refresh.test.ts index 00813ae..ca4b4e2 100644 --- a/test/refresh.test.ts +++ b/test/refresh.test.ts @@ -968,8 +968,40 @@ describe("the workflows that drive the refresh", () => { const refresh = await read("refresh.yml"); expect(refresh).toContain("git rebase"); expect(refresh).toMatch(/for attempt in/); - expect(refresh).not.toContain("--force"); + // Scoped to `git push`, because the bare substring `--force` also matches + // the refresh runner's own `--force` flag, which the dispatch inputs now + // offer and which has nothing to do with rewriting a branch. What must stay + // impossible is a force push — including `--force-with-lease`, which is + // still one. + expect(refresh).not.toMatch(/git push[^\n]*(--force|--force-with-lease|\s-f\b)/); expect(refresh).not.toContain("-f origin"); + expect(refresh).not.toContain("push --force"); + }); + + test("refresh.yml offers both overrides to a dispatch and to no cron", async () => { + // The scheduled run must never be able to pass either one: an override on + // every cycle is the new default with extra steps, and for the robots one a + // cron cannot re-read the permission it would be standing on. Both flags are + // therefore reachable only through `workflow_dispatch` inputs, which are + // empty on a schedule, and each is guarded on the literal string "true". + const refresh = await read("refresh.yml"); + + const dispatch = refresh.slice( + refresh.indexOf("workflow_dispatch:"), + refresh.indexOf("concurrency:"), + ); + expect(dispatch).toContain("force:"); + expect(dispatch).toContain("assume_robots_on_403:"); + + // Every occurrence of either flag is inside a test on its input variable. + expect(refresh).toMatch(/if \[ "\$FORCE" = "true" \]; then\n\s*args\+=\(--force\)/); + expect(refresh).toMatch( + /if \[ "\$ASSUME_ROBOTS_ON_403" = "true" \]; then\n\s*args\+=\(--assume-robots-on-403\)/, + ); + // The cron block itself carries no flags. + const cron = refresh.slice(refresh.indexOf("schedule:"), refresh.indexOf("workflow_dispatch:")); + expect(cron).not.toContain("force"); + expect(cron).not.toContain("assume"); }); test("refresh.yml turns red on a broken source only after committing", async () => {