212 lines
9.0 KiB
YAML
212 lines
9.0 KiB
YAML
name: Refresh sources
|
|
|
|
# Fetch each source at most twice a day and commit the raw snapshots when — and
|
|
# only when — the bytes actually changed. Everything downstream (parse, merge,
|
|
# feed, build, deploy) is CI's existing job; this workflow does not duplicate
|
|
# any of it, it just hands CI fresher input.
|
|
#
|
|
# Twelve hours apart is deliberately well clear of the six-hour-per-source floor
|
|
# in AGENTS.md § Scraping conduct, and the runner enforces that floor itself, so
|
|
# a manual dispatch on top of a scheduled run cannot double up on a wiki.
|
|
on:
|
|
schedule:
|
|
- cron: "27 5,17 * * *"
|
|
workflow_dispatch:
|
|
inputs:
|
|
dry_run:
|
|
description: "Plan only — no requests, no writes"
|
|
type: boolean
|
|
default: false
|
|
only:
|
|
description: "Refresh a single source id (blank = all)"
|
|
type: string
|
|
default: ""
|
|
# `force` is refused on the scheduled run and only works here, on a
|
|
# dispatch, because a dispatch is a person and GitHub records which one —
|
|
# a schedule that forces every cycle is just a shorter interval with extra
|
|
# steps. Prefer pairing it with `only`.
|
|
force:
|
|
description: "Ask sources before their 6h floor is up (one run only)"
|
|
type: boolean
|
|
default: false
|
|
# The scheduled run passes this one anyway (see the Refresh step), so this
|
|
# input exists to turn it OFF for a dispatch — which is how you see which
|
|
# hosts genuinely cannot be read, rather than which ones we assumed.
|
|
assume_robots_on_403:
|
|
description: "Treat a challenged robots.txt 403 as the recorded permission"
|
|
type: boolean
|
|
default: true
|
|
|
|
# Never two refreshes at once: they would both fetch, and the second would race
|
|
# the first's commit. Queue instead of cancelling — a half-finished refresh that
|
|
# has already written snapshots should be allowed to finish and push.
|
|
concurrency:
|
|
group: refresh
|
|
cancel-in-progress: false
|
|
|
|
permissions:
|
|
contents: read
|
|
|
|
jobs:
|
|
refresh:
|
|
name: Fetch sources and rebuild the feed
|
|
runs-on: ubuntu-latest
|
|
# A fork must not point this at the wikis on a schedule. Same shape as the
|
|
# DEPLOY_PAGES gate in ci.yml: off by default for anyone but this repo,
|
|
# while a fork owner can still dispatch it by hand and take responsibility.
|
|
if: >-
|
|
github.event_name == 'workflow_dispatch' ||
|
|
github.repository == 'StereotypicalCat/gacha-event-tracker'
|
|
permissions:
|
|
# Commit the refreshed snapshots.
|
|
contents: write
|
|
# Dispatch ci.yml afterwards: a push made with GITHUB_TOKEN deliberately
|
|
# does not trigger other workflows, so without this the fresh data would
|
|
# sit in the repo undeployed until someone pushed by hand.
|
|
actions: write
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
|
|
- uses: oven-sh/setup-bun@v2
|
|
with:
|
|
bun-version: "1.3"
|
|
|
|
- run: bun install --frozen-lockfile
|
|
|
|
# When each source was last checked. Gitignored on purpose (committing it
|
|
# would mean a commit every cycle saying nothing changed), so it rides in
|
|
# the actions cache instead. A cache miss only means the runner has no
|
|
# record of the last check — the twelve-hour schedule still keeps us well
|
|
# inside the etiquette floor.
|
|
- name: Restore refresh bookkeeping
|
|
uses: actions/cache/restore@v4
|
|
with:
|
|
path: snapshots/*.state.json
|
|
key: refresh-state-
|
|
restore-keys: refresh-state-
|
|
|
|
- name: Refresh
|
|
id: refresh
|
|
env:
|
|
# Identify the crawler with a contact URL, per AGENTS.md.
|
|
REFRESH_CONTACT_URL: ${{ github.server_url }}/${{ github.repository }}
|
|
# Passed through the environment rather than interpolated into the
|
|
# run script, so a dispatch input cannot become shell.
|
|
ONLY: ${{ inputs.only }}
|
|
DRY_RUN: ${{ inputs.dry_run }}
|
|
# Empty on a scheduled run, so --force is never passed by the cron.
|
|
FORCE: ${{ inputs.force }}
|
|
# Also empty on a schedule, which is why EVENT_NAME decides there. The
|
|
# cron stands on the permission recorded in AGENTS.md § Scraping
|
|
# conduct; a dispatch can set the input to false to see the real state.
|
|
ASSUME_ROBOTS_ON_403: ${{ inputs.assume_robots_on_403 }}
|
|
EVENT_NAME: ${{ github.event_name }}
|
|
run: |
|
|
args=()
|
|
if [ "$DRY_RUN" = "true" ]; then
|
|
args+=(--dry-run)
|
|
fi
|
|
if [ -n "$ONLY" ]; then
|
|
args+=(--only "$ONLY")
|
|
fi
|
|
if [ "$FORCE" = "true" ]; then
|
|
args+=(--force)
|
|
fi
|
|
if [ "$ASSUME_ROBOTS_ON_403" = "true" ] || [ "$EVENT_NAME" = "schedule" ]; then
|
|
args+=(--assume-robots-on-403)
|
|
fi
|
|
bun run refresh "${args[@]}"
|
|
|
|
# The key must differ every time this step runs. `run_id` is stable across
|
|
# re-runs, so a re-run's save hits an existing key, is skipped, and the
|
|
# next run restores the bookkeeping from before the re-run — records of
|
|
# requests we did make, lost. `run_attempt` increments per attempt.
|
|
- name: Save refresh bookkeeping
|
|
if: always()
|
|
uses: actions/cache/save@v4
|
|
with:
|
|
path: snapshots/*.state.json
|
|
key: refresh-state-${{ github.run_id }}-${{ github.run_attempt }}
|
|
|
|
# git is the authority on "did anything change" — a 304, an unchanged
|
|
# body, or a rejected parse all leave the working tree clean.
|
|
- name: Detect changes
|
|
id: diff
|
|
run: |
|
|
if [ -n "$(git status --porcelain -- snapshots)" ]; then
|
|
git status --porcelain -- snapshots
|
|
echo "changed=true" >> "$GITHUB_OUTPUT"
|
|
else
|
|
echo "no source changed"
|
|
echo "changed=false" >> "$GITHUB_OUTPUT"
|
|
fi
|
|
|
|
# A human push landing between the checkout and this push makes the push
|
|
# non-fast-forward. Failing there would throw away pages we have already
|
|
# fetched while the bookkeeping above (saved with `if: always()`) has
|
|
# already spent their six-hour budget — the wikis would be asked again for
|
|
# nothing. So rebase onto whatever landed and try again. Never force: this
|
|
# commit is only ever new files under snapshots/, so it has nothing to say
|
|
# about anyone else's work.
|
|
- name: Commit refreshed snapshots
|
|
if: steps.diff.outputs.changed == 'true' && inputs.dry_run != true
|
|
env:
|
|
BRANCH: ${{ github.ref_name }}
|
|
run: |
|
|
set -euo pipefail
|
|
git config user.name "github-actions[bot]"
|
|
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
|
|
git add -- snapshots
|
|
git commit -m "chore(data): refresh source snapshots" \
|
|
-m "Automated fetch from ${{ github.workflow }} run ${{ github.run_id }}."
|
|
|
|
for attempt in 1 2 3; do
|
|
if git push origin "HEAD:$BRANCH"; then
|
|
exit 0
|
|
fi
|
|
echo "push rejected (attempt $attempt); rebasing onto origin/$BRANCH"
|
|
git fetch origin "$BRANCH"
|
|
if ! git rebase "origin/$BRANCH"; then
|
|
git rebase --abort || true
|
|
echo "::error::snapshot commit conflicts with $BRANCH; not force-pushing"
|
|
exit 1
|
|
fi
|
|
sleep $((attempt * 5))
|
|
done
|
|
echo "::error::could not push refreshed snapshots after 3 attempts"
|
|
exit 1
|
|
|
|
# ci.yml owns typecheck, tests, the feed sanity check, the image and the
|
|
# Pages deploy. Dispatching it is how the refreshed data reaches the site
|
|
# without any of that logic being copied here.
|
|
- name: Publish the refreshed feed
|
|
if: steps.diff.outputs.changed == 'true' && inputs.dry_run != true
|
|
env:
|
|
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
run: gh workflow run ci.yml --ref "${{ github.ref_name }}"
|
|
|
|
# A source that has failed three cycles running is broken, not down: that
|
|
# game's calendar has been built from a checked-in fixture for a day and a
|
|
# half while every run showed a green tick. `bun run refresh` exits 0 on
|
|
# this so the steps above still commit and publish what did work; turning
|
|
# the run red is this step's job, and it is last for that reason.
|
|
#
|
|
# `always()` so it still reports when an earlier step failed — but note it
|
|
# cannot report when the Refresh step itself hard-failed, since the output
|
|
# is then unset and the job is already red on its own account.
|
|
- name: Report source health
|
|
if: always()
|
|
env:
|
|
BROKEN: ${{ steps.refresh.outputs.broken }}
|
|
REFRESH_OUTCOME: ${{ steps.refresh.outcome }}
|
|
run: |
|
|
if [ "$REFRESH_OUTCOME" != "success" ]; then
|
|
echo "refresh did not complete ($REFRESH_OUTCOME); no health to report"
|
|
exit 0
|
|
fi
|
|
if [ -n "$BROKEN" ] && [ "$BROKEN" != "0" ]; then
|
|
echo "::error::$BROKEN source(s) have stopped answering; see the job summary"
|
|
exit 1
|
|
fi
|
|
echo "every source is answering"
|