gacha-event-tracker/.github/workflows/refresh.yml

name: Refresh sources

# Fetch each source at most twice a day and commit the raw snapshots when — and
# only when — the bytes actually changed. Everything downstream (parse, merge,
# feed, build, deploy) is CI's existing job; this workflow does not duplicate
# any of it, it just hands CI fresher input.
#
# Twelve hours apart is deliberately well clear of the six-hour-per-source floor
# in AGENTS.md § Scraping conduct, and the runner enforces that floor itself, so
# a manual dispatch on top of a scheduled run cannot double up on a wiki.
on:
  schedule:
    - cron: "27 5,17 * * *"
  workflow_dispatch:
    inputs:
      dry_run:
        description: "Plan only — no requests, no writes"
        type: boolean
        default: false
      only:
        description: "Refresh a single source id (blank = all)"
        type: string
        default: ""
      # Both of these are refused on the scheduled run and only work here, on a
      # dispatch, because a dispatch is a person and GitHub records which one.
      # See AGENTS.md § Scraping conduct: an override on every cycle is just a
      # new default. Prefer pairing either with `only`.
      force:
        description: "Ask sources before their 6h floor is up (one run only)"
        type: boolean
        default: false
      assume_robots_on_403:
        description: "Treat a challenged robots.txt 403 as the recorded permission"
        type: boolean
        default: false

# Never two refreshes at once: they would both fetch, and the second would race
# the first's commit. Queue instead of cancelling — a half-finished refresh that
# has already written snapshots should be allowed to finish and push.
concurrency:
  group: refresh
  cancel-in-progress: false

permissions:
  contents: read

jobs:
  refresh:
    name: Fetch sources and rebuild the feed
    runs-on: [self-hosted, safe-ip]
    # A fork must not point this at the wikis on a schedule. Same shape as the
    # DEPLOY_PAGES gate in ci.yml: off by default for anyone but this repo,
    # while a fork owner can still dispatch it by hand and take responsibility.
    if: >-
      github.event_name == 'workflow_dispatch' ||
      github.repository == 'StereotypicalCat/gacha-event-tracker'
    permissions:
      # Commit the refreshed snapshots.
      contents: write
      # Dispatch ci.yml afterwards: a push made with GITHUB_TOKEN deliberately
      # does not trigger other workflows, so without this the fresh data would
      # sit in the repo undeployed until someone pushed by hand.
      actions: write
    steps:
      - uses: actions/checkout@v4

      - uses: oven-sh/setup-bun@v2
        with:
          bun-version: "1.3"

      - run: bun install --frozen-lockfile

      # When each source was last checked. Gitignored on purpose (committing it
      # would mean a commit every cycle saying nothing changed), so it rides in
      # the actions cache instead. A cache miss only means the runner has no
      # record of the last check — the twelve-hour schedule still keeps us well
      # inside the etiquette floor.
      - name: Restore refresh bookkeeping
        uses: actions/cache/restore@v4
        with:
          path: snapshots/*.state.json
          key: refresh-state-
          restore-keys: refresh-state-

      - name: Refresh
        id: refresh
        env:
          # Identify the crawler with a contact URL, per AGENTS.md.
          REFRESH_CONTACT_URL: ${{ github.server_url }}/${{ github.repository }}
          # Passed through the environment rather than interpolated into the
          # run script, so a dispatch input cannot become shell.
          ONLY: ${{ inputs.only }}
          DRY_RUN: ${{ inputs.dry_run }}
          # Empty on a scheduled run, so neither flag is ever passed by the cron.
          FORCE: ${{ inputs.force }}
          ASSUME_ROBOTS_ON_403: ${{ inputs.assume_robots_on_403 }}
        run: |
          args=()
          if [ "$DRY_RUN" = "true" ]; then
            args+=(--dry-run)
          fi
          if [ -n "$ONLY" ]; then
            args+=(--only "$ONLY")
          fi
          if [ "$FORCE" = "true" ]; then
            args+=(--force)
          fi
          if [ "$ASSUME_ROBOTS_ON_403" = "true" ]; then
            args+=(--assume-robots-on-403)
          fi
          bun run refresh "${args[@]}"

      # The key must differ every time this step runs. `run_id` is stable across
      # re-runs, so a re-run's save hits an existing key, is skipped, and the
      # next run restores the bookkeeping from before the re-run — records of
      # requests we did make, lost. `run_attempt` increments per attempt.
      - name: Save refresh bookkeeping
        if: always()
        uses: actions/cache/save@v4
        with:
          path: snapshots/*.state.json
          key: refresh-state-${{ github.run_id }}-${{ github.run_attempt }}

      # git is the authority on "did anything change" — a 304, an unchanged
      # body, or a rejected parse all leave the working tree clean.
      - name: Detect changes
        id: diff
        run: |
          if [ -n "$(git status --porcelain -- snapshots)" ]; then
            git status --porcelain -- snapshots
            echo "changed=true" >> "$GITHUB_OUTPUT"
          else
            echo "no source changed"
            echo "changed=false" >> "$GITHUB_OUTPUT"
          fi

      # A human push landing between the checkout and this push makes the push
      # non-fast-forward. Failing there would throw away pages we have already
      # fetched while the bookkeeping above (saved with `if: always()`) has
      # already spent their six-hour budget — the wikis would be asked again for
      # nothing. So rebase onto whatever landed and try again. Never force: this
      # commit is only ever new files under snapshots/, so it has nothing to say
      # about anyone else's work.
      - name: Commit refreshed snapshots
        if: steps.diff.outputs.changed == 'true' && inputs.dry_run != true
        env:
          BRANCH: ${{ github.ref_name }}
        run: |
          set -euo pipefail
          git config user.name "github-actions[bot]"
          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
          git add -- snapshots
          git commit -m "chore(data): refresh source snapshots" \
            -m "Automated fetch from ${{ github.workflow }} run ${{ github.run_id }}."

          for attempt in 1 2 3; do
            if git push origin "HEAD:$BRANCH"; then
              exit 0
            fi
            echo "push rejected (attempt $attempt); rebasing onto origin/$BRANCH"
            git fetch origin "$BRANCH"
            if ! git rebase "origin/$BRANCH"; then
              git rebase --abort || true
              echo "::error::snapshot commit conflicts with $BRANCH; not force-pushing"
              exit 1
            fi
            sleep $((attempt * 5))
          done
          echo "::error::could not push refreshed snapshots after 3 attempts"
          exit 1

      # ci.yml owns typecheck, tests, the feed sanity check, the image and the
      # Pages deploy. Dispatching it is how the refreshed data reaches the site
      # without any of that logic being copied here.
      - name: Publish the refreshed feed
        if: steps.diff.outputs.changed == 'true' && inputs.dry_run != true
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: gh workflow run ci.yml --ref "${{ github.ref_name }}"

      # A source that has failed three cycles running is broken, not down: that
      # game's calendar has been built from a checked-in fixture for a day and a
      # half while every run showed a green tick. `bun run refresh` exits 0 on
      # this so the steps above still commit and publish what did work; turning
      # the run red is this step's job, and it is last for that reason.
      #
      # `always()` so it still reports when an earlier step failed — but note it
      # cannot report when the Refresh step itself hard-failed, since the output
      # is then unset and the job is already red on its own account.
      - name: Report source health
        if: always()
        env:
          BROKEN: ${{ steps.refresh.outputs.broken }}
          REFRESH_OUTCOME: ${{ steps.refresh.outcome }}
        run: |
          if [ "$REFRESH_OUTCOME" != "success" ]; then
            echo "refresh did not complete ($REFRESH_OUTCOME); no health to report"
            exit 0
          fi
          if [ -n "$BROKEN" ] && [ "$BROKEN" != "0" ]; then
            echo "::error::$BROKEN source(s) have stopped answering; see the job summary"
            exit 1
          fi
          echo "every source is answering"