gacha-event-tracker/.github/workflows/refresh.yml

name: Refresh sources

# Fetch each source at most twice a day and commit the raw snapshots when — and
# only when — the bytes actually changed. Everything downstream (parse, merge,
# feed, build, deploy) is CI's existing job; this workflow does not duplicate
# any of it, it just hands CI fresher input.
#
# Twelve hours apart is deliberately well clear of the six-hour-per-source floor
# in AGENTS.md § Scraping conduct, and the runner enforces that floor itself, so
# a manual dispatch on top of a scheduled run cannot double up on a wiki.
on:
  schedule:
    - cron: "27 5,17 * * *"
  workflow_dispatch:
    inputs:
      dry_run:
        description: "Plan only — no requests, no writes"
        type: boolean
        default: false
      only:
        description: "Refresh a single source id (blank = all)"
        type: string
        default: ""
      # `force` is refused on the scheduled run and only works here, on a
      # dispatch, because a dispatch is a person and the forge records which one
      # — a schedule that forces every cycle is just a shorter interval with
      # extra steps. Prefer pairing it with `only`.
      force:
        description: "Ask sources before their 6h floor is up (one run only)"
        type: boolean
        default: false
      # The scheduled run passes this one anyway (see the Refresh step), so this
      # input exists to turn it OFF for a dispatch — which is how you see which
      # hosts genuinely cannot be read, rather than which ones we assumed.
      assume_robots_on_403:
        description: "Treat a challenged robots.txt 403 as the recorded permission"
        type: boolean
        default: true

# Never two refreshes at once: they would both fetch, and the second would race
# the first's commit. Queue instead of cancelling — a half-finished refresh that
# has already written snapshots should be allowed to finish and push.
concurrency:
  group: refresh
  cancel-in-progress: false

permissions:
  contents: read

jobs:
  refresh:
    name: Fetch sources and rebuild the feed
    runs-on: ubuntu-latest
    # A fork must not point this at the wikis on a schedule: off by default for
    # anyone but this repo, while a fork owner can still dispatch it by hand and
    # take responsibility.
    if: >-
      github.event_name == 'workflow_dispatch' ||
      github.repository == 'lucasw89/gacha-event-tracker'
    permissions:
      # Commit the refreshed snapshots.
      contents: write
      # Dispatch ci.yml afterwards; see that step for why it is still here on a
      # forge that does not suppress triggers on a token-made push.
      actions: write
    steps:
      - uses: https://github.com/actions/checkout@v4

      - uses: https://github.com/oven-sh/setup-bun@v2
        with:
          bun-version: "1.3"

      - run: bun install --frozen-lockfile

      # When each source was last checked. Gitignored on purpose (committing it
      # would mean a commit every cycle saying nothing changed), so it rides in
      # the actions cache instead. A cache miss only means the runner has no
      # record of the last check — the twelve-hour schedule still keeps us well
      # inside the etiquette floor.
      - name: Restore refresh bookkeeping
        uses: https://github.com/actions/cache/restore@v4
        with:
          path: snapshots/*.state.json
          key: refresh-state-
          restore-keys: refresh-state-

      - name: Refresh
        id: refresh
        env:
          # Identify the crawler with a contact URL, per AGENTS.md.
          REFRESH_CONTACT_URL: ${{ github.server_url }}/${{ github.repository }}
          # Passed through the environment rather than interpolated into the
          # run script, so a dispatch input cannot become shell.
          ONLY: ${{ inputs.only }}
          DRY_RUN: ${{ inputs.dry_run }}
          # Empty on a scheduled run, so --force is never passed by the cron.
          FORCE: ${{ inputs.force }}
          # Also empty on a schedule, which is why EVENT_NAME decides there. The
          # cron stands on the permission recorded in AGENTS.md § Scraping
          # conduct; a dispatch can set the input to false to see the real state.
          ASSUME_ROBOTS_ON_403: ${{ inputs.assume_robots_on_403 }}
          EVENT_NAME: ${{ github.event_name }}
        run: |
          args=()
          if [ "$DRY_RUN" = "true" ]; then
            args+=(--dry-run)
          fi
          if [ -n "$ONLY" ]; then
            args+=(--only "$ONLY")
          fi
          if [ "$FORCE" = "true" ]; then
            args+=(--force)
          fi
          if [ "$ASSUME_ROBOTS_ON_403" = "true" ] || [ "$EVENT_NAME" = "schedule" ]; then
            args+=(--assume-robots-on-403)
          fi
          bun run refresh "${args[@]}"

      # The key must differ every time this step runs. `run_id` is stable across
      # re-runs, so a re-run's save hits an existing key, is skipped, and the
      # next run restores the bookkeeping from before the re-run — records of
      # requests we did make, lost. `run_attempt` increments per attempt.
      - name: Save refresh bookkeeping
        if: always()
        uses: https://github.com/actions/cache/save@v4
        with:
          path: snapshots/*.state.json
          key: refresh-state-${{ github.run_id }}-${{ github.run_attempt }}

      # git is the authority on "did anything change" — a 304, an unchanged
      # body, or a rejected parse all leave the working tree clean.
      - name: Detect changes
        id: diff
        run: |
          if [ -n "$(git status --porcelain -- snapshots)" ]; then
            git status --porcelain -- snapshots
            echo "changed=true" >> "$GITHUB_OUTPUT"
          else
            echo "no source changed"
            echo "changed=false" >> "$GITHUB_OUTPUT"
          fi

      # A human push landing between the checkout and this push makes the push
      # non-fast-forward. Failing there would throw away pages we have already
      # fetched while the bookkeeping above (saved with `if: always()`) has
      # already spent their six-hour budget — the wikis would be asked again for
      # nothing. So rebase onto whatever landed and try again. Never force: this
      # commit is only ever new files under snapshots/, so it has nothing to say
      # about anyone else's work.
      - name: Commit refreshed snapshots
        if: steps.diff.outputs.changed == 'true' && inputs.dry_run != true
        env:
          BRANCH: ${{ github.ref_name }}
        run: |
          set -euo pipefail
          git config user.name "gitea-actions[bot]"
          git config user.email "gitea-actions[bot]@noreply.lucaswinther.info"
          git add -- snapshots
          git commit -m "chore(data): refresh source snapshots" \
            -m "Automated fetch from ${{ github.workflow }} run ${{ github.run_id }}."

          for attempt in 1 2 3; do
            if git push origin "HEAD:$BRANCH"; then
              exit 0
            fi
            echo "push rejected (attempt $attempt); rebasing onto origin/$BRANCH"
            git fetch origin "$BRANCH"
            if ! git rebase "origin/$BRANCH"; then
              git rebase --abort || true
              echo "::error::snapshot commit conflicts with $BRANCH; not force-pushing"
              exit 1
            fi
            sleep $((attempt * 5))
          done
          echo "::error::could not push refreshed snapshots after 3 attempts"
          exit 1

      # ci.yml owns typecheck, tests, the feed sanity check and the image.
      # Dispatching it is how the refreshed data reaches the deployed container
      # without any of that logic being copied here.
      #
      # There is no `gh` on a Gitea runner, so this is the REST dispatch the CLI
      # would have made. Note the difference from GitHub that makes this step
      # look redundant: Gitea does *not* suppress workflow triggers on a push
      # made with the Actions token, so the commit above will usually have
      # started ci.yml already. Dispatching anyway is deliberate — it is the only
      # thing that guarantees publication if that behaviour is ever configured
      # off, and a duplicate run is harmless because ci.yml's concurrency group
      # is per-ref with cancel-in-progress, so the second supersedes the first.
      - name: Publish the refreshed feed
        if: steps.diff.outputs.changed == 'true' && inputs.dry_run != true
        env:
          TOKEN: ${{ secrets.GITHUB_TOKEN }}
          API: ${{ github.server_url }}/api/v1
          REPO: ${{ github.repository }}
          BRANCH: ${{ github.ref_name }}
        run: |
          set -euo pipefail
          curl -sS --fail-with-body -X POST \
            -H "Authorization: token $TOKEN" \
            -H "Content-Type: application/json" \
            "$API/repos/$REPO/actions/workflows/ci.yml/dispatches" \
            -d "{\"ref\":\"$BRANCH\"}"

      # A source that has failed three cycles running is broken, not down: that
      # game's calendar has been built from a checked-in fixture for a day and a
      # half while every run showed a green tick. `bun run refresh` exits 0 on
      # this so the steps above still commit and publish what did work; turning
      # the run red is this step's job, and it is last for that reason.
      #
      # `always()` so it still reports when an earlier step failed — but note it
      # cannot report when the Refresh step itself hard-failed, since the output
      # is then unset and the job is already red on its own account.
      - name: Report source health
        if: always()
        env:
          BROKEN: ${{ steps.refresh.outputs.broken }}
          REFRESH_OUTCOME: ${{ steps.refresh.outcome }}
        run: |
          if [ "$REFRESH_OUTCOME" != "success" ]; then
            echo "refresh did not complete ($REFRESH_OUTCOME); no health to report"
            exit 0
          fi
          if [ -n "$BROKEN" ] && [ "$BROKEN" != "0" ]; then
            echo "::error::$BROKEN source(s) have stopped answering; see the job summary"
            exit 1
          fi
          echo "every source is answering"