Compare commits
3
Commits
b2ec0951f4
...
f32eb93289
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f32eb93289 | ||
|
|
5b8109b556 | ||
|
|
ec80e76cdc |
@@ -78,7 +78,7 @@ body:
|
||||
id: where
|
||||
attributes:
|
||||
label: Where are you running it, and in what browser?
|
||||
placeholder: stereotypicalcat.github.io/gacha-event-tracker — Firefox 141 on Android
|
||||
placeholder: localhost:3000 (container image) — Firefox 141 on Android
|
||||
validations:
|
||||
required: false
|
||||
|
||||
|
||||
+38
-59
@@ -20,9 +20,9 @@ jobs:
|
||||
name: Typecheck, test, feed
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: https://github.com/actions/checkout@v4
|
||||
|
||||
- uses: oven-sh/setup-bun@v2
|
||||
- uses: https://github.com/oven-sh/setup-bun@v2
|
||||
with:
|
||||
bun-version: "1.3"
|
||||
|
||||
@@ -36,7 +36,7 @@ jobs:
|
||||
# Restore-only: refresh.yml owns writing this cache, and a miss just
|
||||
# returns the pre-existing fallback behaviour.
|
||||
- name: Restore refresh bookkeeping
|
||||
uses: actions/cache/restore@v4
|
||||
uses: https://github.com/actions/cache/restore@v4
|
||||
with:
|
||||
path: snapshots/*.state.json
|
||||
key: refresh-state-
|
||||
@@ -105,8 +105,8 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
needs: check
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: oven-sh/setup-bun@v2
|
||||
- uses: https://github.com/actions/checkout@v4
|
||||
- uses: https://github.com/oven-sh/setup-bun@v2
|
||||
with:
|
||||
bun-version: "1.3"
|
||||
- run: bun install --frozen-lockfile
|
||||
@@ -114,19 +114,19 @@ jobs:
|
||||
# This is the job whose output is deployed, so this is the one that must
|
||||
# see the refresh bookkeeping; see the same step in `check`.
|
||||
- name: Restore refresh bookkeeping
|
||||
uses: actions/cache/restore@v4
|
||||
uses: https://github.com/actions/cache/restore@v4
|
||||
with:
|
||||
path: snapshots/*.state.json
|
||||
key: refresh-state-
|
||||
restore-keys: refresh-state-
|
||||
|
||||
- name: Build
|
||||
# Pages serves from /<repo>/, so the app is built with a matching base
|
||||
# href. Built at the domain root it would 404 on every asset.
|
||||
env:
|
||||
BASE_PATH: /${{ github.event.repository.name }}/
|
||||
# Built at the domain root, because the container image is what gets
|
||||
# deployed and `serve.ts` serves from `/`. A BASE_PATH belongs here only
|
||||
# if this artefact is ever hosted under a subpath again — it was set to
|
||||
# /<repo>/ for GitHub Pages, which this pipeline no longer deploys to.
|
||||
run: bun run build
|
||||
- uses: actions/upload-artifact@v4
|
||||
- uses: https://github.com/actions/upload-artifact@v4
|
||||
with:
|
||||
name: site
|
||||
path: public/
|
||||
@@ -143,66 +143,45 @@ jobs:
|
||||
contents: read
|
||||
packages: write
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: https://github.com/actions/checkout@v4
|
||||
|
||||
- uses: docker/setup-buildx-action@v3
|
||||
- uses: https://github.com/docker/setup-buildx-action@v3
|
||||
|
||||
- name: Lowercase image name
|
||||
id: img
|
||||
# GHCR rejects any uppercase in a repository name, and
|
||||
# github.repository preserves the owner's casing verbatim.
|
||||
run: echo "name=ghcr.io/${GITHUB_REPOSITORY,,}" >> "$GITHUB_OUTPUT"
|
||||
# Gitea's container registry, like GHCR before it, rejects any uppercase
|
||||
# in a repository name, and github.repository preserves the owner's
|
||||
# casing verbatim. The host is read from github.server_url rather than
|
||||
# hardcoded, so a fork on another Gitea instance pushes to its own
|
||||
# registry instead of this one.
|
||||
env:
|
||||
SERVER_URL: ${{ github.server_url }}
|
||||
run: |
|
||||
host="${SERVER_URL#https://}"
|
||||
host="${host#http://}"
|
||||
host="${host%%/*}"
|
||||
# Two outputs on purpose: login wants the bare host, the tags want the
|
||||
# full path. Passing the full path as the registry authenticates
|
||||
# against a host that does not exist.
|
||||
echo "host=${host}" >> "$GITHUB_OUTPUT"
|
||||
echo "name=${host}/${GITHUB_REPOSITORY,,}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- uses: docker/login-action@v3
|
||||
- uses: https://github.com/docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
registry: ${{ steps.img.outputs.host }}
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- uses: docker/build-push-action@v6
|
||||
- uses: https://github.com/docker/build-push-action@v6
|
||||
with:
|
||||
context: .
|
||||
push: true
|
||||
tags: |
|
||||
${{ steps.img.outputs.name }}:latest
|
||||
${{ steps.img.outputs.name }}:${{ github.sha }}
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
|
||||
# Deploying to Pages needs two one-time steps that CI cannot do for itself:
|
||||
#
|
||||
# 1. Settings → Pages → Source: "GitHub Actions".
|
||||
# The default GITHUB_TOKEN cannot do this. Creating a Pages site needs
|
||||
# `administration: write`, which is not a permission a workflow can grant
|
||||
# GITHUB_TOKEN, so `configure-pages` with enablement: true fails with
|
||||
# "Resource not accessible by integration".
|
||||
# 2. Settings → Secrets and variables → Actions → Variables:
|
||||
# set DEPLOY_PAGES to "true".
|
||||
#
|
||||
# Gated on that variable so the pipeline stays green for anyone who does not
|
||||
# want Pages, rather than failing on every push forever.
|
||||
pages:
|
||||
name: Deploy to Pages
|
||||
runs-on: ubuntu-latest
|
||||
needs: build
|
||||
if: >-
|
||||
github.ref == 'refs/heads/main' &&
|
||||
github.event_name != 'pull_request' &&
|
||||
vars.DEPLOY_PAGES == 'true'
|
||||
permissions:
|
||||
pages: write
|
||||
id-token: write
|
||||
environment:
|
||||
name: github-pages
|
||||
url: ${{ steps.deploy.outputs.page_url }}
|
||||
steps:
|
||||
- uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: site
|
||||
path: public
|
||||
- uses: actions/configure-pages@v5
|
||||
- uses: actions/upload-pages-artifact@v3
|
||||
with:
|
||||
path: public
|
||||
- id: deploy
|
||||
uses: actions/deploy-pages@v4
|
||||
# Registry cache rather than `type=gha`. Gitea does expose the Actions
|
||||
# cache API, but buildx's gha backend is the least reliable corner of
|
||||
# that compatibility surface, and we are already authenticated to a
|
||||
# registry that can hold the layers.
|
||||
cache-from: type=registry,ref=${{ steps.img.outputs.name }}:buildcache
|
||||
cache-to: type=registry,ref=${{ steps.img.outputs.name }}:buildcache,mode=max
|
||||
|
||||
@@ -22,9 +22,9 @@ on:
|
||||
type: string
|
||||
default: ""
|
||||
# `force` is refused on the scheduled run and only works here, on a
|
||||
# dispatch, because a dispatch is a person and GitHub records which one —
|
||||
# a schedule that forces every cycle is just a shorter interval with extra
|
||||
# steps. Prefer pairing it with `only`.
|
||||
# dispatch, because a dispatch is a person and the forge records which one
|
||||
# — a schedule that forces every cycle is just a shorter interval with
|
||||
# extra steps. Prefer pairing it with `only`.
|
||||
force:
|
||||
description: "Ask sources before their 6h floor is up (one run only)"
|
||||
type: boolean
|
||||
@@ -51,23 +51,22 @@ jobs:
|
||||
refresh:
|
||||
name: Fetch sources and rebuild the feed
|
||||
runs-on: ubuntu-latest
|
||||
# A fork must not point this at the wikis on a schedule. Same shape as the
|
||||
# DEPLOY_PAGES gate in ci.yml: off by default for anyone but this repo,
|
||||
# while a fork owner can still dispatch it by hand and take responsibility.
|
||||
# A fork must not point this at the wikis on a schedule: off by default for
|
||||
# anyone but this repo, while a fork owner can still dispatch it by hand and
|
||||
# take responsibility.
|
||||
if: >-
|
||||
github.event_name == 'workflow_dispatch' ||
|
||||
github.repository == 'StereotypicalCat/gacha-event-tracker'
|
||||
github.repository == 'lucasw89/gacha-event-tracker'
|
||||
permissions:
|
||||
# Commit the refreshed snapshots.
|
||||
contents: write
|
||||
# Dispatch ci.yml afterwards: a push made with GITHUB_TOKEN deliberately
|
||||
# does not trigger other workflows, so without this the fresh data would
|
||||
# sit in the repo undeployed until someone pushed by hand.
|
||||
# Dispatch ci.yml afterwards; see that step for why it is still here on a
|
||||
# forge that does not suppress triggers on a token-made push.
|
||||
actions: write
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: https://github.com/actions/checkout@v4
|
||||
|
||||
- uses: oven-sh/setup-bun@v2
|
||||
- uses: https://github.com/oven-sh/setup-bun@v2
|
||||
with:
|
||||
bun-version: "1.3"
|
||||
|
||||
@@ -79,7 +78,7 @@ jobs:
|
||||
# record of the last check — the twelve-hour schedule still keeps us well
|
||||
# inside the etiquette floor.
|
||||
- name: Restore refresh bookkeeping
|
||||
uses: actions/cache/restore@v4
|
||||
uses: https://github.com/actions/cache/restore@v4
|
||||
with:
|
||||
path: snapshots/*.state.json
|
||||
key: refresh-state-
|
||||
@@ -123,7 +122,7 @@ jobs:
|
||||
# requests we did make, lost. `run_attempt` increments per attempt.
|
||||
- name: Save refresh bookkeeping
|
||||
if: always()
|
||||
uses: actions/cache/save@v4
|
||||
uses: https://github.com/actions/cache/save@v4
|
||||
with:
|
||||
path: snapshots/*.state.json
|
||||
key: refresh-state-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
@@ -154,8 +153,8 @@ jobs:
|
||||
BRANCH: ${{ github.ref_name }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
git config user.name "github-actions[bot]"
|
||||
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
|
||||
git config user.name "gitea-actions[bot]"
|
||||
git config user.email "gitea-actions[bot]@noreply.lucaswinther.info"
|
||||
git add -- snapshots
|
||||
git commit -m "chore(data): refresh source snapshots" \
|
||||
-m "Automated fetch from ${{ github.workflow }} run ${{ github.run_id }}."
|
||||
@@ -176,14 +175,32 @@ jobs:
|
||||
echo "::error::could not push refreshed snapshots after 3 attempts"
|
||||
exit 1
|
||||
|
||||
# ci.yml owns typecheck, tests, the feed sanity check, the image and the
|
||||
# Pages deploy. Dispatching it is how the refreshed data reaches the site
|
||||
# ci.yml owns typecheck, tests, the feed sanity check and the image.
|
||||
# Dispatching it is how the refreshed data reaches the deployed container
|
||||
# without any of that logic being copied here.
|
||||
#
|
||||
# There is no `gh` on a Gitea runner, so this is the REST dispatch the CLI
|
||||
# would have made. Note the difference from GitHub that makes this step
|
||||
# look redundant: Gitea does *not* suppress workflow triggers on a push
|
||||
# made with the Actions token, so the commit above will usually have
|
||||
# started ci.yml already. Dispatching anyway is deliberate — it is the only
|
||||
# thing that guarantees publication if that behaviour is ever configured
|
||||
# off, and a duplicate run is harmless because ci.yml's concurrency group
|
||||
# is per-ref with cancel-in-progress, so the second supersedes the first.
|
||||
- name: Publish the refreshed feed
|
||||
if: steps.diff.outputs.changed == 'true' && inputs.dry_run != true
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: gh workflow run ci.yml --ref "${{ github.ref_name }}"
|
||||
TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
API: ${{ github.server_url }}/api/v1
|
||||
REPO: ${{ github.repository }}
|
||||
BRANCH: ${{ github.ref_name }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
curl -sS --fail-with-body -X POST \
|
||||
-H "Authorization: token $TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
"$API/repos/$REPO/actions/workflows/ci.yml/dispatches" \
|
||||
-d "{\"ref\":\"$BRANCH\"}"
|
||||
|
||||
# A source that has failed three cycles running is broken, not down: that
|
||||
# game's calendar has been built from a checked-in fixture for a day and a
|
||||
|
||||
@@ -97,7 +97,7 @@ bun run parse endfield-wikigg-events fixtures/endfield/wikigg-events-2026-08-15.
|
||||
bun test test/dates.test.ts
|
||||
bun test --test-name-pattern "year-less"
|
||||
|
||||
# Hosting under a subpath (GitHub Pages)
|
||||
# Hosting under a subpath (CI does not set this — the image serves from /)
|
||||
BASE_PATH=/gacha-event-tracker/ bun run build
|
||||
```
|
||||
|
||||
@@ -348,11 +348,11 @@ AI-training crawlers. Our use is a low-rate personal aggregator with attribution
|
||||
training, and no `User-agent: *` rule applies to our paths. Keep it that way: do not raise the fetch
|
||||
rate, and do not add an LLM that consumes page content.
|
||||
|
||||
**game8.co does not answer a GitHub Actions runner** (confirmed 2026-08-17). Its edge returns
|
||||
**game8.co did not answer a GitHub Actions runner** (confirmed 2026-08-17). Its edge returned
|
||||
`202 Accepted` with a bot-management body to every one of the nine game8 sources, from the first
|
||||
scheduled cycle onward — `last confirmed: never` — while the same URLs return `200` and parse
|
||||
cleanly from a normal address. So `robots.txt` permits us and the network does not, and those nine
|
||||
games have only ever been built from checked-in fixtures in CI.
|
||||
cleanly from a normal address. So `robots.txt` permitted us and the network did not, and those nine
|
||||
games had only ever been built from checked-in fixtures in CI.
|
||||
|
||||
The per-host spacing above does not fix this and was not meant to: a 202 on the very first request
|
||||
of a cycle is address reputation, not rate. **Do not work around it.** Browser-shaped headers, a
|
||||
@@ -361,6 +361,16 @@ same reason `uma.moe` was declined below — and unlike `uma.moe` we would be do
|
||||
`robots.txt` was welcoming, which makes it worse, not better. The legitimate options are to run the
|
||||
refresh from an address game8 will serve, or to find those games another source.
|
||||
|
||||
**CI moved to a Gitea runner on 2026-09-16, which is the first of those two options — and its result
|
||||
is not yet known.** A different address is exactly the variable this failure turns on, so the nine
|
||||
game8 sources may now refresh on schedule, or may not. Nothing here should be rewritten on the
|
||||
strength of that hope: the tense above stays past because the GitHub evidence is real and the Gitea
|
||||
evidence does not exist yet. Establish it the same way it was established before — let a cycle run,
|
||||
then read `git log` on `snapshots/` for which hosts actually landed — and only then update this
|
||||
section, `README.md` § Status, `docs/SOURCES.md` § Method and `docs/FEEDBACK.md`'s P0 row together.
|
||||
A single manual `--only` dispatch is enough to answer it for one source; note that `--dry-run`
|
||||
cannot, because it makes no requests at all by design.
|
||||
|
||||
A source whose ToS forbids automated access does not get an adapter. Flag it and ask.
|
||||
|
||||
**Sources assessed and declined** (2026-08-17, extended 2026-08-19), so these are not
|
||||
|
||||
@@ -26,23 +26,33 @@ review queue are specified but not built.
|
||||
| Input sanitization at the ingest boundary | Built, tested |
|
||||
| Scheduled refresh — robots, snapshots, commit-on-change | Built, tested offline |
|
||||
| Web interface, daily checklists, offline support | Built |
|
||||
| Static server, Docker image, GitHub + GitLab CI | Built |
|
||||
| Static server, Docker image, Gitea + GitLab CI | Built |
|
||||
| SQLite, review queue | Specified in `docs/`, not built |
|
||||
|
||||
The feed is still a static JSON file rather than a database read: the refresh job commits the raw
|
||||
pages it fetched, CI rebuilds the feed from them, and a clean checkout with no snapshots falls back
|
||||
to the checked-in fixtures — so the build stays offline and reproducible either way.
|
||||
|
||||
**The scheduled refresh does not reach every source.** game8.co answers the GitHub Actions runner
|
||||
with a bot-management `202` instead of the page, so the nine Game8 sources only move when someone
|
||||
runs `bun run refresh` by hand from an address it will talk to. The wiki sources refresh on schedule
|
||||
**The scheduled refresh does not reach every source.** game8.co answered the GitHub Actions runner
|
||||
with a bot-management `202` instead of the page, so the nine Game8 sources only moved when someone
|
||||
ran `bun run refresh` by hand from an address it will talk to. The wiki sources refresh on schedule
|
||||
as intended. `docs/SOURCES.md` records which hosts answer the runner and which do not; the footer
|
||||
shows when each lane was last refreshed, so a stale one is visible rather than silent.
|
||||
|
||||
Since the move to Gitea the refresh runs from a **different address**, which is one of the two
|
||||
remedies `AGENTS.md` § Scraping conduct allows for this. Whether game8.co serves that address is
|
||||
**not yet established** — it needs one real single-source run on the runner, and until that has
|
||||
happened the paragraph above still describes what we know.
|
||||
|
||||
## Try it
|
||||
|
||||
Live at **<https://stereotypicalcat.github.io/gacha-event-tracker/>**, deployed from `main` by the
|
||||
Pages job below.
|
||||
Deployed from `main` as a container image, published to
|
||||
`gitea.lucaswinther.info/lucasw89/gacha-event-tracker:latest` by the CI job below. Run it with
|
||||
`docker run -p 3000:3000 gitea.lucaswinther.info/lucasw89/gacha-event-tracker:latest` and open
|
||||
<http://localhost:3000>.
|
||||
|
||||
> The previous GitHub Pages deploy at <https://stereotypicalcat.github.io/gacha-event-tracker/> is no
|
||||
> longer updated by this pipeline; it holds whatever `main` looked like when the repo moved.
|
||||
|
||||
Or run it yourself:
|
||||
|
||||
@@ -298,9 +308,9 @@ page is sanitized at the ingest boundary before it reaches the feed, the browser
|
||||
|
||||
## Found a problem, or want something?
|
||||
|
||||
- **[Report a problem](https://github.com/StereotypicalCat/gacha-event-tracker/issues/new?template=bug_report.yml)**
|
||||
- **[Report a problem](https://gitea.lucaswinther.info/lucasw89/gacha-event-tracker/issues/new?template=bug_report.yml)**
|
||||
— a wrong date, a missing event, a lost tick, anything the app got wrong.
|
||||
- **[Request a feature](https://github.com/StereotypicalCat/gacha-event-tracker/issues/new?template=feature_request.yml)**
|
||||
- **[Request a feature](https://gitea.lucaswinther.info/lucasw89/gacha-event-tracker/issues/new?template=feature_request.yml)**
|
||||
— including a game it does not cover yet, which is the most common ask by a distance.
|
||||
|
||||
Both are forms rather than a blank box, for one reason: nothing you mark, type or tick ever leaves
|
||||
@@ -331,15 +341,22 @@ Blank issues are still open for anything that fits neither form. The templates t
|
||||
|
||||
Both `.github/workflows/ci.yml` and `.gitlab-ci.yml` run the same gates on every push — typecheck,
|
||||
tests, and a feed sanity check — then build and publish a container image from the default branch.
|
||||
GitHub Actions can also deploy to Pages, but that needs two one-time steps it cannot do for itself —
|
||||
the default `GITHUB_TOKEN` is not allowed to create a Pages site:
|
||||
|
||||
1. **Settings → Pages → Source: GitHub Actions.**
|
||||
2. **Settings → Secrets and variables → Actions → Variables:** add `DEPLOY_PAGES` = `true`.
|
||||
The workflows run on **Gitea Actions**, not GitHub's. They stay under `.github/workflows/` because
|
||||
Gitea reads that directory natively, and because `test/refresh.test.ts` asserts on those two files by
|
||||
path — three defects that live in YAML and fail silently otherwise. Two consequences worth knowing
|
||||
before editing them:
|
||||
|
||||
Until then the `pages` job is skipped and the pipeline stays green. Pages is unavailable for private
|
||||
repositories on the free plan. Both steps are done here, and the deploy lands at
|
||||
<https://stereotypicalcat.github.io/gacha-event-tracker/>.
|
||||
- **`uses:` steps are written as full URLs** (`https://github.com/actions/checkout@v4`). A bare
|
||||
`actions/checkout@v4` resolves against the instance's `DEFAULT_ACTIONS_URL`, which is configuration
|
||||
we do not control from here. The runner needs outbound access to github.com to fetch them.
|
||||
- **The image is the deploy artefact.** There is no `pages` job: Gitea has no Pages equivalent, so
|
||||
the container image published to Gitea's registry is what gets deployed. `bun run build` therefore
|
||||
runs without `BASE_PATH` — the image serves from `/`.
|
||||
|
||||
The image lands at `gitea.lucaswinther.info/lucasw89/gacha-event-tracker`, tagged `latest` and with
|
||||
the commit SHA. The registry host is derived from `github.server_url` rather than hardcoded, so a
|
||||
fork on another instance pushes to its own registry.
|
||||
|
||||
The feed job fails if the event count collapses, or if any single source parses to nothing — nine
|
||||
healthy sources hide a tenth that has gone quiet, and the total stays comfortably over the floor
|
||||
@@ -354,13 +371,19 @@ pipeline always means the code changed rather than a wiki being down.
|
||||
|
||||
### Refreshing the data
|
||||
|
||||
GitHub Actions only — the GitLab pipeline still runs the gates, but nothing there fetches.
|
||||
Gitea Actions only — the GitLab pipeline still runs the gates, but nothing there fetches.
|
||||
`.github/workflows/refresh.yml` runs `bun run refresh` twice a day (and on demand, with a dry-run
|
||||
input). It fetches each source at most once per cycle, and **commits only when a page's bytes
|
||||
actually changed** — a `304`, an identical body, or a fetch that fails to parse all leave the
|
||||
working tree clean and produce no commit. When something did change it commits the raw snapshots and
|
||||
dispatches `ci.yml`, which typechecks, tests, rebuilds the feed and deploys through the path that
|
||||
already existed; none of that logic is duplicated.
|
||||
dispatches `ci.yml`, which typechecks, tests, rebuilds the feed and republishes the image through the
|
||||
path that already existed; none of that logic is duplicated.
|
||||
|
||||
That dispatch is a REST call rather than `gh workflow run`, there being no `gh` on a Gitea runner.
|
||||
It is also, strictly, belt and braces: unlike GitHub, Gitea does **not** suppress workflow triggers
|
||||
on a push made with the Actions token, so the snapshot commit will usually have started `ci.yml`
|
||||
already. A duplicate run is harmless — `ci.yml`'s concurrency group is per-ref with
|
||||
`cancel-in-progress`, so the second supersedes the first.
|
||||
|
||||
A body that yields zero events is rejected and the previous snapshot kept, so a wiki redesign shows
|
||||
up as a stale timestamp rather than an empty calendar. One source being down is a warning; every
|
||||
|
||||
+10
-6
@@ -37,7 +37,7 @@ are no users.
|
||||
```
|
||||
|
||||
**That is the design, and the scheduler half of it is not built yet.** What runs today is
|
||||
`scripts/refresh-sources.ts` on a GitHub Actions cron, writing snapshots to disk, and a feed built
|
||||
`scripts/refresh-sources.ts` on a Gitea Actions cron, writing snapshots to disk, and a feed built
|
||||
from those files rather than from SQLite — see § Today. The stages, the layering and the review gate
|
||||
are unchanged by that; only what wakes them up and where they land is.
|
||||
|
||||
@@ -230,8 +230,10 @@ gates and publish it.
|
||||
|
||||
Assets resolve against a `<base href>` substituted at build time, the feed URL resolves against
|
||||
`document.baseURI` so deep links work, and the service worker derives its paths from its own
|
||||
registration scope. `BASE_PATH=/gacha-event-tracker/ bun run build` for GitHub Pages; without it a
|
||||
subpath deploy 404s on every asset.
|
||||
registration scope. `BASE_PATH=/gacha-event-tracker/ bun run build` for any host that serves the app
|
||||
from a subpath; without it a subpath deploy 404s on every asset. CI no longer sets it — the container
|
||||
image is what gets deployed and `serve.ts` serves from `/` — but the mechanism stays, because a
|
||||
subpath deploy is a hosting decision rather than a build one.
|
||||
|
||||
### Offline
|
||||
|
||||
@@ -263,9 +265,11 @@ mode of the `fetch()` that follows it, and a mismatch is not a no-op — the bro
|
||||
preload and fetches the feed a second time, which is worse than not preloading. If that link is ever
|
||||
edited, count the feed requests in a real browser; nothing in the test suite can see this.
|
||||
|
||||
Compression is the server's job and both hosts do it: GitHub Pages transparently, and `serve.ts` —
|
||||
which is what the Docker image runs — by negotiating `accept-encoding` per request. It matters more
|
||||
than it sounds: the bundle is 344 KB raw and 100 KB gzipped.
|
||||
Compression is the server's job, and `serve.ts` — which is what the Docker image runs, and so what
|
||||
serves the deployed site — does it by negotiating `accept-encoding` per request. It matters more
|
||||
than it sounds: the bundle is 344 KB raw and 100 KB gzipped. A static host put in front of the built
|
||||
`public/` instead must do its own; GitHub Pages did this transparently while it was the deploy
|
||||
target, which is why nothing here ever had to.
|
||||
|
||||
### The theme, before the bundle arrives
|
||||
|
||||
|
||||
+1
-1
@@ -299,7 +299,7 @@ diagnosis in each item still holds — what changed is whether it has been acted
|
||||
|
||||
| Item | Status |
|
||||
|---|---|
|
||||
| P0 refresh pipeline | **Diagnosed, half acted on.** game8.co answers a GitHub Actions runner with `202` and a bot-management body, so those eight sources have only ever built from fixtures in CI — see `AGENTS.md` § Scraping conduct, including why it is not to be worked around. The `broken` tier now makes a source failing three cycles fail the run. Step 5 (a build assertion on snapshot age) is **not built** |
|
||||
| P0 refresh pipeline | **Diagnosed, half acted on.** game8.co answered a GitHub Actions runner with `202` and a bot-management body, so those eight sources had only ever built from fixtures in CI — see `AGENTS.md` § Scraping conduct, including why it is not to be worked around. CI moved to a Gitea runner on 2026-09-16, which changes the address this failure turns on; whether game8 serves it is **unverified**, and the row stays as-is until a cycle has run. The `broken` tier now makes a source failing three cycles fail the run. Step 5 (a build assertion on snapshot age) is **not built** |
|
||||
| P1a Arknights | **Done.** `arknights-akwiki-events`, via the new `akwiki` parser |
|
||||
| P1b `NextUp` → three | **Done** (2026-08-18). One headline and two behind it, off `nextToExpire` |
|
||||
| P1b cap the long list | **Done** (2026-08-18). Both sections cap at six with "show all N" — truncation of the view, not a re-sort |
|
||||
|
||||
+1
-1
@@ -318,7 +318,7 @@ on the section and leave every `h4` beneath it unable to take it back — publis
|
||||
|
||||
- Send `If-None-Match` / `If-Modified-Since` from `sources.etag` / `last_modified`. A `304` ends
|
||||
the run as `skipped_unchanged`.
|
||||
- `User-Agent: gacha-event-tracker/1.0 (+https://github.com/<owner>/gacha-event-tracker)`.
|
||||
- `User-Agent: gacha-event-tracker/1.0 (+https://gitea.lucaswinther.info/<owner>/gacha-event-tracker)`.
|
||||
- Honor `robots.txt`; cache parsed robots per host for 24h. **Fail closed** — a `robots.txt` that
|
||||
5xxs or times out means "do not fetch", because a permission we could not read is not a
|
||||
permission we have. A 404 means no restrictions.
|
||||
|
||||
+22
-12
@@ -27,23 +27,33 @@ the ruling.
|
||||
## Method
|
||||
|
||||
Every page below was fetched from this machine with the refresh runner's own `User-Agent`
|
||||
(`gacha-event-tracker/1.0 (+https://github.com/StereotypicalCat/gacha-event-tracker)`), one or two
|
||||
requests per host, `robots.txt` first. No browser-shaped headers, no proxy, no JS execution —
|
||||
(`gacha-event-tracker/1.0 (+https://github.com/StereotypicalCat/gacha-event-tracker)` — the contact
|
||||
URL at the time; the repo has since moved to Gitea and the runner now sends that address instead),
|
||||
one or two requests per host, `robots.txt` first. No browser-shaped headers, no proxy, no JS execution —
|
||||
anything that only answers a browser is treated as closed, per the `uma.moe` precedent. Where an
|
||||
existing parser could plausibly read a page, it was run against the fetched bytes offline
|
||||
(`src/ingest/parsers/index.ts`) rather than guessed at.
|
||||
|
||||
Two things this method cannot tell you, and both matter:
|
||||
|
||||
- **Whether a host answers the GitHub Actions runner.** game8.co does not (`AGENTS.md` § Scraping
|
||||
conduct). The only hard evidence we hold is what CI has actually committed: `git log` on
|
||||
`snapshots/` shows `github-actions[bot]` landing **arknights.wiki.gg**, **endfield.wiki.gg** and
|
||||
**bluearchive.wiki** (Miraheze). Fandom sources have never refreshed in CI, because `robots.txt`
|
||||
itself is challenged from a datacentre address and the gate fails closed — and as of 2026-08-19
|
||||
that challenge covers every Fandom wiki from *any* address tried here, not just CI's.
|
||||
Of the 2026-08-19 additions, **Stella Sora** is Miraheze and so is the one likely to refresh in CI;
|
||||
**IOP Wiki** is permissive but unproven; **Chaos Zero Nightmare** and **Umamusume** are game8 and
|
||||
therefore blind in CI by construction.
|
||||
- **Whether a host answers the CI runner.** This is a property of the runner's address, not of the
|
||||
page, so it has to be re-established whenever that address changes — and it just did: CI moved from
|
||||
GitHub Actions to a Gitea runner (`AGENTS.md` § Scraping conduct). **Everything in this bullet
|
||||
below is evidence about the old GitHub address.** Treat it as the prior, not the current state,
|
||||
until a cycle has run on Gitea.
|
||||
|
||||
game8.co did not answer the GitHub runner. The only hard evidence we held was what CI had actually
|
||||
committed: `git log` on `snapshots/` shows `github-actions[bot]` landing **arknights.wiki.gg**,
|
||||
**endfield.wiki.gg** and **bluearchive.wiki** (Miraheze). Fandom sources never refreshed in CI,
|
||||
because `robots.txt` itself is challenged from a datacentre address and the gate fails closed — and
|
||||
as of 2026-08-19 that challenge covered every Fandom wiki from *any* address tried here, not just
|
||||
CI's. Of the 2026-08-19 additions, **Stella Sora** is Miraheze and so was the one likely to refresh
|
||||
in CI; **IOP Wiki** is permissive but unproven; **Chaos Zero Nightmare** and **Umamusume** are
|
||||
game8 and were therefore blind in CI by construction.
|
||||
|
||||
The Gitea runner may well be served where GitHub's was not — that is one of the two remedies
|
||||
`AGENTS.md` allows — but "may well be" is not evidence. The check is the same one that produced the
|
||||
list above: run a cycle, then read `git log` on `snapshots/` for what actually landed.
|
||||
- **Whether a page will still look like this in six weeks.** Each recommendation below names the
|
||||
assertion `canParse` should make, so a redesign fails the source loudly instead of emptying a lane.
|
||||
|
||||
@@ -836,7 +846,7 @@ if it ever prints a year (§ 13). None is worth a pass until something changes u
|
||||
## Appendix — reproducing the checks
|
||||
|
||||
```bash
|
||||
UA='gacha-event-tracker/1.0 (+https://github.com/StereotypicalCat/gacha-event-tracker)'
|
||||
UA='gacha-event-tracker/1.0 (+https://gitea.lucaswinther.info/lucasw89/gacha-event-tracker)'
|
||||
|
||||
# conduct
|
||||
curl -sS -A "$UA" https://iopwiki.com/robots.txt
|
||||
|
||||
@@ -55,7 +55,7 @@ import { RobotsCache, type FetchLike } from "../src/ingest/robots.ts";
|
||||
import { decodeBody, SnapshotStore } from "../src/ingest/snapshots.ts";
|
||||
|
||||
const DEFAULT_CONTACT =
|
||||
"https://github.com/StereotypicalCat/gacha-event-tracker";
|
||||
"https://gitea.lucaswinther.info/lucasw89/gacha-event-tracker";
|
||||
|
||||
export const DEFAULT_USER_AGENT = `gacha-event-tracker/1.0 (+${process.env["REFRESH_CONTACT_URL"] ?? DEFAULT_CONTACT})`;
|
||||
|
||||
|
||||
@@ -23,10 +23,12 @@ const ROOT_DIR = resolve(ROOT);
|
||||
* times the bytes and, on anything slower than a laptop on wifi, three times the
|
||||
* download.
|
||||
*
|
||||
* GitHub Pages compresses for us, so the deployed site never had this problem;
|
||||
* the Docker image serves through this file and did. An image whose payload is
|
||||
* 3x the site's is not a placeholder detail — it is the only thing a self-hoster
|
||||
* ever sees.
|
||||
* This used to be the self-hoster's problem only: GitHub Pages compressed for
|
||||
* us, so the deployed site never had it, while the Docker image serves through
|
||||
* this file and did. Since the move to Gitea the image *is* the deploy, so this
|
||||
* is now the only compression the deployed site gets — there is no host behind
|
||||
* it doing the job silently. Serving 3x the bytes is not a placeholder detail
|
||||
* when it is what every reader gets.
|
||||
*/
|
||||
const COMPRESSIBLE = new Set([
|
||||
".html",
|
||||
|
||||
@@ -142,12 +142,15 @@ export function agentToken(userAgent: string): string {
|
||||
*
|
||||
* RFC 9309 § 2.2.1 matches the *product token* — the header up to the first
|
||||
* `/` — not the header text. Matching anywhere in the header is actively
|
||||
* dangerous here: our contact URL contains the string `StereotypicalCat`, so a
|
||||
* `User-agent: cat` group elsewhere in the file would be treated as naming us,
|
||||
* and because a named group replaces the `*` group outright, that unrelated
|
||||
* group's rules would *discard* every rule the site actually wrote for us.
|
||||
* Erring towards obeying more rules means never letting a coincidence take a
|
||||
* `*` group away.
|
||||
* dangerous here, because our header carries a contact URL and a URL is a long
|
||||
* string of arbitrary substrings. The case that caught it: the contact URL was
|
||||
* once `…/StereotypicalCat/…`, so a `User-agent: cat` group elsewhere in the
|
||||
* file was treated as naming us, and because a named group replaces the `*`
|
||||
* group outright, that unrelated group's rules *discarded* every rule the site
|
||||
* actually wrote for us. The URL has since changed and that particular
|
||||
* coincidence is gone; the hazard is not, since the next contact URL is just as
|
||||
* arbitrary. Erring towards obeying more rules means never letting a
|
||||
* coincidence take a `*` group away.
|
||||
*
|
||||
* A robots.txt that names us with a version (`gacha-event-tracker/1.0`) is
|
||||
* still honoured: the group's own product token is compared too.
|
||||
|
||||
+6
-5
@@ -117,12 +117,13 @@ describe("static server", () => {
|
||||
});
|
||||
|
||||
/**
|
||||
* Compression, which is the whole difference between the Docker image and the
|
||||
* deployed site.
|
||||
* Compression, which used to be the whole difference between the Docker image
|
||||
* and the deployed site.
|
||||
*
|
||||
* GitHub Pages gzips on our behalf, so the bundle crosses the wire at a third of
|
||||
* its size there and did not here — and this file is what the image runs. Three
|
||||
* times the bytes is the only thing a self-hoster would ever have seen.
|
||||
* GitHub Pages gzipped on our behalf, so the bundle crossed the wire at a third
|
||||
* of its size there and did not here — and serve.ts is what the image runs.
|
||||
* Since the move to Gitea the image is the deploy, so these assertions cover
|
||||
* every reader rather than only a self-hoster: nothing else compresses now.
|
||||
*/
|
||||
describe("static server: compression", () => {
|
||||
test("gzips a text asset for a client that asks", async () => {
|
||||
|
||||
Reference in New Issue
Block a user