diff --git a/.github/workflows/connectivity.yml b/.github/workflows/connectivity.yml new file mode 100644 index 0000000..d272d71 --- /dev/null +++ b/.github/workflows/connectivity.yml @@ -0,0 +1,127 @@ +--- +# Diagnostics, for when a deploy job hangs instead of failing. +# +# Two jobs, deliberately separate: +# +# ping uses the real deploy key and asks Ansible to talk to every host. This +# is the right first thing to run after setting the secrets up, and the +# right first thing to run when a deploy misbehaves. It changes nothing. +# probe uses no credentials at all and answers a narrower question: where do +# the packets stop? Run it when ping times out. +# +# The distinction that matters, and the reason `probe` exists: +# +# Permission denied (publickey) the key is wrong — you reached sshd +# Connection refused you reached the host, nothing is listening +# Connection timed out packets are being dropped in transit +# +# Only the third is a network problem, and no amount of re-pasting the key will +# fix it. See docs/connectivity.md. +name: connectivity + +on: + workflow_dispatch: + inputs: + hosts: + description: Space-separated hosts to probe (defaults to the inventory) + required: false + default: edge.example.com app.example.com + +permissions: + contents: read + +jobs: + ping: + name: ansible ping + runs-on: ${{ vars.DEPLOY_RUNNER || 'ubuntu-latest' }} + environment: production + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install the pinned toolchain + run: | + set -eu + scripts/check.sh --print-specs | head -1 | xargs -d '\n' pip install --quiet + - name: Load the deploy key + env: + SSH_KEY_BACKEND: ci + SSH_PRIVATE_KEY_B64: ${{ secrets.SSH_PRIVATE_KEY_B64 }} + run: | + set -eu + SSH_KEY_PATH="$(scripts/load-ssh-key.sh)" + echo "SSH_KEY_PATH=$SSH_KEY_PATH" >> "$GITHUB_ENV" + # Success looks like `SUCCESS => {"ping": "pong"}` for every host. That + # proves the runner decoded the key, reached port 22, authenticated, and + # ran Python on the far side — which is everything a deploy needs except + # the playbook itself. + - name: Ping every host in the inventory + working-directory: ansible + run: ansible all -m ansible.builtin.ping + + probe: + name: network probe + runs-on: ${{ vars.DEPLOY_RUNNER || 'ubuntu-latest' }} + # A report, not a gate. Every section must run even when the thing it probes + # is unreachable, which is the normal case when you are running this. + continue-on-error: true + steps: + - name: Probe + env: + HOSTS: ${{ github.event.inputs.hosts }} + run: | + set +e + + # A bare TCP connect can be answered by a transparent proxy that then + # says nothing, which looks like success and is not. Real sshd greets + # first, so the banner is the only honest proof you reached the server. + ssh_banner() { + b=$(timeout 6 bash -c "exec 3<>/dev/tcp/$1/${2:-22}; head -c 12 <&3" 2>/dev/null || true) + case "$b" in + SSH-*) printf ' %-26s %-5s OK, sshd replied: %s\n' "$1" "${2:-22}" "$b" ;; + '') printf ' %-26s %-5s NO REPLY (dropped, or a silent proxy)\n' "$1" "${2:-22}" ;; + *) printf ' %-26s %-5s answered, but not sshd: %s\n' "$1" "${2:-22}" "$b" ;; + esac + } + + # Distinguishes "refused" (allowed out, nothing listening) from + # "filtered" (dropped in transit). This is the whole point of the job: + # if every port but 443 is filtered, you are behind an egress allowlist + # and no alternative SSH port will help. + port_state() { + # `rc=0; out=$(...) || rc=$?` and not `out=$(...); rc=$?` — the + # latter is a failing simple command, which under set -e ends the + # step before rc is ever read, so a filtered port aborts the report + # instead of describing it. + rc=0 + out=$(timeout 5 bash -c "cat < /dev/null > /dev/tcp/$1/$2" 2>&1) || rc=$? + if [ "$rc" -eq 0 ]; then s='open' + elif printf '%s' "$out" | grep -qi refused; then + s='refused -> port allowed out' + elif [ "$rc" -eq 124 ]; then + s='filtered -> blocked in transit' + else + s="unclear (rc=$rc) $out" + fi + printf ' %-26s %-5s %s\n' "$1" "$2" "$s" + } + + printf '== 0. does this runner have a network at all? ==\n' + printf ' DNS: '; getent hosts github.com >/dev/null 2>&1 && echo resolves || echo 'CANNOT RESOLVE' + printf ' https external: ' + curl -s -o /dev/null -w '%{http_code}\n' --max-time 10 \ + https://github.com || echo 'NO EGRESS' + printf ' egress address: '; curl -s --max-time 10 https://ifconfig.me || echo unknown + + printf '\n\n== 1. SSH to the target hosts ==\n' + for h in $HOSTS; do ssh_banner "$h"; done + + printf '\n== 2. SSH elsewhere: is outbound 22 blocked wholesale? ==\n' + ssh_banner github.com + + printf '\n== 3. Which ports are allowed out at all? ==\n' + for h in $HOSTS; do + port_state "$h" 22 + port_state "$h" 443 + done diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 0000000..ddd64cf --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,151 @@ +--- +# Deploys. Two ways in, and only two: +# +# - a human fills in the form under Actions > deploy > Run workflow +# - another repository asks for a stack by name, via repository_dispatch +# +# Nothing deploys on a push. That is the single most important property of this +# file: a merge to the default branch changes what *would* be deployed, and a +# person still decides when. +# +# GitHub has no per-job manual button, so GitLab's grid of check: and +# deploy: buttons becomes one form with a `stack` dropdown. The list under +# `options:` has to be kept in step with ansible/playbooks/ by hand — see +# skills/adding-a-stack/SKILL.md. Setup: docs/ci-github-actions.md. +name: deploy + +on: + workflow_dispatch: + inputs: + stack: + description: Which stack to act on + required: true + type: choice + options: + - site + - reverse-proxy + - banner + - metrics + - static-site + - webapp + - webapp-staging + mode: + description: check reports what would change; deploy does it + required: true + default: check + type: choice + options: + - check + - deploy + pull: + description: >- + Image pull policy. always fetches newer images and enables per-stack + update work (backups, migrations). + required: true + default: policy + type: choice + options: + - policy + - always + - missing + - never + repository_dispatch: + types: [deploy] + +# Two deploys of the same stack at once would race on the host. +concurrency: + group: deploy-${{ github.event.inputs.stack || github.event.client_payload.stack }} + cancel-in-progress: false + +permissions: + contents: read + +jobs: + ansible: + name: >- + ${{ github.event.inputs.mode || 'deploy' }} + ${{ github.event.inputs.stack || github.event.client_payload.stack }} + # Self-hosted when the hosts are not reachable from GitHub's network — set + # the DEPLOY_RUNNER repository variable to the runner's label. See + # "Runners" in docs/ci-github-actions.md. + runs-on: ${{ vars.DEPLOY_RUNNER || 'ubuntu-latest' }} + # The Environment is what replaces GitLab's Protected variables: the deploy + # key is stored on it, so it exists only for jobs that name it, and its + # protection rules (required reviewers, allowed branches) gate every run. + environment: production + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + # STACK, MODE and PULL reach a shell command, and under repository_dispatch + # they come from another repository. They go through the environment, never + # through ${{ }} interpolation into a run: block — an expression is + # substituted into the script *before* the shell sees it, so a crafted + # value would be executed rather than compared. Validate, then use. + - name: Validate the requested stack and pull policy + env: + STACK: ${{ github.event.inputs.stack || github.event.client_payload.stack }} + MODE: ${{ github.event.inputs.mode || github.event.client_payload.mode || 'deploy' }} + PULL: ${{ github.event.inputs.pull || github.event.client_payload.pull || 'policy' }} + run: | + set -eu + case "$STACK" in + ''|*[!a-z0-9_-]*) echo "refusing STACK='$STACK'"; exit 1 ;; + esac + case "$MODE" in + check|deploy) ;; + *) echo "refusing MODE='$MODE'"; exit 1 ;; + esac + # An unchecked value can smuggle extra `-e` overrides onto the command + # line (stack_dest, ansible_host) and redirect the deploy somewhere + # else entirely. + case "$PULL" in + policy|always|missing|never) ;; + *) echo "refusing PULL='$PULL'"; exit 1 ;; + esac + test -f "ansible/playbooks/${STACK}.yml" \ + || { echo "no playbook for '$STACK'"; exit 1; } + { + echo "STACK=$STACK" + echo "MODE=$MODE" + echo "PULL=$PULL" + } >> "$GITHUB_ENV" + + - name: Install the pinned toolchain + run: | + set -eu + scripts/check.sh --print-specs | head -1 | xargs -d '\n' pip install --quiet + ansible-galaxy install -r ansible/requirements.yml + + # Two lines, not `export SSH_KEY_PATH="$(...)"`. export is a special + # builtin, so the compound command reports *its* exit status and swallows + # the script's — under `set -e` a loader failure goes unnoticed and the job + # dies much later with a per-host "Permission denied (publickey)" instead + # of the loader's own message saying which variable was missing. + - name: Load the deploy key + env: + SSH_KEY_BACKEND: ci + SSH_PRIVATE_KEY_B64: ${{ secrets.SSH_PRIVATE_KEY_B64 }} + run: | + set -eu + SSH_KEY_PATH="$(scripts/load-ssh-key.sh)" + echo "SSH_KEY_PATH=$SSH_KEY_PATH" >> "$GITHUB_ENV" + + - name: Run the playbook + working-directory: ansible + env: + ANSIBLE_FORCE_COLOR: "1" + run: | + set -eu + # An `if`, not `[ ... ] && extra=...`: when the test fails the AND-list + # returns non-zero, and under `set -e` that ends the job right here + # with no error message and a green-looking cancel. + if [ "$MODE" = "check" ]; then + set -- --check --diff + else + set -- + fi + ansible-playbook "playbooks/${STACK}.yml" "$@" -e "pull=$PULL" diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000..223fe9f --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,73 @@ +--- +# The static gate. Runs on every push and pull request, needs no access to any +# host, and is the definition of "done" for a change here. +# +# Everything below is `scripts/check.sh` plus two scans. Keeping the lint steps +# in a script rather than inline is deliberate: a contributor can run the exact +# same gate locally with `make check` before pushing, and the two cannot drift. +name: lint + +on: + push: + branches: ["**"] + pull_request: + +# A second push to the same branch makes the first run pointless. +concurrency: + group: lint-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + ansible: + name: yamllint + ansible-lint + syntax + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + # scripts/check.sh is the single source of the version pins; it prints + # them so that this workflow can install exactly what a contributor runs + # locally. Nothing here can drift from the script. + - name: Install the pinned toolchain + run: | + set -eu + scripts/check.sh --print-specs | tee /dev/stderr | xargs -d '\n' pip install --quiet + + - name: Run the static gate + run: RUNNER=installed scripts/check.sh + + shell: + name: shellcheck + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + # Pre-installed on GitHub-hosted Ubuntu runners; installed explicitly so + # this job also works on a self-hosted runner with a bare image. + - name: Install shellcheck + run: command -v shellcheck || sudo apt-get install -y --no-install-recommends shellcheck + - name: Check the shell scripts + run: shellcheck scripts/*.sh server/*/*/app/*.sh + + secrets: + name: secret scan + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + # detect scans history, not just the working tree. A secret that was + # committed and then removed is still a leaked secret. + fetch-depth: 0 + # Run the upstream image directly rather than the marketplace action: the + # action requires a licence key for organisation-owned repositories, and + # silently no-ops without one. The image has no such condition. + - name: Scan the working tree and history for committed secrets + run: | + docker run --rm -v "$PWD:/repo" zricethezav/gitleaks:latest \ + detect --source=/repo --config=/repo/.gitleaks.toml \ + --redact --verbose --no-banner diff --git a/docs/ci-github-actions.md b/docs/ci-github-actions.md new file mode 100644 index 0000000..7fe87b5 --- /dev/null +++ b/docs/ci-github-actions.md @@ -0,0 +1,145 @@ +# CI on GitHub Actions + +Three workflows. There is a Gitea port of all three in +[`.gitea/workflows/`](../.gitea/workflows) — see [ci-gitea.md](ci-gitea.md), +and note that Gitea reads that directory *instead of* this one. + +| Workflow | Trigger | Needs host access | +| --- | --- | --- | +| [`lint.yml`](../.github/workflows/lint.yml) | every push and pull request | no | +| [`deploy.yml`](../.github/workflows/deploy.yml) | manual, or `repository_dispatch` | yes | +| [`connectivity.yml`](../.github/workflows/connectivity.yml) | manual | yes (`ping` only) | + +## Setup + +### 1. Create the `production` environment + +Settings → Environments → New environment → `production`. + +This is where the deploy key lives, and it is what replaces GitLab's "Protected +variable" concept. A secret on an environment is invisible to any job that does +not name that environment, and the environment's own rules gate the job: + +- **Required reviewers** — a deploy waits for a named person to approve it. + Worth turning on. It makes "press the button" a two-person operation without + any change to this repository. +- **Deployment branches** — restrict to your default branch so a deploy cannot + be run from an arbitrary fork branch. + +### 2. Add the deploy key + +On that environment, add a secret `SSH_PRIVATE_KEY_B64` holding the base64 of +your deploy private key. See [secrets.md](secrets.md) for how to encode it, and +what not to use on Windows. + +The matching public key must be in the deploy user's `authorized_keys` on every +host. + +### 3. Point at a runner + +Deploys need to reach your hosts over SSH. GitHub-hosted runners have general +internet egress, so if your hosts accept SSH from the internet, nothing to do. + +If they do not — a private network, an IP allowlist, or a corporate egress +filter — register a self-hosted runner somewhere that *can* reach them, and set +a repository **variable** (not a secret) `DEPLOY_RUNNER` to its label: + +``` +Settings → Secrets and variables → Actions → Variables +DEPLOY_RUNNER = my-deploy-runner +``` + +Both host-touching workflows read it: + +```yaml +runs-on: ${{ vars.DEPLOY_RUNNER || 'ubuntu-latest' }} +``` + +A runner only makes *outbound* HTTPS connections to GitHub, which is why this +works where an inbound exception would need a policy change. Do not set the +variable before a runner carries that label — a label no runner has leaves jobs +queued forever rather than failing. + +## Deploying + +Actions → **deploy** → Run workflow. Three fields: + +| Field | Values | Meaning | +| --- | --- | --- | +| `stack` | the dropdown | Which playbook to run. `site` is all of them. | +| `mode` | `check`, `deploy` | `check` runs `--check --diff` and touches nothing | +| `pull` | `policy`, `always`, `missing`, `never` | `always` fetches newer images **and** enables per-stack update work | + +Work up from harmless to real the first time: + +| Step | What to run | What it proves | +| --- | --- | --- | +| 1 | **connectivity** → `ping` | The runner can decode the key, reach every host over SSH, and run Python there. Changes nothing. | +| 2 | **deploy** with `mode: check` | What that deploy *would* change. | +| 3 | **deploy** with `mode: deploy` | The real thing. | + +`static-site` is the safest stack to prove this with: one container behind the +proxy, no database and no persistent state, so redeploying it costs nothing if it +goes wrong. + +## Differences from GitLab that shape this file + +**There is no per-job manual button.** GitLab renders a play button for every +`when: manual` job, which is how the `.gitlab-ci.yml` here ends up with a +`check:` and `deploy:` pair per stack. GitHub has one "Run workflow" form per +workflow, so the stack becomes a `choice` input instead. + +The practical cost is that the `options:` list in `deploy.yml` has to be kept in +step with `ansible/playbooks/` by hand. There is no way to populate a `choice` +dynamically. Adding a stack means editing that list — it is step 4 of +[adding a stack](../skills/adding-a-stack/SKILL.md), and forgetting it is how a +service ends up deployable only from a laptop. + +**`repository_dispatch` does not wait.** GitLab's `strategy: depend` makes an +upstream pipeline mirror the downstream result. A `repository_dispatch` returns +as soon as the event is accepted, so the caller learns nothing about whether the +deploy worked. If you need the caller to block, see +[triggering-deploys.md](triggering-deploys.md). + +**Environments replace protected variables.** GitLab hides a Protected variable +from pipelines on unprotected refs, which produces the confusing failure of a +variable that exists but is empty. GitHub's equivalent is scoping the secret to +an environment and restricting which branches may deploy to it — the job is +blocked outright rather than running with an empty key. + +## Why untrusted input is validated + +`deploy.yml` matches `stack` against `[a-z0-9_-]+` and `pull` against a fixed +list before running anything, and confirms the playbook exists. Under +`workflow_dispatch` the values come from a dropdown and this looks redundant. +Under `repository_dispatch` they come from another repository's payload, and they +reach a shell command. + +More importantly, they never go into a `run:` block through `${{ }}`. A workflow +expression is substituted into the script text *before the shell sees it*, so a +value containing shell syntax is executed rather than compared. Values go through +`env:` and are read as `"$STACK"`: + +```yaml + env: + STACK: ${{ github.event.inputs.stack || github.event.client_payload.stack }} + run: | + case "$STACK" in + ''|*[!a-z0-9_-]*) echo "refusing STACK='$STACK'"; exit 1 ;; + esac +``` + +Keep that shape if you restructure the job. It is the difference between a +validated argument and remote code execution on your deploy runner. + +## Troubleshooting + +| Job fails with | Cause | +| --- | --- | +| `set SSH_PRIVATE_KEY_B64, or SSH_PRIVATE_KEY ...` | The secret is not on the `production` environment, or the job does not name that environment | +| `does not look like a private key` | The value was encoded with `certutil` or `Get-Content`, or re-wrapped across lines | +| `Permission denied (publickey)` | The matching public key is not in the deploy user's `authorized_keys` | +| `Connection timed out` | Network path, not credentials — run **connectivity** → `probe`, and read [connectivity.md](connectivity.md) | +| `no playbook for '...'` | The `stack` value does not match a file in `ansible/playbooks/` | +| Job queued forever | `DEPLOY_RUNNER` names a label no runner carries | +| Deploy succeeds, nothing changed | Images are tagged `latest` and `pull` was `policy` |