From 665ac7d52f5b133783e657b69af21c90e009584a Mon Sep 17 00:00:00 2001 From: Lucas Winther Date: Wed, 30 Sep 2026 15:15:04 +0200 Subject: [PATCH] Render stack secrets from OpenBao in compose_stack stack_secrets reads KV v2 on the target host and writes the stack's .env at 0600, failing loudly on a sealed vault, a bad token, a missing field or an unsafe value. Adds an example OpenBao stack on app and moves webapp onto it; metrics keeps a hand-placed .env. Co-Authored-By: Claude Opus 5.5 --- .gitea/workflows/deploy.yml | 6 + .github/workflows/deploy.yml | 6 + .gitlab-ci.yml | 13 + AGENTS.md | 50 ++- CONTRIBUTING.md | 6 +- README.md | 43 ++- ansible/playbooks/metrics.yml | 24 +- ansible/playbooks/openbao.yml | 53 +++ ansible/playbooks/site.yml | 5 +- ansible/playbooks/webapp-staging.yml | 9 + ansible/playbooks/webapp.yml | 36 +-- ansible/roles/compose_stack/defaults/main.yml | 27 ++ ansible/roles/compose_stack/tasks/secrets.yml | 176 ++++++++++ ansible/roles/compose_stack/tasks/sync.yml | 8 + docs/ci-gitea.md | 5 + docs/ci-github-actions.md | 4 + docs/ci-gitlab.md | 3 +- docs/secrets.md | 302 ++++++++++++++++-- server/README.md | 13 +- server/app/openbao/config/openbao.hcl | 21 ++ server/app/openbao/docker-compose.yml | 55 ++++ server/app/webapp/.env.example | 14 +- server/app/webapp/docker-compose.yml | 12 +- skills/adding-a-stack/SKILL.md | 30 +- 24 files changed, 834 insertions(+), 87 deletions(-) create mode 100644 ansible/playbooks/openbao.yml create mode 100644 ansible/roles/compose_stack/tasks/secrets.yml create mode 100644 server/app/openbao/config/openbao.hcl create mode 100644 server/app/openbao/docker-compose.yml diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy.yml index ff32f1a..abe7584 100644 --- a/.gitea/workflows/deploy.yml +++ b/.gitea/workflows/deploy.yml @@ -34,6 +34,7 @@ on: - reverse-proxy - banner - metrics + - openbao - static-site - webapp - webapp-staging @@ -161,6 +162,11 @@ jobs: working-directory: ansible env: ANSIBLE_FORCE_COLOR: "1" + # Read by the compose_stack role through lookup('env', ...) for stacks + # that set stack_secrets. It goes through the environment and never + # through ${{ }} into the run: body, for the reason the Validate step + # above spells out. See docs/secrets.md. + VAULT_TOKEN: ${{ secrets.VAULT_TOKEN }} run: | set -eu # An `if`, not `[ ... ] && extra=...`: when the test fails the AND-list diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index f6d915d..0b9feb4 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -26,6 +26,7 @@ on: - reverse-proxy - banner - metrics + - openbao - static-site - webapp - webapp-staging @@ -145,6 +146,11 @@ jobs: working-directory: ansible env: ANSIBLE_FORCE_COLOR: "1" + # Read by the compose_stack role through lookup('env', ...) for stacks + # that set stack_secrets. It goes through the environment and never + # through ${{ }} into the run: body, for the reason the Validate step + # above spells out. See docs/secrets.md. + VAULT_TOKEN: ${{ secrets.VAULT_TOKEN }} run: | set -eu # An `if`, not `[ ... ] && extra=...`: when the test fails the AND-list diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 11dd99c..1a6cfa2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -63,6 +63,9 @@ lint: tags: - deploy-runner image: python:3.12-slim + # VAULT_TOKEN, for stacks that set stack_secrets, is a CI/CD variable and so + # already in the job's environment, where the compose_stack role reads it. + # Mark it Masked and Protected, like the deploy key. See docs/secrets.md. variables: SSH_KEY_BACKEND: ci ANSIBLE_FORCE_COLOR: '1' @@ -201,6 +204,16 @@ deploy:metrics: variables: STACK_FIXED: metrics +check:openbao: + extends: .check + variables: + STACK_FIXED: openbao + +deploy:openbao: + extends: .deploy + variables: + STACK_FIXED: openbao + check:static-site: extends: .check variables: diff --git a/AGENTS.md b/AGENTS.md index 6e87dd0..3b4c75e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -74,7 +74,8 @@ These are the conventions. Follow them; do not invent per-stack variations. not `sync files`. 8. **Idempotent.** A second run must report zero changes, and `--check --diff` must not error. -9. **No secrets in the repository.** `.env` files stay on the hosts. Never commit +9. **No secrets in the repository.** `.env` files stay on the hosts — rendered + from OpenBao by `stack_secrets`, or placed by hand. Never commit a key, a password, or a token — including inside a compose file, where no exclude can help you. See `docs/secrets.md`. @@ -98,10 +99,16 @@ stack needs something new; do not work around it with loose tasks. | `stack_build` | `policy` | `always` for stacks built from a Dockerfile | | `stack_state` | `present` | `present`, `absent`, or `restarted` | | `stack_start` | `{{ start \| default(true) }}` | `false` syncs and starts nothing | +| `stack_secrets` | `{}` | Secrets to fetch from the vault and render to `.env` | +| `stack_vault_addr` | `{{ vault_addr \| default('http://127.0.0.1:8200') }}` | The vault, as seen from the target host | +| `stack_vault_mount` | `{{ vault_kv_mount \| default('kv') }}` | KV v2 mount point | +| `stack_vault_token` | `$VAULT_TOKEN` | Read from the control node's environment | Worked examples for each: `static-site.yml` (the minimum), `reverse-proxy.yml` (`stack_dirs`, reload), `metrics.yml` (`stack_src`, per-host maps, ownership), -`webapp.yml` (two phases, `stack_files`, `stack_build`), `webapp-staging.yml` +`webapp.yml` (two phases, `stack_files`, `stack_build`, `stack_secrets`), +`openbao.yml` (owner-only escalation; a notice rather than a restart on +`compose_stack_synced`), `webapp-staging.yml` (`stack_src` + `stack_dest` + `stack_env`), `banner.yml` (not a stack at all), `bootstrap.yml` (preparing a host; see `docs/bootstrap.md`). @@ -142,6 +149,45 @@ are bind-mounted configuration, which Compose never recreates a container for: Without the `when`, every run reports a change and the stack is never idempotent. +Not every service survives a restart unattended. `openbao.yml` keys a *notice* +off the fact instead of a restart, because restarting OpenBao seals it and would +fail every stack imported after it in a `site.yml` run. + +#### `stack_secrets` + +A stack's credentials can come from OpenBao on the host rather than from a +`.env` someone placed by hand. Each entry names the environment variable and +where its value lives: + +```yaml + stack_secrets: + SECRET_KEY: + path: stacks/webapp + field: secret_key +``` + +The role reads them at the end of the sync and writes `{{ stack_dest }}/.env` at +`0600`, so `-e start=false` and two-phase playbooks get the file too. Compose +files are unchanged: keep reading the variable with `${VAR:?message}`, which is +still what catches a value that went missing. + +The read runs **on the target host**, against `127.0.0.1:8200`. That is what +lets the vault be published nowhere, and it means a deploy works identically +from a laptop and from a CI runner. It also means a host with no vault of its +own needs `vault_addr` pointing at one it can reach. `VAULT_TOKEN` is read from +the control node's environment and appears in neither this repository nor the +inventory. + +A sealed vault, an unset token, or a path that does not hold the field all fail +before the stack starts, each naming what to fix. No task may print a value: +keep `no_log` on anything that sees one, and keep the asserts that explain a +failure off loops over the reads, where a failed item would carry the plaintext +into the result. See `docs/secrets.md`. + +Removing `stack_secrets` from a playbook stops the render on the next deploy; +it does not delete the `.env` already on the host, which keeps serving its +stale value until something else removes it. + #### Doing maintenance between sync and up `roles: - role: compose_stack` syncs and then starts, which is what almost every diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4062f35..b648637 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -47,13 +47,13 @@ step-by-step version for the most common change. ## Changing the examples -The six example stacks exist to demonstrate specific features of the role +The seven example stacks exist to demonstrate specific features of the role contract — the table in [`server/README.md`](server/README.md) says which. If you change one, check that whatever it was demonstrating is still demonstrated somewhere, or the documentation that points at it stops being true. -New examples are welcome if they show something the current six do not. An -example that shows the same thing a seventh way is a cost, not a contribution. +New examples are welcome if they show something the current seven do not. An +example that shows the same thing an eighth way is a cost, not a contribution. ## Reporting a security issue diff --git a/README.md b/README.md index 400ae3a..18b339a 100644 --- a/README.md +++ b/README.md @@ -4,9 +4,9 @@ A small, complete pattern for deploying Docker Compose stacks to a handful of Linux hosts with Ansible — and for letting CI do it safely, one stack at a time, on purpose rather than on every push. -It is deliberately not a framework. There is **one** Ansible role, about 90 lines -of tasks, and a convention about where files live. Everything else in this -repository is documentation and six worked examples. +It is deliberately not a framework. There is **one** Ansible role, a couple of +hundred lines of tasks, and a convention about where files live. Everything else +in this repository is documentation and seven worked examples. ``` server/app/webapp/ → rsync → /srv/stacks/webapp/ on app @@ -61,7 +61,7 @@ ansible/ ├── inventory/ │ ├── hosts.yml │ └── group_vars/all.yml # connection settings, stack_root, pull/start defaults -├── roles/compose_stack/ # the one shared "sync payload, bring stack up" role +├── roles/compose_stack/ # the one shared role: sync, render secrets, bring up └── playbooks/ ├── site.yml # every stack, in dependency order ├── bootstrap.yml # prepares a host; run once, by an admin @@ -97,6 +97,7 @@ user, its key, the `docker` group, rsync, and a writable `/srv/stacks`. That is cd ansible ansible-galaxy install -r requirements.yml export SSH_KEY_PATH=~/.ssh/deploy_key # see Secrets, below +export VAULT_TOKEN=... # for stacks with stack_secrets ``` Then check what a deploy *would* do before doing it: @@ -204,14 +205,15 @@ to start the stack. ## The examples -Six stacks, each chosen to demonstrate one thing. Read them in this order. +Seven stacks, each chosen to demonstrate one thing. Read them in this order. | Stack | Hosts | What it shows | | --- | --- | --- | | [`static-site`](ansible/playbooks/static-site.yml) | app | The minimum: a name and a network. **Copy this one.** | | [`reverse-proxy`](ansible/playbooks/reverse-proxy.yml) | edge | Pre-creating bind mounts; reloading after a config-only change | -| [`metrics`](ansible/playbooks/metrics.yml) | edge, app | One payload shared by two hosts; per-host variable maps; directories owned by a container's uid | -| [`webapp`](ansible/playbooks/webapp.yml) | app | Backup and migrate *between* sync and up, gated on an update run; a second compose file; building from a Dockerfile; requiring a host-side `.env` | +| [`metrics`](ansible/playbooks/metrics.yml) | edge, app | One payload shared by two hosts; per-host variable maps; directories owned by a container's uid; requiring a hand-placed `.env` | +| [`openbao`](ansible/playbooks/openbao.yml) | app | The secret store, on `127.0.0.1:8200` only. Reports a changed config instead of restarting, because a restart seals it | +| [`webapp`](ansible/playbooks/webapp.yml) | app | Backup and migrate *between* sync and up, gated on an update run; a second compose file; building from a Dockerfile; credentials from the vault with `stack_secrets` | | [`webapp-staging`](ansible/playbooks/webapp-staging.yml) | app | A second instance of the same payload, elsewhere, with different settings | | [`banner`](ansible/playbooks/banner.yml) | edge, app | The escape hatch: something that is not a Compose stack at all | @@ -232,11 +234,25 @@ export SSH_KEY_PATH="$(scripts/load-ssh-key.sh ci)" # from a CI secret export SSH_KEY_PATH="$(scripts/load-ssh-key.sh vault)" # from Vault / OpenBao ``` -**2. Per-stack `.env` files.** Stack secrets live on the hosts, not here. The -role excludes `.env` from every sync, so editing one in the repo has no effect -and a host's `.env` is never overwritten. The example stacks use Compose's -`${VAR:?message}` syntax, so a stack refuses to start with a missing credential -rather than booting with an empty password. +**2. Per-stack secrets.** They live in OpenBao on the host, under +`kv/stacks/`. A playbook names what it needs with `stack_secrets`, and the role +reads it *on the host* during the sync and writes `/srv/stacks//.env` at +`0600`. The vault is published to `127.0.0.1` only, and `VAULT_TOKEN` — a +read-only token — comes from the environment of whoever runs the deploy. +`webapp` works this way. A stack without `stack_secrets` reads a `.env` placed +on the host by hand instead, as `metrics` does; the sync never touches it +either way. Compose files read `${VAR:?message}`, so a stack refuses to start +with a missing credential rather than booting with an empty password. + +Adding or rotating one is a `kv put` and a redeploy. Keep the value off the +command line, and strip the trailing newline — `=-` stores stdin byte for +byte: + +```bash +docker exec -it openbao bao login +openssl rand -hex 32 | tr -d '\n' \ + | docker exec -i openbao bao kv put kv/stacks/ =- +``` **3. Nothing else.** There is no third category, and that is the point. A secret written directly into a compose file cannot be excluded from the sync, cannot be @@ -300,7 +316,8 @@ is in [`skills/adding-a-stack/SKILL.md`](skills/adding-a-stack/SKILL.md). | A deploy succeeds but the service is unchanged | Image tag is `latest` and you did not pass `-e pull=always` | | Config file copied but nothing happens | Bind-mounted config; the stack needs a reload, which the playbook does only when the sync changed something | | `no payload directory at ...` | `stack_src` does not match where the files actually are | -| A stack refuses to start, naming a variable | Its `.env` is missing on the host — that is the design working | +| A secrets task fails naming the vault | OpenBao is sealed (e.g. after a reboot), or `VAULT_TOKEN` is unset or expired — see [`docs/secrets.md`](docs/secrets.md) | +| A stack refuses to start, naming a variable | Its `.env` is missing or incomplete on the host — that is the design working | | `Permission denied (publickey)` | The matching public key is not in the deploy user's `authorized_keys` | | `Connection timed out` | A network path problem, not a key problem — see [`docs/connectivity.md`](docs/connectivity.md) | | `Failed to find required executable "rsync"` | `synchronize` needs rsync at **both** ends; read the PATH in the error to tell the runner from the host | diff --git a/ansible/playbooks/metrics.yml b/ansible/playbooks/metrics.yml index c8a80a9..562977c 100644 --- a/ansible/playbooks/metrics.yml +++ b/ansible/playbooks/metrics.yml @@ -6,7 +6,9 @@ # single payload instead of keeping duplicate copies in step by hand; # - a per-host variable map, which is how a play covers hosts that differ # without growing a second playbook or a `when:` on every task; -# - stack_dirs entries that set ownership, and ones that only set a mode. +# - stack_dirs entries that set ownership, and ones that only set a mode; +# - a .env placed on each host by hand, for a stack that does not use the +# vault: edge runs no OpenBao, and the role only reads from the host's own. - name: Deploy metrics hosts: - edge @@ -48,6 +50,26 @@ - path: "{{ stack_root }}/metrics/prometheus-data" mode: "0777" + pre_tasks: + # The compose file reads the Grafana admin password from .env with no + # default, and the role never ships .env to a host. Fail loudly and say what + # to do, rather than starting a stack with an empty credential. + - name: Look for the metrics .env on the host + ansible.builtin.stat: + path: "{{ stack_root }}/metrics/.env" + register: metrics_env + changed_when: false + + - name: Verify the metrics .env is present on the host + ansible.builtin.assert: + that: + - metrics_env.stat.exists + fail_msg: >- + {{ stack_root }}/metrics/.env is missing on {{ inventory_hostname }}. + It is never synced from this repository; copy .env.example on the + host and fill it in, or restore it from your secret store. + quiet: true + roles: - role: compose_stack vars: diff --git a/ansible/playbooks/openbao.yml b/ansible/playbooks/openbao.yml new file mode 100644 index 0000000..e599a5b --- /dev/null +++ b/ansible/playbooks/openbao.yml @@ -0,0 +1,53 @@ +--- +# OpenBao: the secret store the other stacks read their credentials from, and so +# imported in site.yml before any stack that sets stack_secrets. +# +# It deliberately declares no stack_secrets. It is the root of trust and cannot +# depend on itself. +# +# data/ is the file storage backend. The image runs as uid 100, gid 1000 and +# never as root, so unlike an image that starts as root and chowns its own data, +# it cannot fix the directory from inside the container — a bind mount Docker +# created would be root-owned and OpenBao could not write its storage at all. +# owner and group name that uid outright. No mode: recursing one over live +# storage would rewrite the permissions of the keyring. +# +# Setting owner is what makes the role escalate, so the host has to have been +# bootstrapped with -e deploy_sudo=true. +# +# It runs on app, the only host with a stack that sets stack_secrets. The role +# reads from 127.0.0.1 on the target, so a second host that wants secrets needs +# its own vault or a vault_addr it can reach — see docs/secrets.md. +# +# Init and unseal are manual and stay manual — see docs/secrets.md. After a +# reboot the vault is sealed and every deploy that reads a secret fails until +# somebody unseals it. That is the accepted cost of hosting the vault on the +# host it serves. +- name: Deploy OpenBao + hosts: app + gather_facts: false + tags: [openbao] + + roles: + - role: compose_stack + vars: + stack_name: openbao + stack_dirs: + - path: "{{ stack_dest }}/data" + owner: "100" + group: "1000" + + post_tasks: + # Not a restart. Restarting OpenBao seals it, so an automatic restart in the + # middle of a site.yml run would seal the vault and fail every stack + # imported after this one. Say what happened and let a person pick when. + - name: Report that the payload changed and OpenBao needs a restart + ansible.builtin.debug: + msg: >- + The OpenBao payload changed on {{ inventory_hostname }}, and the + running container is still on the old configuration. Restart it when + you are ready to unseal again: + `docker compose -f {{ stack_root }}/openbao/docker-compose.yml restart`, + then `docker exec -it openbao bao operator unseal` three times. See + docs/secrets.md. + when: compose_stack_synced | default(false) diff --git a/ansible/playbooks/site.yml b/ansible/playbooks/site.yml index b790e59..0dbcd35 100644 --- a/ansible/playbooks/site.yml +++ b/ansible/playbooks/site.yml @@ -8,7 +8,8 @@ # # Order matters. The reverse proxy owns proxy-net and the vhost configuration # that every other stack registers against, so it goes first; banner.yml writes -# into the proxy's html directory, so it goes second. +# into the proxy's html directory, so it goes second. OpenBao comes before any +# stack that sets stack_secrets, since those read it during their sync. - name: Reverse proxy ansible.builtin.import_playbook: reverse-proxy.yml @@ -16,6 +17,8 @@ ansible.builtin.import_playbook: banner.yml - name: Metrics ansible.builtin.import_playbook: metrics.yml +- name: OpenBao + ansible.builtin.import_playbook: openbao.yml - name: Static site ansible.builtin.import_playbook: static-site.yml diff --git a/ansible/playbooks/webapp-staging.yml b/ansible/playbooks/webapp-staging.yml index 0551a54..4b96dc0 100644 --- a/ansible/playbooks/webapp-staging.yml +++ b/ansible/playbooks/webapp-staging.yml @@ -33,3 +33,12 @@ # instance it is. stack_env: COMPOSE_PROJECT_NAME: webapp-staging + # The same variables as production, from a path of its own, so staging + # never holds a production credential. + stack_secrets: + POSTGRES_PASSWORD: + path: stacks/webapp-staging + field: postgres_password + SECRET_KEY: + path: stacks/webapp-staging + field: secret_key diff --git a/ansible/playbooks/webapp.yml b/ansible/playbooks/webapp.yml index b677d36..5d514d0 100644 --- a/ansible/playbooks/webapp.yml +++ b/ansible/playbooks/webapp.yml @@ -1,8 +1,8 @@ --- # The stack that needs more than "sync, then up". # -# Demonstrates the whole heavy end of the contract: an assertion that a host-side -# .env exists, a payload built from a Dockerfile, a second compose file holding +# Demonstrates the whole heavy end of the contract: credentials rendered from the +# vault, a payload built from a Dockerfile, a second compose file holding # maintenance-only services, and work that has to happen *between* the payload # landing and the stack starting. # @@ -48,27 +48,17 @@ # is 0700 or 0750, so it must be left to the image. - path: "{{ stack_root }}/webapp/uploads" - path: "{{ stack_root }}/webapp/backups" - - pre_tasks: - # Every variable in this stack's compose files comes from .env with no - # defaults, and the role never ships .env to a host. Fail loudly and say what - # to do, rather than starting a stack with empty credentials. - - name: Look for the webapp .env on the host - ansible.builtin.stat: - path: "{{ webapp_dir }}/.env" - register: webapp_env - changed_when: false - - - name: Verify the webapp .env is present on the host - ansible.builtin.assert: - that: - - webapp_env.stat.exists - fail_msg: >- - {{ webapp_dir }}/.env is missing on {{ inventory_hostname }}. webapp - reads every credential from it and it is never synced from this - repository; copy .env.example on the host and fill it in, or restore - it from your secret store. - quiet: true + # Every variable in this stack's compose files is required with no default. + # The sync phase reads these from OpenBao on the host and writes .env, and + # fails before anything starts if the vault is sealed or a value is missing. + # Put them in first — see "Adding a secret" in docs/secrets.md. + stack_secrets: + POSTGRES_PASSWORD: + path: stacks/webapp + field: postgres_password + SECRET_KEY: + path: stacks/webapp + field: secret_key tasks: # Phase 1: deliver the payload, including the current Dockerfile and diff --git a/ansible/roles/compose_stack/defaults/main.yml b/ansible/roles/compose_stack/defaults/main.yml index 2c9dfaf..8e02c60 100644 --- a/ansible/roles/compose_stack/defaults/main.yml +++ b/ansible/roles/compose_stack/defaults/main.yml @@ -58,3 +58,30 @@ stack_build: policy # before anything opens them. Defaults to the repo-wide `start` flag, so # `-e start=false` works on every playbook without editing any of them. stack_start: "{{ start | default(true) | bool }}" + +# Secrets to fetch from the vault and render into the stack's .env before the +# stack starts. A map of environment variable name to the path and field that +# hold it: +# +# stack_secrets: +# SECRET_KEY: +# path: stacks/webapp +# field: secret_key +# +# Empty means the role never contacts a vault and never writes a .env, which is +# every stack that has not opted in. Those still read a .env placed on the host +# by hand, since stack_exclude keeps the sync away from it either way. +stack_secrets: {} + +# Where the vault is, as seen from the target host. The read runs on the host, +# which is what lets the vault bind to loopback and be published nowhere. A host +# without its own vault can point vault_addr at one it can reach. See +# docs/secrets.md. +stack_vault_addr: "{{ vault_addr | default('http://127.0.0.1:8200') }}" + +# KV v2 mount point. +stack_vault_mount: "{{ vault_kv_mount | default('kv') }}" + +# Read on the control node, from the operator's shell or a CI secret. Never in +# this repository and never in the inventory. +stack_vault_token: "{{ lookup('env', 'VAULT_TOKEN') }}" diff --git a/ansible/roles/compose_stack/tasks/secrets.yml b/ansible/roles/compose_stack/tasks/secrets.yml new file mode 100644 index 0000000..f9f5ad1 --- /dev/null +++ b/ansible/roles/compose_stack/tasks/secrets.yml @@ -0,0 +1,176 @@ +--- +# Rendering the stack's .env from the vault. Included at the end of sync.yml, +# because the .env is payload delivery: every path that syncs gets it, including +# `-e start=false` and two-phase playbooks that call tasks_from: sync. There is +# deliberately no third entry point for callers to remember. + +- name: Verify a vault token is available + ansible.builtin.assert: + that: + - stack_vault_token | length > 0 + fail_msg: >- + {{ stack_name }} declares stack_secrets, but VAULT_TOKEN is not set in the + environment this deploy is running from. See docs/secrets.md. + quiet: true + +# 200 active, 429 standby, 501 not initialised, 503 sealed. All four are answers +# rather than errors, so accept them and let the assert below say which it was. +- name: Ask the vault whether it is ready + ansible.builtin.uri: + url: "{{ stack_vault_addr }}/v1/sys/health" + status_code: [200, 429, 501, 503] + register: compose_stack_vault_health + # uri does not implement check mode. Without this a --check run skips the read + # and the render below has nothing to write. It is a GET; running it during a + # check is correct. + check_mode: false + changed_when: false + +- name: Verify the vault is initialised and unsealed + ansible.builtin.assert: + that: + - compose_stack_vault_health.json.initialized | default(false) + - not (compose_stack_vault_health.json.sealed | default(true)) + fail_msg: >- + the vault at {{ stack_vault_addr }} on {{ inventory_hostname }} is not + ready to answer (initialised={{ compose_stack_vault_health.json.initialized | default('?') }}, + sealed={{ compose_stack_vault_health.json.sealed | default('?') }}). + Unseal it before deploying — see docs/secrets.md. + quiet: true + +- name: Read the stack's secrets from the vault + ansible.builtin.uri: + url: "{{ stack_vault_addr }}/v1/{{ stack_vault_mount }}/data/{{ item.value.path }}" + headers: + X-Vault-Token: "{{ stack_vault_token }}" + # 404 (wrong path or mount) and 403 (expired, revoked or under-scoped + # token) are the two most likely real failures. Accept them here too and + # let the assert below say which one it was, instead of a bare "censored" + # failure that names neither. + status_code: [200, 403, 404] + register: compose_stack_vault_reads + loop: "{{ stack_secrets | dict2items }}" + loop_control: + label: "{{ item.key }}" + no_log: true + check_mode: false + changed_when: false + +# Reset first: facts persist for the host across every stack in a site.yml run, +# so without this the list accumulates and the second stack inherits the first +# stack's verdict. +- name: Start a fresh list of secrets the vault could not read + ansible.builtin.set_fact: + compose_stack_vault_failed: [] + +- name: Work out which reads the vault rejected + ansible.builtin.set_fact: + compose_stack_vault_failed: >- + {{ compose_stack_vault_failed + + (["%s (path %s, mount %s): HTTP %s" | format( + item.item.key, item.item.value.path, stack_vault_mount, item.status)] + if item.status != 200 else []) }} + loop: "{{ compose_stack_vault_reads.results }}" + loop_control: + label: "{{ item.item.key }}" + no_log: true + +# Deliberately without no_log: this task sees only variable names, paths and +# HTTP status codes, never values, and a censored failure message would be +# useless — a bare "censored" failure names neither the variable nor the cause. +- name: Verify every secret was readable + ansible.builtin.assert: + that: + - compose_stack_vault_failed | length == 0 + fail_msg: >- + the vault rejected: {{ compose_stack_vault_failed | join(', ') }}. A 404 + means the path or mount is wrong; a 403 means stack_vault_token is + expired, revoked or under-scoped. See docs/secrets.md. + quiet: true + +# Reset first, for the same reason as compose_stack_vault_failed above. +- name: Start a fresh list of secrets the vault did not return + ansible.builtin.set_fact: + compose_stack_vault_missing: [] + +- name: Work out which requested fields the vault did not return + ansible.builtin.set_fact: + compose_stack_vault_missing: >- + {{ compose_stack_vault_missing + + ([item.item.key] if item.item.value.field not in (item.json.data.data | default({}, true)) else []) }} + loop: "{{ compose_stack_vault_reads.results }}" + loop_control: + label: "{{ item.item.key }}" + no_log: true + +# Deliberately without no_log: this task sees only variable names, never values, +# and a censored failure message would be useless. +- name: Verify every secret the stack asked for was found + ansible.builtin.assert: + that: + - compose_stack_vault_missing | length == 0 + fail_msg: >- + the vault returned no value for: {{ compose_stack_vault_missing | join(', ') }}. + Check the path and field each one names in stack_secrets against what is + under {{ stack_vault_mount }}/ on {{ inventory_hostname }}. + quiet: true + +# The rendered file is Compose's project .env, whose dotenv parser trims +# unquoted whitespace, treats a whitespace-preceded '#' as a comment, and +# interpolates '$'. A newline corrupts the file outright. Refuse rather than +# silently truncate or expand a credential. +# +# Reset first, for the same reason as compose_stack_vault_failed above. +- name: Start a fresh list of secrets that cannot be safely rendered + ansible.builtin.set_fact: + compose_stack_vault_unsafe: [] + +- name: Work out which values contain a newline or a single quote + ansible.builtin.set_fact: + compose_stack_vault_unsafe: >- + {{ compose_stack_vault_unsafe + + ([item.item.key] + if (item.json.data.data[item.item.value.field] is search('\n') + or item.json.data.data[item.item.value.field] is search('\x27')) + else []) }} + loop: "{{ compose_stack_vault_reads.results }}" + loop_control: + label: "{{ item.item.key }}" + no_log: true + +# Deliberately without no_log, and deliberately not looped: this task sees +# only variable names, never values, and needs no "item" for a failure to +# dump — a per-item assert here would fail with the loop's current item +# (the plaintext value) attached to the result, which is exactly the leak +# this whole check exists to prevent. +- name: Verify no secret value contains a newline or a single quote + ansible.builtin.assert: + that: + - compose_stack_vault_unsafe | length == 0 + fail_msg: >- + {{ compose_stack_vault_unsafe | join(', ') }} cannot be safely rendered + into {{ stack_dest }}/.env: the value contains a newline or a single + quote. Change the value in the vault. See docs/secrets.md. + quiet: true + +- name: Render the stack's .env from the vault + ansible.builtin.copy: + content: | + # Rendered by the compose_stack role from {{ stack_vault_addr }}. + # Edits here are overwritten on the next deploy — change the value in the + # vault instead. See docs/secrets.md. + {% for read in compose_stack_vault_reads.results %} + {{ read.item.key }}='{{ read.json.data.data[read.item.value.field] }}' + {% endfor %} + dest: "{{ stack_dest }}/.env" + mode: "0600" + no_log: true + +# compose_stack_vault_reads holds every plaintext value in host vars for the +# rest of the play, available to a later post_tasks debug or a callback +# plugin. Nothing here dumps it, but there is no reason to leave it lying +# around once the .env is written. +- name: Clear the vault reads now that the .env is rendered + ansible.builtin.set_fact: + compose_stack_vault_reads: {} + no_log: true diff --git a/ansible/roles/compose_stack/tasks/sync.yml b/ansible/roles/compose_stack/tasks/sync.yml index 72ffa57..78e0088 100644 --- a/ansible/roles/compose_stack/tasks/sync.yml +++ b/ansible/roles/compose_stack/tasks/sync.yml @@ -84,3 +84,11 @@ - name: Record whether the payload changed ansible.builtin.set_fact: compose_stack_synced: "{{ compose_stack_sync.changed }}" + +# After the sync, so stack_dest exists, and inside sync rather than up so that +# `-e start=false` and two-phase playbooks get the .env too. Gated here rather +# than on every task in the file: a stack with no stack_secrets never talks to a +# vault and never needs one to exist. +- name: Render the stack's secrets into its .env + ansible.builtin.include_tasks: secrets.yml + when: stack_secrets | length > 0 diff --git a/docs/ci-gitea.md b/docs/ci-gitea.md index ff51897..d39312c 100644 --- a/docs/ci-gitea.md +++ b/docs/ci-gitea.md @@ -83,6 +83,11 @@ over an organisation one of the same name. The matching public key must be in the deploy user's `authorized_keys` on every host. +If any stack sets `stack_secrets`, add `VAULT_TOKEN` the same way, holding the +read-only `stacks-read` token — see step 5 of *Bootstrapping the vault* in +[secrets.md](secrets.md). Every run in the repository can read it, which is why +it must never be the root token. + ### 3. What replaces environments Nothing does, and it is worth being clear-eyed about that before you point this diff --git a/docs/ci-github-actions.md b/docs/ci-github-actions.md index 9a3c5dd..3d502fb 100644 --- a/docs/ci-github-actions.md +++ b/docs/ci-github-actions.md @@ -35,6 +35,10 @@ what not to use on Windows. The matching public key must be in the deploy user's `authorized_keys` on every host. +If any stack sets `stack_secrets`, add `VAULT_TOKEN` beside it, holding the +read-only `stacks-read` token — see step 5 of *Bootstrapping the vault* in +[secrets.md](secrets.md). + ### 3. Point at a runner Deploys need to reach your hosts over SSH. GitHub-hosted runners have general diff --git a/docs/ci-gitlab.md b/docs/ci-gitlab.md index 3e9e61e..0a73f14 100644 --- a/docs/ci-gitlab.md +++ b/docs/ci-gitlab.md @@ -19,7 +19,8 @@ Settings → CI/CD → Variables. Mark every one **Protected**. | `SSH_KEY_BACKEND` | `ci` | Already set as a job default; override only to use `vault` | | `SSH_PRIVATE_KEY_B64` | the deploy key, base64, one line | **Masked** | | `SSH_PRIVATE_KEY` | the raw key | Alternative: type **File**, cannot be masked | -| `VAULT_ADDR` / `VAULT_TOKEN` | a vault address and token | Only for the `vault` backend | +| `VAULT_TOKEN` | the `stacks-read` token | **Masked**. For stacks that set `stack_secrets` — see [secrets.md](secrets.md) §2 | +| `VAULT_ADDR` | a vault address | Only for the `vault` key backend, which reads `VAULT_TOKEN` too — a token used for both needs read on the key's path as well | **Protected matters more than it looks.** A protected variable exists only on protected branches and tags. A pipeline on an ordinary feature branch gets an diff --git a/docs/secrets.md b/docs/secrets.md index 09af76d..a30b1cf 100644 --- a/docs/secrets.md +++ b/docs/secrets.md @@ -87,50 +87,312 @@ repository deploys to, and it seals on reboot, then a deploy that fetches its ke from the vault cannot run — including the deploy that would fix that host. Host it elsewhere, or auto-unseal it. A CI secret is the dependable route. -## 2. Per-stack `.env` files +That is also why **the deploy key does not live in the stacks' OpenBao** (§2), +even when one runs on the host. You need SSH to reach the host the vault runs +on, so a key fetched from that vault could never get you there in the first +place. Two stores, deliberately: CI holds the key that reaches the host, the +vault holds what the stacks need once you are there. -Stack secrets live on the hosts, not here. +## 2. Per-stack secrets -The role excludes `.env` from every sync (`stack_exclude`), so editing one in -this repository has no effect, and a host's `.env` is never overwritten by a -deploy. To change one, edit it on the host and redeploy. +Stack secrets never live in this repository. There are two ways to get them onto +a host, and both end in the same place: a `.env` next to the stack's compose +file, which Compose reads and the sync never touches (`stack_exclude`). -Each example stack that needs credentials ships a `.env.example` listing the -variable names with no values, and reads them in its compose file with Compose's -required-variable syntax: +Either way, compose files read each value with Compose's required-variable +syntax: ```yaml environment: - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?set it in .env on the host} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?missing from .env} ``` That is worth copying. Without the `:?`, a missing variable expands to the empty string and Postgres starts with **no password at all** — a failure that looks like success until someone finds it. With it, Compose refuses to start and names -the variable. +the variable. Each stack also ships a `.env.example` listing the names with no +values, as documentation. -`webapp.yml` goes one step further and asserts the file exists on the host before -doing anything, so the error arrives before the payload is synced rather than -halfway through starting the stack: +### From OpenBao, with `stack_secrets` (recommended) + +A playbook names what it needs, and the `compose_stack` role renders it: + +```yaml + stack_secrets: + SECRET_KEY: + path: stacks/webapp + field: secret_key +``` + +The role reads `kv/stacks/webapp` from `http://127.0.0.1:8200` **on the target +host**, at the end of the sync, and writes `{{ stack_dest }}/.env` at `0600`. +Nothing is placed by hand, and nothing — plaintext or encrypted — is committed. +`webapp.yml` and `webapp-staging.yml` are the worked examples, reading from the +`openbao` stack on the same host. + +A sealed vault, an unset token, a 403 or 404 from the vault, a field that is +not there, and a value that cannot be safely written to a `.env` all fail before +the stack starts, each naming what to fix. No task prints a value: the reads are +`no_log`, and the asserts that explain a failure see only names, paths and +status codes. + +#### Why the read happens on the host + +Because the vault is published to `127.0.0.1:8200` and nowhere else. Ansible +tasks already run on the host, so the store never needs a public hostname, never +needs a certificate, and does not care whether the deploy came from a laptop or +a CI runner. `ansible.cfg` sets `pipelining = True`, so module arguments are +piped to the remote interpreter over stdin and the token is never written to a +file on the host. That covers the token specifically — it does not make every +transfer file-free: `ansible.builtin.copy` still lands its content as a `0600` +temp file under `~/.ansible/tmp` on both ends, removed when the run ends, +regardless of pipelining. + +The consequence is **one vault per host that reads from it**, or a `vault_addr` +the host can reach. In this repository only `app` runs one; `metrics`, which +also runs on `edge`, uses a hand-placed `.env` instead (below). + +#### The token + +`VAULT_TOKEN` is read from the control node's environment: your shell locally, +and a CI secret in a runner. It carries a `stacks-read` policy granting `read` +on `kv/data/stacks/*` and nothing else. It is **not** the root token. + +The name is not configurable. `compose_stack` reads it with +`lookup('env', 'VAULT_TOKEN')`, and the deploy workflows pass it through as an +environment variable rather than interpolating it into a `run:` body — an +expression is substituted before the shell sees it, so a value containing shell +metacharacters would be executed rather than assigned. + +Where to put it, and how long it lasts, are steps 5 and 6 of *Bootstrapping the +vault* below. + +#### Unsealing + +OpenBao has no self-contained auto-unseal; it needs an external KMS or a second +vault. The example stack is unsealed by hand: + +```bash +ssh deploy@app.example.com +docker exec -it openbao bao operator unseal # three times, three different keys +``` + +(`docker exec` inherits the container's environment, including the `BAO_ADDR` +set in `docker-compose.yml` — so none of the commands on this page need their +own `-e BAO_ADDR=...`. Without it the `bao` CLI defaults to +`https://127.0.0.1:8200` and every one of these fails with a TLS handshake +error, since the listener has TLS disabled.) + +After a reboot the vault is sealed and any deploy that reads a secret fails, +naming the vault and pointing here. That is the intended behaviour: a sealed +vault is a loud, safe failure. + +#### Adding a secret + +Putting the root token and the secret value on the command line, as +`-e BAO_TOKEN=` and `=`, lands both in the host's process +table and in the operator's shell history. Log in interactively instead, and +pipe the value in on stdin so it never appears as an argument: + +```bash +ssh deploy@app.example.com +docker exec -it openbao bao login # prompts for the root token + +read -rs SECRET +printf '%s' "$SECRET" | docker exec -i openbao bao kv put kv/stacks/ =- +unset SECRET +``` + +`read -rs` does not echo and does not reach shell history, and `printf '%s'` +adds no trailing newline. **That last part matters.** `=-` stores stdin +byte for byte, including the newline you press Enter for, and the role refuses +to render a value containing a newline or a single quote — a `.env` line it +cannot quote safely is one Compose would misread. The deploy then fails on +*Verify no secret value contains a newline or a single quote*, which is the +guard working, but it is a round trip you can skip. + +Generating a value has the same trap, because every generator ends its output +with a newline: + +```bash +openssl rand -hex 32 | tr -d '\n' | docker exec -i openbao bao kv put kv/stacks/ =- +``` + +`kv put` replaces every field at that path. To add a second field beside an +existing one, use `bao kv patch` instead, or the first is lost. + +Then add the entry to the playbook's `stack_secrets` and deploy. Rotating is the +same `kv put` (or `kv patch`) followed by a redeploy. + +Removing an entry from `stack_secrets` stops the render on the next deploy; it +does not delete the `.env` already on the host, which keeps serving its stale +value until something else removes it. + +#### A changed openbao.hcl needs a manual restart + +Restarting OpenBao seals it. `openbao.yml` therefore reports that its payload +changed rather than restarting the container, because an automatic restart in +the middle of `site.yml` would seal the vault and fail every stack imported +after it. Restart and unseal deliberately. + +#### Bootstrapping the vault + +Hosting the vault on the host it serves is circular, so the order is fixed and +the middle of it is manual. This is the whole sequence, once per vault. + +**1. Prepare the host and deploy the stack.** + +```bash +cd ansible +ansible-playbook playbooks/bootstrap.yml -u -k -K -e deploy_sudo=true +ansible-playbook playbooks/openbao.yml +``` + +`deploy_sudo` is not optional here. `openbao.yml` hands `data/` to uid 100, and a +`stack_dirs` entry with an `owner` is the only thing the role escalates for. +Without it the deploy stops at *Create the extra directories the stack needs* +with `Missing sudo password`. The stack declares no `stack_secrets` of its own — +it is the root of trust and cannot depend on itself — so it comes up with +nothing in place. See [bootstrap.md](bootstrap.md). + +**2. Initialise, once, ever.** + +```bash +ssh deploy@app.example.com +docker exec -it openbao bao operator init +``` + +Five unseal keys and one root token are printed **once**. They are not written +to the host and cannot be recovered. Put all six in your password manager before +doing anything else: three of the five unseal the vault, and losing three makes +the storage unreadable permanently. + +**3. Unseal.** + +```bash +docker exec -it openbao bao operator unseal # three times, three different keys +docker exec openbao bao status # expect: Sealed false +``` + +**4. Create the mount and the read-only policy.** + +Log in interactively rather than passing `-e BAO_TOKEN=`, for the reason +under *Adding a secret* above — an argument lands in the host's process table. + +```bash +docker exec -it openbao bao login # prompts for the root token + +docker exec openbao bao secrets enable -path=kv kv-v2 + +printf 'path "kv/data/stacks/*" {\n capabilities = ["read"]\n}\n' \ + | docker exec -i openbao bao policy write stacks-read - +``` + +**5. Mint the deploy token and store it.** + +```bash +docker exec openbao bao token create \ + -policy=stacks-read -period=768h -display-name=ansible +``` + +This token, never the root one, is `VAULT_TOKEN`. Locally it is an export: + +```bash +export VAULT_TOKEN= +``` + +In CI it is a secret of the same name: + +| Platform | Where | Notes | +| --- | --- | --- | +| GitHub | Settings → Environments → `production` → Secrets | Scoped to the environment, so only jobs naming it can read it | +| GitLab | Settings → CI/CD → Variables | Mark **Masked** and **Protected** | +| Gitea | Settings → Actions → Secrets | No environments and no masking controls: any run in the repository can read it — which is precisely why the token is scoped. See [ci-gitea.md](ci-gitea.md) | + +Unlike the SSH key in §1, this one needs no base64: a vault token is already a +single line with no whitespace, which is the only thing GitLab's masking +requires. + +**6. Diarise the expiry.** + +`-period=768h` makes the token renewable indefinitely; it does not make it +permanent. It lapses 32 days after it was last renewed, and the first sign is a +deploy failing on the vault read after a quiet month. + +Re-minting is the reliable fix — repeat step 5 and replace the secret. To renew +instead, log in as the token rather than passing it as an argument, where the +host's process table would keep it: + +```bash +docker exec -it openbao bao login # paste the stacks-read token +docker exec openbao bao token renew # renews the token you logged in with +docker exec openbao sh -c 'rm -f ~/.bao-token' +``` + +A longer `-period` at mint time is the other lever, traded against how long a +leaked token stays useful. + +**7. Prove the token cannot write.** + +```bash +docker exec -e BAO_TOKEN= openbao bao kv put kv/stacks/canary value=nope +``` + +This must fail with a 403. If it succeeds the policy is wrong and anything +holding the deploy token can write to the store. Nothing else surfaces a policy +typo, so do not skip it. + +**8. Load the secrets and converge.** + +Once per secret, per *Adding a secret* above — mind the trailing newline. For +the examples that is `postgres_password` and `secret_key` under +`kv/stacks/webapp`, and the same two under `kv/stacks/webapp-staging`: + +```bash +openssl rand -hex 32 | tr -d '\n' \ + | docker exec -i openbao bao kv put kv/stacks/webapp postgres_password=- +openssl rand -hex 32 | tr -d '\n' \ + | docker exec -i openbao bao kv patch kv/stacks/webapp secret_key=- +``` + +Then converge with `ansible-playbook playbooks/site.yml` from `ansible/`. + +When you are finished, `bao login` has left the root token in the container at +`~/.bao-token`; remove it with `docker exec openbao sh -c 'rm -f ~/.bao-token'`. + +Steps 2–7 are manual and stay manual. That is the accepted cost of not having a +second machine to hold the keys. + +### By hand, on the host + +A stack with no `stack_secrets` never talks to a vault, and reads a `.env` +someone placed on the host. Editing one in this repository has no effect, and a +host's `.env` is never overwritten by a deploy. To change one, edit it on the +host and redeploy. + +`metrics` works this way, because it runs on `edge` too and `edge` has no +vault. Its playbook asserts the file exists before doing anything, so the error +arrives before the payload is synced rather than halfway through starting the +stack: ```yaml pre_tasks: - - name: Look for the webapp .env on the host + - name: Look for the metrics .env on the host ansible.builtin.stat: - path: "{{ webapp_dir }}/.env" - register: webapp_env + path: "{{ stack_root }}/metrics/.env" + register: metrics_env changed_when: false - - name: Verify the webapp .env is present on the host + - name: Verify the metrics .env is present on the host ansible.builtin.assert: that: - - webapp_env.stat.exists + - metrics_env.stat.exists fail_msg: >- - {{ webapp_dir }}/.env is missing on {{ inventory_hostname }}. + {{ stack_root }}/metrics/.env is missing on {{ inventory_hostname }}. quiet: true ``` -Do this for any stack whose compose file has no safe defaults. +Do this for any hand-placed stack whose compose file has no safe defaults. ## 3. Secrets committed to the repository diff --git a/server/README.md b/server/README.md index 8424e20..faf023f 100644 --- a/server/README.md +++ b/server/README.md @@ -20,7 +20,7 @@ want to be able to diff against upstream. ## The examples -All six are fictional. `example.com` is reserved by RFC 2606 and can never +All seven are fictional. `example.com` is reserved by RFC 2606 and can never resolve to a real service. Replace them with your own; the framework around them is the reusable part. @@ -28,8 +28,9 @@ is the reusable part. | --- | --- | --- | | `edge/reverse-proxy/` | `reverse-proxy.yml` | nginx-proxy + acme-companion. Owns `proxy-net`. Its `proxy-data/` subdirectories are bind mounts holding plain config files. | | `app/static-site/` | `static-site.yml` | One container, one network, no state. The simplest thing that works. | -| `app/webapp/` | `webapp.yml`, `webapp-staging.yml` | Postgres plus an app built from `app/Dockerfile`. A second compose file holds maintenance-only services. Every credential comes from `.env`. | -| `shared/metrics/` | `metrics.yml` | Prometheus and Grafana, identical on both hosts, so it lives here rather than being duplicated per host. | +| `app/webapp/` | `webapp.yml`, `webapp-staging.yml` | Postgres plus an app built from `app/Dockerfile`. A second compose file holds maintenance-only services. Every credential comes from OpenBao, via `stack_secrets`. | +| `app/openbao/` | `openbao.yml` | The vault, on `127.0.0.1:8200` only. `config/openbao.hcl` is bind-mounted; a change needs a manual restart and unseal. | +| `shared/metrics/` | `metrics.yml` | Prometheus and Grafana, identical on both hosts, so it lives here rather than being duplicated per host. Its `.env` is placed on each host by hand. | | `shared/components/banner.html` | `banner.yml` | A single file that belongs to the reverse proxy's directory on every host. | ## Conventions worth copying @@ -37,8 +38,10 @@ is the reusable part. **Take secrets from `.env`, and require them.** Every credential in these payloads is written as `${VAR:?message}`. Compose then refuses to start and names the missing variable, instead of expanding it to an empty string and starting the -service with no password. Ship a `.env.example` listing the names with no values; -the role never syncs `.env` itself. +service with no password. The `.env` itself is never synced: the role renders +it on the host from OpenBao for a stack that sets `stack_secrets`, and otherwise +leaves one placed by hand alone. Ship a `.env.example` listing the names with no +values as documentation. See `docs/secrets.md`. **Pin image tags.** `nginx:1.27-alpine`, not `nginx:latest`. A `latest` tag means a plain deploy will not pick up a new build (Compose keeps the image it has) and diff --git a/server/app/openbao/config/openbao.hcl b/server/app/openbao/config/openbao.hcl new file mode 100644 index 0000000..f151fc9 --- /dev/null +++ b/server/app/openbao/config/openbao.hcl @@ -0,0 +1,21 @@ +# OpenBao's server configuration, bind-mounted read-only. +# +# A change here needs a manual restart *and* a manual unseal afterwards, which +# is why openbao.yml only reports that this file changed instead of restarting +# the container the way reverse-proxy.yml reloads nginx. See docs/secrets.md. + +storage "file" { + path = "/openbao/file" +} + +listener "tcp" { + # 0.0.0.0 inside the container only. docker-compose.yml publishes this to + # 127.0.0.1 on the host, so the socket is reachable from the host and from + # nowhere else. Do not copy this listener anywhere it would be published + # wider without turning TLS back on. + address = "0.0.0.0:8200" + tls_disable = true +} + +api_addr = "http://127.0.0.1:8200" +ui = false diff --git a/server/app/openbao/docker-compose.yml b/server/app/openbao/docker-compose.yml new file mode 100644 index 0000000..980bd3d --- /dev/null +++ b/server/app/openbao/docker-compose.yml @@ -0,0 +1,55 @@ +# OpenBao: the secret store every other stack with a credential reads from. +# +# Loopback only. It is published to 127.0.0.1 on the host, is not on proxy-net, +# and sets no VIRTUAL_HOST and no LETSENCRYPT_HOST, so nothing outside the host +# can reach it. That is the whole reason the compose_stack role does its vault +# read from the host rather than from the control node. See docs/secrets.md. +services: + openbao: + container_name: openbao + # Pinned, per server/README.md — and more firmly than most: a restart seals + # the vault and needs a manual unseal afterwards, so moving this tag is a + # deliberate maintenance window rather than a routine update. + image: docker.io/openbao/openbao:2.6.2 + restart: unless-stopped + ports: + # The bind address matters more than the port. Without the 127.0.0.1 the + # secret store is on every interface the host has. + - "127.0.0.1:8200:8200" + # The image's default command is `server -dev`, which starts an in-memory + # vault with a known root token. Override it — but with the subcommand + # alone: the entrypoint script already appends `-config=/openbao/config` + # whenever the first argument is `server`, so passing that flag again here + # loads the directory twice and OpenBao tries to bind the listener twice, + # failing with "address already in use". + command: server + volumes: + - ./config:/openbao/config:ro + - ./data:/openbao/file + environment: + # The bao CLI defaults to https://127.0.0.1:8200, but this listener has + # TLS disabled (openbao.hcl sets tls_disable = true), so every + # `docker exec ... bao ...` would otherwise fail with a TLS handshake + # error. Set here so every exec'd command inherits it without needing + # its own -e BAO_ADDR. See docs/secrets.md. + BAO_ADDR: http://127.0.0.1:8200 + # ./config is mounted read-only, but the entrypoint's chown of + # /openbao/config runs on every start whenever its uid does not match + # openbao's, which it never will here since the payload is synced as + # the deploy user — and that chown would fail against a read-only + # mount anyway. The config chown is echo-guarded, but the entrypoint's + # matching chowns of /openbao/logs and /openbao/file are not, and the + # script runs under `set -e` as uid 100: an un-runnable-as-non-root + # chown there would exit the container rather than just log noise. This + # stack survives today only because openbao.yml's stack_dirs hands + # data/ to 100:1000 before first start, so the stat short-circuits + # every chown — SKIP_CHOWN makes that not load-bearing. + SKIP_CHOWN: "true" + # mlock, so the unsealed keyring is never paged out to disk. + cap_add: + - IPC_LOCK + logging: + driver: "json-file" + options: + max-size: "1m" + max-file: "1" diff --git a/server/app/webapp/.env.example b/server/app/webapp/.env.example index d692182..1d509a9 100644 --- a/server/app/webapp/.env.example +++ b/server/app/webapp/.env.example @@ -1,11 +1,11 @@ -# Copy to .env ON THE HOST, not here. The compose_stack role excludes .env from -# every sync (stack_exclude), so a .env committed to this repository would be -# ignored, and a host's .env is never overwritten by a deploy. +# Documentation, not a template. The compose_stack role renders .env on the host +# from OpenBao during every sync, from the stack_secrets in +# ansible/playbooks/webapp.yml (and webapp-staging.yml, from its own path). Edits +# to a .env on the host are overwritten by the next deploy. # -# ssh deploy@app.example.com -# cd /srv/stacks/webapp && cp .env.example .env && $EDITOR .env +# POSTGRES_PASSWORD kv/stacks/webapp field postgres_password +# SECRET_KEY kv/stacks/webapp field secret_key # -# webapp.yml refuses to deploy if .env is absent on the host, because every -# variable below is required with no default. +# See "Adding a secret" in docs/secrets.md. POSTGRES_PASSWORD= SECRET_KEY= diff --git a/server/app/webapp/docker-compose.yml b/server/app/webapp/docker-compose.yml index d3b2f2d..5e928bb 100644 --- a/server/app/webapp/docker-compose.yml +++ b/server/app/webapp/docker-compose.yml @@ -2,9 +2,9 @@ # # Every credential comes from .env, with no defaults. `${VAR:?message}` makes # Compose refuse to start and say which variable is missing, which is strictly -# better than booting with an empty password. .env lives on the host and is never -# synced from this repository, so ansible/playbooks/webapp.yml asserts it is -# there before doing anything. +# better than booting with an empty password. .env is never synced from this +# repository: the compose_stack role renders it on the host from OpenBao, from +# the stack_secrets in ansible/playbooks/webapp.yml. services: db: container_name: webapp-db @@ -18,7 +18,7 @@ services: environment: POSTGRES_DB: webapp POSTGRES_USER: webapp - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?set it in .env on the host} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?missing from .env; see stack_secrets in webapp.yml} healthcheck: test: ["CMD-SHELL", "pg_isready -U webapp"] interval: 10s @@ -39,8 +39,8 @@ services: db: condition: service_healthy environment: - DATABASE_URL: postgres://webapp:${POSTGRES_PASSWORD:?set it in .env on the host}@db:5432/webapp - SECRET_KEY: ${SECRET_KEY:?set it in .env on the host} + DATABASE_URL: postgres://webapp:${POSTGRES_PASSWORD:?missing from .env; see stack_secrets in webapp.yml}@db:5432/webapp + SECRET_KEY: ${SECRET_KEY:?missing from .env; see stack_secrets in webapp.yml} # Set by the role to stack_dest, so a path handed to a sibling container is # the path on the host and not wherever the SSH session happened to land. UPLOAD_HOST_DIR: ${PWD}/uploads diff --git a/skills/adding-a-stack/SKILL.md b/skills/adding-a-stack/SKILL.md index 287b0ee..7d03250 100644 --- a/skills/adding-a-stack/SKILL.md +++ b/skills/adding-a-stack/SKILL.md @@ -16,11 +16,23 @@ files and one button. `AGENTS.md` holds the conventions and the full restructure them. Never commit a `.env`, a key, or a secret written inline in a compose file — read `docs/secrets.md` before you decide something is fine to commit. + + If the stack needs a credential, put it in OpenBao now and ship a + `.env.example` listing the variable names. Pipe the value in on stdin rather + than passing `=`, which lands it in the host's process table + and your shell history, and strip the trailing newline — `=-` stores + stdin byte for byte and the role refuses a value containing one: + + ```bash + openssl rand -hex 32 | tr -d '\n' \ + | docker exec -i openbao bao kv put kv/stacks/ =- + ``` 2. **Playbook.** Create `ansible/playbooks/.yml`. Copy `static-site.yml`; most stacks are a name and a network. Tag the play with the stack name. 3. **Register it.** Add an `import_playbook` entry to `ansible/playbooks/site.yml` in dependency order — the reverse proxy owns `proxy-net` and the vhost - configuration, so it stays first. + configuration, so it stays first, and `openbao` must come before any stack + that sets `stack_secrets`. 4. **Give it a button.** All three CI configurations need the stack by name, and none of them can generate the list: - `.github/workflows/deploy.yml` — add it to the `stack` input's `options:`. @@ -35,6 +47,7 @@ files and one button. `AGENTS.md` holds the conventions and the full pipeline triggered with a different one. Delete whichever platforms you are not using rather than letting them rot. + Leave a stack out only if it must never deploy from CI — see the table below. 5. **Verify statically.** `make check` from the repository root. It must pass; this is the definition of done for the playbook. 6. **Deploy.** Run the check mode, read the diff, then deploy. Never run a @@ -46,16 +59,20 @@ files and one button. `AGENTS.md` holds the conventions and the full | If | Do | Example | | --- | --- | --- | | It is not a Compose stack — one file into another stack's directory, host preparation, or a system service | Write plain tasks. Do not force the role | `banner.yml`, `bootstrap.yml` | -| Something must happen between the payload landing and the stack starting (backup, migration) | Use the two entry points, `tasks_from: sync` then `tasks_from: up`, and gate the middle on `pull \| default('policy') == 'always'` | `webapp.yml` | +| Something must happen between the payload landing and the stack starting (backup, migration, a precondition) | Use the two entry points, `tasks_from: sync` then `tasks_from: up`, and gate the middle on `pull \| default('policy') == 'always'` or `compose_stack_synced` | `webapp.yml` | +| It is replacing a hand-deployed instance whose data must be copied in first | `-e start=false`, copy the data into the bind mounts, then deploy normally. Write the steps into the playbook's header comment | none — see "Deploying without starting" in `AGENTS.md` | | The payload is bind-mounted config, which Compose never recreates a container for | Add a `post_tasks` reload keyed on `compose_stack_synced` | `reverse-proxy.yml`, `metrics.yml` | +| …but a restart is itself disruptive (a vault that seals, a runner that kills its own job) | Report on `compose_stack_synced` instead of restarting, and let a person choose when | `openbao.yml` | | The compose file is not named `docker-compose.yml`, or there are several | `stack_files` | `webapp.yml` | | The same payload serves several hosts, or several instances | `stack_src`, and `stack_dest` for the second instance | `metrics.yml`, `webapp-staging.yml` | -| The stack needs a `.env` that is deliberately never shipped | `stat` + `assert` that it exists on the host first | `webapp.yml` | +| The stack needs a credential | `stack_secrets`, naming the vault path and field for each variable. Put the value in OpenBao first | `webapp.yml` — see the contract in `AGENTS.md` and `docs/secrets.md` | +| The stack needs a `.env` placed by hand, on a host with no vault | `stat` + `assert` that it exists on the host first | `metrics.yml` | | It needs an external Docker network | `stack_networks` | any of them | | A bind mount would otherwise be created root-owned by Docker | `stack_dirs`; omit `mode` so an existing directory is left alone | `reverse-proxy.yml` | -| A bind mount must be owned by a uid from inside the container | `stack_dirs` with `owner`/`group` and `recurse`, and no `mode` | `metrics.yml` | +| A bind mount must be owned by a uid from inside the container | `stack_dirs` with `owner`/`group` and `recurse`, and no `mode`. The host needs `bootstrap.yml -e deploy_sudo=true` | `metrics.yml`, `openbao.yml` | | The image is built from a Dockerfile in the payload | `stack_build`, set to `always` only on an update run | `webapp.yml` | | Hosts differ in some small way | A dict keyed by `inventory_hostname`, looked up in the role vars | `metrics.yml` | +| Deploying it from CI would kill the CI job — the runner itself, or whatever the runner depends on | Leave it out of `site.yml` and the deploy form, and assert in `pre_tasks` that `GITHUB_ACTIONS`, `GITEA_ACTIONS` and `GITLAB_CI` are not `true` — the dispatch API accepts any stack with a playbook | none here | Extend the role's variable contract rather than adding a second role or loose tasks. If a stack needs something the contract cannot express, say so instead of @@ -76,7 +93,10 @@ working around it — that is information about the contract. every parent it creates — leaving the payload sync unable to write into the stack's own directory. - **Expecting `.env` to reach the host.** It is excluded from every sync by - design. Secrets live on the host. + design. Use `stack_secrets`, or place it on the host by hand. +- **Storing a vault value with a trailing newline.** `bao kv put ... field=-` + keeps the newline from `echo` or `openssl rand`, and the role refuses to + render it. Pipe through `tr -d '\n'`. - **A reload task with no `when: compose_stack_synced`.** Every run then reports a change, the playbook is never idempotent, and `changed=0` stops meaning anything.