--- # Diagnostics, for when a deploy job hangs instead of failing. # # Two jobs, deliberately separate: # # ping uses the real deploy key and asks Ansible to talk to every host. This # is the right first thing to run after setting the secrets up, and the # right first thing to run when a deploy misbehaves. It changes nothing. # probe uses no credentials at all and answers a narrower question: where do # the packets stop? Run it when ping times out. # # The distinction that matters, and the reason `probe` exists: # # Permission denied (publickey) the key is wrong — you reached sshd # Connection refused you reached the host, nothing is listening # Connection timed out packets are being dropped in transit # # Only the third is a network problem, and no amount of re-pasting the key will # fix it. See docs/connectivity.md. name: connectivity on: workflow_dispatch: inputs: hosts: description: Space-separated hosts to probe (defaults to the inventory) required: false default: edge.example.com app.example.com permissions: contents: read jobs: ping: name: ansible ping runs-on: ${{ vars.DEPLOY_RUNNER || 'ubuntu-latest' }} environment: production steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: python-version: "3.12" - name: Install the pinned toolchain run: | set -eu scripts/check.sh --print-specs | head -1 | xargs -d '\n' pip install --quiet - name: Load the deploy key env: SSH_KEY_BACKEND: ci SSH_PRIVATE_KEY_B64: ${{ secrets.SSH_PRIVATE_KEY_B64 }} run: | set -eu SSH_KEY_PATH="$(scripts/load-ssh-key.sh)" echo "SSH_KEY_PATH=$SSH_KEY_PATH" >> "$GITHUB_ENV" # Success looks like `SUCCESS => {"ping": "pong"}` for every host. That # proves the runner decoded the key, reached port 22, authenticated, and # ran Python on the far side — which is everything a deploy needs except # the playbook itself. - name: Ping every host in the inventory working-directory: ansible run: ansible all -m ansible.builtin.ping probe: name: network probe runs-on: ${{ vars.DEPLOY_RUNNER || 'ubuntu-latest' }} # A report, not a gate. Every section must run even when the thing it probes # is unreachable, which is the normal case when you are running this. continue-on-error: true steps: - name: Probe env: HOSTS: ${{ github.event.inputs.hosts }} run: | set +e # A bare TCP connect can be answered by a transparent proxy that then # says nothing, which looks like success and is not. Real sshd greets # first, so the banner is the only honest proof you reached the server. ssh_banner() { b=$(timeout 6 bash -c "exec 3<>/dev/tcp/$1/${2:-22}; head -c 12 <&3" 2>/dev/null || true) case "$b" in SSH-*) printf ' %-26s %-5s OK, sshd replied: %s\n' "$1" "${2:-22}" "$b" ;; '') printf ' %-26s %-5s NO REPLY (dropped, or a silent proxy)\n' "$1" "${2:-22}" ;; *) printf ' %-26s %-5s answered, but not sshd: %s\n' "$1" "${2:-22}" "$b" ;; esac } # Distinguishes "refused" (allowed out, nothing listening) from # "filtered" (dropped in transit). This is the whole point of the job: # if every port but 443 is filtered, you are behind an egress allowlist # and no alternative SSH port will help. port_state() { # `rc=0; out=$(...) || rc=$?` and not `out=$(...); rc=$?` — the # latter is a failing simple command, which under set -e ends the # step before rc is ever read, so a filtered port aborts the report # instead of describing it. rc=0 out=$(timeout 5 bash -c "cat < /dev/null > /dev/tcp/$1/$2" 2>&1) || rc=$? if [ "$rc" -eq 0 ]; then s='open' elif printf '%s' "$out" | grep -qi refused; then s='refused -> port allowed out' elif [ "$rc" -eq 124 ]; then s='filtered -> blocked in transit' else s="unclear (rc=$rc) $out" fi printf ' %-26s %-5s %s\n' "$1" "$2" "$s" } printf '== 0. does this runner have a network at all? ==\n' printf ' DNS: '; getent hosts github.com >/dev/null 2>&1 && echo resolves || echo 'CANNOT RESOLVE' printf ' https external: ' curl -s -o /dev/null -w '%{http_code}\n' --max-time 10 \ https://github.com || echo 'NO EGRESS' printf ' egress address: '; curl -s --max-time 10 https://ifconfig.me || echo unknown printf '\n\n== 1. SSH to the target hosts ==\n' for h in $HOSTS; do ssh_banner "$h"; done printf '\n== 2. SSH elsewhere: is outbound 22 blocked wholesale? ==\n' ssh_banner github.com printf '\n== 3. Which ports are allowed out at all? ==\n' for h in $HOSTS; do port_state "$h" 22 port_state "$h" 443 done