| 1 | name: Live test |
| 2 | |
| 3 | # End-to-end checks against the real diffinery.com after each deploy: hosted |
| 4 | # Actions jobs in Cloudflare Containers (with Docker builds and services), agent sandboxes, webhooks, scheduled |
| 5 | # workflows, Git LFS, and pull requests (test merge commits, pull_request_target and the merge queue). Each run works in a throwaway private repository that it |
| 6 | # deletes at the end. |
| 7 | # |
| 8 | # Needs the repository secret DIFFINERY_LIVE_TOKEN: a Diffinery token (Settings -> |
| 9 | # Tokens) with the admin scope and access to all repositories. |
| 10 | on: |
| 11 | workflow_dispatch: |
| 12 | workflow_run: |
| 13 | workflows: [Deploy] |
| 14 | types: [completed] |
| 15 | branches: [main] |
| 16 | |
| 17 | permissions: |
| 18 | contents: read |
| 19 | |
| 20 | concurrency: |
| 21 | group: live-test |
| 22 | cancel-in-progress: false |
| 23 | |
| 24 | jobs: |
| 25 | live: |
| 26 | name: Containers, sandboxes, webhooks, schedules, LFS and pull requests |
| 27 | # Runs on the host that deploys production (see the gate in deploy.yml). |
| 28 | if: ${{ github.event_name == 'workflow_dispatch' || (github.server_url == 'https://diffinery.com' && github.event.workflow_run.conclusion == 'success') }} |
| 29 | runs-on: ubuntu-latest |
| 30 | timeout-minutes: 45 |
| 31 | env: |
| 32 | DIFFINERY: https://diffinery.com |
| 33 | TOKEN: ${{ secrets.DIFFINERY_LIVE_TOKEN }} |
| 34 | REPO: live-${{ github.run_id }}-${{ github.run_attempt }} |
| 35 | steps: |
| 36 | - name: Check the token |
| 37 | id: token |
| 38 | run: | |
| 39 | if [ -z "$TOKEN" ]; then |
| 40 | echo "::warning::DIFFINERY_LIVE_TOKEN is not set, so the live test is skipped. Create an admin token at $DIFFINERY/settings/tokens and save it as a repository secret." |
| 41 | echo "skip=true" >> "$GITHUB_OUTPUT" |
| 42 | exit 0 |
| 43 | fi |
| 44 | login=$(curl -fsS "$DIFFINERY/api/v3/user" -H "Authorization: Bearer $TOKEN" | jq -r .login) |
| 45 | echo "OWNER=$login" >> "$GITHUB_ENV" |
| 46 | echo "testing as $login in $login/$REPO" |
| 47 | |
| 48 | - name: Create a throwaway repository with a webhook |
| 49 | if: steps.token.outputs.skip != 'true' |
| 50 | run: | |
| 51 | set -euo pipefail |
| 52 | api() { curl -fsS -X "$1" "$DIFFINERY/api/v3$2" -H "Authorization: Bearer $TOKEN" -H "content-type: application/json" ${3:+--data "$3"}; } |
| 53 | api POST /user/repos "{\"name\":\"$REPO\",\"private\":true}" >/dev/null |
| 54 | api POST "/repos/$OWNER/$REPO/hooks" "{\"config\":{\"url\":\"$DIFFINERY/healthz\"},\"events\":[\"push\"]}" >/dev/null |
| 55 | echo "created $OWNER/$REPO" |
| 56 | |
| 57 | - name: Push workflows and an LFS file |
| 58 | if: steps.token.outputs.skip != 'true' |
| 59 | run: | |
| 60 | set -euo pipefail |
| 61 | mkdir -p src/.github/workflows && cd src |
| 62 | git init -q -b main |
| 63 | git config user.name "Live test" && git config user.email "live-test@diffinery.com" |
| 64 | git lfs install --local >/dev/null |
| 65 | git lfs track "*.bin" >/dev/null |
| 66 | head -c 3145728 /dev/urandom > blob.bin |
| 67 | sha256sum blob.bin | cut -d' ' -f1 > ../blob.sha |
| 68 | echo "# $REPO" > README.md |
| 69 | cat > .github/workflows/ci.yml <<'EOF' |
| 70 | on: push |
| 71 | jobs: |
| 72 | hosted: |
| 73 | runs-on: ubuntu-latest |
| 74 | strategy: |
| 75 | matrix: |
| 76 | n: [1, 2] |
| 77 | steps: |
| 78 | - uses: actions/checkout@v4 |
| 79 | - run: echo "live ok ${{ '${{' }} matrix.n }} $(cat README.md)" |
| 80 | docker: |
| 81 | runs-on: ubuntu-latest |
| 82 | services: |
| 83 | cache: |
| 84 | image: redis:7-alpine |
| 85 | ports: ["6379"] |
| 86 | options: --health-cmd "redis-cli ping" --health-interval 1s --health-retries 30 |
| 87 | steps: |
| 88 | - run: | |
| 89 | printf 'FROM alpine:3.20\nRUN echo built > /msg\nCMD ["cat", "/msg"]\n' > Dockerfile |
| 90 | docker build -q -t live-image . |
| 91 | echo "docker ok $(docker run --rm live-image)" |
| 92 | exec 3<>/dev/tcp/127.0.0.1/${{ '${{' }} job.services.cache.ports['6379'] }} |
| 93 | printf 'PING\r\n' >&3 |
| 94 | read -r line <&3 |
| 95 | echo "service ok ${line%$'\r'}" |
| 96 | EOF |
| 97 | cat > .github/workflows/tick.yml <<'EOF' |
| 98 | on: |
| 99 | schedule: |
| 100 | - cron: "*/5 * * * *" |
| 101 | jobs: |
| 102 | tick: |
| 103 | runs-on: ubuntu-latest |
| 104 | steps: |
| 105 | - run: echo "scheduled ok" |
| 106 | EOF |
| 107 | git add -A && git commit -q -m "Live test" |
| 108 | git push -q "https://x-token:${TOKEN}@${DIFFINERY#https://}/$OWNER/$REPO.git" main |
| 109 | git lfs ls-files |
| 110 | |
| 111 | - name: Hosted Actions jobs (and Docker) run in Cloudflare Containers |
| 112 | if: steps.token.outputs.skip != 'true' |
| 113 | run: | |
| 114 | set -euo pipefail |
| 115 | api() { curl -fsS "$DIFFINERY/api/v3$1" -H "Authorization: Bearer $TOKEN"; } |
| 116 | for i in $(seq 1 90); do |
| 117 | run=$(api "/repos/$OWNER/$REPO/actions/runs?event=push" | jq -c '.workflow_runs[0] // empty') |
| 118 | [ -n "$run" ] && [ "$(jq -r .status <<<"$run")" = completed ] && break |
| 119 | sleep 10 |
| 120 | done |
| 121 | echo "$run" | jq '{id, status, conclusion}' |
| 122 | [ "$(jq -r .conclusion <<<"$run")" = success ] || { echo "::error::the push run did not succeed"; exit 1; } |
| 123 | id=$(jq -r .id <<<"$run") |
| 124 | jobs=$(api "/repos/$OWNER/$REPO/actions/runs/$id/jobs") |
| 125 | for job in $(jq -r '.jobs[] | select(.name | startswith("hosted")) | .id' <<<"$jobs"); do |
| 126 | log=$(api "/repos/$OWNER/$REPO/actions/jobs/$job/logs") |
| 127 | grep -F "live ok" <<<"$log" | grep -F "# $REPO" || { echo "::error::job $job did not print its line (checkout or run step failed)"; exit 1; } |
| 128 | done |
| 129 | job=$(jq -r '.jobs[] | select(.name == "docker") | .id' <<<"$jobs") |
| 130 | log=$(api "/repos/$OWNER/$REPO/actions/jobs/$job/logs") |
| 131 | grep -F "docker ok built" <<<"$log" || { echo "::error::docker build or run failed on a hosted runner"; exit 1; } |
| 132 | grep -F "service ok +PONG" <<<"$log" || { echo "::error::the redis service container was not reachable"; exit 1; } |
| 133 | |
| 134 | - name: Git LFS round trip |
| 135 | if: steps.token.outputs.skip != 'true' |
| 136 | run: | |
| 137 | set -euo pipefail |
| 138 | git clone -q "https://x-token:${TOKEN}@${DIFFINERY#https://}/$OWNER/$REPO.git" clone |
| 139 | [ "$(sha256sum clone/blob.bin | cut -d' ' -f1)" = "$(cat blob.sha)" ] || { echo "::error::the LFS file came back different"; exit 1; } |
| 140 | echo "LFS file matches ($(stat -c %s clone/blob.bin) bytes)" |
| 141 | |
| 142 | - name: Webhook delivered |
| 143 | if: steps.token.outputs.skip != 'true' |
| 144 | run: | |
| 145 | set -euo pipefail |
| 146 | for i in $(seq 1 30); do |
| 147 | hook=$(curl -fsS "$DIFFINERY/api/v3/repos/$OWNER/$REPO/hooks" -H "Authorization: Bearer $TOKEN" | jq -c '.[0]') |
| 148 | [ "$(jq -r '.lastDeliveryAt // empty' <<<"$hook")" != "" ] && break |
| 149 | sleep 5 |
| 150 | done |
| 151 | echo "$hook" | jq '{url, lastStatus, lastDeliveryAt}' |
| 152 | [ "$(jq -r '.lastDeliveryAt // empty' <<<"$hook")" != "" ] || { echo "::error::the push webhook was never delivered"; exit 1; } |
| 153 | |
| 154 | - name: Agent sandbox |
| 155 | if: steps.token.outputs.skip != 'true' |
| 156 | run: | |
| 157 | set -euo pipefail |
| 158 | base="$DIFFINERY/api/v3/repos/$OWNER/$REPO/diffinery/sandboxes" |
| 159 | id=$(curl -fsS -X POST "$base" -H "Authorization: Bearer $TOKEN" -H "content-type: application/json" --data '{}' | jq -r .id) |
| 160 | echo "sandbox $id" |
| 161 | for i in $(seq 1 40); do |
| 162 | out=$(curl -sS -X POST "$base/$id/exec" -H "Authorization: Bearer $TOKEN" -H "content-type: application/json" --data '{"cmd":"cat README.md && git lfs ls-files | wc -l"}' || true) |
| 163 | jq -e '.exitCode == 0' >/dev/null 2>&1 <<<"$out" && break |
| 164 | sleep 6 |
| 165 | done |
| 166 | echo "$out" | jq '{exitCode, stdout}' |
| 167 | curl -sS -X POST "$base/$id/close" -H "Authorization: Bearer $TOKEN" >/dev/null || true |
| 168 | jq -e --arg r "# $REPO" '.exitCode == 0 and (.stdout | contains($r))' >/dev/null <<<"$out" || { echo "::error::the sandbox did not run the command in the repository"; exit 1; } |
| 169 | |
| 170 | - name: Pull requests and the merge queue |
| 171 | if: steps.token.outputs.skip != 'true' |
| 172 | run: | |
| 173 | set -euo pipefail |
| 174 | Q="$REPO-pr" |
| 175 | api() { curl -fsS -X "$1" "$DIFFINERY/api/v3$2" -H "Authorization: Bearer $TOKEN" -H "content-type: application/json" ${3:+--data "$3"}; } |
| 176 | api POST /user/repos "{\"name\":\"$Q\",\"private\":true}" >/dev/null |
| 177 | echo "PR_REPO=$Q" >> "$GITHUB_ENV" |
| 178 | api PUT "/repos/$OWNER/$Q/actions/secrets/TOPSECRET" '{"value":"s3cret"}' >/dev/null |
| 179 | remote="https://x-token:${TOKEN}@${DIFFINERY#https://}/$OWNER/$Q.git" |
| 180 | mkdir -p pr/.github/workflows && cd pr |
| 181 | git init -q -b main |
| 182 | git config user.name "Live test" && git config user.email "live-test@diffinery.com" |
| 183 | printf 'a\nb\nc\n' > app.txt |
| 184 | cat > .github/workflows/ci.yml <<'EOF' |
| 185 | name: CI |
| 186 | on: |
| 187 | pull_request: |
| 188 | merge_group: |
| 189 | jobs: |
| 190 | test: |
| 191 | runs-on: ubuntu-latest |
| 192 | steps: |
| 193 | - uses: actions/checkout@v4 |
| 194 | - run: echo "pr ok event=${{ '${{' }} github.event_name }} ref=${{ '${{' }} github.ref }} sha=${{ '${{' }} github.sha }} head=$(git rev-parse HEAD) app=$(tr '\n' ' ' < app.txt)" |
| 195 | EOF |
| 196 | cat > .github/workflows/target.yml <<'EOF' |
| 197 | name: Target |
| 198 | on: pull_request_target |
| 199 | jobs: |
| 200 | t: |
| 201 | runs-on: ubuntu-latest |
| 202 | steps: |
| 203 | - run: echo "target ok secret=${{ '${{' }} secrets.TOPSECRET == 's3cret' }} ref=${{ '${{' }} github.ref }}" |
| 204 | EOF |
| 205 | git add -A && git commit -q -m "PR test" && git push -q "$remote" main |
| 206 | # The pull request changes app.txt and tries to rewrite the pull_request_target workflow. |
| 207 | git checkout -q -b change |
| 208 | printf 'A\nb\nc\n' > app.txt |
| 209 | sed -i 's/target ok/PWNED/' .github/workflows/target.yml |
| 210 | git commit -q -am "Change" && git push -q "$remote" change |
| 211 | head=$(git rev-parse HEAD) |
| 212 | # Main moves on, so the test merge commit holds both sides. |
| 213 | git checkout -q main |
| 214 | printf 'a\nb\nC\n' > app.txt |
| 215 | git commit -q -am "Main moves" && git push -q "$remote" main |
| 216 | n=$(api POST "/repos/$OWNER/$Q/pulls" '{"title":"Change","head":"change","base":"main"}' | jq -r .number) |
| 217 | echo "opened pull request #$n (head $head)" |
| 218 | |
| 219 | wait_run() { # event -> the completed run JSON |
| 220 | for i in $(seq 1 60); do |
| 221 | run=$(api GET "/repos/$OWNER/$Q/actions/runs?event=$1" | jq -c '.workflow_runs[0] // empty') |
| 222 | [ -n "$run" ] && [ "$(jq -r .status <<<"$run")" = completed ] && { echo "$run"; return; } |
| 223 | sleep 10 |
| 224 | done |
| 225 | echo "::error::no completed $1 run" >&2; exit 1 |
| 226 | } |
| 227 | log_of() { for job in $(api GET "/repos/$OWNER/$Q/actions/runs/$1/jobs" | jq -r '.jobs[].id'); do api GET "/repos/$OWNER/$Q/actions/jobs/$job/logs"; done; } |
| 228 | pr=$(wait_run pull_request) |
| 229 | echo "$pr" | jq '{event, ref, sha, head_sha, conclusion}' |
| 230 | [ "$(jq -r .conclusion <<<"$pr")" = success ] || { echo "::error::the pull_request run failed"; exit 1; } |
| 231 | [ "$(jq -r .ref <<<"$pr")" = "refs/pull/$n/merge" ] && [ "$(jq -r .head_sha <<<"$pr")" = "$head" ] && [ "$(jq -r .sha <<<"$pr")" != "$head" ] || { echo "::error::the pull_request run did not use the test merge commit"; exit 1; } |
| 232 | log_of "$(jq -r .id <<<"$pr")" | grep -F "pr ok event=pull_request ref=refs/pull/$n/merge" | grep -F "app=A b C" || { echo "::error::the pull_request job did not check out the merged code"; exit 1; } |
| 233 | target=$(wait_run pull_request_target) |
| 234 | echo "$target" | jq '{event, ref, sha, head_sha, conclusion}' |
| 235 | tlog=$(log_of "$(jq -r .id <<<"$target")") |
| 236 | grep -F "target ok secret=true ref=refs/heads/main" <<<"$tlog" || { echo "::error::pull_request_target did not run the base workflow with secrets"; exit 1; } |
| 237 | ! grep -q PWNED <<<"$tlog" || { echo "::error::pull_request_target ran the pull request's workflow"; exit 1; } |
| 238 | |
| 239 | api PUT "/repos/$OWNER/$Q/branches/main/protection" '{"requirePullRequest":true,"requiredChecks":["CI / test"],"mergeQueue":{"method":"squash","checkTimeoutMinutes":20}}' >/dev/null |
| 240 | code=$(curl -sS -o /dev/null -w '%{http_code}' -X PUT "$DIFFINERY/api/v3/repos/$OWNER/$Q/pulls/$n/merge" -H "Authorization: Bearer $TOKEN" -H "content-type: application/json" --data '{}') |
| 241 | [ "$code" = 405 ] || { echo "::error::a direct merge into a merge queue branch returned $code"; exit 1; } |
| 242 | api PUT "/repos/$OWNER/$Q/pulls/$n/merge-queue" '{"delete_branch":true}' | jq '{state, position: .entry.position, merge_group: .entry.merge_group}' |
| 243 | for i in $(seq 1 60); do |
| 244 | merged=$(api GET "/repos/$OWNER/$Q/pulls/$n" | jq -r .merged) |
| 245 | [ "$merged" = true ] && break |
| 246 | sleep 10 |
| 247 | done |
| 248 | [ "$merged" = true ] || { echo "::error::the merge queue did not merge #$n"; api GET "/repos/$OWNER/$Q/merge-queue/main" | jq .; exit 1; } |
| 249 | group=$(wait_run merge_group) |
| 250 | echo "$group" | jq '{event, ref, sha, conclusion}' |
| 251 | [[ "$(jq -r .ref <<<"$group")" == refs/heads/gh-readonly-queue/main/pr-$n-* ]] || { echo "::error::the merge_group run is not on a queue branch"; exit 1; } |
| 252 | sha=$(api GET "/repos/$OWNER/$Q/branches/main" | jq -r .commit.sha) |
| 253 | [ "$sha" = "$(jq -r .sha <<<"$group")" ] || { echo "::error::main ($sha) is not the tested merge group"; exit 1; } |
| 254 | git fetch -q "$remote" main && [ "$(git show FETCH_HEAD:app.txt | tr '\n' ' ')" = "A b C " ] || { echo "::error::main does not hold the merged change"; exit 1; } |
| 255 | echo "pull request #$n merged through the queue as $sha" |
| 256 | |
| 257 | - name: Scheduled workflow |
| 258 | if: steps.token.outputs.skip != 'true' |
| 259 | run: | |
| 260 | set -euo pipefail |
| 261 | # The cron trigger fires every 5 minutes, so allow two windows. |
| 262 | for i in $(seq 1 75); do |
| 263 | run=$(curl -fsS "$DIFFINERY/api/v3/repos/$OWNER/$REPO/actions/runs?event=schedule" -H "Authorization: Bearer $TOKEN" | jq -c '.workflow_runs[0] // empty') |
| 264 | [ -n "$run" ] && [ "$(jq -r .status <<<"$run")" = completed ] && break |
| 265 | sleep 10 |
| 266 | done |
| 267 | [ -n "$run" ] || { echo "::error::no scheduled run started"; exit 1; } |
| 268 | echo "$run" | jq '{id, event, status, conclusion}' |
| 269 | [ "$(jq -r .conclusion <<<"$run")" = success ] || { echo "::error::the scheduled run did not succeed"; exit 1; } |
| 270 | |
| 271 | - name: Delete the throwaway repositories |
| 272 | if: always() && steps.token.outputs.skip != 'true' |
| 273 | run: | |
| 274 | curl -sS -o /dev/null -w '%{http_code}\n' -X DELETE "$DIFFINERY/api/v3/repos/$OWNER/$REPO" -H "Authorization: Bearer $TOKEN" |
| 275 | [ -z "${PR_REPO:-}" ] || curl -sS -o /dev/null -w '%{http_code}\n' -X DELETE "$DIFFINERY/api/v3/repos/$OWNER/$PR_REPO" -H "Authorization: Bearer $TOKEN" |