Add a live test of diffinery.com after each deploy (#10)
A throwaway private repository on the real site checks hosted Actions jobs in Cloudflare Containers, Git LFS round trips, webhook delivery, agent sandboxes and scheduled workflows, then deletes itself. It runs after every successful deploy and on demand, and skips with a warning until the DIFFINERY_LIVE_TOKEN secret exists. Claude-Session: https://claude.ai/code/session_01AizKXSsengXZRegJxRATkL Co-authored-by: Claude <noreply@anthropic.com>
Canberk Demirci authored · GitHub committed signed1 parent 55e6c72commit 2fa79bd0708a3c5c3c08494c9dc042b8302aca66Browse files
1 file changed+165−0
added.github/workflows/live-test.yml+165−0View file
| @@ -1,0 +1,165 @@ | ||
| 1 | +name: Live test | |
| 2 | + | |
| 3 | +# End-to-end checks against the real diffinery.com after each deploy: hosted | |
| 4 | +# Actions jobs in Cloudflare Containers, agent sandboxes, webhooks, scheduled | |
| 5 | +# workflows and Git LFS. Each run works in a throwaway private repository that it | |
| 6 | +# deletes at the end. | |
| 7 | +# | |
| 8 | +# Needs the repository secret DIFFINERY_LIVE_TOKEN: a Diffinery token (Settings -> | |
| 9 | +# Tokens) with the admin scope and access to all repositories. | |
| 10 | +on: | |
| 11 | + workflow_dispatch: | |
| 12 | + workflow_run: | |
| 13 | + workflows: [Deploy] | |
| 14 | + types: [completed] | |
| 15 | + branches: [main] | |
| 16 | + | |
| 17 | +permissions: | |
| 18 | + contents: read | |
| 19 | + | |
| 20 | +concurrency: | |
| 21 | + group: live-test | |
| 22 | + cancel-in-progress: false | |
| 23 | + | |
| 24 | +jobs: | |
| 25 | + live: | |
| 26 | + name: Containers, sandboxes, webhooks, schedules and LFS | |
| 27 | + if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }} | |
| 28 | + runs-on: ubuntu-latest | |
| 29 | + timeout-minutes: 30 | |
| 30 | + env: | |
| 31 | + DIFFINERY: https://diffinery.com | |
| 32 | + TOKEN: ${{ secrets.DIFFINERY_LIVE_TOKEN }} | |
| 33 | + REPO: live-${{ github.run_id }}-${{ github.run_attempt }} | |
| 34 | + steps: | |
| 35 | + - name: Check the token | |
| 36 | + id: token | |
| 37 | + run: | | |
| 38 | + if [ -z "$TOKEN" ]; then | |
| 39 | + echo "::warning::DIFFINERY_LIVE_TOKEN is not set, so the live test is skipped. Create an admin token at $DIFFINERY/settings/tokens and save it as a repository secret." | |
| 40 | + echo "skip=true" >> "$GITHUB_OUTPUT" | |
| 41 | + exit 0 | |
| 42 | + fi | |
| 43 | + login=$(curl -fsS "$DIFFINERY/api/v3/user" -H "Authorization: Bearer $TOKEN" | jq -r .login) | |
| 44 | + echo "OWNER=$login" >> "$GITHUB_ENV" | |
| 45 | + echo "testing as $login in $login/$REPO" | |
| 46 | + | |
| 47 | + - name: Create a throwaway repository with a webhook | |
| 48 | + if: steps.token.outputs.skip != 'true' | |
| 49 | + run: | | |
| 50 | + set -euo pipefail | |
| 51 | + api() { curl -fsS -X "$1" "$DIFFINERY/api/v3$2" -H "Authorization: Bearer $TOKEN" -H "content-type: application/json" ${3:+--data "$3"}; } | |
| 52 | + api POST /user/repos "{\"name\":\"$REPO\",\"private\":true}" >/dev/null | |
| 53 | + api POST "/repos/$OWNER/$REPO/hooks" "{\"config\":{\"url\":\"$DIFFINERY/healthz\"},\"events\":[\"push\"]}" >/dev/null | |
| 54 | + echo "created $OWNER/$REPO" | |
| 55 | + | |
| 56 | + - name: Push workflows and an LFS file | |
| 57 | + if: steps.token.outputs.skip != 'true' | |
| 58 | + run: | | |
| 59 | + set -euo pipefail | |
| 60 | + mkdir -p src/.github/workflows && cd src | |
| 61 | + git init -q -b main | |
| 62 | + git config user.name "Live test" && git config user.email "live-test@diffinery.com" | |
| 63 | + git lfs install --local >/dev/null | |
| 64 | + git lfs track "*.bin" >/dev/null | |
| 65 | + head -c 3145728 /dev/urandom > blob.bin | |
| 66 | + sha256sum blob.bin | cut -d' ' -f1 > ../blob.sha | |
| 67 | + echo "# $REPO" > README.md | |
| 68 | + cat > .github/workflows/ci.yml <<'EOF' | |
| 69 | + on: push | |
| 70 | + jobs: | |
| 71 | + hosted: | |
| 72 | + runs-on: ubuntu-latest | |
| 73 | + strategy: | |
| 74 | + matrix: | |
| 75 | + n: [1, 2] | |
| 76 | + steps: | |
| 77 | + - uses: actions/checkout@v4 | |
| 78 | + - run: echo "live ok ${{ '${{' }} matrix.n }} $(cat README.md)" | |
| 79 | + EOF | |
| 80 | + cat > .github/workflows/tick.yml <<'EOF' | |
| 81 | + on: | |
| 82 | + schedule: | |
| 83 | + - cron: "*/5 * * * *" | |
| 84 | + jobs: | |
| 85 | + tick: | |
| 86 | + runs-on: ubuntu-latest | |
| 87 | + steps: | |
| 88 | + - run: echo "scheduled ok" | |
| 89 | + EOF | |
| 90 | + git add -A && git commit -q -m "Live test" | |
| 91 | + git push -q "https://x-token:${TOKEN}@${DIFFINERY#https://}/$OWNER/$REPO.git" main | |
| 92 | + git lfs ls-files | |
| 93 | + | |
| 94 | + - name: Hosted Actions jobs run in Cloudflare Containers | |
| 95 | + if: steps.token.outputs.skip != 'true' | |
| 96 | + run: | | |
| 97 | + set -euo pipefail | |
| 98 | + api() { curl -fsS "$DIFFINERY/api/v3$1" -H "Authorization: Bearer $TOKEN"; } | |
| 99 | + for i in $(seq 1 90); do | |
| 100 | + run=$(api "/repos/$OWNER/$REPO/actions/runs?event=push" | jq -c '.workflow_runs[0] // empty') | |
| 101 | + [ -n "$run" ] && [ "$(jq -r .status <<<"$run")" = completed ] && break | |
| 102 | + sleep 10 | |
| 103 | + done | |
| 104 | + echo "$run" | jq '{id, status, conclusion}' | |
| 105 | + [ "$(jq -r .conclusion <<<"$run")" = success ] || { echo "::error::the push run did not succeed"; exit 1; } | |
| 106 | + id=$(jq -r .id <<<"$run") | |
| 107 | + for job in $(api "/repos/$OWNER/$REPO/actions/runs/$id/jobs" | jq -r '.jobs[].id'); do | |
| 108 | + log=$(api "/repos/$OWNER/$REPO/actions/jobs/$job/logs") | |
| 109 | + grep -F "live ok" <<<"$log" | grep -F "# $REPO" || { echo "::error::job $job did not print its line (checkout or run step failed)"; exit 1; } | |
| 110 | + done | |
| 111 | + | |
| 112 | + - name: Git LFS round trip | |
| 113 | + if: steps.token.outputs.skip != 'true' | |
| 114 | + run: | | |
| 115 | + set -euo pipefail | |
| 116 | + git clone -q "https://x-token:${TOKEN}@${DIFFINERY#https://}/$OWNER/$REPO.git" clone | |
| 117 | + [ "$(sha256sum clone/blob.bin | cut -d' ' -f1)" = "$(cat blob.sha)" ] || { echo "::error::the LFS file came back different"; exit 1; } | |
| 118 | + echo "LFS file matches ($(stat -c %s clone/blob.bin) bytes)" | |
| 119 | + | |
| 120 | + - name: Webhook delivered | |
| 121 | + if: steps.token.outputs.skip != 'true' | |
| 122 | + run: | | |
| 123 | + set -euo pipefail | |
| 124 | + for i in $(seq 1 30); do | |
| 125 | + hook=$(curl -fsS "$DIFFINERY/api/v3/repos/$OWNER/$REPO/hooks" -H "Authorization: Bearer $TOKEN" | jq -c '.[0]') | |
| 126 | + [ "$(jq -r '.lastDeliveryAt // empty' <<<"$hook")" != "" ] && break | |
| 127 | + sleep 5 | |
| 128 | + done | |
| 129 | + echo "$hook" | jq '{url, lastStatus, lastDeliveryAt}' | |
| 130 | + [ "$(jq -r '.lastDeliveryAt // empty' <<<"$hook")" != "" ] || { echo "::error::the push webhook was never delivered"; exit 1; } | |
| 131 | + | |
| 132 | + - name: Agent sandbox | |
| 133 | + if: steps.token.outputs.skip != 'true' | |
| 134 | + run: | | |
| 135 | + set -euo pipefail | |
| 136 | + base="$DIFFINERY/api/v3/repos/$OWNER/$REPO/diffinery/sandboxes" | |
| 137 | + id=$(curl -fsS -X POST "$base" -H "Authorization: Bearer $TOKEN" -H "content-type: application/json" --data '{}' | jq -r .id) | |
| 138 | + echo "sandbox $id" | |
| 139 | + for i in $(seq 1 40); do | |
| 140 | + out=$(curl -sS -X POST "$base/$id/exec" -H "Authorization: Bearer $TOKEN" -H "content-type: application/json" --data '{"cmd":"cat README.md && git lfs ls-files | wc -l"}' || true) | |
| 141 | + jq -e '.exitCode == 0' >/dev/null 2>&1 <<<"$out" && break | |
| 142 | + sleep 6 | |
| 143 | + done | |
| 144 | + echo "$out" | jq '{exitCode, stdout}' | |
| 145 | + curl -sS -X POST "$base/$id/close" -H "Authorization: Bearer $TOKEN" >/dev/null || true | |
| 146 | + jq -e --arg r "# $REPO" '.exitCode == 0 and (.stdout | contains($r))' >/dev/null <<<"$out" || { echo "::error::the sandbox did not run the command in the repository"; exit 1; } | |
| 147 | + | |
| 148 | + - name: Scheduled workflow | |
| 149 | + if: steps.token.outputs.skip != 'true' | |
| 150 | + run: | | |
| 151 | + set -euo pipefail | |
| 152 | + # The cron trigger fires every 5 minutes, so allow two windows. | |
| 153 | + for i in $(seq 1 75); do | |
| 154 | + run=$(curl -fsS "$DIFFINERY/api/v3/repos/$OWNER/$REPO/actions/runs?event=schedule" -H "Authorization: Bearer $TOKEN" | jq -c '.workflow_runs[0] // empty') | |
| 155 | + [ -n "$run" ] && [ "$(jq -r .status <<<"$run")" = completed ] && break | |
| 156 | + sleep 10 | |
| 157 | + done | |
| 158 | + [ -n "$run" ] || { echo "::error::no scheduled run started"; exit 1; } | |
| 159 | + echo "$run" | jq '{id, event, status, conclusion}' | |
| 160 | + [ "$(jq -r .conclusion <<<"$run")" = success ] || { echo "::error::the scheduled run did not succeed"; exit 1; } | |
| 161 | + | |
| 162 | + - name: Delete the throwaway repository | |
| 163 | + if: always() && steps.token.outputs.skip != 'true' | |
| 164 | + run: | | |
| 165 | + curl -sS -o /dev/null -w '%{http_code}\n' -X DELETE "$DIFFINERY/api/v3/repos/$OWNER/$REPO" -H "Authorization: Bearer $TOKEN" |