| 1 | name: Live test |
| 2 | |
| 3 | # End-to-end checks against the real diffinery.com after each deploy: email |
| 4 | # sending, web UI features (syntax highlighting, settings pages, view transitions, milestones, variables), hosted Actions jobs in Cloudflare Containers (with Docker builds and services), the container registry, agent sandboxes, |
| 5 | # webhooks, scheduled workflows, Git LFS, imports from GitHub, that tokens can't mint tokens, and pull requests (test merge commits, pull_request_target and the merge |
| 6 | # queue). Each run works in a throwaway private repository that it |
| 7 | # deletes at the end. |
| 8 | # |
| 9 | # Needs DIFFINERY_LIVE_TOKEN in the production environment: a write-scope token |
| 10 | # (Settings -> Tokens) with access to all repositories, for a dedicated account that isn't |
| 11 | # a site administrator (give it a quota override). The checks that need a site |
| 12 | # administrator are skipped with a notice when the token isn't one's. |
| 13 | on: |
| 14 | workflow_dispatch: |
| 15 | workflow_run: |
| 16 | workflows: [Deploy] |
| 17 | types: [completed] |
| 18 | branches: [main] |
| 19 | |
| 20 | permissions: |
| 21 | contents: read |
| 22 | # To see whether the Deploy run it follows deployed anything. |
| 23 | actions: read |
| 24 | |
| 25 | concurrency: |
| 26 | group: live-test |
| 27 | cancel-in-progress: false |
| 28 | |
| 29 | jobs: |
| 30 | live: |
| 31 | name: Containers, registry, sandboxes, webhooks, schedules, LFS and pull requests |
| 32 | # Runs on the host that deploys production (see the gate in deploy.yml). |
| 33 | if: ${{ github.event_name == 'workflow_dispatch' || (github.server_url == 'https://diffinery.com' && github.event.workflow_run.conclusion == 'success') }} |
| 34 | runs-on: ubuntu-latest |
| 35 | timeout-minutes: 45 |
| 36 | environment: production |
| 37 | env: |
| 38 | DIFFINERY: https://diffinery.com |
| 39 | TOKEN: ${{ secrets.DIFFINERY_LIVE_TOKEN }} |
| 40 | REPO: live-${{ github.run_id }}-${{ github.run_attempt }} |
| 41 | HEAD_SHA: ${{ github.event.workflow_run.head_sha || github.sha }} |
| 42 | steps: |
| 43 | - name: Check the token |
| 44 | id: token |
| 45 | env: |
| 46 | GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} |
| 47 | DEPLOY_RUN: ${{ github.event.workflow_run.id }} |
| 48 | run: | |
| 49 | if [ "$GITHUB_EVENT_NAME" = workflow_run ]; then |
| 50 | # Deploy's gate skips a commit that is no longer main's tip (the newer commit's run deploys it): |
| 51 | # then nothing new is live, and this run would only test the site as it was. |
| 52 | deployed=$(curl -fsS "$GITHUB_API_URL/repos/$GITHUB_REPOSITORY/actions/runs/$DEPLOY_RUN/jobs" -H "Authorization: Bearer $GH_TOKEN" | jq -r '[.jobs[] | select(.name == "Deploy to Cloudflare") | .conclusion][0] // "unknown"' || echo unknown) |
| 53 | if [ "$deployed" = skipped ]; then |
| 54 | echo "::notice::Deploy deployed nothing for $HEAD_SHA (main has moved on), so there is nothing new to test" |
| 55 | echo "skip=true" >> "$GITHUB_OUTPUT" |
| 56 | exit 0 |
| 57 | fi |
| 58 | fi |
| 59 | if [ -z "$TOKEN" ]; then |
| 60 | # After a deploy of diffinery.com a missing token is an error: a green run would mean nothing was tested. |
| 61 | if [ "$GITHUB_EVENT_NAME" = workflow_run ]; then |
| 62 | echo "::error::DIFFINERY_LIVE_TOKEN is not set in the production environment, so nothing was tested. Create a write token at $DIFFINERY/settings/tokens and save it as a production environment secret." |
| 63 | exit 1 |
| 64 | fi |
| 65 | echo "::warning::DIFFINERY_LIVE_TOKEN is not set, so the live test is skipped. Create a write token at $DIFFINERY/settings/tokens and save it as a production environment secret." |
| 66 | echo "skip=true" >> "$GITHUB_OUTPUT" |
| 67 | exit 0 |
| 68 | fi |
| 69 | login=$(curl -fsS "$DIFFINERY/api/v3/user" -H "Authorization: Bearer $TOKEN" | jq -r .login) |
| 70 | echo "OWNER=$login" >> "$GITHUB_ENV" |
| 71 | echo "testing as $login in $login/$REPO" |
| 72 | |
| 73 | - name: Choose the registry check's image sizes |
| 74 | if: steps.token.outputs.skip != 'true' |
| 75 | run: | |
| 76 | set -euo pipefail |
| 77 | # Layers over Cloudflare's 100 MB request limit (which hosted runners push through |
| 78 | # the registry relay) take minutes to push and pull. They're pushed on manual runs and |
| 79 | # whenever the registry, the runner image or this test changed since the last |
| 80 | # successful live test; other deploys push small images through the same path. |
| 81 | full=true |
| 82 | if [ "$GITHUB_EVENT_NAME" = workflow_run ]; then |
| 83 | api() { curl -fsS "$DIFFINERY/api/v3/repos/$GITHUB_REPOSITORY$1" -H "Authorization: Bearer $TOKEN"; } |
| 84 | last=$(api "/actions/runs?workflow=live-test.yml&status=success&per_page=1" | jq -r '.workflow_runs[0].head_sha // empty' || true) |
| 85 | if [ -n "$last" ] && files=$(api "/compare/$last...$HEAD_SHA" | jq -r '.files[].filename'); then |
| 86 | grep -qE '^(apps/diffinery/src/registry/|containers/runner/|\.github/workflows/live-test\.yml$)' <<<"$files" || full=false |
| 87 | echo "changed since $last: $(wc -l <<<"$files") files" |
| 88 | fi |
| 89 | fi |
| 90 | if [ "$full" = true ]; then |
| 91 | echo "BIG=150000000" >> "$GITHUB_ENV" |
| 92 | echo "BUILDX=120000000" >> "$GITHUB_ENV" |
| 93 | else |
| 94 | echo "BIG=5000000" >> "$GITHUB_ENV" |
| 95 | echo "BUILDX=4000000" >> "$GITHUB_ENV" |
| 96 | fi |
| 97 | echo "registry check: full=$full" |
| 98 | |
| 99 | - name: Profile picture upload (WASM resize in the Worker) |
| 100 | if: steps.token.outputs.skip != 'true' |
| 101 | run: | |
| 102 | set -euo pipefail |
| 103 | av="$DIFFINERY/api/avatars/$OWNER" |
| 104 | # Keep the account's current picture, if it has one, to put it back afterwards. |
| 105 | had=$(curl -fsS -o before.webp -w '%{content_type}' "$av?s=256&v=1") |
| 106 | restore() { |
| 107 | if [ "$had" = image/webp ]; then |
| 108 | curl -fsS -X PUT "$DIFFINERY/api/v3/user/avatar" -H "Authorization: Bearer $TOKEN" -H "content-type: image/webp" --data-binary @before.webp | jq -r '"restored the previous picture: \(.avatar_url)"' |
| 109 | else |
| 110 | curl -fsS -o /dev/null -X DELETE "$DIFFINERY/api/v3/user/avatar" -H "Authorization: Bearer $TOKEN" |
| 111 | fi |
| 112 | } |
| 113 | trap restore EXIT |
| 114 | # A 600×400 PNG gradient; the server crops it to a square and makes small WebPs. |
| 115 | python3 - <<'EOF' |
| 116 | import struct, zlib |
| 117 | w, h = 600, 400 |
| 118 | rows = b"".join(b"\0" + bytes(c for x in range(w) for c in (x * 255 // w, y * 255 // h, 128)) for y in range(h)) |
| 119 | chunk = lambda t, d: struct.pack(">I", len(d)) + t + d + struct.pack(">I", zlib.crc32(t + d)) |
| 120 | open("avatar.png", "wb").write(b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", w, h, 8, 2, 0, 0, 0)) + chunk(b"IDAT", zlib.compress(rows)) + chunk(b"IEND", b"")) |
| 121 | EOF |
| 122 | url=$(curl -fsS -X PUT "$DIFFINERY/api/v3/user/avatar" -H "Authorization: Bearer $TOKEN" -H "content-type: image/png" --data-binary @avatar.png | jq -r .avatar_url) |
| 123 | echo "uploaded: $url" |
| 124 | v=${url##*v=} |
| 125 | type=$(curl -fsS -o small.webp -w '%{content_type}' "$av?s=64&v=$v") |
| 126 | [ "$type" = image/webp ] || { echo "::error::expected image/webp, got $type"; exit 1; } |
| 127 | python3 - <<'EOF' |
| 128 | import struct, sys |
| 129 | b = open("small.webp", "rb").read() |
| 130 | ok = b[:4] == b"RIFF" and b[8:12] == b"WEBP" |
| 131 | if b[12:16] == b"VP8 ": |
| 132 | size = (struct.unpack("<H", b[26:28])[0] & 0x3FFF, struct.unpack("<H", b[28:30])[0] & 0x3FFF) |
| 133 | elif b[12:16] == b"VP8X": |
| 134 | size = (int.from_bytes(b[24:27], "little") + 1, int.from_bytes(b[27:30], "little") + 1) |
| 135 | else: |
| 136 | size = None |
| 137 | print(f"64 px avatar: {len(b)} bytes, {size}") |
| 138 | if not ok or size != (64, 64) or len(b) > 5000: |
| 139 | print("::error::the 64 px avatar is not a 64x64 WebP under 5 KB"); sys.exit(1) |
| 140 | EOF |
| 141 | curl -fsS -X DELETE "$DIFFINERY/api/v3/user/avatar" -H "Authorization: Bearer $TOKEN" >/dev/null |
| 142 | # A `v` that is not current bypasses the edge cache, so this sees the removal from any colo. |
| 143 | type=$(curl -fsS -o /dev/null -w '%{content_type}' "$av?s=64&v=1") |
| 144 | [[ "$type" == image/svg+xml* ]] || { echo "::error::after removal expected the initials SVG, got $type"; exit 1; } |
| 145 | echo "removed: initials again ($type)" |
| 146 | |
| 147 | - name: Public docs are served without signing in |
| 148 | run: | |
| 149 | set -euo pipefail |
| 150 | check() { # path, content type, text it must contain |
| 151 | body=$(curl -fsS -D /tmp/h "$DIFFINERY$1") |
| 152 | grep -qi "content-type: $2" /tmp/h || { echo "::error::$1 is not $2"; exit 1; } |
| 153 | grep -qF -- "$3" <<<"$body" || { echo "::error::$1 doesn't contain: $3"; exit 1; } |
| 154 | echo "$1 ok ($(wc -c <<<"$body") bytes)" |
| 155 | } |
| 156 | check /docs "text/html" "Overview" |
| 157 | check /docs/actions/workflow-syntax "text/html" "View as Markdown" |
| 158 | check /docs/quickstart.md "text/markdown" "# Quickstart" |
| 159 | check /llms.txt "text/plain" "/docs/actions/workflow-syntax.md" |
| 160 | check /llms-full.txt "text/plain" "# REST API reference" |
| 161 | check /docs/api/openapi.json "application/json" '"openapi": "3.1.0"' |
| 162 | check /-/cli/diffinery.mjs "text/javascript" "#!/usr/bin/env node" |
| 163 | md=$(curl -fsS -H "Accept: text/markdown" "$DIFFINERY/docs/agents/mcp") |
| 164 | grep -q "^# MCP" <<<"$md" || { echo "::error::Accept: text/markdown didn't return Markdown"; exit 1; } |
| 165 | curl -fsS "$DIFFINERY/-/cli/diffinery.mjs" -o /tmp/diffinery.mjs && node /tmp/diffinery.mjs --version |
| 166 | |
| 167 | - name: Email sending is configured |
| 168 | if: steps.token.outputs.skip != 'true' |
| 169 | run: | |
| 170 | set -euo pipefail |
| 171 | s=$(curl -sS -w '\n%{http_code}' "$DIFFINERY/api/v3/diffinery/admin/email" -H "Authorization: Bearer $TOKEN") |
| 172 | code=${s##*$'\n'}; s=${s%$'\n'*} |
| 173 | # Only a site administrator's token reads the email settings, and the live-test account isn't one. |
| 174 | if [ "$code" = 403 ]; then echo "::notice::skipped: the live-test token isn't a site administrator's"; exit 0; fi |
| 175 | [ "$code" = 200 ] || { echo "::error::/diffinery/admin/email answered $code"; exit 1; } |
| 176 | echo "$s" | jq '{sending, queue, from, last_24h}' |
| 177 | [ "$(jq -r .sending <<<"$s")" = true ] || { echo "::error::the Worker has no Email Sending binding"; exit 1; } |
| 178 | [ "$(jq -r .queue <<<"$s")" = true ] || { echo "::error::the Worker has no mail queue binding"; exit 1; } |
| 179 | failed=$(jq -r .last_24h.failed <<<"$s") |
| 180 | [ "$failed" = 0 ] || echo "::warning::$failed emails failed in the last 24 hours; see Settings -> Email in site administration" |
| 181 | |
| 182 | - name: Create a throwaway repository with a webhook |
| 183 | if: steps.token.outputs.skip != 'true' |
| 184 | run: | |
| 185 | set -euo pipefail |
| 186 | api() { curl -fsS -X "$1" "$DIFFINERY/api/v3$2" -H "Authorization: Bearer $TOKEN" -H "content-type: application/json" ${3:+--data "$3"}; } |
| 187 | api POST /user/repos "{\"name\":\"$REPO\",\"private\":true}" >/dev/null |
| 188 | # Webhooks may only reach public hosts other than Diffinery itself; the push's delivery goes to example.com. |
| 189 | code=$(curl -sS -o /dev/null -w '%{http_code}' -X POST "$DIFFINERY/api/v3/repos/$OWNER/$REPO/hooks" -H "Authorization: Bearer $TOKEN" -H "content-type: application/json" --data "{\"config\":{\"url\":\"$DIFFINERY/healthz\"},\"events\":[\"push\"]}") |
| 190 | [ "$code" = 422 ] || { echo "::error::a webhook to Diffinery itself answered $code, not 422"; exit 1; } |
| 191 | api POST "/repos/$OWNER/$REPO/hooks" "{\"config\":{\"url\":\"https://example.com/diffinery-live-test\"},\"events\":[\"push\"]}" >/dev/null |
| 192 | echo "created $OWNER/$REPO" |
| 193 | |
| 194 | - name: Wait for jobs to start on the deployed runner image |
| 195 | if: steps.token.outputs.skip != 'true' |
| 196 | run: | |
| 197 | set -euo pipefail |
| 198 | # A deploy's container rollout updates instances over a while after it starts, and an |
| 199 | # instance not yet updated starts jobs on the previous runner image. Jobs report their |
| 200 | # image (`runner.image`), so a probe workflow runs until three jobs in a row land on the |
| 201 | # image the deploy set (/meta's runner_image) before the Docker and registry checks. |
| 202 | api() { curl -fsS -X "$1" "$DIFFINERY/api/v3$2" -H "Authorization: Bearer $TOKEN" -H "content-type: application/json" ${3:+--data "$3"}; } |
| 203 | expected=$(api GET /meta | jq -r '.runner_image // empty') |
| 204 | if [ -z "$expected" ]; then |
| 205 | echo "::warning::/meta doesn't name the deployed runner image, so jobs aren't checked against it" |
| 206 | exit 0 |
| 207 | fi |
| 208 | echo "DEPLOYED_RUNNER_IMAGE=$expected" >> "$GITHUB_ENV" |
| 209 | mkdir -p src/.github/workflows && cd src |
| 210 | git init -q -b main |
| 211 | git config user.name "Live test" && git config user.email "live-test@diffinery.com" |
| 212 | cat > .github/workflows/probe.yml <<'EOF' |
| 213 | on: workflow_dispatch |
| 214 | jobs: |
| 215 | probe: |
| 216 | runs-on: ubuntu-latest |
| 217 | strategy: |
| 218 | matrix: |
| 219 | n: [1, 2, 3] |
| 220 | steps: |
| 221 | - run: echo "probe ok" |
| 222 | EOF |
| 223 | git add -A && git commit -q -m "Runner image probe" |
| 224 | git push -q "https://x-token:${TOKEN}@${DIFFINERY#https://}/$OWNER/$REPO.git" main |
| 225 | deadline=$(( $(date +%s) + 15 * 60 )) |
| 226 | while :; do |
| 227 | before=$(api GET "/repos/$OWNER/$REPO/actions/runs?event=workflow_dispatch" | jq '.workflow_runs | length') |
| 228 | api POST "/repos/$OWNER/$REPO/actions/workflows/probe.yml/dispatches" '{"ref":"main"}' >/dev/null |
| 229 | run="" |
| 230 | # Up to 10 minutes: a job turned away from a replaced image starts again after 30 s, 1, 2 and 4 minutes. |
| 231 | for i in $(seq 1 120); do |
| 232 | runs=$(api GET "/repos/$OWNER/$REPO/actions/runs?event=workflow_dispatch") |
| 233 | if [ "$(jq '.workflow_runs | length' <<<"$runs")" -gt "$before" ]; then |
| 234 | run=$(jq -c '.workflow_runs[0]' <<<"$runs") |
| 235 | [ "$(jq -r .status <<<"$run")" = completed ] && break |
| 236 | fi |
| 237 | sleep 5 |
| 238 | done |
| 239 | [ -n "$run" ] || { echo "::error::the probe workflow didn't start"; exit 1; } |
| 240 | probe=$(api GET "/repos/$OWNER/$REPO/actions/runs/$(jq -r .id <<<"$run")/jobs") |
| 241 | images=$(jq -r '.jobs[] | .runner.image // "no image reported"' <<<"$probe") |
| 242 | echo "probe jobs ran on: $(sort <<<"$images" | uniq -c | sed 's/^ *//' | paste -sd ','), started again $(jq '[.jobs[].placementTries // 0] | add // 0' <<<"$probe") time(s) in all" |
| 243 | [ "$(grep -cxF "$expected" <<<"$images" || true)" = 3 ] && { echo "jobs start on $expected"; exit 0; } |
| 244 | if [ "$(date +%s)" -ge "$deadline" ]; then |
| 245 | echo "::error::15 minutes after the deploy, jobs still start on a runner image other than the deployed $expected" |
| 246 | exit 1 |
| 247 | fi |
| 248 | sleep 20 |
| 249 | done |
| 250 | |
| 251 | - name: Push workflows and an LFS file |
| 252 | id: push |
| 253 | if: steps.token.outputs.skip != 'true' |
| 254 | run: | |
| 255 | set -euo pipefail |
| 256 | mkdir -p src/.github/workflows && cd src |
| 257 | git init -q -b main |
| 258 | git config user.name "Live test" && git config user.email "live-test@diffinery.com" |
| 259 | git lfs install --local >/dev/null |
| 260 | git lfs track "*.bin" >/dev/null |
| 261 | head -c 3145728 /dev/urandom > blob.bin |
| 262 | sha256sum blob.bin | cut -d' ' -f1 > ../blob.sha |
| 263 | echo "# $REPO" > README.md |
| 264 | cat > .github/workflows/ci.yml <<'EOF' |
| 265 | on: push |
| 266 | jobs: |
| 267 | hosted: |
| 268 | runs-on: ubuntu-latest |
| 269 | strategy: |
| 270 | matrix: |
| 271 | n: [1, 2] |
| 272 | steps: |
| 273 | - uses: actions/checkout@v4 |
| 274 | - run: echo "live ok ${{ '${{' }} matrix.n }} $(cat README.md)" |
| 275 | # Passwordless sudo, as on GitHub's runners (`sudo apt-get install ...` is common). |
| 276 | - run: sudo -n true && echo "sudo ok $(sudo -n id -u)" |
| 277 | docker: |
| 278 | runs-on: ubuntu-latest |
| 279 | services: |
| 280 | cache: |
| 281 | image: redis:7-alpine |
| 282 | ports: ["6379"] |
| 283 | options: --health-cmd "redis-cli ping" --health-interval 1s --health-retries 30 |
| 284 | steps: |
| 285 | - run: | |
| 286 | printf 'FROM alpine:3.20\nRUN echo built > /msg\nCMD ["cat", "/msg"]\n' > Dockerfile |
| 287 | docker build -q -t live-image . |
| 288 | echo "docker ok $(docker run --rm live-image)" |
| 289 | exec 3<>/dev/tcp/127.0.0.1/${{ '${{' }} job.services.cache.ports['6379'] }} |
| 290 | printf 'PING\r\n' >&3 |
| 291 | read -r line <&3 |
| 292 | echo "service ok ${line%$'\r'}" |
| 293 | # Container registry: a layer pushed with the job's GITHUB_TOKEN (over Cloudflare's |
| 294 | # 100 MB request limit on full runs), plus the docker/*-action route most ghcr.io |
| 295 | # workflows use. __BIG__ and __BUILDX__ are the layer sizes. |
| 296 | publish: |
| 297 | runs-on: ubuntu-latest |
| 298 | permissions: |
| 299 | packages: write |
| 300 | steps: |
| 301 | - run: | |
| 302 | image="diffinery.com/${GITHUB_REPOSITORY,,}" |
| 303 | echo "${{ '${{' }} secrets.GITHUB_TOKEN }}" | docker login diffinery.com -u "${{ '${{' }} github.actor }}" --password-stdin |
| 304 | head -c __BIG__ /dev/urandom > big.bin |
| 305 | sha256sum big.bin | cut -d' ' -f1 > big.sha |
| 306 | printf 'FROM scratch\nCOPY big.bin big.sha /\n' > Dockerfile |
| 307 | docker build -q -t "$image:live" . |
| 308 | docker push "$image:live" |
| 309 | echo "pushed $(cat big.sha)" |
| 310 | buildx: |
| 311 | runs-on: ubuntu-latest |
| 312 | permissions: |
| 313 | packages: write |
| 314 | steps: |
| 315 | - run: | |
| 316 | echo "IMAGE=diffinery.com/${GITHUB_REPOSITORY,,}" >> "$GITHUB_ENV" |
| 317 | head -c __BUILDX__ /dev/urandom > big.bin |
| 318 | printf 'FROM scratch\nCOPY big.bin /\n' > Dockerfile |
| 319 | - uses: docker/setup-buildx-action@v3 |
| 320 | - uses: docker/login-action@v3 |
| 321 | with: |
| 322 | registry: diffinery.com |
| 323 | username: ${{ '${{' }} github.actor }} |
| 324 | password: ${{ '${{' }} secrets.GITHUB_TOKEN }} |
| 325 | - uses: docker/build-push-action@v6 |
| 326 | with: |
| 327 | context: . |
| 328 | push: true |
| 329 | tags: ${{ '${{' }} env.IMAGE }}:buildx |
| 330 | pull: |
| 331 | needs: [publish, buildx] |
| 332 | runs-on: ubuntu-latest |
| 333 | permissions: |
| 334 | packages: read |
| 335 | steps: |
| 336 | - run: | |
| 337 | image="diffinery.com/${GITHUB_REPOSITORY,,}" |
| 338 | echo "${{ '${{' }} secrets.GITHUB_TOKEN }}" | docker login diffinery.com -u "${{ '${{' }} github.actor }}" --password-stdin |
| 339 | docker pull -q "$image:live" |
| 340 | id=$(docker create "$image:live" true) |
| 341 | docker cp "$id:/big.bin" big.bin && docker cp "$id:/big.sha" big.sha |
| 342 | [ "$(sha256sum big.bin | cut -d' ' -f1)" = "$(cat big.sha)" ] && echo "registry ok $(stat -c %s big.bin) bytes" |
| 343 | docker pull -q "$image:buildx" |
| 344 | id=$(docker create "$image:buildx" true) |
| 345 | docker cp "$id:/big.bin" buildx.bin |
| 346 | echo "buildx ok $(stat -c %s buildx.bin) bytes" |
| 347 | EOF |
| 348 | cat > .github/workflows/tick.yml <<'EOF' |
| 349 | on: |
| 350 | schedule: |
| 351 | - cron: "*/5 * * * *" |
| 352 | jobs: |
| 353 | tick: |
| 354 | runs-on: ubuntu-latest |
| 355 | steps: |
| 356 | - run: echo "scheduled ok" |
| 357 | EOF |
| 358 | sed -i "s/__BIG__/$BIG/g; s/__BUILDX__/$BUILDX/g" .github/workflows/ci.yml |
| 359 | git add -A && git commit -q -m "Live test" |
| 360 | git push -q "https://x-token:${TOKEN}@${DIFFINERY#https://}/$OWNER/$REPO.git" main |
| 361 | git lfs ls-files |
| 362 | |
| 363 | - name: Web UI features (highlighting, settings pages, view transitions, base permission, invitations) |
| 364 | if: steps.token.outputs.skip != 'true' |
| 365 | run: | |
| 366 | set -euo pipefail |
| 367 | page() { curl -fsS "$DIFFINERY$1" -H "Authorization: Bearer $TOKEN"; } |
| 368 | has() { grep -qF -- "$2" <<<"$1" || { echo "::error::$3"; exit 1; }; } |
| 369 | html=$(page "/$OWNER/$REPO/blob/main/.github/workflows/ci.yml") |
| 370 | has "$html" '<td class="sh">' "the workflow file is not syntax highlighted" |
| 371 | has "$html" "--shiki-dark" "highlighted code has no dark theme colours" |
| 372 | has "$html" "@view-transition{navigation:auto}" "pages don't opt in to view transitions" |
| 373 | has "$html" "prefers-reduced-motion:no-preference" "view transitions ignore reduced motion" |
| 374 | has "$(page /settings/appearance)" 'name="appearance"' "Settings -> Appearance has no theme choice" |
| 375 | has "$(page /notifications)" 'href="/settings/notifications"' "the notifications page doesn't link to notification settings" |
| 376 | code=$(curl -sS -o /tmp/out -w '%{http_code}' -X POST "$DIFFINERY/api/v3/user/repos" -H "Authorization: Bearer $TOKEN" -H "content-type: application/json" --data "{\"name\":\"$REPO-internal\",\"visibility\":\"internal\"}") |
| 377 | [ "$code" = 422 ] || { echo "::error::visibility internal got $code, expected 422"; curl -sS -o /dev/null -X DELETE "$DIFFINERY/api/v3/repos/$OWNER/$REPO-internal" -H "Authorization: Bearer $TOKEN"; exit 1; } |
| 378 | org=$(page /api/v3/user/orgs | jq -r '.[0].login // empty') |
| 379 | if [ -n "$org" ]; then |
| 380 | perm=$(page "/api/v3/orgs/$org" | jq -r '.default_repository_permission // empty') |
| 381 | case "$perm" in none|read|write|admin) echo "$org's base permission: $perm" ;; *) echo "::error::GET /orgs/$org has no default_repository_permission"; exit 1 ;; esac |
| 382 | # A malformed rename is refused before anything is written (422 for owners, 403 for everyone else). |
| 383 | code=$(curl -sS -o /tmp/out -w '%{http_code}' -X PATCH "$DIFFINERY/api/v3/orgs/$org" -H "Authorization: Bearer $TOKEN" -H "content-type: application/json" --data '{"name":123}') |
| 384 | case "$code" in 422|403) ;; *) echo "::error::PATCH /orgs/$org with a numeric name got $code"; cat /tmp/out; exit 1 ;; esac |
| 385 | fi |
| 386 | page "/api/v3/repos/$OWNER/$REPO/invitations" | jq -e 'type == "array"' >/dev/null || { echo "::error::GET /repos/$OWNER/$REPO/invitations doesn't list pending invitations"; exit 1; } |
| 387 | tools=$(curl -fsS -X POST "$DIFFINERY/mcp" -H "Authorization: Bearer $TOKEN" -H "content-type: application/json" -H "accept: application/json, text/event-stream" --data '{"jsonrpc":"2.0","id":1,"method":"tools/list"}' | jq -r '.result.tools[].name') |
| 388 | for t in get_organization list_invitations revoke_invitation; do grep -qx "$t" <<<"$tools" || { echo "::error::the MCP server has no $t tool"; exit 1; }; done |
| 389 | echo "highlighting, Appearance, notification settings link, view transitions, visibility, organization and invitation checks passed" |
| 390 | |
| 391 | - name: Diffy placeholders, Actions settings, milestones and variables |
| 392 | if: steps.token.outputs.skip != 'true' |
| 393 | run: | |
| 394 | set -euo pipefail |
| 395 | page() { curl -fsS "$DIFFINERY$1" -H "Authorization: Bearer $TOKEN"; } |
| 396 | api() { curl -fsS -X "$1" "$DIFFINERY/api/v3$2" -H "Authorization: Bearer $TOKEN" -H "content-type: application/json" ${3:+--data "$3"}; } |
| 397 | code() { curl -sS -o /tmp/out -w '%{http_code}' -X "$1" "$DIFFINERY/api/v3$2" -H "Authorization: Bearer $TOKEN" -H "content-type: application/json" ${3:+--data "$3"}; } |
| 398 | fail() { echo "::error::$1"; cat /tmp/out 2>/dev/null || true; exit 1; } |
| 399 | has() { grep -qF -- "$2" <<<"$1" || fail "$3"; } |
| 400 | R="/repos/$OWNER/$REPO" |
| 401 | has "$(page /new/import)" 'diffy/hello-world' "the importer's placeholder isn't Diffy" |
| 402 | has "$(page "/$OWNER/$REPO/settings/actions")" 'Allow GitHub Actions to create and approve pull requests' "repository settings have no Actions page" |
| 403 | # Milestones: created closed, checked before an issue edit is saved, shown in the response, and filterable. |
| 404 | [ "$(api POST "$R/milestones" '{"title":"Shipped","state":"closed"}' | jq -r .state)" = closed ] || fail "POST milestones ignored state" |
| 405 | ms=$(api POST "$R/milestones" '{"title":"Live v1"}' | jq -r .number) |
| 406 | n=$(api POST "$R/issues" '{"title":"Planned"}' | jq -r .number) |
| 407 | [ "$(code PATCH "$R/issues/$n" '{"title":"Renamed","milestone":999999}')" = 422 ] || fail "an unknown milestone wasn't refused with 422" |
| 408 | [ "$(api GET "$R/issues/$n" | jq -r .title)" = Planned ] || fail "a refused milestone edit saved the title" |
| 409 | [ "$(api PATCH "$R/issues/$n" "{\"milestone\":$ms}" | jq -r .milestone.number)" = "$ms" ] || fail "PATCH issue doesn't return the new milestone" |
| 410 | [ "$(api GET "$R/issues?milestone=$ms&state=all" | jq -r '[.[].number] | join(",")')" = "$n" ] || fail "issues aren't filtered by milestone" |
| 411 | loc=$(curl -sS -o /dev/null -w '%{redirect_url}' "$DIFFINERY/$OWNER/$REPO/milestone/$ms" -H "Authorization: Bearer $TOKEN") |
| 412 | has "$loc" "/issues?milestone=$ms" "the milestone link doesn't open its issue list" |
| 413 | # Repository variable deletes ignore letter case. |
| 414 | [ "$(code PUT "$R/actions/variables/LIVE_CASE" '{"value":"x"}')" = 204 ] || fail "couldn't set a repository variable" |
| 415 | [ "$(code DELETE "$R/actions/variables/live_case")" = 204 ] || fail "couldn't delete a repository variable" |
| 416 | [ "$(code GET "$R/actions/variables/LIVE_CASE")" = 404 ] || fail "deleting a variable in lower case left it in place" |
| 417 | org=$(page /api/v3/user/orgs | jq -r '.[0].login // empty') |
| 418 | if [ -n "$org" ]; then |
| 419 | # Deleting an organization variable or secret that doesn't exist is a 404 (403 for someone who isn't an admin). |
| 420 | for path in variables secrets; do |
| 421 | c=$(code DELETE "/orgs/$org/actions/$path/LIVE_NEVER_SET_$GITHUB_RUN_ID") |
| 422 | case "$c" in 404|403) ;; *) fail "DELETE of a missing organization $path entry got $c" ;; esac |
| 423 | done |
| 424 | fi |
| 425 | echo "Diffy placeholders, Actions settings page, milestone and variable checks passed" |
| 426 | |
| 427 | - name: Hosted Actions jobs (and Docker) run in Cloudflare Containers |
| 428 | if: steps.token.outputs.skip != 'true' |
| 429 | run: | |
| 430 | set -euo pipefail |
| 431 | api() { curl -fsS "$DIFFINERY/api/v3$1" -H "Authorization: Bearer $TOKEN"; } |
| 432 | for i in $(seq 1 90); do |
| 433 | run=$(api "/repos/$OWNER/$REPO/actions/runs?event=push" | jq -c '.workflow_runs[0] // empty') |
| 434 | [ -n "$run" ] && [ "$(jq -r .status <<<"$run")" = completed ] && break |
| 435 | sleep 10 |
| 436 | done |
| 437 | echo "$run" | jq '{id, status, conclusion}' |
| 438 | id=$(jq -r .id <<<"$run") |
| 439 | jobs=$(api "/repos/$OWNER/$REPO/actions/runs/$id/jobs") |
| 440 | # placementTries: times a job went back in line (no machine, or a runner on a replaced image turned away). |
| 441 | # restarts: times it started over because the platform replaced its runner before its steps ran. |
| 442 | jq -r '.jobs[] | " \(.name): \(.conclusion)\(if .failure then " - " + (.failure.message // "") else "" end)\(try " (\((.completed_at | fromdateiso8601) - (.started_at | fromdateiso8601))s)" catch "")\(if (.placementTries // 0) > 0 then ", started again \(.placementTries) time(s)" else "" end)\(if (.restarts // 0) > 0 then ", restarted \(.restarts) time(s) after its runner was replaced" else "" end)"' <<<"$jobs" |
| 443 | if [ "$(jq -r .conclusion <<<"$run")" != success ]; then |
| 444 | for job in $(jq -r '.jobs[] | select(.conclusion == "failure" or .conclusion == "timed_out") | .id' <<<"$jobs"); do |
| 445 | echo "::group::Last log lines of job $job" |
| 446 | api "/repos/$OWNER/$REPO/actions/jobs/$job/logs" | tail -n 80 || true |
| 447 | echo "::endgroup::" |
| 448 | done |
| 449 | echo "::error::the push run did not succeed" |
| 450 | exit 1 |
| 451 | fi |
| 452 | if [ -n "${DEPLOYED_RUNNER_IMAGE:-}" ]; then |
| 453 | # Every job ran the deployed runner, so these checks test what was just deployed. |
| 454 | stale=$(jq -r --arg want "$DEPLOYED_RUNNER_IMAGE" '.jobs[] | select(.runner.image != $want) | "\(.name) on \(.runner.image // "no image reported")"' <<<"$jobs") |
| 455 | [ -z "$stale" ] || { echo "::error::jobs ran on a runner image other than the deployed $DEPLOYED_RUNNER_IMAGE: $(paste -sd ';' <<<"$stale")"; exit 1; } |
| 456 | echo "every job ran on $DEPLOYED_RUNNER_IMAGE" |
| 457 | fi |
| 458 | for job in $(jq -r '.jobs[] | select(.name | startswith("hosted")) | .id' <<<"$jobs"); do |
| 459 | log=$(api "/repos/$OWNER/$REPO/actions/jobs/$job/logs") |
| 460 | grep -F "live ok" <<<"$log" | grep -F "# $REPO" || { echo "::error::job $job did not print its line (checkout or run step failed)"; exit 1; } |
| 461 | grep -F "sudo ok 0" <<<"$log" || { echo "::error::job $job could not run sudo without a password"; exit 1; } |
| 462 | done |
| 463 | job=$(jq -r '.jobs[] | select(.name == "docker") | .id' <<<"$jobs") |
| 464 | log=$(api "/repos/$OWNER/$REPO/actions/jobs/$job/logs") |
| 465 | grep -F "docker ok built" <<<"$log" || { echo "::error::docker build or run failed on a hosted runner"; exit 1; } |
| 466 | grep -F "service ok +PONG" <<<"$log" || { echo "::error::the redis service container was not reachable"; exit 1; } |
| 467 | job=$(jq -r '.jobs[] | select(.name == "pull") | .id' <<<"$jobs") |
| 468 | log=$(api "/repos/$OWNER/$REPO/actions/jobs/$job/logs") |
| 469 | grep -F "registry ok $BIG bytes" <<<"$log" || { echo "::error::the image pushed with a $BIG-byte layer did not come back intact from the registry"; exit 1; } |
| 470 | grep -F "buildx ok $BUILDX bytes" <<<"$log" || { echo "::error::the image pushed by docker/build-push-action did not come back from the registry"; exit 1; } |
| 471 | api "/users/$OWNER/packages/container/${REPO,,}/versions" | jq -c '.[] | {tags: .metadata.container.tags, size, platforms}' |
| 472 | |
| 473 | - name: An organization's packages page keeps its tabs |
| 474 | if: steps.token.outputs.skip != 'true' |
| 475 | run: | |
| 476 | set -euo pipefail |
| 477 | org=$(curl -fsS "$DIFFINERY/api/v3/user/orgs" -H "Authorization: Bearer $TOKEN" | jq -r '.[0].login // empty') |
| 478 | [ -n "$org" ] || { echo "$OWNER is in no organization; skipped"; exit 0; } |
| 479 | html=$(curl -fsS "$DIFFINERY/$org/-/packages" -H "Authorization: Bearer $TOKEN") |
| 480 | grep -qF "<a href=\"/$org/-/packages\" class=\"on\">" <<<"$html" || { echo "::error::$org's packages page lost the organization's tab bar"; exit 1; } |
| 481 | grep -qF "href=\"/$org?tab=repositories\"" <<<"$html" || { echo "::error::$org's packages page has no Repositories tab"; exit 1; } |
| 482 | echo "$org's packages page shows the organization's tabs with Packages selected" |
| 483 | |
| 484 | - name: Git LFS round trip |
| 485 | if: steps.token.outputs.skip != 'true' |
| 486 | run: | |
| 487 | set -euo pipefail |
| 488 | git clone -q "https://x-token:${TOKEN}@${DIFFINERY#https://}/$OWNER/$REPO.git" clone |
| 489 | [ "$(sha256sum clone/blob.bin | cut -d' ' -f1)" = "$(cat blob.sha)" ] || { echo "::error::the LFS file came back different"; exit 1; } |
| 490 | echo "LFS file matches ($(stat -c %s clone/blob.bin) bytes)" |
| 491 | |
| 492 | - name: Webhook delivered |
| 493 | if: steps.token.outputs.skip != 'true' |
| 494 | run: | |
| 495 | set -euo pipefail |
| 496 | for i in $(seq 1 30); do |
| 497 | hook=$(curl -fsS "$DIFFINERY/api/v3/repos/$OWNER/$REPO/hooks" -H "Authorization: Bearer $TOKEN" | jq -c '.[0]') |
| 498 | # GitHub's hook shape: the target is config.url, the last delivery's status last_response.code. |
| 499 | [ "$(jq -r '.last_response.code // empty' <<<"$hook")" != "" ] && break |
| 500 | sleep 5 |
| 501 | done |
| 502 | echo "$hook" | jq '{target: .config.url, last_response, updated_at}' |
| 503 | [ "$(jq -r '.last_response.code // empty' <<<"$hook")" != "" ] || { echo "::error::the push webhook was never delivered"; exit 1; } |
| 504 | |
| 505 | - name: Tokens can't mint tokens (API or web forms) |
| 506 | if: steps.token.outputs.skip != 'true' |
| 507 | run: | |
| 508 | set -euo pipefail |
| 509 | # Revokes anything a failed check minted, then fails. |
| 510 | minted() { echo "::error::$1"; curl -fsS "$DIFFINERY/api/v3/diffinery/tokens" -H "Authorization: Bearer $TOKEN" | jq -r '.[] | select(.name == "live-test escalation") | .id' | while read -r id; do curl -sS -o /dev/null -X DELETE "$DIFFINERY/api/v3/diffinery/tokens/$id" -H "Authorization: Bearer $TOKEN"; done; exit 1; } |
| 511 | code=$(curl -sS -o /tmp/out -w '%{http_code}' -X POST "$DIFFINERY/api/v3/diffinery/tokens" -H "Authorization: Bearer $TOKEN" -H "content-type: application/json" --data '{"name":"live-test escalation","scopes":["read"],"expires_in_days":1}') |
| 512 | [ "$code" = 403 ] || minted "an admin token minted a token through the API ($code)" |
| 513 | grep -q "browser session" /tmp/out || minted "unexpected refusal: $(cat /tmp/out)" |
| 514 | code=$(curl -sS -o /dev/null -w '%{http_code}' -X POST "$DIFFINERY/api/v3/diffinery/agents" -H "Authorization: Bearer $TOKEN" -H "content-type: application/json" --data '{"login":"live-test-escalation"}') |
| 515 | [ "$code" = 403 ] || { echo "::error::an admin token created an agent ($code); disable live-test-escalation by hand"; exit 1; } |
| 516 | # A web form with a token and a forged double-submit CSRF pair. |
| 517 | code=$(curl -sS -o /dev/null -w '%{http_code}' -X POST "$DIFFINERY/settings/tokens" -H "Authorization: Bearer $TOKEN" -H "cookie: dfy_csrf=forged" --data "_csrf=forged&name=live-test%20escalation&scope=read&days=1") |
| 518 | [ "$code" = 403 ] || minted "a token submitted the web token form ($code)" |
| 519 | echo "tokens can't mint tokens or agents, through the API or the web forms" |
| 520 | |
| 521 | - name: Agent sandbox |
| 522 | if: steps.token.outputs.skip != 'true' |
| 523 | run: | |
| 524 | set -euo pipefail |
| 525 | base="$DIFFINERY/api/v3/repos/$OWNER/$REPO/diffinery/sandboxes" |
| 526 | id=$(curl -fsS --retry 1 --retry-delay 15 -X POST "$base" -H "Authorization: Bearer $TOKEN" -H "content-type: application/json" --data '{}' | jq -r .id) |
| 527 | echo "sandbox $id" |
| 528 | for i in $(seq 1 40); do |
| 529 | out=$(curl -sS -X POST "$base/$id/exec" -H "Authorization: Bearer $TOKEN" -H "content-type: application/json" --data '{"cmd":"cat README.md && git lfs ls-files | wc -l"}' || true) |
| 530 | jq -e '.exitCode == 0' >/dev/null 2>&1 <<<"$out" && break |
| 531 | sleep 6 |
| 532 | done |
| 533 | echo "$out" | jq '{exitCode, stdout}' |
| 534 | curl -sS -X POST "$base/$id/close" -H "Authorization: Bearer $TOKEN" >/dev/null || true |
| 535 | jq -e --arg r "# $REPO" '.exitCode == 0 and (.stdout | contains($r))' >/dev/null <<<"$out" || { echo "::error::the sandbox did not run the command in the repository"; exit 1; } |
| 536 | |
| 537 | # Owned by the agentic thread: proves the agent jails (containers/runner/src/jail.ts) on Cloudflare: a terminal |
| 538 | # task's, and a headless run's (phase 5d), which reaches Diffinery only at its model endpoints. |
| 539 | - name: Agent jails (bubblewrap, no network but the allowlist proxy and the model proxy) |
| 540 | if: steps.token.outputs.skip != 'true' |
| 541 | run: | |
| 542 | set -euo pipefail |
| 543 | base="$DIFFINERY/api/v3/repos/$OWNER/$REPO/diffinery/sandboxes" |
| 544 | id="" |
| 545 | close() { if [ -n "$id" ]; then curl -sS --max-time 60 -X POST "$base/$id/close" -H "Authorization: Bearer $TOKEN" >/dev/null || true; fi; id=""; } |
| 546 | trap close EXIT |
| 547 | run() { curl -sS --max-time "${2:-240}" -X POST "$base/$id/exec" -H "Authorization: Bearer $TOKEN" -H "content-type: application/json" --data "$(jq -n --arg c "$1" '{cmd: $c, timeoutMs: 180000}')"; } |
| 548 | # One create (no retry: a retried POST could open a second sandbox nothing closes), then wait for it to answer. |
| 549 | open() { |
| 550 | close |
| 551 | id=$(curl -sS --max-time 240 -X POST "$base" -H "Authorization: Bearer $TOKEN" -H "content-type: application/json" --data '{}' | jq -r '.id // empty' || true) |
| 552 | [ -n "$id" ] || { echo "::error::could not open a sandbox for the jail check"; exit 1; } |
| 553 | for i in $(seq 1 30); do |
| 554 | jq -e '.exitCode == 0' >/dev/null 2>&1 <<<"$(run true 20 || true)" && return 0 |
| 555 | sleep 6 |
| 556 | done |
| 557 | echo "::error::the sandbox never became ready" |
| 558 | exit 1 |
| 559 | } |
| 560 | # Sandboxes roll out on their own: one opened right after a deploy can still be on the previous image. |
| 561 | jail=no |
| 562 | for attempt in 1 2 3 4 5; do |
| 563 | open |
| 564 | # A runner with headless jails asks for the run's model token; an older one ignores --egress. |
| 565 | has=$(run 'mkdir -p ../probe ../probe.out; [ -n "${DIFFINERY_JAIL:-}" ] && [ -x "$DIFFINERY_JAIL" ] && "$DIFFINERY_JAIL" agent --dir ../probe --out ../probe.out --egress model -- true 2>&1 | grep -q DIFFINERY_MODEL_TOKEN && echo yes || echo no' 20 | jq -r '.stdout // empty' || true) |
| 566 | if [ "$has" = yes ]; then jail=yes; break; fi |
| 567 | echo "sandbox $id is on a runner image without the jail; trying another in a minute" |
| 568 | close |
| 569 | sleep 60 |
| 570 | done |
| 571 | [ "$jail" = yes ] || { echo "::error::the sandbox started on a runner image without the jail (rollout not finished?)"; exit 1; } |
| 572 | inside='echo "direct=$(node -e "require(\"net\").connect(443, \"1.1.1.1\").on(\"connect\", () => (console.log(\"open\"), process.exit())).on(\"error\", (e) => (console.log(e.code), process.exit()))")" |
| 573 | echo "other=$(curl -sS -m 20 -o /dev/null -w "%{http_connect}" https://example.com 2>/dev/null || true)" |
| 574 | echo "sni=$(curl -sS -m 20 -o /dev/null -w "%{http_code}" --connect-to example.com:443:registry.npmjs.org:443 https://example.com 2>/dev/null; true)" |
| 575 | echo "checkout=$(ls -A /workspace/repo | paste -sd,)" |
| 576 | echo "credential=$(cat /workspace/repo/.git/config 2>/dev/null || echo hidden)" |
| 577 | v=$(npm view left-pad version --no-update-notifier --loglevel=error 2>/tmp/npm.err) |
| 578 | echo "npm=$(printf "%s\n" "$v" | grep -E "^[0-9]+\.[0-9]+\.[0-9]+$" | tail -1)" |
| 579 | echo "npmerr=$(tail -3 /tmp/npm.err | paste -sd" " -)"' |
| 580 | out=$(run "mkdir -p ../jail-check ../jail-check.out && \"\$DIFFINERY_JAIL\" agent --dir ../jail-check --out ../jail-check.out -- sh -c $(printf '%q' "$inside")") |
| 581 | echo "$out" | jq -r '"exit \(.exitCode)", .stdout, .stderr' |
| 582 | o=$(jq -r .stdout <<<"$out") |
| 583 | fail=0 |
| 584 | jq -e '.exitCode == 0' >/dev/null <<<"$out" || { echo "::error::the jail didn't run (exit $(jq -r .exitCode <<<"$out"))"; fail=1; } |
| 585 | grep -qE '^direct=(ENETUNREACH|EHOSTUNREACH|ECONNREFUSED|EACCES)$' <<<"$o" || { echo "::error::the jail has a route out of its own"; fail=1; } |
| 586 | grep -qx 'other=403' <<<"$o" || { echo "::error::the proxy didn't refuse example.com"; fail=1; } |
| 587 | grep -qx 'checkout=.git' <<<"$o" || { echo "::error::the jail sees the person's checkout"; fail=1; } |
| 588 | grep -qx 'credential=hidden' <<<"$o" || { echo "::error::the jail sees the checkout's git config"; fail=1; } |
| 589 | grep -qE '^npm=[0-9]+\.[0-9]+' <<<"$o" || { echo "::error::npm view left-pad didn't work through the proxy"; fail=1; } |
| 590 | grep -qx 'sni=000' <<<"$o" || { echo "::error::the proxy let TLS for example.com through a tunnel to registry.npmjs.org"; fail=1; } |
| 591 | |
| 592 | # A headless run's jail: registries only, and Diffinery only at /-/model/ through the model proxy outside, |
| 593 | # which adds the run's token (a stand-in here, so Diffinery answers 401) and drops whatever the agent sends. |
| 594 | host=${DIFFINERY#https://} |
| 595 | headless='echo "model=$(curl -s -o /dev/null -w "%{http_code}" -X POST "$ANTHROPIC_BASE_URL/v1/messages" -H "x-api-key: $ANTHROPIC_API_KEY" -H "content-type: application/json" -d "{}")" |
| 596 | echo "api=$(curl -s -o /dev/null -w "%{http_code}" --path-as-is "$ANTHROPIC_BASE_URL/../../api/v3/user")" |
| 597 | echo "connect=$(curl -sS -m 20 -o /dev/null -w "%{http_connect}" https://'"$host"'/api/v3/user 2>/dev/null || true)" |
| 598 | echo "vendor=$(curl -sS -m 20 -o /dev/null -w "%{http_connect}" https://api.anthropic.com 2>/dev/null || true)" |
| 599 | echo "token=$(env | grep -c dmt_ || true)" |
| 600 | echo "credential=$(cat /workspace/repo/.git/config 2>/dev/null || echo hidden)" |
| 601 | v=$(npm view left-pad version --no-update-notifier --loglevel=error 2>/dev/null) |
| 602 | echo "npm=$(printf "%s\n" "$v" | grep -E "^[0-9]+\.[0-9]+\.[0-9]+$" | tail -1)"' |
| 603 | out=$(run "mkdir -p ../headless-check ../headless-check.out && DIFFINERY_MODEL_TOKEN=dmt_livetest_standin DIFFINERY_MODEL_ORIGIN=$(printf '%q' "$DIFFINERY") ANTHROPIC_BASE_URL=http://127.0.0.1:3129/-/model/anthropic ANTHROPIC_API_KEY=diffinery-jail \"\$DIFFINERY_JAIL\" agent --dir ../headless-check --out ../headless-check.out --egress model -- sh -c $(printf '%q' "$headless")") |
| 604 | echo "$out" | jq -r '"exit \(.exitCode)", .stdout, .stderr' |
| 605 | o=$(jq -r .stdout <<<"$out") |
| 606 | jq -e '.exitCode == 0' >/dev/null <<<"$out" || { echo "::error::the headless jail didn't run (exit $(jq -r .exitCode <<<"$out"))"; fail=1; } |
| 607 | grep -qx 'model=401' <<<"$o" || { echo "::error::the model proxy didn't reach Diffinery's model endpoint with the run's token"; fail=1; } |
| 608 | grep -qx 'api=403' <<<"$o" || { echo "::error::the model proxy forwarded a path outside /-/model/"; fail=1; } |
| 609 | grep -qx 'connect=403' <<<"$o" || { echo "::error::the headless jail could open a tunnel to Diffinery"; fail=1; } |
| 610 | grep -qx 'vendor=403' <<<"$o" || { echo "::error::the headless jail could reach a CLI vendor"; fail=1; } |
| 611 | grep -qx 'token=0' <<<"$o" || { echo "::error::the run's token is inside the headless jail"; fail=1; } |
| 612 | grep -qx 'credential=hidden' <<<"$o" || { echo "::error::the headless jail sees the checkout's git config"; fail=1; } |
| 613 | grep -qE '^npm=[0-9]+\.[0-9]+' <<<"$o" || { echo "::error::npm view left-pad didn't work in the headless jail"; fail=1; } |
| 614 | exit $fail |
| 615 | |
| 616 | - name: Pull requests and the merge queue |
| 617 | if: steps.token.outputs.skip != 'true' |
| 618 | run: | |
| 619 | set -euo pipefail |
| 620 | Q="$REPO-pr" |
| 621 | api() { curl -fsS -X "$1" "$DIFFINERY/api/v3$2" -H "Authorization: Bearer $TOKEN" -H "content-type: application/json" ${3:+--data "$3"}; } |
| 622 | api POST /user/repos "{\"name\":\"$Q\",\"private\":true}" >/dev/null |
| 623 | echo "PR_REPO=$Q" >> "$GITHUB_ENV" |
| 624 | api PUT "/repos/$OWNER/$Q/actions/secrets/TOPSECRET" '{"value":"s3cret"}' >/dev/null |
| 625 | remote="https://x-token:${TOKEN}@${DIFFINERY#https://}/$OWNER/$Q.git" |
| 626 | mkdir -p pr/.github/workflows && cd pr |
| 627 | git init -q -b main |
| 628 | git config user.name "Live test" && git config user.email "live-test@diffinery.com" |
| 629 | printf 'a\nb\nc\n' > app.txt |
| 630 | cat > .github/workflows/ci.yml <<'EOF' |
| 631 | name: CI |
| 632 | on: |
| 633 | pull_request: |
| 634 | merge_group: |
| 635 | jobs: |
| 636 | test: |
| 637 | runs-on: ubuntu-latest |
| 638 | steps: |
| 639 | - uses: actions/checkout@v4 |
| 640 | - run: echo "pr ok event=${{ '${{' }} github.event_name }} ref=${{ '${{' }} github.ref }} sha=${{ '${{' }} github.sha }} head=$(git rev-parse HEAD) app=$(tr '\n' ' ' < app.txt)" |
| 641 | EOF |
| 642 | cat > .github/workflows/target.yml <<'EOF' |
| 643 | name: Target |
| 644 | on: pull_request_target |
| 645 | jobs: |
| 646 | t: |
| 647 | runs-on: ubuntu-latest |
| 648 | steps: |
| 649 | - run: echo "target ok secret=${{ '${{' }} secrets.TOPSECRET == 's3cret' }} ref=${{ '${{' }} github.ref }}" |
| 650 | EOF |
| 651 | git add -A && git commit -q -m "PR test" && git push -q "$remote" main |
| 652 | # The pull request changes app.txt and tries to rewrite the pull_request_target workflow. |
| 653 | git checkout -q -b change |
| 654 | printf 'A\nb\nc\n' > app.txt |
| 655 | sed -i 's/target ok/PWNED/' .github/workflows/target.yml |
| 656 | git commit -q -am "Change" && git push -q "$remote" change |
| 657 | head=$(git rev-parse HEAD) |
| 658 | # Main moves on, so the test merge commit holds both sides. |
| 659 | git checkout -q main |
| 660 | printf 'a\nb\nC\n' > app.txt |
| 661 | git commit -q -am "Main moves" && git push -q "$remote" main |
| 662 | n=$(api POST "/repos/$OWNER/$Q/pulls" '{"title":"Change","head":"change","base":"main"}' | jq -r .number) |
| 663 | echo "opened pull request #$n (head $head)" |
| 664 | |
| 665 | wait_run() { # event -> the completed run JSON |
| 666 | for i in $(seq 1 60); do |
| 667 | run=$(api GET "/repos/$OWNER/$Q/actions/runs?event=$1" | jq -c '.workflow_runs[0] // empty') |
| 668 | [ -n "$run" ] && [ "$(jq -r .status <<<"$run")" = completed ] && { echo "$run"; return; } |
| 669 | sleep 10 |
| 670 | done |
| 671 | echo "::error::no completed $1 run" >&2; exit 1 |
| 672 | } |
| 673 | log_of() { for job in $(api GET "/repos/$OWNER/$Q/actions/runs/$1/jobs" | jq -r '.jobs[].id'); do api GET "/repos/$OWNER/$Q/actions/jobs/$job/logs"; done; } |
| 674 | pr=$(wait_run pull_request) |
| 675 | echo "$pr" | jq '{event, ref, sha, head_sha, conclusion}' |
| 676 | [ "$(jq -r .conclusion <<<"$pr")" = success ] || { echo "::error::the pull_request run failed"; exit 1; } |
| 677 | [ "$(jq -r .ref <<<"$pr")" = "refs/pull/$n/merge" ] && [ "$(jq -r .head_sha <<<"$pr")" = "$head" ] && [ "$(jq -r .sha <<<"$pr")" != "$head" ] || { echo "::error::the pull_request run did not use the test merge commit"; exit 1; } |
| 678 | log_of "$(jq -r .id <<<"$pr")" | grep -F "pr ok event=pull_request ref=refs/pull/$n/merge" | grep -F "app=A b C" || { echo "::error::the pull_request job did not check out the merged code"; exit 1; } |
| 679 | target=$(wait_run pull_request_target) |
| 680 | echo "$target" | jq '{event, ref, sha, head_sha, conclusion}' |
| 681 | tlog=$(log_of "$(jq -r .id <<<"$target")") |
| 682 | grep -F "target ok secret=true ref=refs/heads/main" <<<"$tlog" || { echo "::error::pull_request_target did not run the base workflow with secrets"; exit 1; } |
| 683 | ! grep -q PWNED <<<"$tlog" || { echo "::error::pull_request_target ran the pull request's workflow"; exit 1; } |
| 684 | |
| 685 | api PUT "/repos/$OWNER/$Q/branches/main/protection" '{"requirePullRequest":true,"requiredChecks":["CI / test"],"mergeQueue":{"method":"squash","checkTimeoutMinutes":20}}' >/dev/null |
| 686 | code=$(curl -sS -o /dev/null -w '%{http_code}' -X PUT "$DIFFINERY/api/v3/repos/$OWNER/$Q/pulls/$n/merge" -H "Authorization: Bearer $TOKEN" -H "content-type: application/json" --data '{}') |
| 687 | [ "$code" = 405 ] || { echo "::error::a direct merge into a merge queue branch returned $code"; exit 1; } |
| 688 | api PUT "/repos/$OWNER/$Q/pulls/$n/merge-queue" '{"delete_branch":true}' | jq '{state, position: .entry.position, merge_group: .entry.merge_group}' |
| 689 | for i in $(seq 1 60); do |
| 690 | merged=$(api GET "/repos/$OWNER/$Q/pulls/$n" | jq -r .merged) |
| 691 | [ "$merged" = true ] && break |
| 692 | sleep 10 |
| 693 | done |
| 694 | [ "$merged" = true ] || { echo "::error::the merge queue did not merge #$n"; api GET "/repos/$OWNER/$Q/merge-queue/main" | jq .; exit 1; } |
| 695 | group=$(wait_run merge_group) |
| 696 | echo "$group" | jq '{event, ref, sha, conclusion}' |
| 697 | [[ "$(jq -r .ref <<<"$group")" == refs/heads/gh-readonly-queue/main/pr-$n-* ]] || { echo "::error::the merge_group run is not on a queue branch"; exit 1; } |
| 698 | sha=$(api GET "/repos/$OWNER/$Q/branches/main" | jq -r .commit.sha) |
| 699 | [ "$sha" = "$(jq -r .sha <<<"$group")" ] || { echo "::error::main ($sha) is not the tested merge group"; exit 1; } |
| 700 | git fetch -q "$remote" main && [ "$(git show FETCH_HEAD:app.txt | tr '\n' ' ')" = "A b C " ] || { echo "::error::main does not hold the merged change"; exit 1; } |
| 701 | echo "pull request #$n merged through the queue as $sha" |
| 702 | |
| 703 | - name: Import a public GitHub repository |
| 704 | if: steps.token.outputs.skip != 'true' |
| 705 | env: |
| 706 | # A small public repository with issues and pull requests; a repository variable can change it. |
| 707 | SOURCE: ${{ vars.LIVE_IMPORT_SOURCE || 'octocat/git-consortium' }} |
| 708 | # A read-only GitHub token (copied here by the sync), so GitHub's rate limit doesn't skip the check. Never printed. |
| 709 | GITHUB_TOKEN_FOR_IMPORT: ${{ secrets.IMPORT_GITHUB_TOKEN }} |
| 710 | run: | |
| 711 | set -euo pipefail |
| 712 | api() { curl -fsS -X "$1" "$DIFFINERY/api/v3$2" -H "Authorization: Bearer $TOKEN" -H "content-type: application/json" ${3:+--data "$3"}; } |
| 713 | name="$REPO-import" |
| 714 | body=$(jq -nc --arg s "$SOURCE" --arg o "$OWNER" --arg n "$name" --arg t "${GITHUB_TOKEN_FOR_IMPORT:-}" \ |
| 715 | '{source: $s, owner: $o, name: $n, visibility: "private"} + (if $t == "" then {} else {github_token: $t} end)') |
| 716 | if [ -n "${GITHUB_TOKEN_FOR_IMPORT:-}" ]; then echo "importing with IMPORT_GITHUB_TOKEN"; else echo "::warning::IMPORT_GITHUB_TOKEN is not set, so the import runs token-free"; fi |
| 717 | # Token-free, GitHub allows 60 requests an hour per address: a rate limit skips this check with a warning. |
| 718 | # Starting the import reads the repository from GitHub; a GitHub outage (502 "could not reach GitHub") |
| 719 | # is retried, and skips this check with a warning if it outlasts the retries. While other imports hold |
| 720 | # every slot (an import campaign), it waits up to 15 minutes for one, so a busy account doesn't hide this check. |
| 721 | tries=0 waits=0 |
| 722 | while :; do |
| 723 | code=$(curl -sS -o import.json -w '%{http_code}' -X POST "$DIFFINERY/api/v3/diffinery/imports" -H "Authorization: Bearer $TOKEN" -H "content-type: application/json" \ |
| 724 | --data-binary @- <<<"$body" || true) |
| 725 | unreachable=$(jq -r '.message // "" | startswith("could not reach GitHub")' import.json 2>/dev/null || echo false) |
| 726 | busy=$(jq -r '.message // "" | startswith("too many repository imports are already running")' import.json 2>/dev/null || echo false) |
| 727 | if [ "$code" = 429 ] && [ "$busy" = true ]; then |
| 728 | waits=$((waits + 1)) |
| 729 | [ "$waits" -le 30 ] || break |
| 730 | [ "$waits" -gt 1 ] || echo "every import slot is taken ($(jq -r .message import.json)); waiting for one, up to 15 minutes" |
| 731 | sleep 30 |
| 732 | continue |
| 733 | fi |
| 734 | { [ "$code" = 502 ] && [ "$unreachable" = true ]; } || [ "$code" = 000 ] || break |
| 735 | tries=$((tries + 1)) |
| 736 | [ "$tries" -lt 4 ] || break |
| 737 | echo "starting the import returned $code ($(jq -r .message import.json 2>/dev/null)); trying again in 30 seconds" |
| 738 | sleep 30 |
| 739 | done |
| 740 | if [ "$code" = 429 ] && [ "$busy" = true ]; then echo "::warning::no import slot came free in 15 minutes, so the import check is skipped: $(jq -r .message import.json)"; exit 0; fi |
| 741 | if [ "$code" = 429 ]; then echo "::warning::GitHub's rate limit stopped the import check: $(jq -r .message import.json)"; exit 0; fi |
| 742 | if [ "$code" = 502 ] && [ "$unreachable" = true ]; then echo "::warning::GitHub could not be reached, so the import check is skipped: $(jq -r .message import.json)"; exit 0; fi |
| 743 | [ "$code" = 201 ] || { echo "::error::starting the import returned $code: $(cat import.json)"; exit 1; } |
| 744 | echo "IMPORT_REPO=$name" >> "$GITHUB_ENV" |
| 745 | id=$(jq -r .id import.json) |
| 746 | echo "import $id: $SOURCE -> $OWNER/$name" |
| 747 | for i in $(seq 1 90); do |
| 748 | imp=$(api GET "/diffinery/imports/$id") |
| 749 | status=$(jq -r .status <<<"$imp") |
| 750 | case "$status" in succeeded|failed|cancelled) break ;; esac |
| 751 | if jq -e '[.steps[].message // ""] | any(test("rate limit"))' >/dev/null <<<"$imp"; then |
| 752 | api POST "/diffinery/imports/$id/cancel" >/dev/null || true |
| 753 | echo "::warning::GitHub's rate limit paused the import, so the import check is skipped: $(jq -r '[.steps[].message // ""] | map(select(test("rate limit")))[0]' <<<"$imp")" |
| 754 | exit 0 |
| 755 | fi |
| 756 | sleep 10 |
| 757 | done |
| 758 | jq '{status, error, warnings, steps: [.steps[] | "\(.name): \(.status) \(.message // "")"], git}' <<<"$imp" |
| 759 | [ "$status" = succeeded ] || { echo "::error::the import ended $status"; exit 1; } |
| 760 | |
| 761 | # The same refs, issue and pull request numbers as on GitHub (when GitHub still answers this runner). |
| 762 | R="/repos/$OWNER/$name" |
| 763 | [ "$(api GET "$R" | jq -r .private)" = true ] || { echo "::error::the imported repository is not private"; exit 1; } |
| 764 | gh() { curl -fsS "https://api.github.com/repos/$SOURCE$1" -H "Accept: application/vnd.github+json" ${GITHUB_TOKEN_FOR_IMPORT:+-H "Authorization: Bearer $GITHUB_TOKEN_FOR_IMPORT"}; } |
| 765 | if src=$(gh "") && branches=$(gh "/branches?per_page=100") && items=$(gh "/issues?state=all&per_page=30&sort=created&direction=asc"); then |
| 766 | def=$(jq -r .default_branch <<<"$src") |
| 767 | want=$(jq -r --arg b "$def" '.[] | select(.name == $b) | .commit.sha' <<<"$branches") |
| 768 | got=$(api GET "$R/branches/$def" | jq -r .commit.sha) |
| 769 | [ "$got" = "$want" ] || { echo "::error::$def is $got on Diffinery, $want on GitHub"; exit 1; } |
| 770 | diff <(jq -r '.[].name' <<<"$branches" | sort) <(api GET "$R/branches?per_page=100" | jq -r '.[].name' | sort) || { echo "::error::the branches differ"; exit 1; } |
| 771 | n=0 |
| 772 | for row in $(jq -r '.[] | "\(.number):\(if .pull_request then "pulls" else "issues" end)"' <<<"$items"); do |
| 773 | num=${row%%:*} kind=${row#*:} |
| 774 | want=$(jq -r --argjson n "$num" '.[] | select(.number == $n) | .title' <<<"$items") |
| 775 | got=$(api GET "$R/$kind/$num" | jq -r .title) |
| 776 | [ "$got" = "$want" ] || { echo "::error::$kind #$num is \"$got\" on Diffinery, \"$want\" on GitHub"; exit 1; } |
| 777 | n=$((n + 1)) |
| 778 | done |
| 779 | echo "$def at $got, $(jq length <<<"$branches") branches and $n issues and pull requests match GitHub" |
| 780 | else |
| 781 | echo "::warning::GitHub did not answer this runner (rate limit?), so the import is checked without comparing to GitHub" |
| 782 | fi |
| 783 | # No side effects: importing runs no workflows and sends no notifications. |
| 784 | runs=$(api GET "$R/actions/runs" | jq .total_count) |
| 785 | [ "$runs" = 0 ] || { echo "::error::the import started $runs workflow runs"; exit 1; } |
| 786 | |
| 787 | - name: Scheduled workflow |
| 788 | # Runs even when the import check failed: the two are independent. It needs the scheduled |
| 789 | # workflow pushed, though, so a failed setup doesn't add a 13-minute wait for a run that can't come. |
| 790 | if: ${{ !cancelled() && steps.token.outputs.skip != 'true' && steps.push.outcome == 'success' }} |
| 791 | run: | |
| 792 | set -euo pipefail |
| 793 | # The cron trigger fires every 5 minutes, so allow two windows. |
| 794 | for i in $(seq 1 75); do |
| 795 | run=$(curl -fsS "$DIFFINERY/api/v3/repos/$OWNER/$REPO/actions/runs?event=schedule" -H "Authorization: Bearer $TOKEN" | jq -c '.workflow_runs[0] // empty') |
| 796 | [ -n "$run" ] && [ "$(jq -r .status <<<"$run")" = completed ] && break |
| 797 | sleep 10 |
| 798 | done |
| 799 | if [ -z "$run" ]; then |
| 800 | # Whether the scheduler stopped ticking (last_tick_at is old) or ticked without starting the run |
| 801 | # (site administrators only: another token prints the 403). |
| 802 | echo "scheduler: $(curl -sS "$DIFFINERY/api/v3/diffinery/admin/scheduler" -H "Authorization: Bearer $TOKEN") (now $(date -u +%FT%TZ))" |
| 803 | echo "::error::no scheduled run started" |
| 804 | exit 1 |
| 805 | fi |
| 806 | echo "$run" | jq '{id, event, status, conclusion}' |
| 807 | [ "$(jq -r .conclusion <<<"$run")" = success ] || { echo "::error::the scheduled run did not succeed"; exit 1; } |
| 808 | |
| 809 | - name: Delete the throwaway repositories |
| 810 | if: always() && steps.token.outputs.skip != 'true' |
| 811 | run: | |
| 812 | curl -sS -o /dev/null -w 'package %{http_code}\n' -X DELETE "$DIFFINERY/api/v3/users/$OWNER/packages/container/${REPO,,}" -H "Authorization: Bearer $TOKEN" |
| 813 | curl -sS -o /dev/null -w '%{http_code}\n' -X DELETE "$DIFFINERY/api/v3/repos/$OWNER/$REPO" -H "Authorization: Bearer $TOKEN" |
| 814 | [ -z "${PR_REPO:-}" ] || curl -sS -o /dev/null -w '%{http_code}\n' -X DELETE "$DIFFINERY/api/v3/repos/$OWNER/$PR_REPO" -H "Authorization: Bearer $TOKEN" |
| 815 | [ -z "${IMPORT_REPO:-}" ] || curl -sS -o /dev/null -w '%{http_code}\n' -X DELETE "$DIFFINERY/api/v3/repos/$OWNER/$IMPORT_REPO" -H "Authorization: Bearer $TOKEN" |