Diffinery

codician/diffinerypublic

Open-source, agent-first GitHub and GitHub Actions alternative that runs on your own Cloudflare account: git hosting, pull requests, code review and drop-in GitHub Actions YAML on Workers, Durable Objects, R2 and Containers.

617 lines · 40.0 KBRaw History
1name: Live test
2
3# End-to-end checks against the real diffinery.com after each deploy: email
4# sending, web UI features (syntax highlighting, settings pages, view transitions, milestones, variables), hosted Actions jobs in Cloudflare Containers (with Docker builds and services), the container registry, agent sandboxes,
5# webhooks, scheduled workflows, Git LFS, imports from GitHub, that tokens can't mint tokens, and pull requests (test merge commits, pull_request_target and the merge
6# queue). Each run works in a throwaway private repository that it
7# deletes at the end.
8#
9# Needs the repository secret DIFFINERY_LIVE_TOKEN: a Diffinery token (Settings ->
10# Tokens) with the admin scope and access to all repositories.
11on:
12 workflow_dispatch:
13 workflow_run:
14 workflows: [Deploy]
15 types: [completed]
16 branches: [main]
17
18permissions:
19 contents: read
20
21concurrency:
22 group: live-test
23 cancel-in-progress: false
24
25jobs:
26 live:
27 name: Containers, registry, sandboxes, webhooks, schedules, LFS and pull requests
28 # Runs on the host that deploys production (see the gate in deploy.yml).
29 if: ${{ github.event_name == 'workflow_dispatch' || (github.server_url == 'https://diffinery.com' && github.event.workflow_run.conclusion == 'success') }}
30 runs-on: ubuntu-latest
31 timeout-minutes: 45
32 env:
33 DIFFINERY: https://diffinery.com
34 TOKEN: ${{ secrets.DIFFINERY_LIVE_TOKEN }}
35 REPO: live-${{ github.run_id }}-${{ github.run_attempt }}
36 HEAD_SHA: ${{ github.event.workflow_run.head_sha || github.sha }}
37 steps:
38 - name: Check the token
39 id: token
40 run: |
41 if [ -z "$TOKEN" ]; then
42 echo "::warning::DIFFINERY_LIVE_TOKEN is not set, so the live test is skipped. Create an admin token at $DIFFINERY/settings/tokens and save it as a repository secret."
43 echo "skip=true" >> "$GITHUB_OUTPUT"
44 exit 0
45 fi
46 login=$(curl -fsS "$DIFFINERY/api/v3/user" -H "Authorization: Bearer $TOKEN" | jq -r .login)
47 echo "OWNER=$login" >> "$GITHUB_ENV"
48 echo "testing as $login in $login/$REPO"
49
50 - name: Choose the registry check's image sizes
51 if: steps.token.outputs.skip != 'true'
52 run: |
53 set -euo pipefail
54 # Layers over Cloudflare's 100 MB request limit (which hosted runners push through
55 # the registry relay) take minutes to push and pull. They're pushed on manual runs and
56 # whenever the registry, the runner image or this test changed since the last
57 # successful live test; other deploys push small images through the same path.
58 full=true
59 if [ "$GITHUB_EVENT_NAME" = workflow_run ]; then
60 api() { curl -fsS "$DIFFINERY/api/v3/repos/$GITHUB_REPOSITORY$1" -H "Authorization: Bearer $TOKEN"; }
61 last=$(api "/actions/runs?workflow=live-test.yml&status=success&per_page=1" | jq -r '.workflow_runs[0].head_sha // empty' || true)
62 if [ -n "$last" ] && files=$(api "/compare/$last...$HEAD_SHA" | jq -r '.files[].filename'); then
63 grep -qE '^(apps/diffinery/src/registry/|containers/runner/|\.github/workflows/live-test\.yml$)' <<<"$files" || full=false
64 echo "changed since $last: $(wc -l <<<"$files") files"
65 fi
66 fi
67 if [ "$full" = true ]; then
68 echo "BIG=150000000" >> "$GITHUB_ENV"
69 echo "BUILDX=120000000" >> "$GITHUB_ENV"
70 else
71 echo "BIG=5000000" >> "$GITHUB_ENV"
72 echo "BUILDX=4000000" >> "$GITHUB_ENV"
73 fi
74 echo "registry check: full=$full"
75
76 - name: Profile picture upload (WASM resize in the Worker)
77 if: steps.token.outputs.skip != 'true'
78 run: |
79 set -euo pipefail
80 av="$DIFFINERY/api/avatars/$OWNER"
81 # Keep the account's current picture, if it has one, to put it back afterwards.
82 had=$(curl -fsS -o before.webp -w '%{content_type}' "$av?s=256&v=1")
83 restore() {
84 if [ "$had" = image/webp ]; then
85 curl -fsS -X PUT "$DIFFINERY/api/v3/user/avatar" -H "Authorization: Bearer $TOKEN" -H "content-type: image/webp" --data-binary @before.webp | jq -r '"restored the previous picture: \(.avatar_url)"'
86 else
87 curl -fsS -o /dev/null -X DELETE "$DIFFINERY/api/v3/user/avatar" -H "Authorization: Bearer $TOKEN"
88 fi
89 }
90 trap restore EXIT
91 # A 600×400 PNG gradient; the server crops it to a square and makes small WebPs.
92 python3 - <<'EOF'
93 import struct, zlib
94 w, h = 600, 400
95 rows = b"".join(b"\0" + bytes(c for x in range(w) for c in (x * 255 // w, y * 255 // h, 128)) for y in range(h))
96 chunk = lambda t, d: struct.pack(">I", len(d)) + t + d + struct.pack(">I", zlib.crc32(t + d))
97 open("avatar.png", "wb").write(b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", w, h, 8, 2, 0, 0, 0)) + chunk(b"IDAT", zlib.compress(rows)) + chunk(b"IEND", b""))
98 EOF
99 url=$(curl -fsS -X PUT "$DIFFINERY/api/v3/user/avatar" -H "Authorization: Bearer $TOKEN" -H "content-type: image/png" --data-binary @avatar.png | jq -r .avatar_url)
100 echo "uploaded: $url"
101 v=${url##*v=}
102 type=$(curl -fsS -o small.webp -w '%{content_type}' "$av?s=64&v=$v")
103 [ "$type" = image/webp ] || { echo "::error::expected image/webp, got $type"; exit 1; }
104 python3 - <<'EOF'
105 import struct, sys
106 b = open("small.webp", "rb").read()
107 ok = b[:4] == b"RIFF" and b[8:12] == b"WEBP"
108 if b[12:16] == b"VP8 ":
109 size = (struct.unpack("<H", b[26:28])[0] & 0x3FFF, struct.unpack("<H", b[28:30])[0] & 0x3FFF)
110 elif b[12:16] == b"VP8X":
111 size = (int.from_bytes(b[24:27], "little") + 1, int.from_bytes(b[27:30], "little") + 1)
112 else:
113 size = None
114 print(f"64 px avatar: {len(b)} bytes, {size}")
115 if not ok or size != (64, 64) or len(b) > 5000:
116 print("::error::the 64 px avatar is not a 64x64 WebP under 5 KB"); sys.exit(1)
117 EOF
118 curl -fsS -X DELETE "$DIFFINERY/api/v3/user/avatar" -H "Authorization: Bearer $TOKEN" >/dev/null
119 # A `v` that is not current bypasses the edge cache, so this sees the removal from any colo.
120 type=$(curl -fsS -o /dev/null -w '%{content_type}' "$av?s=64&v=1")
121 [[ "$type" == image/svg+xml* ]] || { echo "::error::after removal expected the initials SVG, got $type"; exit 1; }
122 echo "removed: initials again ($type)"
123
124 - name: Public docs are served without signing in
125 run: |
126 set -euo pipefail
127 check() { # path, content type, text it must contain
128 body=$(curl -fsS -D /tmp/h "$DIFFINERY$1")
129 grep -qi "content-type: $2" /tmp/h || { echo "::error::$1 is not $2"; exit 1; }
130 grep -qF -- "$3" <<<"$body" || { echo "::error::$1 doesn't contain: $3"; exit 1; }
131 echo "$1 ok ($(wc -c <<<"$body") bytes)"
132 }
133 check /docs "text/html" "Overview"
134 check /docs/actions/workflow-syntax "text/html" "View as Markdown"
135 check /docs/quickstart.md "text/markdown" "# Quickstart"
136 check /llms.txt "text/plain" "/docs/actions/workflow-syntax.md"
137 check /llms-full.txt "text/plain" "# REST API reference"
138 check /docs/api/openapi.json "application/json" '"openapi": "3.1.0"'
139 check /-/cli/diffinery.mjs "text/javascript" "#!/usr/bin/env node"
140 md=$(curl -fsS -H "Accept: text/markdown" "$DIFFINERY/docs/agents/mcp")
141 grep -q "^# MCP" <<<"$md" || { echo "::error::Accept: text/markdown didn't return Markdown"; exit 1; }
142 curl -fsS "$DIFFINERY/-/cli/diffinery.mjs" -o /tmp/diffinery.mjs && node /tmp/diffinery.mjs --version
143
144 - name: Email sending is configured
145 if: steps.token.outputs.skip != 'true'
146 run: |
147 set -euo pipefail
148 s=$(curl -fsS "$DIFFINERY/api/v3/diffinery/admin/email" -H "Authorization: Bearer $TOKEN")
149 echo "$s" | jq '{sending, queue, from, last_24h}'
150 [ "$(jq -r .sending <<<"$s")" = true ] || { echo "::error::the Worker has no Email Sending binding"; exit 1; }
151 [ "$(jq -r .queue <<<"$s")" = true ] || { echo "::error::the Worker has no mail queue binding"; exit 1; }
152 failed=$(jq -r .last_24h.failed <<<"$s")
153 [ "$failed" = 0 ] || echo "::warning::$failed emails failed in the last 24 hours; see Settings -> Email in site administration"
154
155 - name: Create a throwaway repository with a webhook
156 if: steps.token.outputs.skip != 'true'
157 run: |
158 set -euo pipefail
159 api() { curl -fsS -X "$1" "$DIFFINERY/api/v3$2" -H "Authorization: Bearer $TOKEN" -H "content-type: application/json" ${3:+--data "$3"}; }
160 api POST /user/repos "{\"name\":\"$REPO\",\"private\":true}" >/dev/null
161 api POST "/repos/$OWNER/$REPO/hooks" "{\"config\":{\"url\":\"$DIFFINERY/healthz\"},\"events\":[\"push\"]}" >/dev/null
162 echo "created $OWNER/$REPO"
163
164 - name: Push workflows and an LFS file
165 if: steps.token.outputs.skip != 'true'
166 run: |
167 set -euo pipefail
168 mkdir -p src/.github/workflows && cd src
169 git init -q -b main
170 git config user.name "Live test" && git config user.email "live-test@diffinery.com"
171 git lfs install --local >/dev/null
172 git lfs track "*.bin" >/dev/null
173 head -c 3145728 /dev/urandom > blob.bin
174 sha256sum blob.bin | cut -d' ' -f1 > ../blob.sha
175 echo "# $REPO" > README.md
176 cat > .github/workflows/ci.yml <<'EOF'
177 on: push
178 jobs:
179 hosted:
180 runs-on: ubuntu-latest
181 strategy:
182 matrix:
183 n: [1, 2]
184 steps:
185 - uses: actions/checkout@v4
186 - run: echo "live ok ${{ '${{' }} matrix.n }} $(cat README.md)"
187 docker:
188 runs-on: ubuntu-latest
189 services:
190 cache:
191 image: redis:7-alpine
192 ports: ["6379"]
193 options: --health-cmd "redis-cli ping" --health-interval 1s --health-retries 30
194 steps:
195 - run: |
196 printf 'FROM alpine:3.20\nRUN echo built > /msg\nCMD ["cat", "/msg"]\n' > Dockerfile
197 docker build -q -t live-image .
198 echo "docker ok $(docker run --rm live-image)"
199 exec 3<>/dev/tcp/127.0.0.1/${{ '${{' }} job.services.cache.ports['6379'] }}
200 printf 'PING\r\n' >&3
201 read -r line <&3
202 echo "service ok ${line%$'\r'}"
203 # Container registry: a layer pushed with the job's GITHUB_TOKEN (over Cloudflare's
204 # 100 MB request limit on full runs), plus the docker/*-action route most ghcr.io
205 # workflows use. __BIG__ and __BUILDX__ are the layer sizes.
206 publish:
207 runs-on: ubuntu-latest
208 permissions:
209 packages: write
210 steps:
211 - run: |
212 image="diffinery.com/${GITHUB_REPOSITORY,,}"
213 echo "${{ '${{' }} secrets.GITHUB_TOKEN }}" | docker login diffinery.com -u "${{ '${{' }} github.actor }}" --password-stdin
214 head -c __BIG__ /dev/urandom > big.bin
215 sha256sum big.bin | cut -d' ' -f1 > big.sha
216 printf 'FROM scratch\nCOPY big.bin big.sha /\n' > Dockerfile
217 docker build -q -t "$image:live" .
218 docker push "$image:live"
219 echo "pushed $(cat big.sha)"
220 buildx:
221 runs-on: ubuntu-latest
222 permissions:
223 packages: write
224 steps:
225 - run: |
226 echo "IMAGE=diffinery.com/${GITHUB_REPOSITORY,,}" >> "$GITHUB_ENV"
227 head -c __BUILDX__ /dev/urandom > big.bin
228 printf 'FROM scratch\nCOPY big.bin /\n' > Dockerfile
229 - uses: docker/setup-buildx-action@v3
230 - uses: docker/login-action@v3
231 with:
232 registry: diffinery.com
233 username: ${{ '${{' }} github.actor }}
234 password: ${{ '${{' }} secrets.GITHUB_TOKEN }}
235 - uses: docker/build-push-action@v6
236 with:
237 context: .
238 push: true
239 tags: ${{ '${{' }} env.IMAGE }}:buildx
240 pull:
241 needs: [publish, buildx]
242 runs-on: ubuntu-latest
243 permissions:
244 packages: read
245 steps:
246 - run: |
247 image="diffinery.com/${GITHUB_REPOSITORY,,}"
248 echo "${{ '${{' }} secrets.GITHUB_TOKEN }}" | docker login diffinery.com -u "${{ '${{' }} github.actor }}" --password-stdin
249 docker pull -q "$image:live"
250 id=$(docker create "$image:live" true)
251 docker cp "$id:/big.bin" big.bin && docker cp "$id:/big.sha" big.sha
252 [ "$(sha256sum big.bin | cut -d' ' -f1)" = "$(cat big.sha)" ] && echo "registry ok $(stat -c %s big.bin) bytes"
253 docker pull -q "$image:buildx"
254 id=$(docker create "$image:buildx" true)
255 docker cp "$id:/big.bin" buildx.bin
256 echo "buildx ok $(stat -c %s buildx.bin) bytes"
257 EOF
258 cat > .github/workflows/tick.yml <<'EOF'
259 on:
260 schedule:
261 - cron: "*/5 * * * *"
262 jobs:
263 tick:
264 runs-on: ubuntu-latest
265 steps:
266 - run: echo "scheduled ok"
267 EOF
268 sed -i "s/__BIG__/$BIG/g; s/__BUILDX__/$BUILDX/g" .github/workflows/ci.yml
269 git add -A && git commit -q -m "Live test"
270 git push -q "https://x-token:${TOKEN}@${DIFFINERY#https://}/$OWNER/$REPO.git" main
271 git lfs ls-files
272
273 - name: Web UI features (highlighting, settings pages, view transitions, base permission, invitations)
274 if: steps.token.outputs.skip != 'true'
275 run: |
276 set -euo pipefail
277 page() { curl -fsS "$DIFFINERY$1" -H "Authorization: Bearer $TOKEN"; }
278 has() { grep -qF -- "$2" <<<"$1" || { echo "::error::$3"; exit 1; }; }
279 html=$(page "/$OWNER/$REPO/blob/main/.github/workflows/ci.yml")
280 has "$html" '<td class="sh">' "the workflow file is not syntax highlighted"
281 has "$html" "--shiki-dark" "highlighted code has no dark theme colours"
282 has "$html" "@view-transition{navigation:auto}" "pages don't opt in to view transitions"
283 has "$html" "prefers-reduced-motion:no-preference" "view transitions ignore reduced motion"
284 has "$(page /settings/appearance)" 'name="appearance"' "Settings -> Appearance has no theme choice"
285 has "$(page /notifications)" 'href="/settings/notifications"' "the notifications page doesn't link to notification settings"
286 code=$(curl -sS -o /tmp/out -w '%{http_code}' -X POST "$DIFFINERY/api/v3/user/repos" -H "Authorization: Bearer $TOKEN" -H "content-type: application/json" --data "{\"name\":\"$REPO-internal\",\"visibility\":\"internal\"}")
287 [ "$code" = 422 ] || { echo "::error::visibility internal got $code, expected 422"; curl -sS -o /dev/null -X DELETE "$DIFFINERY/api/v3/repos/$OWNER/$REPO-internal" -H "Authorization: Bearer $TOKEN"; exit 1; }
288 org=$(page /api/v3/user/orgs | jq -r '.[0].login // empty')
289 if [ -n "$org" ]; then
290 perm=$(page "/api/v3/orgs/$org" | jq -r '.default_repository_permission // empty')
291 case "$perm" in none|read|write|admin) echo "$org's base permission: $perm" ;; *) echo "::error::GET /orgs/$org has no default_repository_permission"; exit 1 ;; esac
292 # A malformed rename is refused before anything is written (422 for owners, 403 for everyone else).
293 code=$(curl -sS -o /tmp/out -w '%{http_code}' -X PATCH "$DIFFINERY/api/v3/orgs/$org" -H "Authorization: Bearer $TOKEN" -H "content-type: application/json" --data '{"name":123}')
294 case "$code" in 422|403) ;; *) echo "::error::PATCH /orgs/$org with a numeric name got $code"; cat /tmp/out; exit 1 ;; esac
295 fi
296 page "/api/v3/repos/$OWNER/$REPO/invitations" | jq -e 'type == "array"' >/dev/null || { echo "::error::GET /repos/$OWNER/$REPO/invitations doesn't list pending invitations"; exit 1; }
297 tools=$(curl -fsS -X POST "$DIFFINERY/mcp" -H "Authorization: Bearer $TOKEN" -H "content-type: application/json" -H "accept: application/json, text/event-stream" --data '{"jsonrpc":"2.0","id":1,"method":"tools/list"}' | jq -r '.result.tools[].name')
298 for t in get_organization list_invitations revoke_invitation; do grep -qx "$t" <<<"$tools" || { echo "::error::the MCP server has no $t tool"; exit 1; }; done
299 echo "highlighting, Appearance, notification settings link, view transitions, visibility, organization and invitation checks passed"
300
301 - name: Diffy placeholders, Actions settings, milestones and variables
302 if: steps.token.outputs.skip != 'true'
303 run: |
304 set -euo pipefail
305 page() { curl -fsS "$DIFFINERY$1" -H "Authorization: Bearer $TOKEN"; }
306 api() { curl -fsS -X "$1" "$DIFFINERY/api/v3$2" -H "Authorization: Bearer $TOKEN" -H "content-type: application/json" ${3:+--data "$3"}; }
307 code() { curl -sS -o /tmp/out -w '%{http_code}' -X "$1" "$DIFFINERY/api/v3$2" -H "Authorization: Bearer $TOKEN" -H "content-type: application/json" ${3:+--data "$3"}; }
308 fail() { echo "::error::$1"; cat /tmp/out 2>/dev/null || true; exit 1; }
309 has() { grep -qF -- "$2" <<<"$1" || fail "$3"; }
310 R="/repos/$OWNER/$REPO"
311 has "$(page /new/import)" 'diffy/hello-world' "the importer's placeholder isn't Diffy"
312 has "$(page "/$OWNER/$REPO/settings/actions")" 'Allow GitHub Actions to create and approve pull requests' "repository settings have no Actions page"
313 # Milestones: created closed, checked before an issue edit is saved, shown in the response, and filterable.
314 [ "$(api POST "$R/milestones" '{"title":"Shipped","state":"closed"}' | jq -r .state)" = closed ] || fail "POST milestones ignored state"
315 ms=$(api POST "$R/milestones" '{"title":"Live v1"}' | jq -r .number)
316 n=$(api POST "$R/issues" '{"title":"Planned"}' | jq -r .number)
317 [ "$(code PATCH "$R/issues/$n" '{"title":"Renamed","milestone":999999}')" = 422 ] || fail "an unknown milestone wasn't refused with 422"
318 [ "$(api GET "$R/issues/$n" | jq -r .title)" = Planned ] || fail "a refused milestone edit saved the title"
319 [ "$(api PATCH "$R/issues/$n" "{\"milestone\":$ms}" | jq -r .milestone.number)" = "$ms" ] || fail "PATCH issue doesn't return the new milestone"
320 [ "$(api GET "$R/issues?milestone=$ms&state=all" | jq -r '[.[].number] | join(",")')" = "$n" ] || fail "issues aren't filtered by milestone"
321 loc=$(curl -sS -o /dev/null -w '%{redirect_url}' "$DIFFINERY/$OWNER/$REPO/milestone/$ms" -H "Authorization: Bearer $TOKEN")
322 has "$loc" "/issues?milestone=$ms" "the milestone link doesn't open its issue list"
323 # Repository variable deletes ignore letter case.
324 [ "$(code PUT "$R/actions/variables/LIVE_CASE" '{"value":"x"}')" = 204 ] || fail "couldn't set a repository variable"
325 [ "$(code DELETE "$R/actions/variables/live_case")" = 204 ] || fail "couldn't delete a repository variable"
326 [ "$(code GET "$R/actions/variables/LIVE_CASE")" = 404 ] || fail "deleting a variable in lower case left it in place"
327 org=$(page /api/v3/user/orgs | jq -r '.[0].login // empty')
328 if [ -n "$org" ]; then
329 # Deleting an organization variable or secret that doesn't exist is a 404 (403 for someone who isn't an admin).
330 for path in variables secrets; do
331 c=$(code DELETE "/orgs/$org/actions/$path/LIVE_NEVER_SET_$GITHUB_RUN_ID")
332 case "$c" in 404|403) ;; *) fail "DELETE of a missing organization $path entry got $c" ;; esac
333 done
334 fi
335 echo "Diffy placeholders, Actions settings page, milestone and variable checks passed"
336
337 - name: Hosted Actions jobs (and Docker) run in Cloudflare Containers
338 if: steps.token.outputs.skip != 'true'
339 run: |
340 set -euo pipefail
341 api() { curl -fsS "$DIFFINERY/api/v3$1" -H "Authorization: Bearer $TOKEN"; }
342 for i in $(seq 1 90); do
343 run=$(api "/repos/$OWNER/$REPO/actions/runs?event=push" | jq -c '.workflow_runs[0] // empty')
344 [ -n "$run" ] && [ "$(jq -r .status <<<"$run")" = completed ] && break
345 sleep 10
346 done
347 echo "$run" | jq '{id, status, conclusion}'
348 id=$(jq -r .id <<<"$run")
349 jobs=$(api "/repos/$OWNER/$REPO/actions/runs/$id/jobs")
350 jq -r '.jobs[] | " \(.name): \(.conclusion)\(if .failure then " - " + (.failure.message // "") else "" end)\(try " (\((.completed_at | fromdateiso8601) - (.started_at | fromdateiso8601))s)" catch "")"' <<<"$jobs"
351 if [ "$(jq -r .conclusion <<<"$run")" != success ]; then
352 for job in $(jq -r '.jobs[] | select(.conclusion == "failure" or .conclusion == "timed_out") | .id' <<<"$jobs"); do
353 echo "::group::Last log lines of job $job"
354 api "/repos/$OWNER/$REPO/actions/jobs/$job/logs" | tail -n 80 || true
355 echo "::endgroup::"
356 done
357 echo "::error::the push run did not succeed"
358 exit 1
359 fi
360 for job in $(jq -r '.jobs[] | select(.name | startswith("hosted")) | .id' <<<"$jobs"); do
361 log=$(api "/repos/$OWNER/$REPO/actions/jobs/$job/logs")
362 grep -F "live ok" <<<"$log" | grep -F "# $REPO" || { echo "::error::job $job did not print its line (checkout or run step failed)"; exit 1; }
363 done
364 job=$(jq -r '.jobs[] | select(.name == "docker") | .id' <<<"$jobs")
365 log=$(api "/repos/$OWNER/$REPO/actions/jobs/$job/logs")
366 grep -F "docker ok built" <<<"$log" || { echo "::error::docker build or run failed on a hosted runner"; exit 1; }
367 grep -F "service ok +PONG" <<<"$log" || { echo "::error::the redis service container was not reachable"; exit 1; }
368 job=$(jq -r '.jobs[] | select(.name == "pull") | .id' <<<"$jobs")
369 log=$(api "/repos/$OWNER/$REPO/actions/jobs/$job/logs")
370 grep -F "registry ok $BIG bytes" <<<"$log" || { echo "::error::the image pushed with a $BIG-byte layer did not come back intact from the registry"; exit 1; }
371 grep -F "buildx ok $BUILDX bytes" <<<"$log" || { echo "::error::the image pushed by docker/build-push-action did not come back from the registry"; exit 1; }
372 api "/users/$OWNER/packages/container/${REPO,,}/versions" | jq -c '.[] | {tags: .metadata.container.tags, size, platforms}'
373
374 - name: An organization's packages page keeps its tabs
375 if: steps.token.outputs.skip != 'true'
376 run: |
377 set -euo pipefail
378 org=$(curl -fsS "$DIFFINERY/api/v3/user/orgs" -H "Authorization: Bearer $TOKEN" | jq -r '.[0].login // empty')
379 [ -n "$org" ] || { echo "$OWNER is in no organization; skipped"; exit 0; }
380 html=$(curl -fsS "$DIFFINERY/$org/-/packages" -H "Authorization: Bearer $TOKEN")
381 grep -qF "<a href=\"/$org/-/packages\" class=\"on\">" <<<"$html" || { echo "::error::$org's packages page lost the organization's tab bar"; exit 1; }
382 grep -qF "href=\"/$org?tab=repositories\"" <<<"$html" || { echo "::error::$org's packages page has no Repositories tab"; exit 1; }
383 echo "$org's packages page shows the organization's tabs with Packages selected"
384
385 - name: Git LFS round trip
386 if: steps.token.outputs.skip != 'true'
387 run: |
388 set -euo pipefail
389 git clone -q "https://x-token:${TOKEN}@${DIFFINERY#https://}/$OWNER/$REPO.git" clone
390 [ "$(sha256sum clone/blob.bin | cut -d' ' -f1)" = "$(cat blob.sha)" ] || { echo "::error::the LFS file came back different"; exit 1; }
391 echo "LFS file matches ($(stat -c %s clone/blob.bin) bytes)"
392
393 - name: Webhook delivered
394 if: steps.token.outputs.skip != 'true'
395 run: |
396 set -euo pipefail
397 for i in $(seq 1 30); do
398 hook=$(curl -fsS "$DIFFINERY/api/v3/repos/$OWNER/$REPO/hooks" -H "Authorization: Bearer $TOKEN" | jq -c '.[0]')
399 # GitHub's hook shape: the target is config.url, the last delivery's status last_response.code.
400 [ "$(jq -r '.last_response.code // empty' <<<"$hook")" != "" ] && break
401 sleep 5
402 done
403 echo "$hook" | jq '{target: .config.url, last_response, updated_at}'
404 [ "$(jq -r '.last_response.code // empty' <<<"$hook")" != "" ] || { echo "::error::the push webhook was never delivered"; exit 1; }
405
406 - name: Tokens can't mint tokens (API or web forms)
407 if: steps.token.outputs.skip != 'true'
408 run: |
409 set -euo pipefail
410 # Revokes anything a failed check minted, then fails.
411 minted() { echo "::error::$1"; curl -fsS "$DIFFINERY/api/v3/diffinery/tokens" -H "Authorization: Bearer $TOKEN" | jq -r '.[] | select(.name == "live-test escalation") | .id' | while read -r id; do curl -sS -o /dev/null -X DELETE "$DIFFINERY/api/v3/diffinery/tokens/$id" -H "Authorization: Bearer $TOKEN"; done; exit 1; }
412 code=$(curl -sS -o /tmp/out -w '%{http_code}' -X POST "$DIFFINERY/api/v3/diffinery/tokens" -H "Authorization: Bearer $TOKEN" -H "content-type: application/json" --data '{"name":"live-test escalation","scopes":["read"],"expires_in_days":1}')
413 [ "$code" = 403 ] || minted "an admin token minted a token through the API ($code)"
414 grep -q "browser session" /tmp/out || minted "unexpected refusal: $(cat /tmp/out)"
415 code=$(curl -sS -o /dev/null -w '%{http_code}' -X POST "$DIFFINERY/api/v3/diffinery/agents" -H "Authorization: Bearer $TOKEN" -H "content-type: application/json" --data '{"login":"live-test-escalation"}')
416 [ "$code" = 403 ] || { echo "::error::an admin token created an agent ($code); disable live-test-escalation by hand"; exit 1; }
417 # A web form with a token and a forged double-submit CSRF pair.
418 code=$(curl -sS -o /dev/null -w '%{http_code}' -X POST "$DIFFINERY/settings/tokens" -H "Authorization: Bearer $TOKEN" -H "cookie: dfy_csrf=forged" --data "_csrf=forged&name=live-test%20escalation&scope=read&days=1")
419 [ "$code" = 403 ] || minted "a token submitted the web token form ($code)"
420 echo "tokens can't mint tokens or agents, through the API or the web forms"
421
422 - name: Agent sandbox
423 if: steps.token.outputs.skip != 'true'
424 run: |
425 set -euo pipefail
426 base="$DIFFINERY/api/v3/repos/$OWNER/$REPO/diffinery/sandboxes"
427 id=$(curl -fsS --retry 1 --retry-delay 15 -X POST "$base" -H "Authorization: Bearer $TOKEN" -H "content-type: application/json" --data '{}' | jq -r .id)
428 echo "sandbox $id"
429 for i in $(seq 1 40); do
430 out=$(curl -sS -X POST "$base/$id/exec" -H "Authorization: Bearer $TOKEN" -H "content-type: application/json" --data '{"cmd":"cat README.md && git lfs ls-files | wc -l"}' || true)
431 jq -e '.exitCode == 0' >/dev/null 2>&1 <<<"$out" && break
432 sleep 6
433 done
434 echo "$out" | jq '{exitCode, stdout}'
435 curl -sS -X POST "$base/$id/close" -H "Authorization: Bearer $TOKEN" >/dev/null || true
436 jq -e --arg r "# $REPO" '.exitCode == 0 and (.stdout | contains($r))' >/dev/null <<<"$out" || { echo "::error::the sandbox did not run the command in the repository"; exit 1; }
437
438 - name: Pull requests and the merge queue
439 if: steps.token.outputs.skip != 'true'
440 run: |
441 set -euo pipefail
442 Q="$REPO-pr"
443 api() { curl -fsS -X "$1" "$DIFFINERY/api/v3$2" -H "Authorization: Bearer $TOKEN" -H "content-type: application/json" ${3:+--data "$3"}; }
444 api POST /user/repos "{\"name\":\"$Q\",\"private\":true}" >/dev/null
445 echo "PR_REPO=$Q" >> "$GITHUB_ENV"
446 api PUT "/repos/$OWNER/$Q/actions/secrets/TOPSECRET" '{"value":"s3cret"}' >/dev/null
447 remote="https://x-token:${TOKEN}@${DIFFINERY#https://}/$OWNER/$Q.git"
448 mkdir -p pr/.github/workflows && cd pr
449 git init -q -b main
450 git config user.name "Live test" && git config user.email "live-test@diffinery.com"
451 printf 'a\nb\nc\n' > app.txt
452 cat > .github/workflows/ci.yml <<'EOF'
453 name: CI
454 on:
455 pull_request:
456 merge_group:
457 jobs:
458 test:
459 runs-on: ubuntu-latest
460 steps:
461 - uses: actions/checkout@v4
462 - run: echo "pr ok event=${{ '${{' }} github.event_name }} ref=${{ '${{' }} github.ref }} sha=${{ '${{' }} github.sha }} head=$(git rev-parse HEAD) app=$(tr '\n' ' ' < app.txt)"
463 EOF
464 cat > .github/workflows/target.yml <<'EOF'
465 name: Target
466 on: pull_request_target
467 jobs:
468 t:
469 runs-on: ubuntu-latest
470 steps:
471 - run: echo "target ok secret=${{ '${{' }} secrets.TOPSECRET == 's3cret' }} ref=${{ '${{' }} github.ref }}"
472 EOF
473 git add -A && git commit -q -m "PR test" && git push -q "$remote" main
474 # The pull request changes app.txt and tries to rewrite the pull_request_target workflow.
475 git checkout -q -b change
476 printf 'A\nb\nc\n' > app.txt
477 sed -i 's/target ok/PWNED/' .github/workflows/target.yml
478 git commit -q -am "Change" && git push -q "$remote" change
479 head=$(git rev-parse HEAD)
480 # Main moves on, so the test merge commit holds both sides.
481 git checkout -q main
482 printf 'a\nb\nC\n' > app.txt
483 git commit -q -am "Main moves" && git push -q "$remote" main
484 n=$(api POST "/repos/$OWNER/$Q/pulls" '{"title":"Change","head":"change","base":"main"}' | jq -r .number)
485 echo "opened pull request #$n (head $head)"
486
487 wait_run() { # event -> the completed run JSON
488 for i in $(seq 1 60); do
489 run=$(api GET "/repos/$OWNER/$Q/actions/runs?event=$1" | jq -c '.workflow_runs[0] // empty')
490 [ -n "$run" ] && [ "$(jq -r .status <<<"$run")" = completed ] && { echo "$run"; return; }
491 sleep 10
492 done
493 echo "::error::no completed $1 run" >&2; exit 1
494 }
495 log_of() { for job in $(api GET "/repos/$OWNER/$Q/actions/runs/$1/jobs" | jq -r '.jobs[].id'); do api GET "/repos/$OWNER/$Q/actions/jobs/$job/logs"; done; }
496 pr=$(wait_run pull_request)
497 echo "$pr" | jq '{event, ref, sha, head_sha, conclusion}'
498 [ "$(jq -r .conclusion <<<"$pr")" = success ] || { echo "::error::the pull_request run failed"; exit 1; }
499 [ "$(jq -r .ref <<<"$pr")" = "refs/pull/$n/merge" ] && [ "$(jq -r .head_sha <<<"$pr")" = "$head" ] && [ "$(jq -r .sha <<<"$pr")" != "$head" ] || { echo "::error::the pull_request run did not use the test merge commit"; exit 1; }
500 log_of "$(jq -r .id <<<"$pr")" | grep -F "pr ok event=pull_request ref=refs/pull/$n/merge" | grep -F "app=A b C" || { echo "::error::the pull_request job did not check out the merged code"; exit 1; }
501 target=$(wait_run pull_request_target)
502 echo "$target" | jq '{event, ref, sha, head_sha, conclusion}'
503 tlog=$(log_of "$(jq -r .id <<<"$target")")
504 grep -F "target ok secret=true ref=refs/heads/main" <<<"$tlog" || { echo "::error::pull_request_target did not run the base workflow with secrets"; exit 1; }
505 ! grep -q PWNED <<<"$tlog" || { echo "::error::pull_request_target ran the pull request's workflow"; exit 1; }
506
507 api PUT "/repos/$OWNER/$Q/branches/main/protection" '{"requirePullRequest":true,"requiredChecks":["CI / test"],"mergeQueue":{"method":"squash","checkTimeoutMinutes":20}}' >/dev/null
508 code=$(curl -sS -o /dev/null -w '%{http_code}' -X PUT "$DIFFINERY/api/v3/repos/$OWNER/$Q/pulls/$n/merge" -H "Authorization: Bearer $TOKEN" -H "content-type: application/json" --data '{}')
509 [ "$code" = 405 ] || { echo "::error::a direct merge into a merge queue branch returned $code"; exit 1; }
510 api PUT "/repos/$OWNER/$Q/pulls/$n/merge-queue" '{"delete_branch":true}' | jq '{state, position: .entry.position, merge_group: .entry.merge_group}'
511 for i in $(seq 1 60); do
512 merged=$(api GET "/repos/$OWNER/$Q/pulls/$n" | jq -r .merged)
513 [ "$merged" = true ] && break
514 sleep 10
515 done
516 [ "$merged" = true ] || { echo "::error::the merge queue did not merge #$n"; api GET "/repos/$OWNER/$Q/merge-queue/main" | jq .; exit 1; }
517 group=$(wait_run merge_group)
518 echo "$group" | jq '{event, ref, sha, conclusion}'
519 [[ "$(jq -r .ref <<<"$group")" == refs/heads/gh-readonly-queue/main/pr-$n-* ]] || { echo "::error::the merge_group run is not on a queue branch"; exit 1; }
520 sha=$(api GET "/repos/$OWNER/$Q/branches/main" | jq -r .commit.sha)
521 [ "$sha" = "$(jq -r .sha <<<"$group")" ] || { echo "::error::main ($sha) is not the tested merge group"; exit 1; }
522 git fetch -q "$remote" main && [ "$(git show FETCH_HEAD:app.txt | tr '\n' ' ')" = "A b C " ] || { echo "::error::main does not hold the merged change"; exit 1; }
523 echo "pull request #$n merged through the queue as $sha"
524
525 - name: Import a public GitHub repository
526 if: steps.token.outputs.skip != 'true'
527 env:
528 # A small public repository with issues and pull requests; a repository variable can change it.
529 SOURCE: ${{ vars.LIVE_IMPORT_SOURCE || 'octocat/git-consortium' }}
530 run: |
531 set -euo pipefail
532 api() { curl -fsS -X "$1" "$DIFFINERY/api/v3$2" -H "Authorization: Bearer $TOKEN" -H "content-type: application/json" ${3:+--data "$3"}; }
533 name="$REPO-import"
534 # Token-free, so GitHub allows 60 requests an hour per address: a rate limit skips this check with a warning.
535 # Starting the import reads the repository from GitHub; a GitHub outage (502 "could not reach GitHub")
536 # is retried, and skips this check with a warning if it outlasts the retries.
537 for try in 1 2 3 4; do
538 code=$(curl -sS -o import.json -w '%{http_code}' -X POST "$DIFFINERY/api/v3/diffinery/imports" -H "Authorization: Bearer $TOKEN" -H "content-type: application/json" \
539 --data "{\"source\":\"$SOURCE\",\"owner\":\"$OWNER\",\"name\":\"$name\",\"visibility\":\"private\"}" || true)
540 unreachable=$(jq -r '.message // "" | startswith("could not reach GitHub")' import.json 2>/dev/null || echo false)
541 { [ "$code" = 502 ] && [ "$unreachable" = true ]; } || [ "$code" = 000 ] || break
542 echo "starting the import returned $code ($(jq -r .message import.json 2>/dev/null)); trying again in 30 seconds"
543 sleep 30
544 done
545 if [ "$code" = 429 ]; then echo "::warning::GitHub's rate limit stopped the import check: $(jq -r .message import.json)"; exit 0; fi
546 if [ "$code" = 502 ] && [ "$unreachable" = true ]; then echo "::warning::GitHub could not be reached, so the import check is skipped: $(jq -r .message import.json)"; exit 0; fi
547 [ "$code" = 201 ] || { echo "::error::starting the import returned $code: $(cat import.json)"; exit 1; }
548 echo "IMPORT_REPO=$name" >> "$GITHUB_ENV"
549 id=$(jq -r .id import.json)
550 echo "import $id: $SOURCE -> $OWNER/$name"
551 for i in $(seq 1 90); do
552 imp=$(api GET "/diffinery/imports/$id")
553 status=$(jq -r .status <<<"$imp")
554 case "$status" in succeeded|failed|cancelled) break ;; esac
555 if jq -e '[.steps[].message // ""] | any(test("rate limit"))' >/dev/null <<<"$imp"; then
556 api POST "/diffinery/imports/$id/cancel" >/dev/null || true
557 echo "::warning::GitHub's rate limit paused the import, so the import check is skipped: $(jq -r '[.steps[].message // ""] | map(select(test("rate limit")))[0]' <<<"$imp")"
558 exit 0
559 fi
560 sleep 10
561 done
562 jq '{status, error, warnings, steps: [.steps[] | "\(.name): \(.status) \(.message // "")"], git}' <<<"$imp"
563 [ "$status" = succeeded ] || { echo "::error::the import ended $status"; exit 1; }
564
565 # The same refs, issue and pull request numbers as on GitHub (when GitHub still answers this runner).
566 R="/repos/$OWNER/$name"
567 [ "$(api GET "$R" | jq -r .private)" = true ] || { echo "::error::the imported repository is not private"; exit 1; }
568 gh() { curl -fsS "https://api.github.com/repos/$SOURCE$1" -H "Accept: application/vnd.github+json"; }
569 if src=$(gh "") && branches=$(gh "/branches?per_page=100") && items=$(gh "/issues?state=all&per_page=30&sort=created&direction=asc"); then
570 def=$(jq -r .default_branch <<<"$src")
571 want=$(jq -r --arg b "$def" '.[] | select(.name == $b) | .commit.sha' <<<"$branches")
572 got=$(api GET "$R/branches/$def" | jq -r .commit.sha)
573 [ "$got" = "$want" ] || { echo "::error::$def is $got on Diffinery, $want on GitHub"; exit 1; }
574 diff <(jq -r '.[].name' <<<"$branches" | sort) <(api GET "$R/branches?per_page=100" | jq -r '.[].name' | sort) || { echo "::error::the branches differ"; exit 1; }
575 n=0
576 for row in $(jq -r '.[] | "\(.number):\(if .pull_request then "pulls" else "issues" end)"' <<<"$items"); do
577 num=${row%%:*} kind=${row#*:}
578 want=$(jq -r --argjson n "$num" '.[] | select(.number == $n) | .title' <<<"$items")
579 got=$(api GET "$R/$kind/$num" | jq -r .title)
580 [ "$got" = "$want" ] || { echo "::error::$kind #$num is \"$got\" on Diffinery, \"$want\" on GitHub"; exit 1; }
581 n=$((n + 1))
582 done
583 echo "$def at $got, $(jq length <<<"$branches") branches and $n issues and pull requests match GitHub"
584 else
585 echo "::warning::GitHub did not answer this runner (rate limit?), so the import is checked without comparing to GitHub"
586 fi
587 # No side effects: importing runs no workflows and sends no notifications.
588 runs=$(api GET "$R/actions/runs" | jq .total_count)
589 [ "$runs" = 0 ] || { echo "::error::the import started $runs workflow runs"; exit 1; }
590
591 - name: Scheduled workflow
592 # Runs even when the import check failed: the two are independent.
593 if: ${{ !cancelled() && steps.token.outputs.skip != 'true' }}
594 run: |
595 set -euo pipefail
596 # The cron trigger fires every 5 minutes, so allow two windows.
597 for i in $(seq 1 75); do
598 run=$(curl -fsS "$DIFFINERY/api/v3/repos/$OWNER/$REPO/actions/runs?event=schedule" -H "Authorization: Bearer $TOKEN" | jq -c '.workflow_runs[0] // empty')
599 [ -n "$run" ] && [ "$(jq -r .status <<<"$run")" = completed ] && break
600 sleep 10
601 done
602 if [ -z "$run" ]; then
603 # Whether the scheduler stopped ticking (last_tick_at is old) or ticked without starting the run.
604 echo "scheduler: $(curl -sS "$DIFFINERY/api/v3/diffinery/admin/scheduler" -H "Authorization: Bearer $TOKEN") (now $(date -u +%FT%TZ))"
605 echo "::error::no scheduled run started"
606 exit 1
607 fi
608 echo "$run" | jq '{id, event, status, conclusion}'
609 [ "$(jq -r .conclusion <<<"$run")" = success ] || { echo "::error::the scheduled run did not succeed"; exit 1; }
610
611 - name: Delete the throwaway repositories
612 if: always() && steps.token.outputs.skip != 'true'
613 run: |
614 curl -sS -o /dev/null -w 'package %{http_code}\n' -X DELETE "$DIFFINERY/api/v3/users/$OWNER/packages/container/${REPO,,}" -H "Authorization: Bearer $TOKEN"
615 curl -sS -o /dev/null -w '%{http_code}\n' -X DELETE "$DIFFINERY/api/v3/repos/$OWNER/$REPO" -H "Authorization: Bearer $TOKEN"
616 [ -z "${PR_REPO:-}" ] || curl -sS -o /dev/null -w '%{http_code}\n' -X DELETE "$DIFFINERY/api/v3/repos/$OWNER/$PR_REPO" -H "Authorization: Bearer $TOKEN"
617 [ -z "${IMPORT_REPO:-}" ] || curl -sS -o /dev/null -w '%{http_code}\n' -X DELETE "$DIFFINERY/api/v3/repos/$OWNER/$IMPORT_REPO" -H "Authorization: Bearer $TOKEN"