2026-10-22T15:34:34.608Z Diffinery runner: job live (Containers, registry, sandboxes, webhooks, schedules, LFS and pull requests) for codician/diffinery@f454e336a703 2026-10-22T15:34:34.718Z Runner: image diffinery-runner:4d3bf17f06575e64ab4155b3, in ams21, WEUR, container db9a23704a7ebebd4d37c5ca08378ccdfaf41eeb875e15b44065de1fbbf6e61e 2026-10-22T15:34:34.718Z Runner size: large (4 vCPU, 12 GiB of memory, 20 GB of disk), this job uses what a smaller runner doesn't have; runs-on: diffinery-large (or -medium, -small, -tiny) sets a size 2026-10-22T15:34:34.718Z Host: AMD EPYC (family 25, model 17), 4 vCPUs, 2546 MHz 2026-10-22T15:34:34.720Z Workspace: /workspace/diffinery 2026-10-22T15:34:34.821Z ##[group]Run if [ "$GITHUB_EVENT_NAME" = workflow_run ]; then 2026-10-22T15:34:34.821Z # Deploy's gate skips a commit that is no longer main's tip (the newer commit's run deploys it): 2026-10-22T15:34:34.821Z # then nothing new is live, and this run would only test the site as it was. 2026-10-22T15:34:34.821Z deployed=$(curl -fsS "$GITHUB_API_URL/repos/$GITHUB_REPOSITORY/actions/runs/$DEPLOY_RUN/jobs" -H "Authorization: Bearer $GH_TOKEN" | jq -r '[.jobs[] | select(.name == "Deploy to Cloudflare") | .conclusion][0] // "unknown"' || echo unknown) 2026-10-22T15:34:34.821Z if [ "$deployed" = skipped ]; then 2026-10-22T15:34:34.821Z echo "::notice::Deploy deployed nothing for $HEAD_SHA (main has moved on), so there is nothing new to test" 2026-10-22T15:34:34.821Z echo "skip=true" >> "$GITHUB_OUTPUT" 2026-10-22T15:34:34.821Z exit 0 2026-10-22T15:34:34.821Z fi 2026-10-22T15:34:34.821Z fi 2026-10-22T15:34:34.821Z if [ -z "$TOKEN" ]; then 2026-10-22T15:34:34.821Z # After a deploy of diffinery.com a missing token is an error: a green run would mean nothing was tested. 2026-10-22T15:34:34.821Z if [ "$GITHUB_EVENT_NAME" = workflow_run ]; then 2026-10-22T15:34:34.821Z echo "::error::DIFFINERY_LIVE_TOKEN is not set in the production environment, so nothing was tested. Create a write token at $DIFFINERY/settings/tokens and save it as a production environment secret." 2026-10-22T15:34:34.821Z exit 1 2026-10-22T15:34:34.821Z fi 2026-10-22T15:34:34.821Z echo "::warning::DIFFINERY_LIVE_TOKEN is not set, so the live test is skipped. Create a write token at $DIFFINERY/settings/tokens and save it as a production environment secret." 2026-10-22T15:34:34.821Z echo "skip=true" >> "$GITHUB_OUTPUT" 2026-10-22T15:34:34.821Z exit 0 2026-10-22T15:34:34.821Z fi 2026-10-22T15:34:34.821Z login=$(curl -fsS "$DIFFINERY/api/v3/user" -H "Authorization: Bearer $TOKEN" | jq -r .login) 2026-10-22T15:34:34.821Z echo "OWNER=$login" >> "$GITHUB_ENV" 2026-10-22T15:34:34.821Z echo "testing as $login in $login/$REPO" 2026-10-22T15:34:34.821Z env: 2026-10-22T15:34:34.821Z GH_TOKEN: *** 2026-10-22T15:34:34.821Z DEPLOY_RUN: 586 2026-10-22T15:34:34.821Z ##[endgroup] 2026-10-22T15:34:36.341Z testing as cnbrkdmrci in cnbrkdmrci/live-587-1 2026-10-22T15:34:36.767Z ##[group]Run set -euo pipefail 2026-10-22T15:34:36.767Z # Layers over Cloudflare's 100 MB request limit (which hosted runners push through 2026-10-22T15:34:36.767Z # the registry relay) take minutes to push and pull. They're pushed on manual runs and 2026-10-22T15:34:36.767Z # whenever the registry, the runner image or this test changed since the last 2026-10-22T15:34:36.767Z # successful live test; other deploys push small images through the same path. 2026-10-22T15:34:36.767Z full=true 2026-10-22T15:34:36.767Z if [ "$GITHUB_EVENT_NAME" = workflow_run ]; then 2026-10-22T15:34:36.767Z api() { curl -fsS "$DIFFINERY/api/v3/repos/$GITHUB_REPOSITORY$1" -H "Authorization: Bearer $TOKEN"; } 2026-10-22T15:34:36.767Z last=$(api "/actions/runs?workflow=live-test.yml&status=success&per_page=1" | jq -r '.workflow_runs[0].head_sha // empty' || true) 2026-10-22T15:34:36.767Z if [ -n "$last" ] && files=$(api "/compare/$last...$HEAD_SHA" | jq -r '.files[].filename'); then 2026-10-22T15:34:36.767Z grep -qE '^(apps/diffinery/src/registry/|containers/runner/|\.github/workflows/live-test\.yml$)' <<<"$files" || full=false 2026-10-22T15:34:36.767Z echo "changed since $last: $(wc -l <<<"$files") files" 2026-10-22T15:34:36.767Z fi 2026-10-22T15:34:36.767Z fi 2026-10-22T15:34:36.767Z if [ "$full" = true ]; then 2026-10-22T15:34:36.767Z echo "BIG=150000000" >> "$GITHUB_ENV" 2026-10-22T15:34:36.767Z echo "BUILDX=120000000" >> "$GITHUB_ENV" 2026-10-22T15:34:36.767Z else 2026-10-22T15:34:36.767Z echo "BIG=5000000" >> "$GITHUB_ENV" 2026-10-22T15:34:36.767Z echo "BUILDX=4000000" >> "$GITHUB_ENV" 2026-10-22T15:34:36.767Z fi 2026-10-22T15:34:36.767Z echo "registry check: full=$full" 2026-10-22T15:34:36.767Z ##[endgroup] 2026-10-22T15:34:38.564Z changed since 4d9ceaac2e5ea85369b9b00d7912f159d9bd329f: 17 files 2026-10-22T15:34:38.564Z registry check: full=false 2026-10-22T15:34:38.688Z ##[group]Run set -euo pipefail 2026-10-22T15:34:38.688Z av="$DIFFINERY/api/avatars/$OWNER" 2026-10-22T15:34:38.688Z # Keep the account's current picture, if it has one, to put it back afterwards. 2026-10-22T15:34:38.688Z had=$(curl -fsS -o before.webp -w '%{content_type}' "$av?s=256&v=1") 2026-10-22T15:34:38.688Z restore() { 2026-10-22T15:34:38.688Z if [ "$had" = image/webp ]; then 2026-10-22T15:34:38.688Z curl -fsS -X PUT "$DIFFINERY/api/v3/user/avatar" -H "Authorization: Bearer $TOKEN" -H "content-type: image/webp" --data-binary @before.webp | jq -r '"restored the previous picture: \(.avatar_url)"' 2026-10-22T15:34:38.688Z else 2026-10-22T15:34:38.688Z curl -fsS -o /dev/null -X DELETE "$DIFFINERY/api/v3/user/avatar" -H "Authorization: Bearer $TOKEN" 2026-10-22T15:34:38.688Z fi 2026-10-22T15:34:38.688Z } 2026-10-22T15:34:38.688Z trap restore EXIT 2026-10-22T15:34:38.688Z # A 600×400 PNG gradient; the server crops it to a square and makes small WebPs. 2026-10-22T15:34:38.688Z python3 - <<'EOF' 2026-10-22T15:34:38.688Z import struct, zlib 2026-10-22T15:34:38.688Z w, h = 600, 400 2026-10-22T15:34:38.688Z rows = b"".join(b"\0" + bytes(c for x in range(w) for c in (x * 255 // w, y * 255 // h, 128)) for y in range(h)) 2026-10-22T15:34:38.688Z chunk = lambda t, d: struct.pack(">I", len(d)) + t + d + struct.pack(">I", zlib.crc32(t + d)) 2026-10-22T15:34:38.688Z open("avatar.png", "wb").write(b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", w, h, 8, 2, 0, 0, 0)) + chunk(b"IDAT", zlib.compress(rows)) + chunk(b"IEND", b"")) 2026-10-22T15:34:38.688Z EOF 2026-10-22T15:34:38.688Z url=$(curl -fsS -X PUT "$DIFFINERY/api/v3/user/avatar" -H "Authorization: Bearer $TOKEN" -H "content-type: image/png" --data-binary @avatar.png | jq -r .avatar_url) 2026-10-22T15:34:38.688Z echo "uploaded: $url" 2026-10-22T15:34:38.688Z v=${url##*v=} 2026-10-22T15:34:38.688Z type=$(curl -fsS -o small.webp -w '%{content_type}' "$av?s=64&v=$v") 2026-10-22T15:34:38.688Z [ "$type" = image/webp ] || { echo "::error::expected image/webp, got $type"; exit 1; } 2026-10-22T15:34:38.688Z python3 - <<'EOF' 2026-10-22T15:34:38.688Z import struct, sys 2026-10-22T15:34:38.688Z b = open("small.webp", "rb").read() 2026-10-22T15:34:38.688Z ok = b[:4] == b"RIFF" and b[8:12] == b"WEBP" 2026-10-22T15:34:38.688Z if b[12:16] == b"VP8 ": 2026-10-22T15:34:38.688Z size = (struct.unpack(" 5000: 2026-10-22T15:34:38.688Z print("::error::the 64 px avatar is not a 64x64 WebP under 5 KB"); sys.exit(1) 2026-10-22T15:34:38.688Z EOF 2026-10-22T15:34:38.688Z curl -fsS -X DELETE "$DIFFINERY/api/v3/user/avatar" -H "Authorization: Bearer $TOKEN" >/dev/null 2026-10-22T15:34:38.688Z # A `v` that is not current bypasses the edge cache, so this sees the removal from any colo. 2026-10-22T15:34:38.688Z type=$(curl -fsS -o /dev/null -w '%{content_type}' "$av?s=64&v=1") 2026-10-22T15:34:38.688Z [[ "$type" == image/svg+xml* ]] || { echo "::error::after removal expected the initials SVG, got $type"; exit 1; } 2026-10-22T15:34:38.688Z echo "removed: initials again ($type)" 2026-10-22T15:34:38.688Z ##[endgroup] 2026-10-22T15:34:42.011Z uploaded: https://diffinery.com/api/avatars/cnbrkdmrci?v=1792683280 2026-10-22T15:34:42.571Z 64 px avatar: 256 bytes, (64, 64) 2026-10-22T15:34:43.798Z removed: initials again (image/svg+xml; charset=utf-8) 2026-10-22T15:34:47.376Z restored the previous picture: https://diffinery.com/api/avatars/cnbrkdmrci?v=1792683284 2026-10-22T15:34:47.579Z ##[group]Run set -euo pipefail 2026-10-22T15:34:47.579Z check() { # path, content type, text it must contain 2026-10-22T15:34:47.579Z body=$(curl -fsS -D /tmp/h "$DIFFINERY$1") 2026-10-22T15:34:47.579Z grep -qi "content-type: $2" /tmp/h || { echo "::error::$1 is not $2"; exit 1; } 2026-10-22T15:34:47.579Z grep -qF -- "$3" <<<"$body" || { echo "::error::$1 doesn't contain: $3"; exit 1; } 2026-10-22T15:34:47.579Z echo "$1 ok ($(wc -c <<<"$body") bytes)" 2026-10-22T15:34:47.579Z } 2026-10-22T15:34:47.579Z check /docs "text/html" "Overview" 2026-10-22T15:34:47.579Z check /docs/actions/workflow-syntax "text/html" "View as Markdown" 2026-10-22T15:34:47.579Z check /docs/quickstart.md "text/markdown" "# Quickstart" 2026-10-22T15:34:47.579Z check /llms.txt "text/plain" "/docs/actions/workflow-syntax.md" 2026-10-22T15:34:47.579Z check /llms-full.txt "text/plain" "# REST API reference" 2026-10-22T15:34:47.579Z check /docs/api/openapi.json "application/json" '"openapi": "3.1.0"' 2026-10-22T15:34:47.579Z check /-/cli/diffinery.mjs "text/javascript" "#!/usr/bin/env node" 2026-10-22T15:34:47.579Z md=$(curl -fsS -H "Accept: text/markdown" "$DIFFINERY/docs/agents/mcp") 2026-10-22T15:34:47.579Z grep -q "^# MCP" <<<"$md" || { echo "::error::Accept: text/markdown didn't return Markdown"; exit 1; } 2026-10-22T15:34:47.579Z curl -fsS "$DIFFINERY/-/cli/diffinery.mjs" -o /tmp/diffinery.mjs && node /tmp/diffinery.mjs --version 2026-10-22T15:34:47.579Z ##[endgroup] 2026-10-22T15:34:47.797Z /docs ok (91341 bytes) 2026-10-22T15:34:48.306Z /docs/actions/workflow-syntax ok (177167 bytes) 2026-10-22T15:34:48.442Z /docs/quickstart.md ok (17010 bytes) 2026-10-22T15:34:48.574Z /llms.txt ok (22991 bytes) 2026-10-22T15:34:49.043Z /llms-full.txt ok (2534517 bytes) 2026-10-22T15:34:49.268Z /docs/api/openapi.json ok (818665 bytes) 2026-10-22T15:34:49.380Z /-/cli/diffinery.mjs ok (250493 bytes) 2026-10-22T15:34:49.674Z diffinery 0.1.0 2026-10-22T15:34:49.788Z ##[group]Run set -euo pipefail 2026-10-22T15:34:49.788Z s=$(curl -sS -w '\n%{http_code}' "$DIFFINERY/api/v3/diffinery/admin/email" -H "Authorization: Bearer $TOKEN") 2026-10-22T15:34:49.788Z code=${s##*$'\n'}; s=${s%$'\n'*} 2026-10-22T15:34:49.788Z # Only a site administrator's token reads the email settings, and the live-test account isn't one. 2026-10-22T15:34:49.788Z if [ "$code" = 403 ]; then echo "::notice::skipped: the live-test token isn't a site administrator's"; exit 0; fi 2026-10-22T15:34:49.788Z [ "$code" = 200 ] || { echo "::error::/diffinery/admin/email answered $code"; exit 1; } 2026-10-22T15:34:49.788Z echo "$s" | jq '{sending, queue, from, last_24h}' 2026-10-22T15:34:49.788Z [ "$(jq -r .sending <<<"$s")" = true ] || { echo "::error::the Worker has no Email Sending binding"; exit 1; } 2026-10-22T15:34:49.788Z [ "$(jq -r .queue <<<"$s")" = true ] || { echo "::error::the Worker has no mail queue binding"; exit 1; } 2026-10-22T15:34:49.788Z failed=$(jq -r .last_24h.failed <<<"$s") 2026-10-22T15:34:49.788Z [ "$failed" = 0 ] || echo "::warning::$failed emails failed in the last 24 hours; see Settings -> Email in site administration" 2026-10-22T15:34:49.788Z ##[endgroup] 2026-10-22T15:34:51.475Z { 2026-10-22T15:34:51.475Z "sending": true, 2026-10-22T15:34:51.475Z "queue": true, 2026-10-22T15:34:51.475Z "from": "notifications@diffinery.com", 2026-10-22T15:34:51.475Z "last_24h": { 2026-10-22T15:34:51.475Z "sent": 39, 2026-10-22T15:34:51.475Z "failed": 0, 2026-10-22T15:34:51.475Z "skipped": 0 2026-10-22T15:34:51.475Z } 2026-10-22T15:34:51.475Z } 2026-10-22T15:34:51.609Z ##[group]Run set -euo pipefail 2026-10-22T15:34:51.609Z api() { curl -fsS -X "$1" "$DIFFINERY/api/v3$2" -H "Authorization: Bearer $TOKEN" -H "content-type: application/json" ${3:+--data "$3"}; } 2026-10-22T15:34:51.609Z api POST /user/repos "{\"name\":\"$REPO\",\"private\":true}" >/dev/null 2026-10-22T15:34:51.609Z # Webhooks may only reach public hosts other than Diffinery itself; the push's delivery goes to example.com. 2026-10-22T15:34:51.609Z code=$(curl -sS -o /dev/null -w '%{http_code}' -X POST "$DIFFINERY/api/v3/repos/$OWNER/$REPO/hooks" -H "Authorization: Bearer $TOKEN" -H "content-type: application/json" --data "{\"config\":{\"url\":\"$DIFFINERY/healthz\"},\"events\":[\"push\"]}") 2026-10-22T15:34:51.609Z [ "$code" = 422 ] || { echo "::error::a webhook to Diffinery itself answered $code, not 422"; exit 1; } 2026-10-22T15:34:51.609Z api POST "/repos/$OWNER/$REPO/hooks" "{\"config\":{\"url\":\"https://example.com/diffinery-live-test\"},\"events\":[\"push\"]}" >/dev/null 2026-10-22T15:34:51.609Z echo "created $OWNER/$REPO" 2026-10-22T15:34:51.609Z ##[endgroup] 2026-10-22T15:34:54.835Z created cnbrkdmrci/live-587-1 2026-10-22T15:34:54.929Z ##[group]Run set -euo pipefail 2026-10-22T15:34:54.929Z # A deploy's container rollout updates instances over a while after it starts, and an 2026-10-22T15:34:54.929Z # instance not yet updated starts jobs on the previous runner image. Jobs report their 2026-10-22T15:34:54.929Z # image (`runner.image`), so a probe workflow runs until three jobs in a row land on the 2026-10-22T15:34:54.929Z # image the deploy set (/meta's runner_image) before the Docker and registry checks. 2026-10-22T15:34:54.929Z api() { curl -fsS -X "$1" "$DIFFINERY/api/v3$2" -H "Authorization: Bearer $TOKEN" -H "content-type: application/json" ${3:+--data "$3"}; } 2026-10-22T15:34:54.929Z expected=$(api GET /meta | jq -r '.runner_image // empty') 2026-10-22T15:34:54.929Z if [ -z "$expected" ]; then 2026-10-22T15:34:54.929Z echo "::warning::/meta doesn't name the deployed runner image, so jobs aren't checked against it" 2026-10-22T15:34:54.929Z exit 0 2026-10-22T15:34:54.929Z fi 2026-10-22T15:34:54.929Z echo "DEPLOYED_RUNNER_IMAGE=$expected" >> "$GITHUB_ENV" 2026-10-22T15:34:54.929Z mkdir -p src/.github/workflows && cd src 2026-10-22T15:34:54.929Z git init -q -b main 2026-10-22T15:34:54.929Z git config user.name "Live test" && git config user.email "live-test@diffinery.com" 2026-10-22T15:34:54.929Z cat > .github/workflows/probe.yml <<'EOF' 2026-10-22T15:34:54.929Z on: workflow_dispatch 2026-10-22T15:34:54.929Z jobs: 2026-10-22T15:34:54.929Z probe: 2026-10-22T15:34:54.929Z runs-on: ubuntu-latest 2026-10-22T15:34:54.929Z strategy: 2026-10-22T15:34:54.929Z matrix: 2026-10-22T15:34:54.929Z n: [1, 2, 3] 2026-10-22T15:34:54.929Z steps: 2026-10-22T15:34:54.929Z - run: echo "probe ok" 2026-10-22T15:34:54.929Z EOF 2026-10-22T15:34:54.929Z git add -A && git commit -q -m "Runner image probe" 2026-10-22T15:34:54.929Z git push -q "https://x-token:${TOKEN}@${DIFFINERY#https://}/$OWNER/$REPO.git" main 2026-10-22T15:34:54.929Z deadline=$(( $(date +%s) + 15 * 60 )) 2026-10-22T15:34:54.929Z while :; do 2026-10-22T15:34:54.929Z before=$(api GET "/repos/$OWNER/$REPO/actions/runs?event=workflow_dispatch" | jq '.workflow_runs | length') 2026-10-22T15:34:54.929Z api POST "/repos/$OWNER/$REPO/actions/workflows/probe.yml/dispatches" '{"ref":"main"}' >/dev/null 2026-10-22T15:34:54.929Z run="" 2026-10-22T15:34:54.929Z # Up to 10 minutes: a job turned away from a replaced image starts again after 30 s, 1, 2 and 4 minutes. 2026-10-22T15:34:54.929Z for i in $(seq 1 120); do 2026-10-22T15:34:54.929Z runs=$(api GET "/repos/$OWNER/$REPO/actions/runs?event=workflow_dispatch") 2026-10-22T15:34:54.929Z if [ "$(jq '.workflow_runs | length' <<<"$runs")" -gt "$before" ]; then 2026-10-22T15:34:54.929Z run=$(jq -c '.workflow_runs[0]' <<<"$runs") 2026-10-22T15:34:54.929Z [ "$(jq -r .status <<<"$run")" = completed ] && break 2026-10-22T15:34:54.929Z fi 2026-10-22T15:34:54.929Z sleep 5 2026-10-22T15:34:54.929Z done 2026-10-22T15:34:54.929Z [ -n "$run" ] || { echo "::error::the probe workflow didn't start"; exit 1; } 2026-10-22T15:34:54.929Z probe=$(api GET "/repos/$OWNER/$REPO/actions/runs/$(jq -r .id <<<"$run")/jobs") 2026-10-22T15:34:54.929Z images=$(jq -r '.jobs[] | .runner.image // "no image reported"' <<<"$probe") 2026-10-22T15:34:54.929Z echo "probe jobs ran on: $(sort <<<"$images" | uniq -c | sed 's/^ *//' | paste -sd ','), started again $(jq '[.jobs[].placementTries // 0] | add // 0' <<<"$probe") time(s) in all" 2026-10-22T15:34:54.929Z [ "$(grep -cxF "$expected" <<<"$images" || true)" = 3 ] && { echo "jobs start on $expected"; exit 0; } 2026-10-22T15:34:54.929Z if [ "$(date +%s)" -ge "$deadline" ]; then 2026-10-22T15:34:54.929Z echo "::error::15 minutes after the deploy, jobs still start on a runner image other than the deployed $expected" 2026-10-22T15:34:54.929Z exit 1 2026-10-22T15:34:54.929Z fi 2026-10-22T15:34:54.929Z sleep 20 2026-10-22T15:34:54.929Z done 2026-10-22T15:34:54.929Z ##[endgroup] 2026-10-22T15:35:13.088Z curl: (22) The requested URL returned error: 500 2026-10-22T15:35:13.094Z ##[error]Process completed with exit code 22. 2026-10-22T15:35:13.980Z ##[group]Run curl -sS -o /dev/null -w 'package %{http_code}\n' -X DELETE "$DIFFINERY/api/v3/users/$OWNER/packages/container/${REPO,,}" -H "Authorization: Bearer $TOKEN" 2026-10-22T15:35:13.980Z curl -sS -o /dev/null -w '%{http_code}\n' -X DELETE "$DIFFINERY/api/v3/repos/$OWNER/$REPO" -H "Authorization: Bearer $TOKEN" 2026-10-22T15:35:13.980Z [ -z "${PR_REPO:-}" ] || curl -sS -o /dev/null -w '%{http_code}\n' -X DELETE "$DIFFINERY/api/v3/repos/$OWNER/$PR_REPO" -H "Authorization: Bearer $TOKEN" 2026-10-22T15:35:13.980Z [ -z "${IMPORT_REPO:-}" ] || curl -sS -o /dev/null -w '%{http_code}\n' -X DELETE "$DIFFINERY/api/v3/repos/$OWNER/$IMPORT_REPO" -H "Authorization: Bearer $TOKEN" 2026-10-22T15:35:13.980Z ##[endgroup] 2026-10-22T15:35:14.776Z package 404 2026-10-22T15:35:16.664Z 204 2026-10-22T15:35:16.720Z Host: 3% of the vCPUs' time went to other work on the host (steal) 2026-10-22T15:35:16.720Z Job completed with conclusion: failure 2026-10-22T15:35:16.721Z Resources: 0.1 of 4 vCPUs busy on average (0.7 at the 95th percentile), peak memory 0.4 GiB of 11.9 GiB, peak disk 1.1 GB of 19.2 GB