Live test: push big registry layers only when the registry could have changed (#35)

The 150 MB and 120 MB registry images are pushed on manual runs and when the registry, the runner image or the live test changed since the last successful live test; other deploys push small images through the same paths. The sync and the live test print each Diffinery job's duration.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Ruyx3Qr4jYrZEpPi466deb

Canberk Demirci authored 13 days ago· GitHub committed 13 days agosigned1 parent 7bbb8edcommit c9005de0717221b0906e0151f518b4051a416da4Browse files

2 files changed+41−9

modified.github/workflows/live-test.yml+36−7View file

modifiedscripts/diffinery-wait-runs.sh+5−2View file

		@@ -33,6 +33,7 @@
33	33	DIFFINERY: https://diffinery.com
34	34	TOKEN: ${{ secrets.DIFFINERY_LIVE_TOKEN }}
35	35	REPO: live-${{ github.run_id }}-${{ github.run_attempt }}
	36	+ HEAD_SHA: ${{ github.event.workflow_run.head_sha \|\| github.sha }}
36	37	steps:
37	38	- name: Check the token
38	39	id: token
		@@ -46,6 +47,32 @@
46	47	echo "OWNER=$login" >> "$GITHUB_ENV"
47	48	echo "testing as $login in $login/$REPO"
48	49
	50	+ - name: Choose the registry check's image sizes
	51	+ if: steps.token.outputs.skip != 'true'
	52	+ run: \|
	53	+ set -euo pipefail
	54	+ # Layers over Cloudflare's 100 MB request limit (which hosted runners push through
	55	+ # the registry relay) take minutes to push and pull. They're pushed on manual runs and
	56	+ # whenever the registry, the runner image or this test changed since the last
	57	+ # successful live test; other deploys push small images through the same path.
	58	+ full=true
	59	+ if [ "$GITHUB_EVENT_NAME" = workflow_run ]; then
	60	+ api() { curl -fsS "$DIFFINERY/api/v3/repos/$GITHUB_REPOSITORY$1" -H "Authorization: Bearer $TOKEN"; }
	61	+ last=$(api "/actions/runs?workflow=live-test.yml&status=success&per_page=1" \| jq -r '.workflow_runs[0].head_sha // empty' \|\| true)
	62	+ if [ -n "$last" ] && files=$(api "/compare/$last...$HEAD_SHA" \| jq -r '.files[].filename'); then
	63	+ grep -qE '^(apps/diffinery/src/registry/\|containers/runner/\|\.github/workflows/live-test\.yml$)' <<<"$files" \|\| full=false
	64	+ echo "changed since $last: $(wc -l <<<"$files") files"
	65	+ fi
	66	+ fi
	67	+ if [ "$full" = true ]; then
	68	+ echo "BIG=150000000" >> "$GITHUB_ENV"
	69	+ echo "BUILDX=120000000" >> "$GITHUB_ENV"
	70	+ else
	71	+ echo "BIG=5000000" >> "$GITHUB_ENV"
	72	+ echo "BUILDX=4000000" >> "$GITHUB_ENV"
	73	+ fi
	74	+ echo "registry check: full=$full"
	75	+
49	76	- name: Profile picture upload (WASM resize in the Worker)
50	77	if: steps.token.outputs.skip != 'true'
51	78	run: \|
		@@ -173,8 +200,9 @@
173	200	printf 'PING\r\n' >&3
174	201	read -r line <&3
175	202	echo "service ok ${line%$'\r'}"
176		- # Container registry: a layer over Cloudflare's 100 MB request limit, pushed with
177		- # the job's GITHUB_TOKEN, plus the docker/*-action route most ghcr.io workflows use.
	203	+ # Container registry: a layer pushed with the job's GITHUB_TOKEN (over Cloudflare's
	204	+ # 100 MB request limit on full runs), plus the docker/*-action route most ghcr.io
	205	+ # workflows use. __BIG__ and __BUILDX__ are the layer sizes.
178	206	publish:
179	207	runs-on: ubuntu-latest
180	208	permissions:
		@@ -183,7 +211,7 @@
183	211	- run: \|
184	212	image="diffinery.com/${GITHUB_REPOSITORY,,}"
185	213	echo "${{ '${{' }} secrets.GITHUB_TOKEN }}" \| docker login diffinery.com -u "${{ '${{' }} github.actor }}" --password-stdin
186		- head -c 150000000 /dev/urandom > big.bin
	214	+ head -c __BIG__ /dev/urandom > big.bin
187	215	sha256sum big.bin \| cut -d' ' -f1 > big.sha
188	216	printf 'FROM scratch\nCOPY big.bin big.sha /\n' > Dockerfile
189	217	docker build -q -t "$image:live" .
		@@ -196,7 +224,7 @@
196	224	steps:
197	225	- run: \|
198	226	echo "IMAGE=diffinery.com/${GITHUB_REPOSITORY,,}" >> "$GITHUB_ENV"
199		- head -c 120000000 /dev/urandom > big.bin
	227	+ head -c __BUILDX__ /dev/urandom > big.bin
200	228	printf 'FROM scratch\nCOPY big.bin /\n' > Dockerfile
201	229	- uses: docker/setup-buildx-action@v3
202	230	- uses: docker/login-action@v3
		@@ -237,6 +265,7 @@
237	265	steps:
238	266	- run: echo "scheduled ok"
239	267	EOF
	268	+ sed -i "s/__BIG__/$BIG/g; s/__BUILDX__/$BUILDX/g" .github/workflows/ci.yml
240	269	git add -A && git commit -q -m "Live test"
241	270	git push -q "https://x-token:${TOKEN}@${DIFFINERY#https://}/$OWNER/$REPO.git" main
242	271	git lfs ls-files
		@@ -276,7 +305,7 @@
276	305	echo "$run" \| jq '{id, status, conclusion}'
277	306	id=$(jq -r .id <<<"$run")
278	307	jobs=$(api "/repos/$OWNER/$REPO/actions/runs/$id/jobs")
279		- jq -r '.jobs[] \| " \(.name): \(.conclusion)\(if .failure then " - " + (.failure.message // "") else "" end)"' <<<"$jobs"
	308	+ jq -r '.jobs[] \| " \(.name): \(.conclusion)\(if .failure then " - " + (.failure.message // "") else "" end)\(try " (\((.completed_at \| fromdateiso8601) - (.started_at \| fromdateiso8601))s)" catch "")"' <<<"$jobs"
280	309	if [ "$(jq -r .conclusion <<<"$run")" != success ]; then
281	310	for job in $(jq -r '.jobs[] \| select(.conclusion == "failure" or .conclusion == "timed_out") \| .id' <<<"$jobs"); do
282	311	echo "::group::Last log lines of job $job"
		@@ -296,8 +325,8 @@
296	325	grep -F "service ok +PONG" <<<"$log" \|\| { echo "::error::the redis service container was not reachable"; exit 1; }
297	326	job=$(jq -r '.jobs[] \| select(.name == "pull") \| .id' <<<"$jobs")
298	327	log=$(api "/repos/$OWNER/$REPO/actions/jobs/$job/logs")
299		- grep -F "registry ok 150000000 bytes" <<<"$log" \|\| { echo "::error::the image pushed with a 150 MB layer did not come back intact from the registry"; exit 1; }
300		- grep -F "buildx ok 120000000 bytes" <<<"$log" \|\| { echo "::error::the image pushed by docker/build-push-action did not come back from the registry"; exit 1; }
	328	+ grep -F "registry ok $BIG bytes" <<<"$log" \|\| { echo "::error::the image pushed with a $BIG-byte layer did not come back intact from the registry"; exit 1; }
	329	+ grep -F "buildx ok $BUILDX bytes" <<<"$log" \|\| { echo "::error::the image pushed by docker/build-push-action did not come back from the registry"; exit 1; }
301	330	api "/users/$OWNER/packages/container/${REPO,,}/versions" \| jq -c '.[] \| {tags: .metadata.container.tags, size, platforms}'
302	331
303	332	- name: An organization's packages page keeps its tabs

		@@ -32,9 +32,12 @@
32	32	failed=0
33	33	for id in $(jq -r '.workflow_runs[].id' <<<"$runs"); do
34	34	run=$(api "/actions/runs/$id")
35		- jq -r '"\(.name) [\(.event)]: \(.conclusion) \(.html_url)"' <<<"$run"
	35	+ jq -r '"\(.name) [\(.event)]: \(.conclusion)\(try " in \((.updated_at \| fromdateiso8601) - (.created_at \| fromdateiso8601))s" catch "") \(.html_url)"' <<<"$run"
36	36	jobs=$(api "/actions/runs/$id/jobs")
37		- jq -r '.jobs[] \| " \(.name): \(.conclusion)\(if .failure then " - " + (.failure.message // "") else "" end)"' <<<"$jobs"
	37	+ # Each job's time, and its steps that took over 20 seconds, to see where a sync spends it.
	38	+ jq -r 'def secs: try ((.completed_at \| fromdateiso8601) - (.started_at \| fromdateiso8601)) catch null;
	39	+ .jobs[] \| " \(.name): \(.conclusion)\(if .failure then " - " + (.failure.message // "") else "" end)\(if secs then " (\(secs)s)" else "" end)",
	40	+ (.steps[]? \| select((secs // 0) > 20) \| " \(secs)s \(.name)")' <<<"$jobs"
38	41	if [ "$(jq -r .conclusion <<<"$run")" != success ]; then
39	42	failed=1
40	43	for job in $(jq -r '.jobs[] \| select(.conclusion == "failure" or .conclusion == "timed_out") \| .id' <<<"$jobs"); do

codician/diffinerypublic

Live test: push big registry layers only when the registry could have changed (#35)