Fix docker/setup-buildx-action on hosted runners (#20)
The docker-container driver reaches BuildKit through docker exec, which can't join BuildKit's cgroup inside a Cloudflare Container. The hosted runner's docker wrapper now starts BuildKit itself and registers it as a remote builder; docker buildx rm removes it. The live test prints failed jobs' logs. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Ruyx3Qr4jYrZEpPi466deb
Canberk Demirci authored · GitHub committed signed1 parent 7cf0a49commit 70b8cade504ed5b9aaa95d8d9ec7773f72da054cBrowse files
2 files changed+105−28
modified.github/workflows/live-test.yml+10−1View file
| @@ -184,9 +184,18 @@ | ||
| 184 | 184 | sleep 10 |
| 185 | 185 | done |
| 186 | 186 | echo "$run" | jq '{id, status, conclusion}' |
| 187 | - [ "$(jq -r .conclusion <<<"$run")" = success ] || { echo "::error::the push run did not succeed"; exit 1; } | |
| 188 | 187 | id=$(jq -r .id <<<"$run") |
| 189 | 188 | jobs=$(api "/repos/$OWNER/$REPO/actions/runs/$id/jobs") |
| 189 | + jq -r '.jobs[] | " \(.name): \(.conclusion)\(if .failure then " - " + (.failure.message // "") else "" end)"' <<<"$jobs" | |
| 190 | + if [ "$(jq -r .conclusion <<<"$run")" != success ]; then | |
| 191 | + for job in $(jq -r '.jobs[] | select(.conclusion == "failure" or .conclusion == "timed_out") | .id' <<<"$jobs"); do | |
| 192 | + echo "::group::Last log lines of job $job" | |
| 193 | + api "/repos/$OWNER/$REPO/actions/jobs/$job/logs" | tail -n 80 || true | |
| 194 | + echo "::endgroup::" | |
| 195 | + done | |
| 196 | + echo "::error::the push run did not succeed" | |
| 197 | + exit 1 | |
| 198 | + fi | |
| 190 | 199 | for job in $(jq -r '.jobs[] | select(.name | startswith("hosted")) | .id' <<<"$jobs"); do |
| 191 | 200 | log=$(api "/repos/$OWNER/$REPO/actions/jobs/$job/logs") |
| 192 | 201 | grep -F "live ok" <<<"$log" | grep -F "# $REPO" || { echo "::error::job $job did not print its line (checkout or run step failed)"; exit 1; } |
modifiedcontainers/runner/image/docker+95−27View file
| @@ -5,8 +5,8 @@ | ||
| 5 | 5 | # |
| 6 | 6 | # The daemon reaches the Diffinery container registry through the runner's registry |
| 7 | 7 | # relay (agent.mjs --registry-relay), which splits image layers into chunks under |
| 8 | -# Cloudflare's 100 MB request limit. Builders that `docker buildx create` starts | |
| 9 | -# (docker/setup-buildx-action) are pointed at the relay too. | |
| 8 | +# Cloudflare's 100 MB request limit. BuildKit builders that `docker buildx create` | |
| 9 | +# starts (docker/setup-buildx-action) are set up here too, see below. | |
| 10 | 10 | real=/usr/bin/docker |
| 11 | 11 | ready=/run/diffinery-dockerd.ready |
| 12 | 12 | state=/run/diffinery |
| @@ -64,35 +64,103 @@ | ||
| 64 | 64 | exec 9>&- |
| 65 | 65 | fi |
| 66 | 66 | |
| 67 | -# docker buildx create (docker-container driver): run BuildKit on the host network with | |
| 68 | -# the relay as its HTTPS proxy, trusting the relay for the Diffinery host. | |
| 69 | -if [ "${1:-}" = buildx ] && [ "${2:-}" = create ] && [ -s "$state/relay.host" ]; then | |
| 70 | - host=$(cat "$state/relay.host") | |
| 71 | - args=("$@") | |
| 72 | - driver=docker-container | |
| 73 | - cfg=-1 | |
| 74 | - for ((i = 2; i < ${#args[@]}; i++)); do | |
| 75 | - case "${args[i]}" in | |
| 76 | - --driver) driver=${args[i + 1]:-} ;; | |
| 77 | - --driver=*) driver=${args[i]#--driver=} ;; | |
| 78 | - --buildkitd-config | --config) cfg=$((i + 1)) ;; | |
| 79 | - --driver-opt) [[ "${args[i + 1]:-}" == network=* ]] && driver=custom-network ;; | |
| 80 | - --driver-opt=network=*) driver=custom-network ;; | |
| 67 | +# BuildKit builders. The docker-container driver talks to its BuildKit container through | |
| 68 | +# `docker exec`, which can't join that container's cgroup inside a Cloudflare Container. | |
| 69 | +# So `docker buildx create` (docker-container driver, as docker/setup-buildx-action runs it) | |
| 70 | +# starts the BuildKit container itself, on the host network with its socket on the host, | |
| 71 | +# and registers it as a remote builder. When the registry relay runs, BuildKit uses it as | |
| 72 | +# its HTTPS proxy and trusts it for the Diffinery host. `docker buildx rm` removes it. | |
| 73 | +buildkit_dir=/run/diffinery-buildkit | |
| 74 | + | |
| 75 | +buildx_create() { | |
| 76 | + local name="" driver=docker-container image=moby/buildkit:buildx-stable-1 cfg="" use=0 bootstrap=0 | |
| 77 | + local -a flags=() env=() keep=() | |
| 78 | + while [ $# -gt 0 ]; do | |
| 79 | + case "$1" in | |
| 80 | + --name) name=${2:-}; shift ;; | |
| 81 | + --name=*) name=${1#--name=} ;; | |
| 82 | + --driver) driver=${2:-}; shift ;; | |
| 83 | + --driver=*) driver=${1#--driver=} ;; | |
| 84 | + --driver-opt | --driver-opt=*) | |
| 85 | + local opt | |
| 86 | + if [ "$1" = --driver-opt ]; then opt=${2:-}; shift; else opt=${1#--driver-opt=}; fi | |
| 87 | + local o | |
| 88 | + IFS=, read -ra o <<<"$opt" | |
| 89 | + for o in "${o[@]}"; do | |
| 90 | + case "$o" in | |
| 91 | + image=*) image=${o#image=} ;; | |
| 92 | + env.*=*) o=${o#env.}; env+=(-e "$o") ;; | |
| 93 | + network=* | cgroup-parent=* | restart-policy=* | memory=* | memory-swap=* | cpu-*=* | default-load=* | provenance-add-gha=*) ;; | |
| 94 | + *) return 2 ;; | |
| 95 | + esac | |
| 96 | + done | |
| 97 | + ;; | |
| 98 | + --buildkitd-flags) read -ra flags <<<"${2:-}"; shift ;; | |
| 99 | + --buildkitd-flags=*) read -ra flags <<<"${1#--buildkitd-flags=}" ;; | |
| 100 | + --buildkitd-config | --config) cfg=${2:-}; shift ;; | |
| 101 | + --buildkitd-config=* | --config=*) cfg=${1#*=} ;; | |
| 102 | + --use) use=1 ;; | |
| 103 | + --bootstrap) bootstrap=1 ;; | |
| 104 | + --platform) keep+=("$1" "${2:-}"); shift ;; | |
| 105 | + --platform=*) keep+=("$1") ;; | |
| 106 | + --node | --node=*) [ "$1" = --node ] && shift ;; | |
| 107 | + # Appending nodes, endpoints and anything else: leave it to buildx. | |
| 108 | + *) return 2 ;; | |
| 81 | 109 | esac |
| 110 | + shift | |
| 82 | 111 | done |
| 83 | - if [ "$driver" = docker-container ]; then | |
| 84 | - if [ "$cfg" -ge 0 ] && [ -f "${args[cfg]}" ]; then | |
| 85 | - if ! grep -qF "registry.\"$host\"" "${args[cfg]}"; then | |
| 86 | - merged=$(mktemp "$state/buildkitd-XXXXXX.toml") | |
| 87 | - { cat "${args[cfg]}"; echo; cat "$state/buildkitd.toml"; } >"$merged" | |
| 88 | - args[cfg]=$merged | |
| 89 | - fi | |
| 90 | - elif [ "$cfg" -lt 0 ]; then | |
| 91 | - args+=(--buildkitd-config "$state/buildkitd.toml") | |
| 112 | + [ "$driver" = docker-container ] || return 2 | |
| 113 | + [ -n "$name" ] || name="builder-$(cat /proc/sys/kernel/random/uuid)" | |
| 114 | + local dir="$buildkit_dir/$name" box="buildx_buildkit_${name}0" | |
| 115 | + mkdir -p "$dir" "$state" | |
| 116 | + local -a args=(--config /etc/buildkit/buildkitd.toml) | |
| 117 | + : >"$dir/buildkitd.toml" | |
| 118 | + [ -n "$cfg" ] && cat "$cfg" >>"$dir/buildkitd.toml" | |
| 119 | + if [ -s "$state/relay.host" ]; then | |
| 120 | + local host | |
| 121 | + host=$(cat "$state/relay.host") | |
| 122 | + if ! grep -qF "registry.\"$host\"" "$dir/buildkitd.toml"; then | |
| 123 | + { echo; cat "$state/buildkitd.toml"; } >>"$dir/buildkitd.toml" | |
| 92 | 124 | fi |
| 93 | - args+=(--driver-opt network=host --driver-opt "env.HTTPS_PROXY=http://127.0.0.1:$relay_port" --driver-opt "env.https_proxy=http://127.0.0.1:$relay_port") | |
| 94 | - exec "$real" "${args[@]}" | |
| 125 | + env+=(-e "HTTPS_PROXY=http://127.0.0.1:$relay_port" -e "https_proxy=http://127.0.0.1:$relay_port") | |
| 95 | 126 | fi |
| 127 | + "$real" rm -f "$box" >/dev/null 2>&1 | |
| 128 | + "$real" run -d --name "$box" --label "diffinery.buildkit=$name" --privileged --network host \ | |
| 129 | + -v "$dir:/run/buildkit" -v "$dir/buildkitd.toml:/etc/buildkit/buildkitd.toml:ro" -v "$state:$state:ro" \ | |
| 130 | + -v "${box}_state:/var/lib/buildkit" "${env[@]}" "$image" "${args[@]}" "${flags[@]}" >/dev/null || return 1 | |
| 131 | + for _ in $(seq 1 120); do | |
| 132 | + [ -S "$dir/buildkitd.sock" ] && break | |
| 133 | + if [ "$("$real" inspect -f '{{.State.Running}}' "$box" 2>/dev/null)" != true ]; then | |
| 134 | + echo "BuildKit did not start:" >&2 | |
| 135 | + "$real" logs "$box" 2>&1 | tail -n 20 >&2 | |
| 136 | + return 1 | |
| 137 | + fi | |
| 138 | + sleep 0.25 | |
| 139 | + done | |
| 140 | + local -a create=(buildx create --name "$name" --driver remote "${keep[@]}") | |
| 141 | + [ "$use" = 1 ] && create+=(--use) | |
| 142 | + [ "$bootstrap" = 1 ] && create+=(--bootstrap) | |
| 143 | + exec "$real" "${create[@]}" "unix://$dir/buildkitd.sock" | |
| 144 | +} | |
| 145 | + | |
| 146 | +if [ "${1:-}" = buildx ] && [ "${2:-}" = create ] && [ -z "${DOCKER_HOST:-}" ] && [ "$(id -u)" = 0 ]; then | |
| 147 | + buildx_create "${@:3}" | |
| 148 | + [ $? = 2 ] || exit 1 | |
| 96 | 149 | fi |
| 97 | 150 | |
| 151 | +# docker buildx rm: also remove the BuildKit containers buildx_create started. | |
| 152 | +if [ "${1:-}" = buildx ] && [ "${2:-}" = rm ] && [ -d "$buildkit_dir" ]; then | |
| 153 | + "$real" "$@" | |
| 154 | + status=$? | |
| 155 | + for arg in "${@:3}"; do | |
| 156 | + case "$arg" in -*) continue ;; esac | |
| 157 | + if [ -d "$buildkit_dir/$arg" ]; then | |
| 158 | + "$real" rm -f -v "buildx_buildkit_${arg}0" >/dev/null 2>&1 | |
| 159 | + "$real" volume rm "buildx_buildkit_${arg}0_state" >/dev/null 2>&1 | |
| 160 | + rm -rf "${buildkit_dir:?}/$arg" | |
| 161 | + fi | |
| 162 | + done | |
| 163 | + exit "$status" | |
| 164 | +fi | |
| 165 | + | |
| 98 | 166 | exec "$real" "$@" |