Docker on hosted runners, graceful rollouts, and a workflow to move this repo to Diffinery (#12)
* Docker probe: use the image's existing ubuntu user for rootless Docker Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01AizKXSsengXZRegJxRATkL * Docker probe: check build DNS with a daemon resolver and time daemon startup Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01AizKXSsengXZRegJxRATkL * Docker probe: test the runner image's Docker wrapper from a cold start Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01AizKXSsengXZRegJxRATkL * Runner image: Docker Engine, started on first use Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01AizKXSsengXZRegJxRATkL * Docker on hosted runners, graceful rollouts, and a workflow to move this repo to Diffinery Hosted jobs can now use job containers, services, Docker actions and plain docker build/run/compose: the runner image ships Docker Engine, started on first use, with Docker Hub pulled through mirror.gcr.io. A runner that gets SIGTERM during a container rollout finishes its job (up to 14 minutes), and rollouts leave jobs younger than an hour alone. Production deploys are pinned to github.com until the cutover, and a manual workflow copies branches, tags, secrets and variables to diffinery.com/codician-team/diffinery. The live test now checks a Docker build and a service container on a hosted runner. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01AizKXSsengXZRegJxRATkL * Leave containers younger than an hour alone during rollouts Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01AizKXSsengXZRegJxRATkL --------- Co-authored-by: Claude <noreply@anthropic.com>
Canberk Demirci authored · GitHub committed signed1 parent 36b475fcommit 014d046c679cdb0dcfbf9714cb85a3dc992d1391Browse files
18 files changed+260−55
Files (18)
- .github/workflows/deploy.yml+4−0
- .github/workflows/dind-probe.yml+2−0
- .github/workflows/live-test.yml+24−3
- .github/workflows/move-to-diffinery.yml+95−0
- README.md+8−7
- apps/diffinery/scripts/dind-probe/.gitignore+3−0
- apps/diffinery/scripts/dind-probe/Dockerfile+4−4
- apps/diffinery/scripts/dind-probe/probe.sh+27−11
- apps/diffinery/src/repo/actions.ts+0−3
- apps/diffinery/test/e2e/e2e.test.mjs+1−13
- apps/diffinery/wrangler.jsonc+2−2
- containers/runner/.dockerignore+1−0
- containers/runner/Dockerfile+14−0
- containers/runner/image/daemon.json+5−0
- containers/runner/image/docker+21−0
- containers/runner/src/agent.ts+15−1
- containers/runner/src/main.ts+2−0
- containers/runner/test/job.test.ts+32−11
modified.github/workflows/deploy.yml+4−0View file
| @@ -18,6 +18,10 @@ | ||
| 18 | 18 | jobs: |
| 19 | 19 | deploy: |
| 20 | 20 | name: Deploy to Cloudflare |
| 21 | + # Production deploys from one host only. This repository is mirrored to | |
| 22 | + # diffinery.com (see move-to-diffinery.yml); switch this to | |
| 23 | + # 'https://diffinery.com' to make Diffinery the one that deploys. | |
| 24 | + if: ${{ github.server_url == 'https://github.com' }} | |
| 21 | 25 | runs-on: ubuntu-latest |
| 22 | 26 | timeout-minutes: 45 |
| 23 | 27 | environment: |
modified.github/workflows/dind-probe.yml+2−0View file
| @@ -31,6 +31,8 @@ | ||
| 31 | 31 | working-directory: apps/diffinery/scripts/dind-probe |
| 32 | 32 | run: | |
| 33 | 33 | set -euo pipefail |
| 34 | + # Test the runner image's real Docker wrapper and daemon config. | |
| 35 | + cp ../../../../containers/runner/image/docker ../../../../containers/runner/image/daemon.json . | |
| 34 | 36 | cat > wrangler.json <<JSON |
| 35 | 37 | {"name":"$NAME","main":"worker.js","compatibility_date":"2026-10-01","workers_dev":true, |
| 36 | 38 | "containers":[{"class_name":"Probe","image":"./Dockerfile","instance_type":"standard-2","max_instances":1}], |
modified.github/workflows/live-test.yml+24−3View file
| @@ -1,7 +1,7 @@ | ||
| 1 | 1 | name: Live test |
| 2 | 2 | |
| 3 | 3 | # End-to-end checks against the real diffinery.com after each deploy: hosted |
| 4 | -# Actions jobs in Cloudflare Containers, agent sandboxes, webhooks, scheduled | |
| 4 | +# Actions jobs in Cloudflare Containers (with Docker builds and services), agent sandboxes, webhooks, scheduled | |
| 5 | 5 | # workflows and Git LFS. Each run works in a throwaway private repository that it |
| 6 | 6 | # deletes at the end. |
| 7 | 7 | # |
| @@ -76,6 +76,22 @@ | ||
| 76 | 76 | steps: |
| 77 | 77 | - uses: actions/checkout@v4 |
| 78 | 78 | - run: echo "live ok ${{ '${{' }} matrix.n }} $(cat README.md)" |
| 79 | + docker: | |
| 80 | + runs-on: ubuntu-latest | |
| 81 | + services: | |
| 82 | + cache: | |
| 83 | + image: redis:7-alpine | |
| 84 | + ports: ["6379"] | |
| 85 | + options: --health-cmd "redis-cli ping" --health-interval 1s --health-retries 30 | |
| 86 | + steps: | |
| 87 | + - run: | | |
| 88 | + printf 'FROM alpine:3.20\nRUN echo built > /msg\nCMD ["cat", "/msg"]\n' > Dockerfile | |
| 89 | + docker build -q -t live-image . | |
| 90 | + echo "docker ok $(docker run --rm live-image)" | |
| 91 | + exec 3<>/dev/tcp/127.0.0.1/${{ '${{' }} job.services.cache.ports['6379'] }} | |
| 92 | + printf 'PING\r\n' >&3 | |
| 93 | + read -r line <&3 | |
| 94 | + echo "service ok ${line%$'\r'}" | |
| 79 | 95 | EOF |
| 80 | 96 | cat > .github/workflows/tick.yml <<'EOF' |
| 81 | 97 | on: |
| @@ -91,7 +107,7 @@ | ||
| 91 | 107 | git push -q "https://x-token:${TOKEN}@${DIFFINERY#https://}/$OWNER/$REPO.git" main |
| 92 | 108 | git lfs ls-files |
| 93 | 109 | |
| 94 | - - name: Hosted Actions jobs run in Cloudflare Containers | |
| 110 | + - name: Hosted Actions jobs (and Docker) run in Cloudflare Containers | |
| 95 | 111 | if: steps.token.outputs.skip != 'true' |
| 96 | 112 | run: | |
| 97 | 113 | set -euo pipefail |
| @@ -104,10 +120,15 @@ | ||
| 104 | 120 | echo "$run" | jq '{id, status, conclusion}' |
| 105 | 121 | [ "$(jq -r .conclusion <<<"$run")" = success ] || { echo "::error::the push run did not succeed"; exit 1; } |
| 106 | 122 | id=$(jq -r .id <<<"$run") |
| 107 | - for job in $(api "/repos/$OWNER/$REPO/actions/runs/$id/jobs" | jq -r '.jobs[].id'); do | |
| 123 | + jobs=$(api "/repos/$OWNER/$REPO/actions/runs/$id/jobs") | |
| 124 | + for job in $(jq -r '.jobs[] | select(.name | startswith("hosted")) | .id' <<<"$jobs"); do | |
| 108 | 125 | log=$(api "/repos/$OWNER/$REPO/actions/jobs/$job/logs") |
| 109 | 126 | grep -F "live ok" <<<"$log" | grep -F "# $REPO" || { echo "::error::job $job did not print its line (checkout or run step failed)"; exit 1; } |
| 110 | 127 | done |
| 128 | + job=$(jq -r '.jobs[] | select(.name == "docker") | .id' <<<"$jobs") | |
| 129 | + log=$(api "/repos/$OWNER/$REPO/actions/jobs/$job/logs") | |
| 130 | + grep -F "docker ok built" <<<"$log" || { echo "::error::docker build or run failed on a hosted runner"; exit 1; } | |
| 131 | + grep -F "service ok +PONG" <<<"$log" || { echo "::error::the redis service container was not reachable"; exit 1; } | |
| 111 | 132 | |
| 112 | 133 | - name: Git LFS round trip |
| 113 | 134 | if: steps.token.outputs.skip != 'true' |
added.github/workflows/move-to-diffinery.yml+95−0View file
| @@ -1,0 +1,95 @@ | ||
| 1 | +name: Move to Diffinery | |
| 2 | + | |
| 3 | +# Copies this repository to diffinery.com/codician-team/diffinery: every branch | |
| 4 | +# and tag, plus the Actions secrets and variables our workflows use. Safe to | |
| 5 | +# re-run; each run brings Diffinery up to date. | |
| 6 | +# | |
| 7 | +# Secrets go straight from GitHub's secret store to Diffinery's over HTTPS and are | |
| 8 | +# never printed. Needs DIFFINERY_LIVE_TOKEN (an admin token on diffinery.com). | |
| 9 | +on: | |
| 10 | + workflow_dispatch: | |
| 11 | + | |
| 12 | +permissions: | |
| 13 | + contents: read | |
| 14 | + | |
| 15 | +concurrency: | |
| 16 | + group: move-to-diffinery | |
| 17 | + cancel-in-progress: false | |
| 18 | + | |
| 19 | +jobs: | |
| 20 | + move: | |
| 21 | + name: Copy code, secrets and variables | |
| 22 | + runs-on: ubuntu-latest | |
| 23 | + timeout-minutes: 20 | |
| 24 | + environment: production | |
| 25 | + env: | |
| 26 | + DIFFINERY: https://diffinery.com | |
| 27 | + TOKEN: ${{ secrets.DIFFINERY_LIVE_TOKEN }} | |
| 28 | + ORG: codician-team | |
| 29 | + NAME: diffinery | |
| 30 | + steps: | |
| 31 | + - name: Check the token | |
| 32 | + run: | | |
| 33 | + [ -n "$TOKEN" ] || { echo "::error::Add DIFFINERY_LIVE_TOKEN (an admin token from $DIFFINERY/settings/tokens) as a repository secret first."; exit 1; } | |
| 34 | + curl -fsS "$DIFFINERY/api/v3/user" -H "Authorization: Bearer $TOKEN" | jq '{login, site_admin}' | |
| 35 | + | |
| 36 | + - name: Create the organization and repository | |
| 37 | + env: | |
| 38 | + DESC: ${{ github.event.repository.description }} | |
| 39 | + run: | | |
| 40 | + set -euo pipefail | |
| 41 | + api() { curl -sS -o /tmp/out -w '%{http_code}' -X "$1" "$DIFFINERY/api/v3$2" -H "Authorization: Bearer $TOKEN" -H "content-type: application/json" ${3:+--data "$3"}; } | |
| 42 | + if [ "$(api GET "/users/$ORG")" = 404 ]; then | |
| 43 | + code=$(api POST /user/orgs "{\"login\":\"$ORG\",\"name\":\"Codician\"}"); cat /tmp/out; echo | |
| 44 | + [ "$code" = 201 ] || { echo "::error::could not create the $ORG organization ($code)"; exit 1; } | |
| 45 | + fi | |
| 46 | + if [ "$(api GET "/repos/$ORG/$NAME")" = 404 ]; then | |
| 47 | + desc=$(jq -n --arg d "$DESC" '$d') | |
| 48 | + code=$(api POST "/orgs/$ORG/repos" "{\"name\":\"$NAME\",\"private\":true,\"default_branch\":\"main\",\"description\":$desc}"); cat /tmp/out; echo | |
| 49 | + [ "$code" = 201 ] || { echo "::error::could not create $ORG/$NAME ($code)"; exit 1; } | |
| 50 | + fi | |
| 51 | + echo "$DIFFINERY/$ORG/$NAME is ready" | |
| 52 | + | |
| 53 | + - name: Copy Actions secrets and variables | |
| 54 | + env: | |
| 55 | + S_CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} | |
| 56 | + S_CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} | |
| 57 | + S_DIFFINERY_LIVE_TOKEN: ${{ secrets.DIFFINERY_LIVE_TOKEN }} | |
| 58 | + V_ACCESS_TEAM_DOMAIN: ${{ vars.ACCESS_TEAM_DOMAIN }} | |
| 59 | + run: | | |
| 60 | + set -euo pipefail | |
| 61 | + put() { curl -fsS -o /dev/null -X PUT "$DIFFINERY/api/v3/repos/$ORG/$NAME/actions/$1/$2" -H "Authorization: Bearer $TOKEN" -H "content-type: application/json" --data-binary @-; } | |
| 62 | + for name in CLOUDFLARE_API_TOKEN CLOUDFLARE_ACCOUNT_ID DIFFINERY_LIVE_TOKEN; do | |
| 63 | + v="S_$name" | |
| 64 | + if [ -n "${!v}" ]; then jq -n --arg v "${!v}" '{value: $v}' | put secrets "$name"; echo "secret $name copied"; else echo "::warning::secret $name is empty here, so it was not copied"; fi | |
| 65 | + done | |
| 66 | + for name in ACCESS_TEAM_DOMAIN; do | |
| 67 | + v="V_$name" | |
| 68 | + if [ -n "${!v}" ]; then jq -n --arg v "${!v}" '{value: $v}' | put variables "$name"; echo "variable $name = ${!v}"; fi | |
| 69 | + done | |
| 70 | + | |
| 71 | + - uses: actions/checkout@v4 | |
| 72 | + with: | |
| 73 | + fetch-depth: 0 | |
| 74 | + lfs: true | |
| 75 | + | |
| 76 | + - name: Push every branch and tag | |
| 77 | + run: | | |
| 78 | + set -euo pipefail | |
| 79 | + git fetch -q --tags origin '+refs/heads/*:refs/remotes/origin/*' | |
| 80 | + remote="https://x-token:${TOKEN}@${DIFFINERY#https://}/$ORG/$NAME.git" | |
| 81 | + refs=$(git for-each-ref --format='+%(refname):refs/heads/%(refname:lstrip=3)' refs/remotes/origin | grep -v ':refs/heads/HEAD$') | |
| 82 | + git push -q "$remote" $refs '+refs/tags/*:refs/tags/*' | |
| 83 | + if git lfs ls-files | grep -q .; then git lfs push --all "$remote"; fi | |
| 84 | + echo "branches on Diffinery:" | |
| 85 | + curl -fsS "$DIFFINERY/api/v3/repos/$ORG/$NAME/branches" -H "Authorization: Bearer $TOKEN" | jq -r '.[] | " \(.name) \(.commit.sha)"' | |
| 86 | + want=$(git rev-parse refs/remotes/origin/main) | |
| 87 | + got=$(curl -fsS "$DIFFINERY/api/v3/repos/$ORG/$NAME/branches" -H "Authorization: Bearer $TOKEN" | jq -r '.[] | select(.name=="main") | .commit.sha') | |
| 88 | + [ "$want" = "$got" ] || { echo "::error::main on Diffinery is $got, expected $want"; exit 1; } | |
| 89 | + echo "main matches GitHub at $want" | |
| 90 | + | |
| 91 | + - name: Check Actions on Diffinery | |
| 92 | + run: | | |
| 93 | + set -euo pipefail | |
| 94 | + curl -fsS "$DIFFINERY/api/v3/repos/$ORG/$NAME/actions/workflows" -H "Authorization: Bearer $TOKEN" | jq -r '.workflows[] | "\(.path): \(.state) \(.errors | join("; "))"' | |
| 95 | + curl -fsS "$DIFFINERY/api/v3/repos/$ORG/$NAME/actions/secrets" -H "Authorization: Bearer $TOKEN" | jq -r '.secrets[] | "secret \(.name)"' |
modifiedREADME.md+8−7View file
| @@ -144,13 +144,14 @@ | ||
| 144 | 144 | owner). `with`, `secrets` (a mapping or `inherit`), `on.workflow_call` |
| 145 | 145 | outputs and nesting up to four levels deep all work. A matrix on the calling |
| 146 | 146 | job isn't supported yet. |
| 147 | -- Docker, on self-hosted runners with Docker installed: a job `container:` | |
| 148 | - (steps and JavaScript actions run inside it), `services:` (reachable by | |
| 149 | - name from the job container, or on `localhost` through their mapped ports, | |
| 150 | - with health checks awaited), and Docker container actions (`docker://image` | |
| 151 | - steps and actions with `runs.using: docker`, built from their Dockerfile). | |
| 152 | - Hosted runners can't run Docker, so these jobs need | |
| 153 | - `runs-on: [self-hosted, linux]`. | |
| 147 | +- Docker, on hosted runners and on self-hosted runners with Docker installed: | |
| 148 | + a job `container:` (steps and JavaScript actions run inside it), `services:` | |
| 149 | + (reachable by name from the job container, or on `localhost` through their | |
| 150 | + mapped ports, with health checks awaited), Docker container actions | |
| 151 | + (`docker://image` steps and actions with `runs.using: docker`, built from | |
| 152 | + their Dockerfile), and plain `docker build` / `docker run` / `docker compose` | |
| 153 | + in steps. Hosted runners start the Docker daemon the first time a job uses | |
| 154 | + it and pull Docker Hub images through `mirror.gcr.io`. | |
| 154 | 155 | |
| 155 | 156 | **Not supported yet:** `actions/github-script`, |
| 156 | 157 | Windows and macOS hosted runners, `pull_request_target`, `merge_group`, and |
addedapps/diffinery/scripts/dind-probe/.gitignore+3−0View file
| @@ -1,0 +1,3 @@ | ||
| 1 | +docker | |
| 2 | +daemon.json | |
| 3 | +wrangler.json |
modifiedapps/diffinery/scripts/dind-probe/Dockerfile+4−4View file
| @@ -2,14 +2,14 @@ | ||
| 2 | 2 | FROM ubuntu:24.04 |
| 3 | 3 | ENV DEBIAN_FRONTEND=noninteractive |
| 4 | 4 | RUN apt-get update \ |
| 5 | - && apt-get install -y --no-install-recommends ca-certificates curl gnupg uidmap iproute2 slirp4netns fuse-overlayfs dbus-user-session kmod nodejs procps iptables \ | |
| 5 | + && apt-get install -y --no-install-recommends ca-certificates curl gnupg uidmap iproute2 slirp4netns fuse-overlayfs dbus-user-session kmod nodejs procps iptables bc \ | |
| 6 | 6 | && install -m 0755 -d /etc/apt/keyrings \ |
| 7 | 7 | && curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc \ |
| 8 | 8 | && echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu noble stable" > /etc/apt/sources.list.d/docker.list \ |
| 9 | 9 | && apt-get update \ |
| 10 | - && apt-get install -y --no-install-recommends docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-ce-rootless-extras \ | |
| 11 | - && useradd -m -u 1000 rootless \ | |
| 10 | + && apt-get install -y --no-install-recommends docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin \ | |
| 11 | + && (grep -q "^ubuntu:" /etc/subuid || usermod --add-subuids 100000-165535 --add-subgids 100000-165535 ubuntu) \ | |
| 12 | 12 | && rm -rf /var/lib/apt/lists/* |
| 13 | -COPY probe.sh server.mjs /opt/probe/ | |
| 13 | +COPY probe.sh server.mjs docker daemon.json /opt/probe/ | |
| 14 | 14 | EXPOSE 8080 |
| 15 | 15 | ENTRYPOINT ["node", "/opt/probe/server.mjs"] |
modifiedapps/diffinery/scripts/dind-probe/probe.sh+27−11View file
| @@ -4,6 +4,7 @@ | ||
| 4 | 4 | echo "=== environment" |
| 5 | 5 | uname -a; id; grep -E '^(CapEff|CapBnd|NoNewPrivs|Seccomp)' /proc/self/status; nproc; free -m | head -2; df -h / | tail -1 |
| 6 | 6 | grep -E 'overlay|fuse' /proc/filesystems; ls -l /dev/fuse /dev/net/tun 2>&1; mount | grep -E 'cgroup' | head -3 |
| 7 | +cat /etc/resolv.conf | |
| 7 | 8 | r userns "unshare -Ur id" |
| 8 | 9 | r netns "unshare -n true" |
| 9 | 10 | |
| @@ -23,19 +24,34 @@ | ||
| 23 | 24 | docker rm -f svc pub >/dev/null 2>&1; docker network rm probe-net >/dev/null 2>&1 |
| 24 | 25 | } |
| 25 | 26 | |
| 26 | -echo "=== rootful dockerd" | |
| 27 | -(dockerd --iptables=false --ip6tables=false > /tmp/dockerd.log 2>&1 &) | |
| 28 | -if wait_docker; then echo "RESULT rootful-daemon ok"; checks rootful; else echo "RESULT rootful-daemon fail"; tail -n 25 /tmp/dockerd.log; fi | |
| 29 | -pkill dockerd; sleep 3 | |
| 30 | - | |
| 31 | 27 | echo "=== rootful dockerd with iptables" |
| 28 | +start=$(date +%s.%N) | |
| 32 | 29 | (dockerd > /tmp/dockerd2.log 2>&1 &) |
| 33 | -if wait_docker; then echo "RESULT rootful-iptables-daemon ok"; checks rootful-ipt; else echo "RESULT rootful-iptables-daemon fail"; tail -n 25 /tmp/dockerd2.log; fi | |
| 30 | +if wait_docker; then echo "RESULT rootful-iptables-daemon ok (ready in $(echo "$(date +%s.%N) - $start" | bc)s)"; checks rootful-ipt; else echo "RESULT rootful-iptables-daemon fail"; tail -n 25 /tmp/dockerd2.log; fi | |
| 34 | 31 | pkill dockerd; sleep 3 |
| 35 | 32 | |
| 36 | -echo "=== rootless dockerd" | |
| 37 | -mkdir -p /run/user/1000 && chown rootless /run/user/1000 && chmod 700 /run/user/1000 | |
| 38 | -su rootless -c 'export XDG_RUNTIME_DIR=/run/user/1000 HOME=/home/rootless PATH=/usr/bin:/usr/sbin:/bin; dockerd-rootless.sh --iptables=false --ip6tables=false > /tmp/rootless.log 2>&1 &' | |
| 39 | -export DOCKER_HOST=unix:///run/user/1000/docker.sock | |
| 40 | -if wait_docker; then echo "RESULT rootless-daemon ok"; checks rootless; else echo "RESULT rootless-daemon fail"; tail -n 25 /tmp/rootless.log; fi | |
| 33 | +echo "=== rootful dockerd with iptables and dns in daemon.json" | |
| 34 | +mkdir -p /etc/docker && echo '{"dns":["1.1.1.1","8.8.8.8"]}' > /etc/docker/daemon.json | |
| 35 | +(dockerd > /tmp/dockerd3.log 2>&1 &) | |
| 36 | +if wait_docker; then | |
| 37 | + echo "RESULT dns-daemon ok" | |
| 38 | + r dns-build-default "docker build --progress=plain --no-cache -t probe-dns /tmp/ctx" | |
| 39 | + r dns-run-default "docker run --rm mirror.gcr.io/library/alpine:3.20 sh -c 'wget -T 10 -qO- https://example.com | head -c 60'" | |
| 40 | + r dns-bridge-dns "docker network create probe-net2 >/dev/null && docker run -d --name svc2 --network probe-net2 --network-alias svc mirror.gcr.io/library/nginx:alpine >/dev/null && sleep 3 && docker run --rm --network probe-net2 mirror.gcr.io/library/alpine:3.20 sh -c 'wget -T 10 -qO- http://svc | head -c 40 && wget -T 10 -qO- https://example.com | head -c 40'" | |
| 41 | + r dns-buildx "docker buildx build --progress=plain --no-cache --load -t probe-bx /tmp/ctx" | |
| 42 | +else echo "RESULT dns-daemon fail"; tail -n 25 /tmp/dockerd3.log; fi | |
| 43 | +pkill dockerd; sleep 3 | |
| 44 | + | |
| 45 | +echo "=== runner image wrapper (cold start, Docker Hub through the mirror)" | |
| 46 | +rm -f /run/diffinery-dockerd.ready /etc/docker/daemon.json | |
| 47 | +if [ -f /opt/probe/docker ]; then | |
| 48 | + cp /opt/probe/daemon.json /etc/docker/daemon.json && install -m 0755 /opt/probe/docker /usr/local/bin/docker | |
| 49 | + mkdir -p /tmp/ctx2 && printf 'FROM alpine:3.20\nRUN wget -qO- https://example.com | head -c 20 && echo\n' > /tmp/ctx2/Dockerfile | |
| 50 | + start=$(date +%s.%N) | |
| 51 | + r wrapper-cold "/usr/local/bin/docker run --rm alpine:3.20 echo wrapped" | |
| 52 | + echo "cold start and first run took $(echo "$(date +%s.%N) - $start" | bc)s" | |
| 53 | + r wrapper-build "/usr/local/bin/docker build --progress=plain -t probe-w /tmp/ctx2" | |
| 54 | + r wrapper-compose "/usr/local/bin/docker compose version" | |
| 55 | + r wrapper-hub-pull "/usr/local/bin/docker pull redis:7-alpine" | |
| 56 | +fi | |
| 41 | 57 | echo "PROBE DONE" |
modifiedapps/diffinery/src/repo/actions.ts+0−3View file
| @@ -701,9 +701,6 @@ | ||
| 701 | 701 | const unsupported = labels.filter((l) => !SUPPORTED_LABEL.test(l)); |
| 702 | 702 | const selfHosted = labels.some((l) => l.toLowerCase() === "self-hosted"); |
| 703 | 703 | if (unsupported.length && !selfHosted) return fail(label(job.name ?? short), `No Diffinery runner matches runs-on: ${unsupported.join(", ")}. Linux (ubuntu-*) runners are hosted; anything else needs runs-on: [self-hosted, ...] and a registered runner.`); |
| 704 | - // Hosted runners are Cloudflare Containers, which cannot run Docker themselves. | |
| 705 | - const docker = job.container ? "Job containers (jobs.<id>.container)" : job.services && Object.keys(job.services).length ? "Service containers (jobs.<id>.services)" : null; | |
| 706 | - if (docker && !selfHosted) return fail(label(job.name ?? short), `${docker} need Docker, which hosted Diffinery runners don't have. Use runs-on: [self-hosted, linux] with a runner on a machine with Docker.`); | |
| 707 | 704 | |
| 708 | 705 | let combos: Array<Record<string, unknown>>; |
| 709 | 706 | try { |
modifiedapps/diffinery/test/e2e/e2e.test.mjs+1−13View file
| @@ -512,12 +512,6 @@ | ||
| 512 | 512 | join(src, ".github/workflows/ci.yml"), |
| 513 | 513 | `on: push |
| 514 | 514 | jobs: |
| 515 | - hosted: | |
| 516 | - runs-on: ubuntu-latest | |
| 517 | - services: | |
| 518 | - redis: mirror.gcr.io/library/redis:7-alpine | |
| 519 | - steps: | |
| 520 | - - run: echo never | |
| 521 | 515 | boxed: |
| 522 | 516 | runs-on: [self-hosted, linux] |
| 523 | 517 | container: |
| @@ -551,13 +545,7 @@ | ||
| 551 | 545 | const r = await ok("GET", `/repos/${owner}/boxes/actions/runs/${run.id}`); |
| 552 | 546 | return r.status === "completed" ? r : null; |
| 553 | 547 | }); |
| 554 | - // Hosted runners can't run Docker, and say so. | |
| 555 | - assert.equal(done.conclusion, "failure"); | |
| 556 | - const jobs = (await ok("GET", `/repos/${owner}/boxes/actions/runs/${run.id}/jobs`)).jobs; | |
| 557 | - const hosted = jobs.find((j) => j.name === "hosted"); | |
| 558 | - assert.equal(hosted.conclusion, "failure"); | |
| 559 | - assert.match(JSON.stringify(hosted), /hosted Diffinery runners don't have/); | |
| 560 | - assert.equal(jobs.find((j) => j.name === "boxed").conclusion, "success"); | |
| 548 | + assert.equal(done.conclusion, "success"); | |
| 561 | 549 | }); |
| 562 | 550 | |
| 563 | 551 | test("D1 sessions: bookmarks for token clients and browsers", async () => { |
modifiedapps/diffinery/wrangler.jsonc+2−2View file
| @@ -42,8 +42,8 @@ | ||
| 42 | 42 | "migrations": [{ "tag": "v1", "new_sqlite_classes": ["RepoDO", "RunnerContainer", "SandboxContainer"] }], |
| 43 | 43 | "workflows": [{ "name": "diffinery-run", "binding": "RUN_WORKFLOW", "class_name": "RunWorkflow" }], |
| 44 | 44 | "containers": [ |
| 45 | - { "class_name": "RunnerContainer", "image": "../../containers/runner/Dockerfile", "image_build_context": "../../containers/runner", "instance_type": "standard-2", "max_instances": 100 }, | |
| 46 | - { "class_name": "SandboxContainer", "image": "../../containers/runner/Dockerfile", "image_build_context": "../../containers/runner", "instance_type": "standard-1", "max_instances": 50 } | |
| 45 | + { "class_name": "RunnerContainer", "image": "../../containers/runner/Dockerfile", "image_build_context": "../../containers/runner", "instance_type": "standard-2", "max_instances": 100, "rollout_active_grace_period": 3600 }, | |
| 46 | + { "class_name": "SandboxContainer", "image": "../../containers/runner/Dockerfile", "image_build_context": "../../containers/runner", "instance_type": "standard-1", "max_instances": 50, "rollout_active_grace_period": 3600 } | |
| 47 | 47 | ], |
| 48 | 48 | "triggers": { "crons": ["*/5 * * * *"] }, |
| 49 | 49 |
modifiedcontainers/runner/.dockerignore+1−0View file
| @@ -1,2 +1,3 @@ | ||
| 1 | 1 | * |
| 2 | 2 | !dist/agent.mjs |
| 3 | +!image/ |
modifiedcontainers/runner/Dockerfile+14−0View file
| @@ -44,6 +44,20 @@ | ||
| 44 | 44 | npm cache clean --force; \ |
| 45 | 45 | rm -rf /root/.npm /tmp/* |
| 46 | 46 | |
| 47 | +# Docker Engine, started on first use by the /usr/local/bin/docker wrapper. Cloudflare | |
| 48 | +# Containers run it rootful with iptables, so bridge networks, service DNS, port | |
| 49 | +# publishing and builds all work like on GitHub's hosted runners. | |
| 50 | +RUN set -eux; \ | |
| 51 | + install -m 0755 -d /etc/apt/keyrings; \ | |
| 52 | + curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc; \ | |
| 53 | + echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu noble stable" > /etc/apt/sources.list.d/docker.list; \ | |
| 54 | + apt-get update; \ | |
| 55 | + apt-get install -y --no-install-recommends docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin iptables; \ | |
| 56 | + apt-get clean; \ | |
| 57 | + rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/* | |
| 58 | +COPY image/daemon.json /etc/docker/daemon.json | |
| 59 | +COPY image/docker /usr/local/bin/docker | |
| 60 | + | |
| 47 | 61 | RUN mkdir -p /workspace /opt/hostedtoolcache /opt/diffinery |
| 48 | 62 | COPY dist/agent.mjs /opt/diffinery/agent.mjs |
| 49 | 63 |
addedcontainers/runner/image/daemon.json+5−0View file
| @@ -1,0 +1,5 @@ | ||
| 1 | +{ | |
| 2 | + "dns": ["1.1.1.1", "8.8.8.8"], | |
| 3 | + "registry-mirrors": ["https://mirror.gcr.io"], | |
| 4 | + "log-driver": "local" | |
| 5 | +} |
addedcontainers/runner/image/docker+21−0View file
| @@ -1,0 +1,21 @@ | ||
| 1 | +#!/bin/bash | |
| 2 | +# Hosted Diffinery runners: starts the Docker daemon the first time anything needs | |
| 3 | +# it (a step running `docker`, job containers, services, Docker actions), then runs | |
| 4 | +# the real CLI. The daemon runs in its own session so step cleanup leaves it alone. | |
| 5 | +real=/usr/bin/docker | |
| 6 | +ready=/run/diffinery-dockerd.ready | |
| 7 | +if [ ! -e "$ready" ] && [ -z "${DOCKER_HOST:-}" ] && [ "$(id -u)" = 0 ]; then | |
| 8 | + exec 9>/run/diffinery-dockerd.lock | |
| 9 | + flock 9 | |
| 10 | + if [ ! -e "$ready" ]; then | |
| 11 | + if ! "$real" version >/dev/null 2>&1; then | |
| 12 | + echo "Starting the Docker daemon" >&2 | |
| 13 | + env -i PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin setsid -f dockerd >/var/log/dockerd.log 2>&1 </dev/null | |
| 14 | + for _ in $(seq 1 240); do "$real" version >/dev/null 2>&1 && break; sleep 0.25; done | |
| 15 | + "$real" version >/dev/null 2>&1 || { echo "The Docker daemon did not start:" >&2; tail -n 20 /var/log/dockerd.log >&2; exit 1; } | |
| 16 | + fi | |
| 17 | + touch "$ready" | |
| 18 | + fi | |
| 19 | + exec 9>&- | |
| 20 | +fi | |
| 21 | +exec "$real" "$@" |
modifiedcontainers/runner/src/agent.ts+15−1View file
| @@ -10,6 +10,12 @@ | ||
| 10 | 10 | echoLogs?: boolean; |
| 11 | 11 | /** Install a SIGTERM/SIGINT handler that cancels the job. */ |
| 12 | 12 | handleSignals?: boolean; |
| 13 | + /** | |
| 14 | + * Hosted runners: on SIGTERM (a platform rollout replacing this container), keep | |
| 15 | + * running the job for up to this long before cancelling it. SIGINT or a second | |
| 16 | + * SIGTERM still cancels at once. | |
| 17 | + */ | |
| 18 | + drainOnSigtermMs?: number; | |
| 13 | 19 | /** Exposes the runner (e.g. so tests can cancel). */ |
| 14 | 20 | onRunner?: (r: JobRunner) => void; |
| 15 | 21 | logIntervalMs?: number; |
| @@ -36,7 +42,14 @@ | ||
| 36 | 42 | runner.logger.line(-1, `##[error]Received ${sig}; cancelling the job.`); |
| 37 | 43 | runner.cancel("cancelled"); |
| 38 | 44 | }; |
| 39 | - const sigterm = () => onSignal("SIGTERM"); | |
| 45 | + let draining: NodeJS.Timeout | null = null; | |
| 46 | + const sigterm = () => { | |
| 47 | + if (!o.drainOnSigtermMs || draining) return onSignal("SIGTERM"); | |
| 48 | + const minutes = Math.round(o.drainOnSigtermMs / 60_000); | |
| 49 | + runner.logger.line(-1, `##[warning]This runner is being replaced by a new version. The job keeps running for up to ${minutes} minutes before it is cancelled.`); | |
| 50 | + draining = setTimeout(() => onSignal("SIGTERM"), o.drainOnSigtermMs); | |
| 51 | + draining.unref(); | |
| 52 | + }; | |
| 40 | 53 | const sigint = () => onSignal("SIGINT"); |
| 41 | 54 | if (o.handleSignals) { |
| 42 | 55 | process.on("SIGTERM", sigterm); |
| @@ -57,6 +70,7 @@ | ||
| 57 | 70 | } catch (e) { |
| 58 | 71 | process.stderr.write(`[diffinery-runner] failed to report completion: ${errMsg(e)}\n`); |
| 59 | 72 | } |
| 73 | + if (draining) clearTimeout(draining); | |
| 60 | 74 | if (o.handleSignals) { |
| 61 | 75 | process.off("SIGTERM", sigterm); |
| 62 | 76 | process.off("SIGINT", sigint); |
modifiedcontainers/runner/src/main.ts+2−0View file
| @@ -98,6 +98,8 @@ | ||
| 98 | 98 | toolCache: env.RUNNER_TOOL_CACHE, |
| 99 | 99 | echoLogs: env.DIFFINERY_ECHO_LOGS === "1" || apiBase === null, |
| 100 | 100 | handleSignals: true, |
| 101 | + // Cloudflare gives a container 15 minutes after SIGTERM during a rollout. | |
| 102 | + drainOnSigtermMs: args.spec ? undefined : 14 * 60_000, | |
| 101 | 103 | }); |
| 102 | 104 | process.stdout.write(`[diffinery-runner] job finished: ${completion.conclusion}\n`); |
| 103 | 105 | process.exit(completion.conclusion === "success" ? 0 : 1); |
modifiedcontainers/runner/test/job.test.ts+32−11View file
| @@ -588,18 +588,39 @@ | ||
| 588 | 588 | expect(api.completion).toMatchObject({ conclusion: "failure", failure: { step: 0 } }); |
| 589 | 589 | expect(readFileSync(path.join(workRoot, "w2/_temp/_github_workflow/event.json"), "utf8")).toContain("pusher"); |
| 590 | 590 | |
| 591 | - // SIGTERM: kill the running step's process group and report cancelled | |
| 592 | - api.completion = null; | |
| 593 | - api.steps = []; | |
| 594 | - api.specs.set("job-3", { ...spec, jobId: "job-3", steps: [{ run: "echo started; sleep 60" }, { run: "echo never" }] }); | |
| 595 | - const child = spawn(process.execPath, [out], { | |
| 596 | - env: { ...process.env, DIFFINERY_WORK_ROOT: path.join(workRoot, "w3"), DIFFINERY_API_URL: api.url, DIFFINERY_JOB_ID: "job-3", DIFFINERY_JOB_TOKEN: api.token }, | |
| 597 | - }); | |
| 598 | - const exited = new Promise<number | null>((r) => child.on("close", (code) => r(code))); | |
| 591 | + const startJob = (id: string, steps: StepSpec[]) => { | |
| 592 | + api.completion = null; | |
| 593 | + api.steps = []; | |
| 594 | + api.logs = []; | |
| 595 | + api.specs.set(id, { ...spec, jobId: id, steps }); | |
| 596 | + const child = spawn(process.execPath, [out], { | |
| 597 | + env: { ...process.env, DIFFINERY_WORK_ROOT: path.join(workRoot, id), DIFFINERY_API_URL: api.url, DIFFINERY_JOB_ID: id, DIFFINERY_JOB_TOKEN: api.token }, | |
| 598 | + }); | |
| 599 | + const exited = new Promise<number | null>((r) => child.on("close", (code) => r(code))); | |
| 600 | + return { child, exited }; | |
| 601 | + }; | |
| 602 | + const waitForLog = async (text: string) => { | |
| 603 | + const t0 = Date.now(); | |
| 604 | + while (!api.logs.some((l) => l.text === text) && Date.now() - t0 < 15_000) await new Promise((r) => setTimeout(r, 50)); | |
| 605 | + }; | |
| 606 | + | |
| 607 | + // One SIGTERM (a container rollout): the job keeps running and finishes. | |
| 608 | + const drain = startJob("job-3", [{ run: "echo started; sleep 2" }, { run: "echo after" }]); | |
| 609 | + await waitForLog("started"); | |
| 610 | + drain.child.kill("SIGTERM"); | |
| 611 | + expect(await drain.exited).toBe(0); | |
| 612 | + expect(api.completion).toMatchObject({ conclusion: "success" }); | |
| 613 | + expect(api.logText()).toContain("being replaced by a new version"); | |
| 614 | + expect(api.logText(1)).toContain("after"); | |
| 615 | + | |
| 616 | + // A second SIGTERM: kill the running step's process group and report cancelled. | |
| 599 | 617 | const t0 = Date.now(); |
| 600 | - while (!api.logs.some((l) => l.text === "started") && Date.now() - t0 < 15_000) await new Promise((r) => setTimeout(r, 50)); | |
| 601 | - child.kill("SIGTERM"); | |
| 602 | - expect(await exited).toBe(1); | |
| 618 | + const stop = startJob("job-4", [{ run: "echo started; sleep 60" }, { run: "echo never" }]); | |
| 619 | + await waitForLog("started"); | |
| 620 | + stop.child.kill("SIGTERM"); | |
| 621 | + await new Promise((r) => setTimeout(r, 200)); | |
| 622 | + stop.child.kill("SIGTERM"); | |
| 623 | + expect(await stop.exited).toBe(1); | |
| 603 | 624 | expect(Date.now() - t0).toBeLessThan(15_000); |
| 604 | 625 | expect(api.completion).toMatchObject({ conclusion: "cancelled" }); |
| 605 | 626 | expect(api.lastReport(0)?.conclusion).toBe("cancelled"); |