Diffinery

codician/diffinerypublic

Open-source, agent-first GitHub and GitHub Actions alternative that runs on your own Cloudflare account: git hosting, pull requests, code review and drop-in GitHub Actions YAML on Workers, Durable Objects, R2 and Containers.

Runner image has the libraries Ruby needs; passing container start errors are retried; the campaign can inspect a run (#147)

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01AizKXSsengXZRegJxRATkL
Canberk Demirci authored · GitHub committed signed1 parent 01989b4commit 8e39f86ebb5afc766be1bfa46d16c8571a9302ecBrowse files
8 files changed+67−18
Files (8)
modified.github/workflows/import-campaign.yml+29−11View file
@@ -39,13 +39,17 @@
3939 description: Instead of starting a campaign, re-run the failed jobs of this Diffinery run of an imported repository ("<repository> <run id>") and print how they end
4040 type: string
4141 default: ""
42+ inspect_run:
43+ description: Instead of starting a campaign, print the jobs of this Diffinery run of an imported repository ("<repository> [<run id>]", the newest run without an id), with each failed job's runner, steps and log
44+ type: string
45+ default: ""
4246
4347 permissions:
4448 contents: read
4549
46-# A peek runs next to the campaign it looks at; campaigns queue behind each other.
50+# A peek or an inspection runs next to the campaign it looks at; campaigns queue behind each other.
4751 concurrency:
48- group: ${{ inputs.peek && 'import-campaign-peek' || 'import-campaign' }}
52+ group: ${{ (inputs.peek || inputs.inspect_run != '') && 'import-campaign-peek' || 'import-campaign' }}
4953 cancel-in-progress: false
5054
5155 jobs:
@@ -61,7 +65,7 @@
6165 R: codician-team/diffinery
6266 steps:
6367 - name: Copy IMPORT_GITHUB_TOKEN to Diffinery
64- if: ${{ inputs.follow_run == '' && !inputs.peek && inputs.rerun_failed == '' }}
68+ if: ${{ inputs.follow_run == '' && !inputs.peek && inputs.rerun_failed == '' && inputs.inspect_run == '' }}
6569 env:
6670 IMPORT_GITHUB_TOKEN: ${{ secrets.IMPORT_GITHUB_TOKEN }}
6771 run: |
@@ -80,21 +84,35 @@
8084 FOLLOW_RUN: ${{ inputs.follow_run }}
8185 PEEK: ${{ inputs.peek }}
8286 RERUN_FAILED: ${{ inputs.rerun_failed }}
87+ INSPECT_RUN: ${{ inputs.inspect_run }}
8388 run: |
8489 set -euo pipefail
8590 api() { curl -fsS --retry 3 -X "$1" "$DIFFINERY/api/v3$2" -H "Authorization: Bearer $TOKEN" -H "content-type: application/json" ${3:+--data "$3"}; }
86- if [ -n "$RERUN_FAILED" ]; then
87- read -r repo run <<<"$RERUN_FAILED"
88- [[ "$repo" =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ && "$run" =~ ^[A-Za-z0-9_-]+$ ]] || { echo "::error::rerun_failed must be \"<owner>/<repository> <run id>\""; exit 1; }
89- api POST "/repos/$repo/actions/runs/$run/rerun-failed-jobs" >/dev/null
90- echo "Re-running the failed jobs of $(api GET "/repos/$repo/actions/runs/$run" | jq -r .html_url)"
91- sleep 30
92- while [ "$(api GET "/repos/$repo/actions/runs/$run" | jq -r .status)" != completed ]; do sleep 30; done
91+ if [ -n "$RERUN_FAILED$INSPECT_RUN" ]; then
92+ read -r repo run <<<"${RERUN_FAILED:-$INSPECT_RUN}"
93+ if [ -n "$INSPECT_RUN" ] && [ -z "$run" ]; then
94+ # No run id: list the repository's latest runs and inspect the newest.
95+ runs=$(api GET "/repos/$repo/actions/runs?per_page=10")
96+ jq -r '.workflow_runs[] | "\(.id) \(.name) \(.created_at) \(.status) \(.conclusion // "")"' <<<"$runs"
97+ run=$(jq -r '.workflow_runs[0].id // ""' <<<"$runs")
98+ fi
99+ [[ "$repo" =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ && "$run" =~ ^[A-Za-z0-9_-]+$ ]] || { echo "::error::rerun_failed takes \"<owner>/<repository> <run id>\", inspect_run \"<owner>/<repository> [<run id>]\""; exit 1; }
100+ if [ -n "$RERUN_FAILED" ]; then
101+ # Diffinery answers with the run that re-runs it ({run_id}); GitHub re-runs in place.
102+ run=$(api POST "/repos/$repo/actions/runs/$run/rerun-failed-jobs" | jq -r --arg run "$run" '.run_id // $run')
103+ echo "Re-running the failed jobs as $(api GET "/repos/$repo/actions/runs/$run" | jq -r .html_url)"
104+ sleep 30
105+ while [ "$(api GET "/repos/$repo/actions/runs/$run" | jq -r .status)" != completed ]; do sleep 30; done
106+ else
107+ echo "Jobs of $(api GET "/repos/$repo/actions/runs/$run" | jq -r '"\(.html_url) (\(.status) \(.conclusion // ""), attempt \(.run_attempt // 1))"')"
108+ fi
93109 jobs=$(api GET "/repos/$repo/actions/runs/$run/jobs?per_page=100")
94110 jq -r '.jobs[] | "\(.name): \(.conclusion // .status)\(if .failure then " (\(.failure.message? // .failure))" else "" end)"' <<<"$jobs" | tee -a "$GITHUB_STEP_SUMMARY"
95111 for job in $(jq -r '.jobs[] | select(.conclusion == "failure") | .id' <<<"$jobs"); do
96112 echo "::group::$(jq -r --arg id "$job" '.jobs[] | select((.id | tostring) == $id) | .name' <<<"$jobs")"
97- { api GET "/repos/$repo/actions/jobs/$job/logs" || true; } | sed -E 's/^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9:.]+Z //' | tail -80
113+ # Where it ran and how far it got, for a job that stopped without a log.
114+ jq -r --arg id "$job" '.jobs[] | select((.id | tostring) == $id) | "runner: \(.runner // .runner_name // "none")", "started \(.started_at // "-"), completed \(.completed_at // "-")", (.steps[]? | " step \(.number) \(.name): \(.status) \(.conclusion // "")")' <<<"$jobs"
115+ { api GET "/repos/$repo/actions/jobs/$job/logs" || echo "(no log: the request failed)"; } | sed -E 's/^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9:.]+Z //' | tail -80
98116 echo "::endgroup::"
99117 done
100118 exit 0
addedAGENTS.md+10−0View file
@@ -1,0 +1,10 @@
1+<!-- BEGIN:turborepo-agent-rules -->
2+
3+# This is NOT the Turborepo you know
4+
5+Turborepo configuration, task behavior, and CLI commands can vary between installed versions and may differ from your training data. Resolve the `turbo` package from this file's directory or relevant workspace; in monorepos, it may not be visible from the repository root. For example, run `node -p "require.resolve('turbo/package.json')"` from a workspace that depends on `turbo`.
6+
7+Read `docs/README.md` inside that installed package first, then read the relevant pages from its `docs/` directory before changing Turborepo configuration or commands. Heed deprecation notices. These bundled docs match the installed package version and are available without network access.
8+
9+This block is written and re-added by `turbo` before repository-scoped commands when an AI agent is detected. In the Turborepo source repository, its template is defined in `crates/turborepo-cli/src/cli/agent_guidance.rs`. Removing the managed block while updates are enabled means a later qualifying invocation will add it again. Set `"agentGuidance": false` in the root `turbo.json` or `turbo.jsonc` to opt out; this does not remove an existing block. Keep the block committed with your work to avoid an uncommitted change on the next agent invocation.
10+<!-- END:turborepo-agent-rules -->
modifiedapps/diffinery/docs/actions/overview.md+1−1View file
@@ -433,7 +433,7 @@
433433 | A hosted job stays queued, then fails with `The job waited 6 hours for one of this account's hosted jobs to finish, and was not started.` | The account already runs as many hosted jobs at once as its quota allows. Queued jobs start as running ones finish. See [Account quotas](/docs/reference/limits#account-quotas). |
434434 | `The job was not started: Actions are suspended for this repository by a site administrator: <reason>.` (or `for every repository of <account>`) | A site administrator suspended Actions here. Nothing runs until they lift it, and a red banner on the Actions page shows the reason. See [Kill switches](/docs/admin/site-administration#kill-switches). |
435435 | `The job was not started: Hosted runners are paused on this instance by a site administrator: <reason>. Self-hosted runners still work.` | Hosted compute is paused on the whole instance. Use a self-hosted runner, or re-run once it is resumed. See [Kill switches](/docs/admin/site-administration#kill-switches). |
436-| `Could not start a runner: ...` | The job's container failed to start. Re-run the run. |
436+| `Could not start a runner: ...` | The job's container failed to start with an error that trying again won't fix (passing platform errors are retried first; see [Limits](/docs/reference/limits)). Re-run the run. |
437437 | `The runner stopped unexpectedly (exit code <n>).` | The job's container exited before reporting a result. Re-run the run; if it repeats, check that the job doesn't exhaust memory or disk. |
438438 | `This runner is being replaced by a new version. The job keeps running for up to 13 minutes before it is cancelled, but a step the platform's stop signal ends fails.` | The platform is replacing this job's container. Deploys leave running jobs alone, so this is rare. The platform's stop signal reaches the running step as well, which then fails with "The platform stopped this step's processes while replacing the runner with a new version." Re-run the job. |
439439 | `Error evaluating 'if': ...` | The job's `if:` expression failed at runtime. See [Expressions](/docs/actions/expressions). |
modifiedapps/diffinery/docs/actions/runners.md+1−0View file
@@ -82,6 +82,7 @@
8282 | Base tools | `ca-certificates`, `curl`, `git`, `git-lfs`, `jq`, `zip`, `unzip`, `xz-utils`, `tar`, `gzip`, `zstd`, `pkg-config`, `openssh-client`, `gnupg` |
8383 | GitHub CLI | `gh`, from GitHub's apt repository |
8484 | Build tools | `build-essential` (gcc, g++, make and the C library headers) |
85+| Libraries for Ruby and native extensions | `libyaml-dev`, `libssl-dev`, `libffi-dev`, `zlib1g-dev`, `libreadline-dev`, `libgmp-dev`, so `ruby/setup-ruby`'s prebuilt Rubies run and gems with C extensions build |
8586 | Python | Ubuntu's `python3` (3.12), `python3-pip`, `python3-venv`, `python-is-python3` (so `python` runs `python3`) |
8687 | Node.js | Node.js 22.22.0 with npm, plus corepack shims activated for pnpm 10.28.0 and yarn 1.22.22; nvm 0.40.7 in `/usr/local/nvm` (`NVM_DIR`), loaded in login shells such as `shell: bash -l {0}`, as on GitHub |
8788 | Docker | Docker Engine (`docker-ce`), the Docker CLI, `containerd.io`, the Buildx plugin, the Compose plugin, `iptables` |
modifiedapps/diffinery/docs/reference/limits.md+1−1View file
@@ -210,7 +210,7 @@
210210 | Hosted runner size | Cloudflare Containers `standard-2` | Each `runs-on: ubuntu-latest` job runs in its own container of this instance type. |
211211 | Hosted job containers at once | 100 | `max_instances` of the runner container class. |
212212 | Hosted jobs at once per repository | 20 | Further jobs stay queued and start, oldest first, as the repository's own jobs finish, so one repository can't take every container. Self-hosted jobs don't count. |
213-| No free container | Wait, then retry, 8 times | When the platform has no container free for a job (many jobs starting at once, or just after a deploy), or the job's container stops before its runner fetched the job, the job goes back to queued and starts again after 30 seconds, then 1, 2, 4 and 5 minutes. After 8 tries, about half an hour, it fails with `No hosted runner could be started`. Only jobs whose runner never started are retried, so no step runs twice. |
213+| No free container | Wait, then retry, 8 times | When the platform has no container free for a job (many jobs starting at once, or just after a deploy), the start fails on the platform's side in a way that passes ("Container sidecar is shutting down", a dropped connection), or the job's container stops before its runner fetched the job, the job goes back to queued and starts again after 30 seconds, then 1, 2, 4 and 5 minutes. After 8 tries, about half an hour, it fails with `No hosted runner could be started`. Only jobs whose runner never started are retried, so no step runs twice. |
214214 | Unfinished runs per repository | 500 | Past it, a `workflow_dispatch` or `repository_dispatch` is refused with `429 this repository has 500 unfinished workflow runs, the most allowed; wait for some to finish`, and other events start no run. |
215215 | Self-hosted runners per repository | 100, each with at most 20 labels | Registering one more gets `429 this repository has 100 self-hosted runners, the most allowed; remove one first`. A registration token works for an hour. |
216216 | Purging unreachable objects | Repositories of up to 300,000 objects | A larger repository gets `422`; ask a site administrator. See [Removing a secret that was pushed](/docs/security/push-protection#removing-a-secret-that-was-pushed). |
modifiedapps/diffinery/src/repo/actions.ts+7−2View file
@@ -66,7 +66,12 @@
6666 * many times; after that it runs on whatever image it got rather than wait on a stuck rollout.
6767 */
6868 export const STALE_IMAGE_TRIES = 4;
69-const NO_INSTANCE = "no container instance";
69+/**
70+ * A container start that failed on the platform's side and works when tried again: no machine free,
71+ * a container sidecar or Durable Object restarting under it (seen during rollouts as "Container
72+ * sidecar is shutting down"), or a dropped connection to it.
73+ */
74+const RETRYABLE_START = /no container instance|is shutting down|network connection lost|durable object reset|overloaded/i;
7075 /** Runs one repository may have unfinished at once; past it, new ones aren't started. */
7176 export const MAX_ACTIVE_RUNS = 500;
7277 /** Self-hosted runners one repository may have registered. */
@@ -1061,7 +1066,7 @@
10611066 await stub.startJob({ repoId: m.repoId, jobId, token, apiUrl: this.env.PUBLIC_URL, egress: await egressFor(this.env, m.ownerId) });
10621067 } catch (e) {
10631068 const message = (e as Error).message;
1064- if (message.includes(NO_INSTANCE)) return this.placementFailed(jobId, message);
1069+ if (RETRYABLE_START.test(message)) return this.placementFailed(jobId, message);
10651070 await this.completeJob(jobId, { conclusion: "failure", outputs: {}, failure: { message: `Could not start a runner: ${message}` } });
10661071 }
10671072 }
modifiedapps/diffinery/test/unit/actions-runtime.test.ts+15−3View file
@@ -686,6 +686,7 @@
686686 sql.exec(REPO_SCHEMA);
687687 migrateRepoSchema(sql as unknown as SqlStorage);
688688 let full = true;
689+ let startError = "there is no container instance that can be provided to this durable object";
689690 const events: { type: string; payload: Record<string, unknown> }[] = [];
690691 const repo = {
691692 sql,
@@ -706,7 +707,7 @@
706707 idFromName: (n: string) => n,
707708 get: () => ({
708709 startJob: async () => {
709- if (full) throw new Error("there is no container instance that can be provided to this durable object");
710+ if (full) throw new Error(startError);
710711 },
711712 stopJob: async () => {},
712713 }),
@@ -748,9 +749,20 @@
748749 await actions.containerStopped("j2", null);
749750 expect(job("j2")).toMatchObject({ status: "completed", conclusion: "failure" });
750751
751- // Past the limit it fails, and says why.
752+ // A container sidecar restarting under the start counts the same way; an error that won't go away fails the job.
752753 full = true;
753- for (let i = 3; i < MAX_PLACEMENT_TRIES; i++) await actions.relaunchJob("j1");
754+ startError = "Container sidecar is shutting down";
755+ await actions.relaunchJob("j1");
756+ expect(job("j1")).toMatchObject({ status: "queued", conclusion: null, placement_tries: 4 });
757+ sql.exec("INSERT INTO jobs (id, run_id, job_key, name, status) VALUES ('j3', 'run1', 'k3', 'k3', 'queued')");
758+ startError = "image not found";
759+ await launch("j3");
760+ expect(job("j3")).toMatchObject({ status: "completed", conclusion: "failure", placement_tries: 0 });
761+ expect(sql.exec("SELECT failure FROM jobs WHERE id = 'j3'").one().failure).toMatch(/Could not start a runner: image not found/);
762+
763+ // Past the limit it fails, and says why.
764+ startError = "there is no container instance that can be provided to this durable object";
765+ for (let i = 4; i < MAX_PLACEMENT_TRIES; i++) await actions.relaunchJob("j1");
754766 expect(job("j1")).toMatchObject({ status: "queued", placement_tries: MAX_PLACEMENT_TRIES });
755767 await actions.relaunchJob("j1");
756768 expect(job("j1")).toMatchObject({ status: "completed", conclusion: "failure" });
modifiedcontainers/runner/Dockerfile+3−0View file
@@ -21,11 +21,14 @@
2121 # CI scripts commonly `pip install` directly, so allow it like hosted runners do.
2222 PIP_BREAK_SYSTEM_PACKAGES=1
2323
24+# The -dev libraries are what ruby/setup-ruby's prebuilt Rubies link against (psych needs
25+# libyaml) and what gems' native extensions build with, as on hosted runners.
2426 RUN apt-get update \
2527 && apt-get install -y --no-install-recommends \
2628 ca-certificates curl git git-lfs jq zip unzip xz-utils tar gzip zstd \
2729 build-essential pkg-config openssh-client gnupg \
2830 python3 python3-pip python3-venv python-is-python3 \
31+ libyaml-dev libssl-dev libffi-dev zlib1g-dev libreadline-dev libgmp-dev \
2932 && git config --system --add safe.directory '*' \
3033 && git lfs install --system --skip-repo \
3134 && apt-get clean \