Diffinery

codician/diffinerypublic

Open-source, agent-first GitHub and GitHub Actions alternative that runs on your own Cloudflare account: git hosting, pull requests, code review and drop-in GitHub Actions YAML on Workers, Durable Objects, R2 and Containers.

Support job containers, service containers and Docker actions (#7)

Self-hosted runners with Docker now run jobs.<id>.container (steps and
JavaScript actions execute inside it), jobs.<id>.services (on a per-job
network, health checks awaited, ports exposed via job.services) and Docker
container actions (docker:// steps and runs.using: docker, including
Dockerfile builds and pre/post entrypoints). Hosted jobs that ask for
containers fail with a message pointing at self-hosted runners.


Claude-Session: https://claude.ai/code/session_01AizKXSsengXZRegJxRATkL

Co-authored-by: Claude <noreply@anthropic.com>
Canberk Demirci authored · GitHub committed signed1 parent e104acdcommit f4c20e79412a6ca39d638e3f66a01c152f7a6ca9Browse files
10 files changed+764−28
Files (10)
modifiedREADME.md+8−1View file
@@ -144,8 +144,15 @@
144144 owner). `with`, `secrets` (a mapping or `inherit`), `on.workflow_call`
145145 outputs and nesting up to four levels deep all work. A matrix on the calling
146146 job isn't supported yet.
147+- Docker, on self-hosted runners with Docker installed: a job `container:`
148+ (steps and JavaScript actions run inside it), `services:` (reachable by
149+ name from the job container, or on `localhost` through their mapped ports,
150+ with health checks awaited), and Docker container actions (`docker://image`
151+ steps and actions with `runs.using: docker`, built from their Dockerfile).
152+ Hosted runners can't run Docker, so these jobs need
153+ `runs-on: [self-hosted, linux]`.
147154
148-**Not supported yet:** job `container:` and `services:`, Docker container actions, `actions/github-script`,
155+**Not supported yet:** `actions/github-script`,
149156 Windows and macOS hosted runners, `pull_request_target`, `merge_group`, and
150157 `deepen-not` / partial clone filters. Pull request runs use the head commit
151158 rather than a test merge commit. A job that uses an unsupported feature fails
modifiedapps/diffinery/src/repo/actions.ts+5−2View file
@@ -700,8 +700,9 @@
700700 const unsupported = labels.filter((l) => !SUPPORTED_LABEL.test(l));
701701 const selfHosted = labels.some((l) => l.toLowerCase() === "self-hosted");
702702 if (unsupported.length && !selfHosted) return fail(label(job.name ?? short), `No Diffinery runner matches runs-on: ${unsupported.join(", ")}. Linux (ubuntu-*) runners are hosted; anything else needs runs-on: [self-hosted, ...] and a registered runner.`);
703- if (job.container) return fail(label(job.name ?? short), "Job containers (jobs.<id>.container) are not supported yet; steps run directly on the Ubuntu runner image.");
704- if (job.services && Object.keys(job.services).length) return fail(label(job.name ?? short), "Service containers (jobs.<id>.services) are not supported yet on Diffinery runners.");
703+ // Hosted runners are Cloudflare Containers, which cannot run Docker themselves.
704+ const docker = job.container ? "Job containers (jobs.<id>.container)" : job.services && Object.keys(job.services).length ? "Service containers (jobs.<id>.services)" : null;
705+ if (docker && !selfHosted) return fail(label(job.name ?? short), `${docker} need Docker, which hosted Diffinery runners don't have. Use runs-on: [self-hosted, linux] with a runner on a machine with Docker.`);
705706
706707 let combos: Array<Record<string, unknown>>;
707708 try {
@@ -947,6 +948,8 @@
947948 secrets,
948949 outputs: def.outputs,
949950 steps: def.steps.map((s) => ({ ...s, "timeout-minutes": s["timeout-minutes"] === undefined ? undefined : Number(s["timeout-minutes"]) })),
951+ container: def.container,
952+ services: def.services,
950953 timeoutMinutes: job.timeout_minutes,
951954 continueOnError: def.continueOnError === true || def.continueOnError === "true",
952955 idToken: !!idToken,
modifiedapps/diffinery/test/e2e/e2e.test.mjs+78−2View file
@@ -87,6 +87,21 @@
8787 assert.fail(`timed out waiting for ${what}\n--- wrangler log tail ---\n${devLog.slice(-4000)}`);
8888 }
8989
90+function spawnAsync(cmd, args, env, timeoutMs) {
91+ return new Promise((done) => {
92+ const child = spawn(cmd, args, { env: { ...process.env, ...env } });
93+ let stdout = "";
94+ let stderr = "";
95+ child.stdout.on("data", (d) => (stdout += d));
96+ child.stderr.on("data", (d) => (stderr += d));
97+ const t = setTimeout(() => child.kill("SIGKILL"), timeoutMs);
98+ child.on("close", (status) => {
99+ clearTimeout(t);
100+ done({ status, stdout, stderr });
101+ });
102+ });
103+}
104+
90105 /** Claim one waiting self-hosted job and run it with the real agent. */
91106 async function runOneJob(owner, repo) {
92107 const claim = await until("a job to claim", async () => {
@@ -96,9 +111,10 @@
96111 const jobEnv = { DIFFINERY_API_URL: claim.api_url, DIFFINERY_JOB_ID: claim.job_id, DIFFINERY_JOB_TOKEN: claim.token };
97112 // E2E_RUNNER_IMAGE runs the job inside the real runner image instead of the local agent bundle.
98113 const image = process.env.E2E_RUNNER_IMAGE;
114+ // Spawned asynchronously: a blocked event loop leaves undici holding keep-alive sockets the server already closed.
99115 const r = image
100- ? spawnSync("docker", ["run", "--rm", "--network", "host", ...Object.entries(jobEnv).flatMap(([k, v]) => ["-e", `${k}=${v}`]), image], { encoding: "utf8", timeout: 300_000 })
101- : spawnSync(process.execPath, [AGENT], { encoding: "utf8", env: { ...process.env, ...jobEnv, DIFFINERY_WORK_ROOT: join(work, `job-${claim.job_id}`) }, timeout: 120_000 });
116+ ? await spawnAsync("docker", ["run", "--rm", "--network", "host", ...Object.entries(jobEnv).flatMap(([k, v]) => ["-e", `${k}=${v}`]), image], {}, 300_000)
117+ : await spawnAsync(process.execPath, [AGENT], { ...process.env, ...jobEnv, DIFFINERY_WORK_ROOT: join(work, `job-${claim.job_id}`) }, 180_000);
102118 const out = `${r.stdout}${r.stderr}`;
103119 if (r.status !== 0) {
104120 const log = await fetch(`${base}/api/v3/repos/${owner}/${repo}/actions/jobs/${claim.job_id}/logs`, { headers: { authorization: `Bearer ${token}` } }).then((x) => x.text(), (e) => String(e));
@@ -435,3 +451,63 @@
435451 const jobs = (await ok("GET", `/repos/${owner}/reuse/actions/runs/${run.id}/jobs`)).jobs;
436452 assert.deepEqual(jobs.filter((j) => j.conclusion === "skipped").map((j) => j.name).sort(), ["nope / say", "nope / shout"]);
437453 });
454+
455+const hasDocker = spawnSync("docker", ["info"], { stdio: "ignore", timeout: 15_000 }).status === 0;
456+
457+test("containers: a job container and service containers on a self-hosted runner", { skip: !hasDocker && "Docker is not available" }, async () => {
458+ const owner = (await ok("GET", "/user")).login;
459+ await ok("POST", "/user/repos", { name: "boxes" });
460+ const src = join(work, "boxes");
461+ mkdirSync(join(src, ".github/workflows"), { recursive: true });
462+ // Docker Hub rate-limits anonymous pulls; mirror.gcr.io serves the same official images.
463+ writeFileSync(
464+ join(src, ".github/workflows/ci.yml"),
465+ `on: push
466+jobs:
467+ hosted:
468+ runs-on: ubuntu-latest
469+ services:
470+ redis: mirror.gcr.io/library/redis:7-alpine
471+ steps:
472+ - run: echo never
473+ boxed:
474+ runs-on: [self-hosted, linux]
475+ container:
476+ image: mirror.gcr.io/library/debian:\${{ github.event_name == 'push' && 'bookworm-slim' || 'nope' }}
477+ services:
478+ cache:
479+ image: mirror.gcr.io/library/redis:7-alpine
480+ options: --health-cmd "redis-cli ping" --health-interval 1s --health-retries 30
481+ steps:
482+ - run: |
483+ grep -q bookworm /etc/os-release && echo "inside debian"
484+ exec 3<>/dev/tcp/cache/6379
485+ printf 'PING\\r\\n' >&3
486+ read -r line <&3
487+ echo "cache answered \${line%$'\\r'}"
488+`,
489+ );
490+ git(src, ["init", "-q", "-b", "main"]);
491+ git(src, ["add", "."]);
492+ git(src, ["commit", "-q", "-m", "containers"]);
493+ git(src, ["remote", "add", "origin", `${base}/${owner}/boxes.git`]);
494+ git(src, ["push", "-q", "origin", "main"]);
495+
496+ const run = await until("boxes run", async () => (await ok("GET", `/repos/${owner}/boxes/actions/runs`)).workflow_runs[0]);
497+ const boxed = await runOneJob(owner, "boxes");
498+ assert.equal(boxed.claim.name, "boxed");
499+ const log = await (await fetch(`${base}/api/v3/repos/${owner}/boxes/actions/jobs/${boxed.claim.job_id}/logs`, { headers: { authorization: `Bearer ${token}` } })).text();
500+ assert.match(log, /inside debian/);
501+ assert.match(log, /cache answered \+PONG/);
502+ const done = await until("boxes run to complete", async () => {
503+ const r = await ok("GET", `/repos/${owner}/boxes/actions/runs/${run.id}`);
504+ return r.status === "completed" ? r : null;
505+ });
506+ // Hosted runners can't run Docker, and say so.
507+ assert.equal(done.conclusion, "failure");
508+ const jobs = (await ok("GET", `/repos/${owner}/boxes/actions/runs/${run.id}/jobs`)).jobs;
509+ const hosted = jobs.find((j) => j.name === "hosted");
510+ assert.equal(hosted.conclusion, "failure");
511+ assert.match(JSON.stringify(hosted), /hosted Diffinery runners don't have/);
512+ assert.equal(jobs.find((j) => j.name === "boxed").conclusion, "success");
513+});
addedcontainers/runner/src/docker.ts+295−0View file
@@ -1,0 +1,295 @@
1+import { createHash } from "node:crypto";
2+import { existsSync } from "node:fs";
3+import { mkdir } from "node:fs/promises";
4+import path from "node:path";
5+import type { ContainerSpec } from "@diffinery/shared";
6+import { failure, type JobRunner, type StepIO } from "./job.js";
7+import { shellWords, sleep, which } from "./util.js";
8+
9+/**
10+ * Docker support for self-hosted runners: job containers (`jobs.<id>.container`),
11+ * service containers (`jobs.<id>.services`) and Docker container actions.
12+ *
13+ * Like GitHub's runner, every container of a job joins one bridge network, the job
14+ * container mounts the work directory at the same path it has on the host (so file
15+ * commands, scripts and the tool cache need no path mapping), and Docker actions run
16+ * as sibling containers with the workspace at /github/workspace.
17+ */
18+
19+/** Env vars that would change how the docker CLI itself behaves, so they are passed inline. */
20+const INLINE_ENV = /^(PATH|HOME|TMPDIR|DOCKER_\w*|LD_\w*|SSL_CERT_\w*)$/;
21+const SOCKET = "/var/run/docker.sock";
22+/** Inside a job container, the runner's own node binary (for JavaScript actions). */
23+export const CONTAINER_NODE = "/__e/node";
24+const CONTAINER_HOME = "/github/home";
25+
26+export interface RunningContainer {
27+ id: string;
28+ name: string;
29+ network: string;
30+ ports: Record<string, string>;
31+}
32+
33+export interface JobContainer extends RunningContainer {
34+ /** PATH from the image, used when the workflow doesn't set one. */
35+ path: string;
36+ /** Shells and interpreters found in the image. */
37+ tools: Set<string>;
38+}
39+
40+export type Wrap = (argv: string[], env: Record<string, string>, cwd: string) => { argv: string[]; env: NodeJS.ProcessEnv; cwd: string };
41+
42+const sanitize = (s: string) => s.replace(/[^A-Za-z0-9_.-]/g, "_").replace(/^[^A-Za-z0-9]+/, "") || "x";
43+const hash = (s: string) => createHash("sha256").update(s).digest("hex").slice(0, 12);
44+
45+export class Docker {
46+ readonly label: string;
47+ network: string | null = null;
48+ container: JobContainer | null = null;
49+ readonly services: Record<string, RunningContainer> = {};
50+ private bin: string | null = null;
51+ private configDir: string | null = null;
52+ private seq = 0;
53+
54+ constructor(private readonly job: JobRunner) {
55+ this.label = `diffinery.job=${sanitize(job.spec.jobId)}`;
56+ }
57+
58+ // ------------------------------------------------------------ CLI
59+
60+ /** Fails with a clear message when the runner has no usable Docker. */
61+ async ensure(io: StepIO, what: string): Promise<string> {
62+ if (this.bin) return this.bin;
63+ const bin = which("docker", this.job.hostEnv().PATH);
64+ if (!bin) throw failure(`${what} need Docker, and this runner has no docker CLI. Use runs-on: [self-hosted, ...] on a machine with Docker installed.`);
65+ const r = await this.job.tool(bin, ["version", "--format", "{{.Server.Version}}"], { io, capture: true, silent: true, env: this.job.hostEnv() });
66+ if (r.exitCode !== 0) throw failure(`${what} need Docker, and the Docker daemon on this runner is not reachable: ${(r.stderr || r.stdout).trim().split("\n")[0]}`);
67+ this.job.logger.line(io.logIndex, `Docker ${r.stdout.trim()}`);
68+ this.bin = bin;
69+ return bin;
70+ }
71+
72+ private async docker(io: StepIO, args: string[], o: { capture?: boolean; silent?: boolean; input?: string; env?: NodeJS.ProcessEnv; config?: boolean } = {}) {
73+ const bin = this.bin!;
74+ const full = o.config && this.configDir ? ["--config", this.configDir, ...args] : args;
75+ return this.job.tool(bin, full, { io, capture: o.capture, silent: o.silent, input: o.input, env: o.env ?? this.job.hostEnv() });
76+ }
77+
78+ private async must(io: StepIO, args: string[], o: Parameters<Docker["docker"]>[2] = {}): Promise<string> {
79+ const r = await this.docker(io, args, { capture: true, ...o });
80+ if (r.exitCode !== 0) throw failure(`docker ${args[0]} failed (exit code ${r.exitCode}): ${(r.stderr || r.stdout).trim().split("\n").slice(-3).join(" ")}`);
81+ return r.stdout.trim();
82+ }
83+
84+ /** `-e NAME` flags with the values carried in the docker CLI's own environment (keeps secrets out of argv). */
85+ envArgs(env: Record<string, string>): { args: string[]; env: NodeJS.ProcessEnv } {
86+ const args: string[] = [];
87+ const cli: NodeJS.ProcessEnv = this.job.hostEnv();
88+ for (const [k, v] of Object.entries(env)) {
89+ if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(k)) continue;
90+ if (INLINE_ENV.test(k)) args.push("-e", `${k}=${v}`);
91+ else {
92+ args.push("-e", k);
93+ cli[k] = v;
94+ }
95+ }
96+ return { args, env: cli };
97+ }
98+
99+ async ensureNetwork(io: StepIO): Promise<string> {
100+ if (this.network) return this.network;
101+ const name = `diffinery_${sanitize(this.job.spec.jobId)}_${hash(`${Date.now()}${Math.random()}`).slice(0, 6)}`;
102+ await this.must(io, ["network", "create", "--label", this.label, name]);
103+ this.network = name;
104+ return name;
105+ }
106+
107+ private async login(io: StepIO, image: string, creds: ContainerSpec["credentials"]): Promise<void> {
108+ if (!creds?.username && !creds?.password) return;
109+ if (!creds.username || !creds.password) throw failure(`Registry credentials for ${image} need both a username and a password.`);
110+ this.configDir ??= path.join(this.job.tempDir, "_docker_config");
111+ await mkdir(this.configDir, { recursive: true, mode: 0o700 });
112+ this.job.masker.add(creds.password);
113+ const first = image.split("/")[0]!;
114+ const registry = image.includes("/") && (first.includes(".") || first.includes(":") || first === "localhost") ? first : "";
115+ await this.must(io, ["login", "--username", creds.username, "--password-stdin", ...(registry ? [registry] : [])], { input: creds.password, config: true });
116+ }
117+
118+ async pull(io: StepIO, image: string, creds?: ContainerSpec["credentials"]): Promise<void> {
119+ await this.login(io, image, creds);
120+ this.job.logger.line(io.logIndex, `##[command]docker pull ${image}`);
121+ const r = await this.docker(io, ["pull", image], { config: !!creds?.username });
122+ if (r.exitCode !== 0) throw failure(`Could not pull ${image} (docker pull exited with ${r.exitCode}).`);
123+ }
124+
125+ // ------------------------------------------------------------ services and the job container
126+
127+ private specArgs(spec: ContainerSpec, where: string): string[] {
128+ const args: string[] = [];
129+ for (const p of spec.ports ?? []) args.push("-p", String(p));
130+ for (const v of spec.volumes ?? []) args.push("-v", v);
131+ const options = spec.options ? shellWords(spec.options) : [];
132+ for (const o of options) {
133+ if (/^--(network|net)(=|$)/.test(o)) throw failure(`${where}: the --network option is not supported; every container of the job shares one network.`);
134+ }
135+ return [...args, ...options];
136+ }
137+
138+ private async ports(io: StepIO, id: string): Promise<Record<string, string>> {
139+ const out = await this.must(io, ["port", id]).catch(() => "");
140+ const ports: Record<string, string> = {};
141+ for (const line of out.split("\n")) {
142+ const m = /^(\d+)\/\w+\s*->\s*.*:(\d+)$/.exec(line.trim());
143+ if (m && !ports[m[1]!]) ports[m[1]!] = m[2]!;
144+ }
145+ return ports;
146+ }
147+
148+ /** Waits for a container's health check, if it has one. */
149+ private async healthy(io: StepIO, id: string, name: string): Promise<void> {
150+ const deadline = Date.now() + 15 * 60_000;
151+ let delay = 1000;
152+ for (;;) {
153+ const status = await this.must(io, ["inspect", "--format", "{{if .State.Health}}{{.State.Health.Status}}{{end}}", id]);
154+ if (status === "" || status === "healthy") {
155+ if (status) this.job.logger.line(io.logIndex, `${name} is healthy`);
156+ return;
157+ }
158+ if (status === "unhealthy") {
159+ await this.docker(io, ["logs", "--tail", "50", id]);
160+ throw failure(`Service container ${name} failed its health check.`);
161+ }
162+ const running = await this.must(io, ["inspect", "--format", "{{.State.Status}}", id]);
163+ if (running === "exited" || running === "dead") {
164+ await this.docker(io, ["logs", "--tail", "50", id]);
165+ throw failure(`Service container ${name} exited before it became healthy.`);
166+ }
167+ if (Date.now() > deadline || this.job.isStopping) throw failure(`Service container ${name} did not become healthy in time (status: ${status}).`);
168+ this.job.logger.line(io.logIndex, `Waiting for ${name} to be healthy (${status})...`);
169+ await sleep(delay);
170+ delay = Math.min(delay * 2, 10_000);
171+ }
172+ }
173+
174+ async startService(io: StepIO, key: string, spec: ContainerSpec): Promise<void> {
175+ const network = await this.ensureNetwork(io);
176+ await this.pull(io, spec.image, spec.credentials);
177+ const name = `${sanitize(key)}_${hash(network)}`;
178+ const { args: envArgs, env } = this.envArgs(spec.env ?? {});
179+ const args = ["run", "-d", "--name", name, "--label", this.label, "--network", network, "--network-alias", key, ...envArgs, ...this.specArgs(spec, `services.${key}`), spec.image];
180+ this.job.logger.line(io.logIndex, `##[command]docker ${["run", "-d", "--name", name, "--network", network, "--network-alias", key, ...this.specArgs(spec, `services.${key}`), spec.image].join(" ")}`);
181+ const id = await this.must(io, args, { env });
182+ await this.healthy(io, id, key);
183+ const ports = await this.ports(io, id);
184+ this.services[key] = { id, name, network, ports };
185+ const shown = Object.entries(ports).map(([c, h]) => `${c} -> localhost:${h}`);
186+ this.job.logger.line(io.logIndex, `Service ${key} is running${shown.length ? ` (${shown.join(", ")})` : ""}`);
187+ }
188+
189+ async startJobContainer(io: StepIO, spec: ContainerSpec): Promise<void> {
190+ const network = await this.ensureNetwork(io);
191+ await this.pull(io, spec.image, spec.credentials);
192+ const job = this.job;
193+ const name = `job_${hash(`${network}${job.spec.jobId}`)}`;
194+ const workRoot = path.resolve(job.opts.workRoot);
195+ const home = path.join(job.tempDir, "_github_home");
196+ await mkdir(home, { recursive: true });
197+ const mounts = ["-v", `${workRoot}:${workRoot}`, "-v", `${home}:${CONTAINER_HOME}`];
198+ const toolCache = path.resolve(job.toolCache);
199+ if (!toolCache.startsWith(`${workRoot}/`)) mounts.push("-v", `${toolCache}:${toolCache}`);
200+ const tempDir = path.resolve(job.tempDir);
201+ if (!tempDir.startsWith(`${workRoot}/`)) mounts.push("-v", `${tempDir}:${tempDir}`);
202+ mounts.push("-v", `${process.execPath}:${CONTAINER_NODE}:ro`);
203+ if (existsSync(SOCKET)) mounts.push("-v", `${SOCKET}:${SOCKET}`);
204+ const { args: envArgs, env } = this.envArgs(spec.env ?? {});
205+ const extra = this.specArgs(spec, "container");
206+ const args = ["run", "-d", "--name", name, "--label", this.label, "--network", network, "--workdir", job.workspace, ...mounts, ...envArgs, ...extra, "--entrypoint", "tail", spec.image, "-f", "/dev/null"];
207+ job.logger.line(io.logIndex, `##[command]docker run -d --name ${name} --network ${network} --workdir ${job.workspace} ${[...mounts, ...extra].join(" ")} --entrypoint tail ${spec.image} -f /dev/null`);
208+ const id = await this.must(io, args, { env });
209+ const running = await this.must(io, ["inspect", "--format", "{{.State.Running}}", id]);
210+ if (running !== "true") {
211+ await this.docker(io, ["logs", "--tail", "50", id]);
212+ throw failure(`The job container (${spec.image}) exited right after starting. The image needs a 'tail' command to stay up.`);
213+ }
214+ const imageEnv = await this.must(io, ["inspect", "--format", "{{range .Config.Env}}{{println .}}{{end}}", id]);
215+ const pathLine = imageEnv.split("\n").find((l) => l.startsWith("PATH="));
216+ const probe = await this.docker(io, ["exec", id, "sh", "-c", 'for c in bash sh python python3 pwsh powershell; do command -v "$c" >/dev/null 2>&1 && echo "$c"; done'], { capture: true, silent: true });
217+ const tools = new Set(probe.stdout.split("\n").map((s) => s.trim()).filter(Boolean));
218+ this.container = {
219+ id,
220+ name,
221+ network,
222+ ports: await this.ports(io, id),
223+ path: pathLine ? pathLine.slice(5) : "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
224+ tools,
225+ };
226+ job.logger.line(io.logIndex, `Steps run in the job container ${spec.image} (${name})`);
227+ }
228+
229+ /** Runs a step's process inside the job container. */
230+ execWrap(): Wrap | undefined {
231+ const c = this.container;
232+ if (!c) return undefined;
233+ return (argv, env, cwd) => {
234+ const inner = { ...env, HOME: CONTAINER_HOME, PATH: [...this.job.pathAccum, env.PATH ?? c.path].filter(Boolean).join(":") };
235+ const e = this.envArgs(inner);
236+ return { argv: [this.bin!, "exec", "-w", cwd, ...e.args, c.id, ...argv], env: e.env, cwd };
237+ };
238+ }
239+
240+ // ------------------------------------------------------------ Docker actions
241+
242+ /** Pulls `docker://image` or builds an action's Dockerfile; returns the image to run. */
243+ async prepareAction(io: StepIO, image: string, actionDir: string | null): Promise<string> {
244+ await this.ensure(io, "Docker container actions");
245+ if (image.startsWith("docker://")) {
246+ const ref = image.slice("docker://".length);
247+ await this.pull(io, ref);
248+ return ref;
249+ }
250+ if (!actionDir) throw failure(`Invalid Docker image '${image}'`);
251+ const dockerfile = path.resolve(actionDir, image);
252+ if (!dockerfile.startsWith(`${path.resolve(actionDir)}/`) || !existsSync(dockerfile)) throw failure(`Dockerfile '${image}' not found in the action at ${actionDir}`);
253+ const tag = `diffinery-action-${hash(dockerfile)}:latest`;
254+ this.job.logger.line(io.logIndex, `##[command]docker build -t ${tag} -f ${dockerfile} ${actionDir}`);
255+ const r = await this.docker(io, ["build", "-t", tag, "-f", dockerfile, actionDir]);
256+ if (r.exitCode !== 0) throw failure(`Building the action's Dockerfile failed (exit code ${r.exitCode}).`);
257+ return tag;
258+ }
259+
260+ /** Runs an action image as a sibling container on the job network. */
261+ async runWrap(io: StepIO, image: string, o: { entrypoint?: string; args: string[] }): Promise<Wrap> {
262+ const network = await this.ensureNetwork(io);
263+ const job = this.job;
264+ const home = path.join(job.tempDir, "_github_home");
265+ await mkdir(home, { recursive: true });
266+ const tempDir = path.resolve(job.tempDir);
267+ const name = `action_${hash(`${network}${++this.seq}`)}_${this.seq}`;
268+ const mounts = ["-v", `${path.resolve(job.workspace)}:/github/workspace`, "-v", `${tempDir}:${tempDir}`, "-v", `${home}:${CONTAINER_HOME}`];
269+ if (existsSync(SOCKET)) mounts.push("-v", `${SOCKET}:${SOCKET}`);
270+ return (_argv, env) => {
271+ const inner: Record<string, string> = { ...env, GITHUB_WORKSPACE: "/github/workspace", HOME: CONTAINER_HOME };
272+ // The image keeps its own PATH unless the workflow set one.
273+ if (env.PATH === undefined) delete inner.PATH;
274+ const e = this.envArgs(inner);
275+ const argv = [this.bin!, "run", "--rm", "--name", name, "--label", this.label, "--network", network, "--workdir", "/github/workspace", ...mounts, ...e.args];
276+ if (o.entrypoint) argv.push("--entrypoint", o.entrypoint);
277+ argv.push(image, ...o.args);
278+ return { argv, env: e.env, cwd: job.workspace };
279+ };
280+ }
281+
282+ // ------------------------------------------------------------ cleanup
283+
284+ async cleanup(io: StepIO): Promise<void> {
285+ if (!this.bin) return;
286+ const ids = (await this.docker(io, ["ps", "-aq", "--filter", `label=${this.label}`], { capture: true, silent: true })).stdout.split("\n").filter(Boolean);
287+ if (ids.length) {
288+ this.job.logger.line(io.logIndex, `Stopping ${ids.length} container${ids.length === 1 ? "" : "s"}`);
289+ await this.docker(io, ["rm", "-f", "-v", ...ids], { capture: true, silent: true });
290+ }
291+ if (this.network) await this.docker(io, ["network", "rm", this.network], { capture: true, silent: true });
292+ this.network = null;
293+ this.container = null;
294+ }
295+}
modifiedcontainers/runner/src/job.ts+83−6View file
@@ -3,10 +3,11 @@
33 import { existsSync } from "node:fs";
44 import { mkdir, readFile, writeFile } from "node:fs/promises";
55 import path from "node:path";
6-import type { Conclusion, JobCompletion, JobSpec, StepReport, StepSpec } from "@diffinery/shared";
6+import type { Conclusion, ContainerSpec, JobCompletion, JobSpec, StepReport, StepSpec } from "@diffinery/shared";
77 import { evaluate, evaluateIf, interpolate, type StatusFns } from "@diffinery/workflow-yaml";
88 import type { RunnerApi } from "./api.js";
99 import { parseKeyValueFile, parsePathFile } from "./commands.js";
10+import { Docker, type Wrap } from "./docker.js";
1011 import { JobLogger, OutputProcessor, type Annotation, type OutputSink } from "./log.js";
1112 import { Masker } from "./mask.js";
1213 import { runAction } from "./steps/actions.js";
@@ -86,6 +87,8 @@
8687 readonly children = new Set<ChildProcess>();
8788 readonly stepErrors = new Map<number, Annotation[]>();
8889 readonly stepReasons = new Map<number, string>();
90+ /** Job and service containers, and Docker actions (self-hosted runners with Docker). */
91+ readonly docker: Docker;
8992
9093 private stopping: null | "cancelled" | "timeout" = null;
9194 private firstFailure: { step?: number; message: string } | null = null;
@@ -114,8 +117,14 @@
114117 for (const k of SENSITIVE_ENV) delete base[k];
115118 this.baseEnv = base;
116119 for (const v of Object.values(this.secrets)) this.masker.add(v);
120+ this.docker = new Docker(this);
117121 }
118122
123+ /** The runner's own environment (for internal tools such as the docker CLI). */
124+ hostEnv(): NodeJS.ProcessEnv {
125+ return { ...this.baseEnv };
126+ }
127+
119128 // ------------------------------------------------------------ public
120129
121130 get isStopping(): boolean {
@@ -142,6 +151,7 @@
142151 const top: Scope = { steps: {}, inputs: spec.inputs ?? {}, failed: false, inheritedEnv: {}, composite: false, depth: 0 };
143152 try {
144153 await this.setup();
154+ await this.initContainers(top);
145155 for (let i = 0; i < spec.steps.length; i++) {
146156 const step = spec.steps[i]!;
147157 const fallbackName = defaultStepName(step);
@@ -171,6 +181,11 @@
171181 this.firstFailure ??= { message: msg };
172182 } finally {
173183 if (jobTimer) clearTimeout(jobTimer);
184+ try {
185+ await this.docker.cleanup({ logIndex: -1, outputs: {}, state: {} });
186+ } catch (e) {
187+ this.logger.line(-1, `##[warning]Could not remove the job's containers: ${errMsg(e)}`);
188+ }
174189 }
175190
176191 let conclusion: Conclusion = top.failed ? "failure" : "success";
@@ -281,6 +296,51 @@
281296 }
282297 }
283298
299+ // ------------------------------------------------------------ containers
300+
301+ private evalContainer(spec: ContainerSpec, ctx: Record<string, unknown>): ContainerSpec {
302+ const i = (v: unknown) => this.interp(v === undefined || v === null ? "" : String(v), ctx);
303+ const out: ContainerSpec = { image: i(spec.image).trim() };
304+ if (spec.credentials) out.credentials = { username: i(spec.credentials.username), password: i(spec.credentials.password) };
305+ if (spec.env) out.env = Object.fromEntries(Object.entries(spec.env).map(([k, v]) => [k, i(v)]));
306+ if (spec.ports) out.ports = spec.ports.map((p) => i(p)).filter(Boolean);
307+ if (spec.volumes) out.volumes = spec.volumes.map((v) => i(v)).filter(Boolean);
308+ if (spec.options) out.options = i(spec.options);
309+ return out;
310+ }
311+
312+ /** Starts service containers and the job container, like GitHub's "Initialize containers". */
313+ private async initContainers(top: Scope): Promise<void> {
314+ const spec = this.spec;
315+ if (!spec.container && !Object.keys(spec.services ?? {}).length) return;
316+ const ctx = this.contexts(top, { ...this.jobEnv });
317+ const container = spec.container ? this.evalContainer(spec.container, ctx) : null;
318+ // A service whose image evaluates to an empty string is skipped (GitHub does the same).
319+ const services = Object.entries(spec.services ?? {})
320+ .map(([k, s]) => [k, this.evalContainer(s, ctx)] as const)
321+ .filter(([, s]) => s.image);
322+ if (!container?.image && !services.length) return;
323+ const io: StepIO = { logIndex: -1, outputs: {}, state: {} };
324+ this.logger.line(-1, "##[group]Initialize containers");
325+ try {
326+ await this.docker.ensure(io, container?.image ? "Job containers (jobs.<id>.container)" : "Service containers (jobs.<id>.services)");
327+ for (const [key, s] of services) await this.docker.startService(io, key, s);
328+ if (container?.image) await this.docker.startJobContainer(io, container);
329+ } finally {
330+ this.logger.line(-1, "##[endgroup]");
331+ }
332+ }
333+
334+ /** Wraps a step process so it runs in the job container, when there is one. */
335+ containerWrap(): Wrap | undefined {
336+ return this.docker.execWrap();
337+ }
338+
339+ /** The node binary JavaScript actions run with (the runner's own, mounted into a job container). */
340+ nodePath(): string {
341+ return this.docker.container ? "/__e/node" : process.execPath;
342+ }
343+
284344 // ------------------------------------------------------------ contexts
285345
286346 statusFns(scope: Scope): StatusFns {
@@ -328,7 +388,11 @@
328388 environment: "diffinery-hosted",
329389 },
330390 inputs: scope.inputs,
331- job: { status: this.stopping ? "cancelled" : scope.failed ? "failure" : "success", container: {}, services: {} },
391+ job: {
392+ status: this.stopping ? "cancelled" : scope.failed ? "failure" : "success",
393+ container: this.docker.container ? { id: this.docker.container.id, network: this.docker.container.network } : {},
394+ services: Object.fromEntries(Object.entries(this.docker.services).map(([k, v]) => [k, { id: v.id, network: v.network, ports: v.ports }])),
395+ },
332396 strategy: spec.strategy ?? {},
333397 };
334398 }
@@ -488,7 +552,16 @@
488552 */
489553 async runStepProcess(
490554 argv: string[],
491- o: { cwd: string; envCtx: Record<string, string>; io: StepIO; timeoutMs?: number; extraEnv?: Record<string, string>; timeoutLabel?: string },
555+ o: {
556+ cwd: string;
557+ envCtx: Record<string, string>;
558+ io: StepIO;
559+ timeoutMs?: number;
560+ extraEnv?: Record<string, string>;
561+ timeoutLabel?: string;
562+ /** Run the process somewhere else (the job container, or a Docker action's container). */
563+ wrap?: Wrap;
564+ },
492565 ): Promise<ExecResult> {
493566 const id = randomUUID();
494567 const files = {
@@ -504,9 +577,13 @@
504577 if (!existsSync(o.cwd)) {
505578 throw new StepFailure(`An error occurred trying to start process '${argv[0]}' with working directory '${o.cwd}'. No such file or directory`);
506579 }
507- const res = await exec(argv[0]!, argv.slice(1), {
508- cwd: o.cwd,
509- env: this.processEnv(o.envCtx, { ...(o.extraEnv ?? {}), ...files }),
580+ let cmd = argv;
581+ let cwd = o.cwd;
582+ let env = this.processEnv(o.envCtx, { ...(o.extraEnv ?? {}), ...files });
583+ if (o.wrap) ({ argv: cmd, cwd, env } = o.wrap(argv, { ...o.envCtx, ...this.defaultEnv, ...(o.extraEnv ?? {}), ...files }, o.cwd));
584+ const res = await exec(cmd[0]!, cmd.slice(1), {
585+ cwd,
586+ env,
510587 timeoutMs: o.timeoutMs,
511588 onLine: (l) => proc.handle(l),
512589 onSpawn: (c) => {
modifiedcontainers/runner/src/steps/actions.ts+104−4View file
@@ -4,7 +4,7 @@
44 import type { StepSpec } from "@diffinery/shared";
55 import { getBuiltin, makeBuiltinContext } from "../builtins/index.js";
66 import { errMsg, failure, StepFailure, type JobRunner, type Scope, type StepIO } from "../job.js";
7-import { exec } from "../util.js";
7+import { exec, shellWords } from "../util.js";
88 import { parseYaml } from "../yaml.js";
99 import type { StepExec } from "./run.js";
1010
@@ -46,6 +46,10 @@
4646 steps?: StepSpec[];
4747 image?: string;
4848 env?: Record<string, string>;
49+ entrypoint?: string;
50+ args?: unknown[];
51+ "pre-entrypoint"?: string;
52+ "post-entrypoint"?: string;
4953 };
5054 }
5155
@@ -137,7 +141,21 @@
137141 export async function runAction(job: JobRunner, x: StepExec): Promise<void> {
138142 const uses = job.interp(String(x.step.uses), x.ctx, x.scope);
139143 const ref = parseUses(uses);
140- if (ref.kind === "docker") throw failure("Docker actions are not supported on Diffinery runners yet");
144+ if (ref.kind === "docker") {
145+ // `uses: docker://image`: `with.args` and `with.entrypoint` drive the container; the rest become INPUT_*.
146+ const given = computeInputs(job, x, null);
147+ const inputs: Record<string, string> = {};
148+ let args: string[] = [];
149+ let entrypoint: string | undefined;
150+ for (const [k, v] of Object.entries(given)) {
151+ if (k.toLowerCase() === "args") args = shellWords(v);
152+ else if (k.toLowerCase() === "entrypoint") entrypoint = v || undefined;
153+ else inputs[k] = v;
154+ }
155+ logActionHeader(job, x.io, uses, given);
156+ await runDockerAction(job, x, { image: `docker://${ref.image}`, dir: null, args, entrypoint, env: {}, inputs, action: { path: "", repository: "", ref: "" } });
157+ return;
158+ }
141159
142160 if (ref.kind === "remote") {
143161 const builtin = getBuiltin(ref.name, ref.ref);
@@ -169,7 +187,29 @@
169187 logActionHeader(job, x.io, uses, inputs);
170188 const actionInfo = { path: dir, repository, ref: actionRef };
171189
172- if (using === "docker" || meta.runs.image) throw failure("Docker actions are not supported on Diffinery runners yet");
190+ if (using === "docker" || meta.runs.image) {
191+ const runs = meta.runs;
192+ if (!runs.image) throw failure(`Action '${uses}' uses Docker but has no 'runs.image'`);
193+ const actx = job.contexts({ ...x.scope, inputs }, x.envCtx);
194+ const ev = (v: unknown) => job.interp(v === undefined || v === null ? "" : String(v), actx, x.scope);
195+ const args = Array.isArray(runs.args) ? runs.args.map(ev) : [];
196+ const env: Record<string, string> = {};
197+ for (const [k, v] of Object.entries(runs.env ?? {})) env[k] = ev(v);
198+ await runDockerAction(job, x, {
199+ image: runs.image,
200+ dir,
201+ args,
202+ entrypoint: runs.entrypoint ? ev(runs.entrypoint) : undefined,
203+ pre: runs["pre-entrypoint"] ? ev(runs["pre-entrypoint"]) : undefined,
204+ post: runs["post-entrypoint"] ? ev(runs["post-entrypoint"]) : undefined,
205+ preIf: runs["pre-if"],
206+ postIf: runs["post-if"],
207+ env,
208+ inputs,
209+ action: actionInfo,
210+ });
211+ return;
212+ }
173213 if (using === "composite") {
174214 await runComposite(job, x, meta, inputs, actionInfo);
175215 return;
@@ -211,12 +251,13 @@
211251 };
212252 const state: Record<string, string> = {};
213253 const runScriptFile = async (file: string, io: StepIO, envCtx: Record<string, string>, extra: Record<string, string>, label: string) => {
214- const res = await job.runStepProcess([process.execPath, path.join(action.path, file)], {
254+ const res = await job.runStepProcess([job.nodePath(), path.join(action.path, file)], {
215255 cwd: job.workspace,
216256 envCtx,
217257 io,
218258 timeoutMs: x.timeoutMs,
219259 extraEnv: extra,
260+ wrap: job.containerWrap(),
220261 });
221262 if (job.isStopping) return;
222263 if (res.timedOut) throw failure(io.reason ?? "The step timed out.", true);
@@ -267,6 +308,65 @@
267308 }
268309 }
269310
311+/** Docker container actions: `uses: docker://image`, or an action with `runs.using: docker`. */
312+async function runDockerAction(
313+ job: JobRunner,
314+ x: StepExec,
315+ d: {
316+ image: string;
317+ dir: string | null;
318+ args: string[];
319+ entrypoint?: string;
320+ pre?: string;
321+ post?: string;
322+ preIf?: string;
323+ postIf?: string;
324+ env: Record<string, string>;
325+ inputs: Record<string, string>;
326+ action: { path: string; repository: string; ref: string };
327+ },
328+): Promise<void> {
329+ const image = await job.docker.prepareAction(x.io, d.image, d.dir);
330+ const extra: Record<string, string> = {
331+ GITHUB_ACTION: x.step.id ?? "__" + (d.action.repository || "docker").replace(/[^A-Za-z0-9]/g, "_"),
332+ GITHUB_ACTION_REPOSITORY: d.action.repository,
333+ GITHUB_ACTION_REF: d.action.ref,
334+ };
335+ for (const [k, v] of Object.entries(d.inputs)) extra[inputEnvName(k)] = v;
336+ const runIn = async (io: StepIO, envCtx: Record<string, string>, o: { entrypoint?: string; args: string[] }, label: string) => {
337+ const wrap = await job.docker.runWrap(io, image, o);
338+ const res = await job.runStepProcess([], { cwd: job.workspace, envCtx: { ...envCtx, ...d.env }, io, timeoutMs: x.timeoutMs, extraEnv: extra, wrap });
339+ if (job.isStopping) return;
340+ if (res.timedOut) throw failure(io.reason ?? "The step timed out.", true);
341+ if (res.exitCode !== 0) {
342+ const msg = `${label}Docker run failed with exit code ${res.exitCode}`;
343+ job.logger.line(io.logIndex, `##[error]${msg}`);
344+ job.addAnnotation(io.logIndex, { level: "error", message: msg });
345+ io.reason = msg;
346+ throw failure(msg, true);
347+ }
348+ };
349+ if (d.pre) {
350+ let ok = true;
351+ try {
352+ ok = job.evalIf(d.preIf ?? "always()", x.ctx, x.scope);
353+ } catch (e) {
354+ throw failure(`Error evaluating pre-if: ${errMsg(e)}`);
355+ }
356+ if (ok) await runIn({ logIndex: x.io.logIndex, outputs: {}, state: {} }, x.envCtx, { entrypoint: d.pre, args: [] }, "pre: ");
357+ }
358+ if (d.post) {
359+ const post = d.post;
360+ const envAtMain = { ...x.envCtx };
361+ job.registerPost({
362+ name: `Post ${x.name}`,
363+ condition: d.postIf ?? "always()",
364+ run: async (logIndex) => runIn({ logIndex, outputs: {}, state: {} }, { ...envAtMain, ...job.envAccum }, { entrypoint: post, args: [] }, "post: "),
365+ });
366+ }
367+ await runIn(x.io, x.envCtx, { entrypoint: d.entrypoint, args: d.args }, "");
368+}
369+
270370 function stateEnvOf(state: Record<string, string>): Record<string, string> {
271371 const out: Record<string, string> = {};
272372 for (const [k, v] of Object.entries(state)) out[`STATE_${k}`] = v;
modifiedcontainers/runner/src/steps/run.ts+17−10View file
@@ -20,15 +20,18 @@
2020 ext: string;
2121 }
2222
23-/** Resolves a `shell:` value to an argv (with `{0}` replaced by the script path). */
24-export function resolveShell(shell: string | undefined, scriptPath: string, pathEnv: string | undefined): ShellPlan {
23+/**
24+ * Resolves a `shell:` value to an argv (with `{0}` replaced by the script path).
25+ * `has` says whether a command exists (on the runner, or in the job container).
26+ */
27+export function resolveShell(shell: string | undefined, scriptPath: string, pathEnv: string | undefined, has: (cmd: string) => boolean = (c) => !!which(c, pathEnv)): ShellPlan {
2528 const s = (shell ?? "").trim();
2629 const sub = (tpl: string, ext: string): ShellPlan => {
2730 const words = shellWords(tpl).map((w) => w.split("{0}").join(scriptPath));
2831 return { argv: words, ext };
2932 };
3033 if (s === "" ) {
31- if (which("bash", pathEnv)) return sub("bash --noprofile --norc -eo pipefail {0}", ".sh");
34+ if (has("bash")) return sub("bash --noprofile --norc -eo pipefail {0}", ".sh");
3235 return sub("sh -e {0}", ".sh");
3336 }
3437 switch (s) {
@@ -37,13 +40,13 @@
3740 case "sh":
3841 return sub("sh -e {0}", ".sh");
3942 case "python": {
40- const py = which("python", pathEnv) ? "python" : "python3";
43+ const py = has("python") ? "python" : "python3";
4144 return sub(`${py} {0}`, ".py");
4245 }
4346 case "pwsh":
4447 return sub(`pwsh -command ". '{0}'"`, ".ps1");
4548 case "powershell":
46- return sub(`${which("powershell", pathEnv) ? "powershell" : "pwsh"} -command ". '{0}'"`, ".ps1");
49+ return sub(`${has("powershell") ? "powershell" : "pwsh"} -command ". '{0}'"`, ".ps1");
4750 case "cmd":
4851 throw failure("The `cmd` shell is not supported on Linux runners.");
4952 }
@@ -84,16 +87,20 @@
8487
8588 const tmpBase = path.join(job.tempDir, randomUUID());
8689 const pathEnv = job.processEnv(envCtx).PATH;
87- const probe = resolveShell(shell, tmpBase, pathEnv);
90+ const container = job.docker.container;
91+ // In a job container, only the probed shells are known; anything else is left to the container to resolve.
92+ const has = container ? (c: string) => container.tools.has(c) : (c: string) => !!which(c, pathEnv);
93+ const probe = resolveShell(shell, tmpBase, pathEnv, has);
8894 const scriptPath = tmpBase + probe.ext;
89- const plan = resolveShell(shell, scriptPath, pathEnv);
95+ const plan = resolveShell(shell, scriptPath, pathEnv, has);
9096 let content = script;
9197 if (probe.ext === ".ps1") {
9298 content = `$ErrorActionPreference = 'stop'\n${script}\nif ((Test-Path -LiteralPath variable:\\LASTEXITCODE)) { exit $LASTEXITCODE }\n`;
9399 }
94100 await writeFile(scriptPath, content, { mode: 0o755 });
95- if (!which(plan.argv[0]!, pathEnv)) {
96- throw failure(`${plan.argv[0]}: command not found. The shell '${shell ?? "bash"}' is not available on this runner.`);
101+ const known = container ? !container.tools.size || container.tools.has(plan.argv[0]!) || !["bash", "sh", "python", "python3", "pwsh", "powershell"].includes(plan.argv[0]!) : !!which(plan.argv[0]!, pathEnv);
102+ if (!known) {
103+ throw failure(`${plan.argv[0]}: command not found. The shell '${shell ?? "bash"}' is not available ${container ? "in the job container" : "on this runner"}.`);
97104 }
98105 const extraEnv: Record<string, string> = { GITHUB_ACTION: step.id ?? "__run" };
99106 if (scope.action) {
@@ -101,7 +108,7 @@
101108 extraEnv.GITHUB_ACTION_REPOSITORY = scope.action.repository;
102109 extraEnv.GITHUB_ACTION_REF = scope.action.ref;
103110 }
104- const res = await job.runStepProcess(plan.argv, { cwd: wd, envCtx, io, timeoutMs: x.timeoutMs, extraEnv });
111+ const res = await job.runStepProcess(plan.argv, { cwd: wd, envCtx, io, timeoutMs: x.timeoutMs, extraEnv, wrap: job.containerWrap() });
105112 if (job.isStopping) return;
106113 if (res.timedOut) throw failure(io.reason ?? "The step timed out.", true);
107114 if (res.exitCode !== 0) {
addedcontainers/runner/test/docker.test.ts+159−0View file
@@ -1,0 +1,159 @@
1+import { execFileSync } from "node:child_process";
2+import { mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
3+import path from "node:path";
4+import { afterEach, beforeEach, describe, expect, it } from "vitest";
5+import type { JobSpec } from "@diffinery/shared";
6+import { apiBaseFor, runAgentJob } from "../src/agent.js";
7+import { makeSpec, startFakeApi, tmp, type FakeApi } from "./helpers.js";
8+
9+/** Job containers, service containers and Docker actions; skipped where Docker isn't available. */
10+function dockerWorks(): boolean {
11+ try {
12+ execFileSync("docker", ["info", "--format", "{{.ServerVersion}}"], { stdio: "ignore", timeout: 15_000 });
13+ return true;
14+ } catch {
15+ return false;
16+ }
17+}
18+
19+// Docker Hub rate-limits anonymous pulls; its official images are mirrored here.
20+const ALPINE = "mirror.gcr.io/library/alpine:3.20";
21+const DEBIAN = "mirror.gcr.io/library/debian:bookworm-slim";
22+const REDIS = "mirror.gcr.io/library/redis:7-alpine";
23+
24+let api: FakeApi;
25+let workRoot: string;
26+
27+beforeEach(async () => {
28+ api = await startFakeApi();
29+ workRoot = tmp();
30+});
31+afterEach(async () => {
32+ await api.close();
33+ // Containers may leave root-owned files behind; clean up through Docker.
34+ if (dockerWorks()) execFileSync("docker", ["run", "--rm", "-v", `${workRoot}:/w`, ALPINE, "sh", "-c", "rm -rf /w/* /w/.[!.]*"], { stdio: "ignore" });
35+ rmSync(workRoot, { recursive: true, force: true });
36+});
37+
38+function run(spec: JobSpec, baseEnv?: NodeJS.ProcessEnv) {
39+ return runAgentJob(spec, { apiBase: apiBaseFor(spec.apiUrl, spec.jobId), workRoot, toolCache: path.join(workRoot, "_tool"), logIntervalMs: 50, echoLogs: false, baseEnv });
40+}
41+
42+const leftovers = (jobId: string) =>
43+ execFileSync("docker", ["ps", "-aq", "--filter", `label=diffinery.job=${jobId}`], { encoding: "utf8" }).trim() +
44+ execFileSync("docker", ["network", "ls", "-q", "--filter", `label=diffinery.job=${jobId}`], { encoding: "utf8" }).trim();
45+
46+const redisService = { image: REDIS, ports: [6379], options: "--health-cmd 'redis-cli ping' --health-interval 1s --health-timeout 2s --health-retries 20" };
47+
48+describe.skipIf(!dockerWorks())("docker", () => {
49+ it("runs steps and JavaScript actions in a job container next to a service container", async () => {
50+ const action = `name: probe
51+outputs:
52+ version: { description: node }
53+runs:
54+ using: node20
55+ main: index.js
56+`;
57+ const js = `const fs = require("fs");
58+const os = fs.readFileSync("/etc/os-release", "utf8").includes("bookworm");
59+fs.appendFileSync(process.env.GITHUB_OUTPUT, "version=" + process.version + "\\n");
60+console.log("node action in debian=" + os + " input=" + process.env.INPUT_WHO);
61+`;
62+ const spec = makeSpec(
63+ api,
64+ [
65+ {
66+ run: [
67+ 'grep -q bookworm /etc/os-release && echo "in debian, home=$HOME, ws=$GITHUB_WORKSPACE, greeting=$GREETING"',
68+ "exec 3<>/dev/tcp/redis/6379",
69+ "printf 'PING\\r\\n' >&3",
70+ "read -r line <&3",
71+ 'echo "redis says ${line%$\'\\r\'}"',
72+ 'echo "FROM_STEP=yes" >> "$GITHUB_ENV"',
73+ "mkdir -p .github/probe",
74+ `cat > .github/probe/action.yml <<'EOF'\n${action}EOF`,
75+ `cat > .github/probe/index.js <<'EOF'\n${js}EOF`,
76+ ].join("\n"),
77+ },
78+ { id: "probe", uses: "./.github/probe", with: { who: "octo" } },
79+ { run: 'echo "env=$FROM_STEP node=${{ steps.probe.outputs.version }} container=${{ job.container.id != \'\' }} port=${{ job.services.redis.ports[\'6379\'] != \'\' }}"' },
80+ ],
81+ { jobId: "dkr-container", container: { image: DEBIAN, env: { GREETING: "hi" } }, services: { redis: redisService } },
82+ );
83+ const c = await run(spec);
84+ const log = api.logText();
85+ expect(c.conclusion, log).toBe("success");
86+ expect(log).toContain(`in debian, home=/github/home, ws=${path.join(workRoot, "widgets")}, greeting=hi`);
87+ expect(log).toContain("redis says +PONG");
88+ expect(log).toContain("node action in debian=true input=octo");
89+ expect(log).toMatch(/env=yes node=v\d+\.\d+\.\d+ container=true port=true/);
90+ expect(leftovers("dkr-container")).toBe("");
91+ }, 180_000);
92+
93+ it("runs docker:// steps and Dockerfile actions with inputs, outputs, args and post entrypoints", async () => {
94+ // Written up front: a `run:` script would expand the action's own ${{ inputs.who }}.
95+ const dir = path.join(workRoot, "widgets", ".github", "hello");
96+ mkdirSync(dir, { recursive: true });
97+ writeFileSync(
98+ path.join(dir, "action.yml"),
99+ `name: hello
100+inputs:
101+ who: { description: who, default: world }
102+runs:
103+ using: docker
104+ image: Dockerfile
105+ args: ["\${{ inputs.who }}", "two words"]
106+ post-entrypoint: /post.sh
107+`,
108+ );
109+ writeFileSync(path.join(dir, "Dockerfile"), `FROM ${ALPINE}\nCOPY entrypoint.sh post.sh /\nRUN chmod +x /entrypoint.sh /post.sh\nENTRYPOINT ["/entrypoint.sh"]\n`);
110+ writeFileSync(
111+ path.join(dir, "entrypoint.sh"),
112+ '#!/bin/sh\necho "hello $1 [$2] input=$INPUT_WHO ws=$GITHUB_WORKSPACE pwd=$(pwd)"\necho "said=hello-$1" >> "$GITHUB_OUTPUT"\necho "FROM_DOCKER=1" >> "$GITHUB_ENV"\nprintf "PING\\r\\n" | nc -w 2 redis 6379\n',
113+ );
114+ writeFileSync(path.join(dir, "post.sh"), '#!/bin/sh\necho "post ran"\n');
115+ const spec = makeSpec(
116+ api,
117+ [
118+ { id: "hello", uses: "./.github/hello", with: { who: "octo" } },
119+ { id: "raw", uses: `docker://${ALPINE}`, with: { args: "sh -c 'echo raw $INPUT_COLOR > /github/workspace/raw.txt; echo color=$INPUT_COLOR >> $GITHUB_OUTPUT'", color: "blue" } },
120+ { uses: `docker://${ALPINE}`, with: { entrypoint: "/bin/false" }, "continue-on-error": true },
121+ { run: 'echo "said=${{ steps.hello.outputs.said }} color=${{ steps.raw.outputs.color }} env=$FROM_DOCKER file=$(cat raw.txt)"' },
122+ ],
123+ { jobId: "dkr-actions", services: { redis: redisService } },
124+ );
125+ const c = await run(spec);
126+ const log = api.logText();
127+ expect(c.conclusion, log).toBe("success");
128+ expect(log).toContain("hello octo [two words] input=octo ws=/github/workspace pwd=/github/workspace");
129+ expect(log).toContain("+PONG");
130+ expect(log).toContain("Docker run failed with exit code 1");
131+ expect(log).toContain("said=hello-octo color=blue env=1 file=raw blue");
132+ expect(log).toContain("post ran");
133+ expect(readFileSync(path.join(workRoot, "widgets", "raw.txt"), "utf8")).toBe("raw blue\n");
134+ expect(leftovers("dkr-actions")).toBe("");
135+ }, 180_000);
136+
137+ it("fails a service that never gets healthy, and cleans up", async () => {
138+ const spec = makeSpec(api, [{ run: "echo never" }], {
139+ jobId: "dkr-unhealthy",
140+ services: { bad: { image: REDIS, options: "--health-cmd false --health-interval 1s --health-retries 2" } },
141+ });
142+ const c = await run(spec);
143+ expect(c.conclusion).toBe("failure");
144+ expect(c.failure?.message).toContain("Service container bad failed its health check");
145+ expect(api.logText()).not.toContain("never");
146+ expect(leftovers("dkr-unhealthy")).toBe("");
147+ }, 120_000);
148+});
149+
150+describe("docker missing", () => {
151+ it("explains that containers need Docker", async () => {
152+ const spec = makeSpec(api, [{ run: "echo hi" }], { jobId: "no-docker", services: { redis: { image: REDIS } } });
153+ const c = await run(spec, { PATH: path.join(workRoot, "empty-bin") });
154+ expect(c.conclusion).toBe("failure");
155+ expect(c.failure?.message).toContain("Service containers (jobs.<id>.services) need Docker");
156+ const c2 = await run(makeSpec(api, [{ uses: `docker://${ALPINE}` }], { jobId: "no-docker-2" }), { PATH: path.join(workRoot, "empty-bin") });
157+ expect(c2.failure?.message).toContain("Docker container actions need Docker, and this runner has no docker CLI");
158+ });
159+});
modifiedcontainers/runner/test/job.test.ts+1−3View file
@@ -419,9 +419,7 @@
419419 }
420420 });
421421
422- it("rejects docker actions and github-script with clear errors", async () => {
423- const c = await run(makeSpec(api, [{ uses: "docker://alpine:3" }]));
424- expect(c.failure?.message).toContain("Docker actions are not supported on Diffinery runners yet");
422+ it("rejects github-script with a clear error", async () => {
425423 const c2 = await run(makeSpec(api, [{ uses: "actions/github-script@v7", with: { script: "1" } }]));
426424 expect(c2.failure?.message).toContain("actions/github-script is not supported");
427425 });
modifiedpackages/shared/src/runner.ts+14−0View file
@@ -31,6 +31,16 @@
3131 "timeout-minutes"?: number;
3232 };
3333
34+/** `jobs.<id>.container` or one of `jobs.<id>.services`; values may contain `${{ }}`. */
35+export type ContainerSpec = {
36+ image: string;
37+ credentials?: { username?: string; password?: string };
38+ env?: Record<string, string>;
39+ ports?: Array<string | number>;
40+ volumes?: string[];
41+ options?: string;
42+};
43+
3444 export interface NeedsResult {
3545 result: Conclusion;
3646 outputs: Record<string, string>;
@@ -62,6 +72,10 @@
6272 /** Job `outputs:` map; values are expressions evaluated after the steps. */
6373 outputs: Record<string, string>;
6474 steps: StepSpec[];
75+ /** Job container: steps run inside it (needs Docker on the runner). */
76+ container?: ContainerSpec;
77+ /** Service containers, reachable by name from a job container or via mapped ports. */
78+ services?: Record<string, ContainerSpec>;
6579 timeoutMinutes: number;
6680 continueOnError: boolean;
6781 /** Whether `permissions: id-token: write` was granted. */