| 1 | import { spawn } from "node:child_process"; |
| 2 | import { createHash } from "node:crypto"; |
| 3 | import { existsSync } from "node:fs"; |
| 4 | import { mkdir } from "node:fs/promises"; |
| 5 | import path from "node:path"; |
| 6 | import type { ContainerSpec } from "@diffinery/shared"; |
| 7 | import { failure, type JobRunner, type StepIO } from "./job.js"; |
| 8 | import { shellWords, sleep, which } from "./util.js"; |
| 9 | |
| 10 | /** |
| 11 | * Docker support for self-hosted runners: job containers (`jobs.<id>.container`), |
| 12 | * service containers (`jobs.<id>.services`) and Docker container actions. |
| 13 | * |
| 14 | * Like GitHub's runner, every container of a job joins one bridge network, the job |
| 15 | * container mounts the work directory at the same path it has on the host (so file |
| 16 | * commands, scripts and the tool cache need no path mapping), and Docker actions run |
| 17 | * as sibling containers with the workspace at /github/workspace. |
| 18 | */ |
| 19 | |
| 20 | /** Env vars that would change how the docker CLI itself behaves, so they are passed inline. */ |
| 21 | const INLINE_ENV = /^(PATH|HOME|TMPDIR|DOCKER_\w*|LD_\w*|SSL_CERT_\w*)$/; |
| 22 | const SOCKET = "/var/run/docker.sock"; |
| 23 | /** Inside a job container, the runner's own node binary (for JavaScript actions). */ |
| 24 | export const CONTAINER_NODE = "/__e/node"; |
| 25 | const CONTAINER_HOME = "/github/home"; |
| 26 | |
| 27 | export interface RunningContainer { |
| 28 | id: string; |
| 29 | name: string; |
| 30 | network: string; |
| 31 | ports: Record<string, string>; |
| 32 | } |
| 33 | |
| 34 | export interface JobContainer extends RunningContainer { |
| 35 | /** PATH from the image, used when the workflow doesn't set one. */ |
| 36 | path: string; |
| 37 | /** Shells and interpreters found in the image. */ |
| 38 | tools: Set<string>; |
| 39 | } |
| 40 | |
| 41 | export type Wrap = (argv: string[], env: Record<string, string>, cwd: string) => { argv: string[]; env: NodeJS.ProcessEnv; cwd: string }; |
| 42 | |
| 43 | const sanitize = (s: string) => s.replace(/[^A-Za-z0-9_.-]/g, "_").replace(/^[^A-Za-z0-9]+/, "") || "x"; |
| 44 | const hash = (s: string) => createHash("sha256").update(s).digest("hex").slice(0, 12); |
| 45 | |
| 46 | export class Docker { |
| 47 | readonly label: string; |
| 48 | network: string | null = null; |
| 49 | container: JobContainer | null = null; |
| 50 | readonly services: Record<string, RunningContainer> = {}; |
| 51 | private bin: string | null = null; |
| 52 | private configDir: string | null = null; |
| 53 | private seq = 0; |
| 54 | |
| 55 | constructor(private readonly job: JobRunner) { |
| 56 | this.label = `diffinery.job=${sanitize(job.spec.jobId)}`; |
| 57 | } |
| 58 | |
| 59 | // ------------------------------------------------------------ CLI |
| 60 | |
| 61 | /** |
| 62 | * Hosted runners start the Docker daemon the first time anything runs `docker` (image/docker), which |
| 63 | * takes 2-5 s. A job that will use Docker starts it as the job begins instead, while checkout and |
| 64 | * setup run; the first real `docker` call then waits only for what is left (the wrapper's lock). |
| 65 | * If this start fails, that first call starts the daemon again and reports why it can't. |
| 66 | */ |
| 67 | prewarm(): void { |
| 68 | const env = this.job.hostEnv(); |
| 69 | const bin = which("docker", env.PATH); |
| 70 | if (!bin) return; |
| 71 | try { |
| 72 | const child = spawn(bin, ["version"], { env, stdio: "ignore", detached: true }); |
| 73 | child.on("error", () => {}); |
| 74 | child.unref(); |
| 75 | } catch { |
| 76 | /* the first real docker call starts it */ |
| 77 | } |
| 78 | } |
| 79 | |
| 80 | /** Fails with a clear message when the runner has no usable Docker. */ |
| 81 | async ensure(io: StepIO, what: string): Promise<string> { |
| 82 | if (this.bin) return this.bin; |
| 83 | const bin = which("docker", this.job.hostEnv().PATH); |
| 84 | if (!bin) throw failure(`${what} need Docker, and this runner has no docker CLI. Use runs-on: [self-hosted, ...] on a machine with Docker installed.`); |
| 85 | const r = await this.job.tool(bin, ["version", "--format", "{{.Server.Version}}"], { io, capture: true, silent: true, env: this.job.hostEnv() }); |
| 86 | if (r.exitCode !== 0) throw failure(`${what} need Docker, and the Docker daemon on this runner is not reachable: ${(r.stderr || r.stdout).trim().split("\n")[0]}`); |
| 87 | this.job.logger.line(io.logIndex, `Docker ${r.stdout.trim()}`); |
| 88 | this.bin = bin; |
| 89 | return bin; |
| 90 | } |
| 91 | |
| 92 | private async docker(io: StepIO, args: string[], o: { capture?: boolean; silent?: boolean; input?: string; env?: NodeJS.ProcessEnv; config?: boolean } = {}) { |
| 93 | const bin = this.bin!; |
| 94 | const full = o.config && this.configDir ? ["--config", this.configDir, ...args] : args; |
| 95 | return this.job.tool(bin, full, { io, capture: o.capture, silent: o.silent, input: o.input, env: o.env ?? this.job.hostEnv() }); |
| 96 | } |
| 97 | |
| 98 | private async must(io: StepIO, args: string[], o: Parameters<Docker["docker"]>[2] = {}): Promise<string> { |
| 99 | const r = await this.docker(io, args, { capture: true, ...o }); |
| 100 | if (r.exitCode !== 0) throw failure(`docker ${args[0]} failed (exit code ${r.exitCode}): ${(r.stderr || r.stdout).trim().split("\n").slice(-3).join(" ")}`); |
| 101 | return r.stdout.trim(); |
| 102 | } |
| 103 | |
| 104 | /** `-e NAME` flags with the values carried in the docker CLI's own environment (keeps secrets out of argv). */ |
| 105 | envArgs(env: Record<string, string>): { args: string[]; env: NodeJS.ProcessEnv } { |
| 106 | const args: string[] = []; |
| 107 | const cli: NodeJS.ProcessEnv = this.job.hostEnv(); |
| 108 | for (const [k, v] of Object.entries(env)) { |
| 109 | if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(k)) continue; |
| 110 | if (INLINE_ENV.test(k)) args.push("-e", `${k}=${v}`); |
| 111 | else { |
| 112 | args.push("-e", k); |
| 113 | cli[k] = v; |
| 114 | } |
| 115 | } |
| 116 | return { args, env: cli }; |
| 117 | } |
| 118 | |
| 119 | async ensureNetwork(io: StepIO): Promise<string> { |
| 120 | if (this.network) return this.network; |
| 121 | const name = `diffinery_${sanitize(this.job.spec.jobId)}_${hash(`${Date.now()}${Math.random()}`).slice(0, 6)}`; |
| 122 | await this.must(io, ["network", "create", "--label", this.label, name]); |
| 123 | this.network = name; |
| 124 | return name; |
| 125 | } |
| 126 | |
| 127 | private async login(io: StepIO, image: string, creds: ContainerSpec["credentials"]): Promise<void> { |
| 128 | if (!creds?.username && !creds?.password) return; |
| 129 | if (!creds.username || !creds.password) throw failure(`Registry credentials for ${image} need both a username and a password.`); |
| 130 | this.configDir ??= path.join(this.job.tempDir, "_docker_config"); |
| 131 | await mkdir(this.configDir, { recursive: true, mode: 0o700 }); |
| 132 | this.job.masker.add(creds.password); |
| 133 | const first = image.split("/")[0]!; |
| 134 | const registry = image.includes("/") && (first.includes(".") || first.includes(":") || first === "localhost") ? first : ""; |
| 135 | await this.must(io, ["login", "--username", creds.username, "--password-stdin", ...(registry ? [registry] : [])], { input: creds.password, config: true }); |
| 136 | } |
| 137 | |
| 138 | async pull(io: StepIO, image: string, creds?: ContainerSpec["credentials"]): Promise<void> { |
| 139 | await this.login(io, image, creds); |
| 140 | this.job.logger.line(io.logIndex, `##[command]docker pull ${image}`); |
| 141 | const r = await this.docker(io, ["pull", image], { config: !!creds?.username }); |
| 142 | if (r.exitCode !== 0) throw failure(`Could not pull ${image} (docker pull exited with ${r.exitCode}).`); |
| 143 | } |
| 144 | |
| 145 | // ------------------------------------------------------------ services and the job container |
| 146 | |
| 147 | private specArgs(spec: ContainerSpec, where: string): string[] { |
| 148 | const args: string[] = []; |
| 149 | for (const p of spec.ports ?? []) args.push("-p", String(p)); |
| 150 | for (const v of spec.volumes ?? []) args.push("-v", v); |
| 151 | const options = spec.options ? shellWords(spec.options) : []; |
| 152 | for (const o of options) { |
| 153 | if (/^--(network|net)(=|$)/.test(o)) throw failure(`${where}: the --network option is not supported; every container of the job shares one network.`); |
| 154 | } |
| 155 | return [...args, ...options]; |
| 156 | } |
| 157 | |
| 158 | private async ports(io: StepIO, id: string): Promise<Record<string, string>> { |
| 159 | const out = await this.must(io, ["port", id]).catch(() => ""); |
| 160 | const ports: Record<string, string> = {}; |
| 161 | for (const line of out.split("\n")) { |
| 162 | const m = /^(\d+)\/\w+\s*->\s*.*:(\d+)$/.exec(line.trim()); |
| 163 | if (m && !ports[m[1]!]) ports[m[1]!] = m[2]!; |
| 164 | } |
| 165 | return ports; |
| 166 | } |
| 167 | |
| 168 | /** Waits for a container's health check, if it has one. */ |
| 169 | private async healthy(io: StepIO, id: string, name: string): Promise<void> { |
| 170 | const deadline = Date.now() + 15 * 60_000; |
| 171 | let delay = 1000; |
| 172 | for (;;) { |
| 173 | const status = await this.must(io, ["inspect", "--format", "{{if .State.Health}}{{.State.Health.Status}}{{end}}", id]); |
| 174 | if (status === "" || status === "healthy") { |
| 175 | if (status) this.job.logger.line(io.logIndex, `${name} is healthy`); |
| 176 | return; |
| 177 | } |
| 178 | if (status === "unhealthy") { |
| 179 | await this.docker(io, ["logs", "--tail", "50", id]); |
| 180 | throw failure(`Service container ${name} failed its health check.`); |
| 181 | } |
| 182 | const running = await this.must(io, ["inspect", "--format", "{{.State.Status}}", id]); |
| 183 | if (running === "exited" || running === "dead") { |
| 184 | await this.docker(io, ["logs", "--tail", "50", id]); |
| 185 | throw failure(`Service container ${name} exited before it became healthy.`); |
| 186 | } |
| 187 | if (Date.now() > deadline || this.job.isStopping) throw failure(`Service container ${name} did not become healthy in time (status: ${status}).`); |
| 188 | this.job.logger.line(io.logIndex, `Waiting for ${name} to be healthy (${status})...`); |
| 189 | await sleep(delay); |
| 190 | delay = Math.min(delay * 2, 10_000); |
| 191 | } |
| 192 | } |
| 193 | |
| 194 | async startService(io: StepIO, key: string, spec: ContainerSpec): Promise<void> { |
| 195 | const network = await this.ensureNetwork(io); |
| 196 | await this.pull(io, spec.image, spec.credentials); |
| 197 | const name = `${sanitize(key)}_${hash(network)}`; |
| 198 | const { args: envArgs, env } = this.envArgs(spec.env ?? {}); |
| 199 | const args = ["run", "-d", "--name", name, "--label", this.label, "--network", network, "--network-alias", key, ...envArgs, ...this.specArgs(spec, `services.${key}`), spec.image]; |
| 200 | this.job.logger.line(io.logIndex, `##[command]docker ${["run", "-d", "--name", name, "--network", network, "--network-alias", key, ...this.specArgs(spec, `services.${key}`), spec.image].join(" ")}`); |
| 201 | const id = await this.must(io, args, { env }); |
| 202 | await this.healthy(io, id, key); |
| 203 | const ports = await this.ports(io, id); |
| 204 | this.services[key] = { id, name, network, ports }; |
| 205 | const shown = Object.entries(ports).map(([c, h]) => `${c} -> localhost:${h}`); |
| 206 | this.job.logger.line(io.logIndex, `Service ${key} is running${shown.length ? ` (${shown.join(", ")})` : ""}`); |
| 207 | } |
| 208 | |
| 209 | async startJobContainer(io: StepIO, spec: ContainerSpec): Promise<void> { |
| 210 | const network = await this.ensureNetwork(io); |
| 211 | await this.pull(io, spec.image, spec.credentials); |
| 212 | const job = this.job; |
| 213 | const name = `job_${hash(`${network}${job.spec.jobId}`)}`; |
| 214 | const workRoot = path.resolve(job.opts.workRoot); |
| 215 | const home = path.join(job.tempDir, "_github_home"); |
| 216 | await mkdir(home, { recursive: true }); |
| 217 | const mounts = ["-v", `${workRoot}:${workRoot}`, "-v", `${home}:${CONTAINER_HOME}`]; |
| 218 | const toolCache = path.resolve(job.toolCache); |
| 219 | if (!toolCache.startsWith(`${workRoot}/`)) mounts.push("-v", `${toolCache}:${toolCache}`); |
| 220 | const tempDir = path.resolve(job.tempDir); |
| 221 | if (!tempDir.startsWith(`${workRoot}/`)) mounts.push("-v", `${tempDir}:${tempDir}`); |
| 222 | mounts.push("-v", `${process.execPath}:${CONTAINER_NODE}:ro`); |
| 223 | if (existsSync(SOCKET)) mounts.push("-v", `${SOCKET}:${SOCKET}`); |
| 224 | const { args: envArgs, env } = this.envArgs(spec.env ?? {}); |
| 225 | const extra = this.specArgs(spec, "container"); |
| 226 | const args = ["run", "-d", "--name", name, "--label", this.label, "--network", network, "--workdir", job.workspace, ...mounts, ...envArgs, ...extra, "--entrypoint", "tail", spec.image, "-f", "/dev/null"]; |
| 227 | job.logger.line(io.logIndex, `##[command]docker run -d --name ${name} --network ${network} --workdir ${job.workspace} ${[...mounts, ...extra].join(" ")} --entrypoint tail ${spec.image} -f /dev/null`); |
| 228 | const id = await this.must(io, args, { env }); |
| 229 | const running = await this.must(io, ["inspect", "--format", "{{.State.Running}}", id]); |
| 230 | if (running !== "true") { |
| 231 | await this.docker(io, ["logs", "--tail", "50", id]); |
| 232 | throw failure(`The job container (${spec.image}) exited right after starting. The image needs a 'tail' command to stay up.`); |
| 233 | } |
| 234 | const imageEnv = await this.must(io, ["inspect", "--format", "{{range .Config.Env}}{{println .}}{{end}}", id]); |
| 235 | const pathLine = imageEnv.split("\n").find((l) => l.startsWith("PATH=")); |
| 236 | const probe = await this.docker(io, ["exec", id, "sh", "-c", 'for c in bash sh python python3 pwsh powershell; do command -v "$c" >/dev/null 2>&1 && echo "$c"; done'], { capture: true, silent: true }); |
| 237 | const tools = new Set(probe.stdout.split("\n").map((s) => s.trim()).filter(Boolean)); |
| 238 | this.container = { |
| 239 | id, |
| 240 | name, |
| 241 | network, |
| 242 | ports: await this.ports(io, id), |
| 243 | path: pathLine ? pathLine.slice(5) : "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", |
| 244 | tools, |
| 245 | }; |
| 246 | job.logger.line(io.logIndex, `Steps run in the job container ${spec.image} (${name})`); |
| 247 | } |
| 248 | |
| 249 | /** Runs a step's process inside the job container. */ |
| 250 | execWrap(): Wrap | undefined { |
| 251 | const c = this.container; |
| 252 | if (!c) return undefined; |
| 253 | return (argv, env, cwd) => { |
| 254 | const inner = { ...env, HOME: CONTAINER_HOME, PATH: [...this.job.pathAccum, env.PATH ?? c.path].filter(Boolean).join(":") }; |
| 255 | const e = this.envArgs(inner); |
| 256 | return { argv: [this.bin!, "exec", "-w", cwd, ...e.args, c.id, ...argv], env: e.env, cwd }; |
| 257 | }; |
| 258 | } |
| 259 | |
| 260 | // ------------------------------------------------------------ Docker actions |
| 261 | |
| 262 | /** Pulls `docker://image` or builds an action's Dockerfile; returns the image to run. */ |
| 263 | async prepareAction(io: StepIO, image: string, actionDir: string | null): Promise<string> { |
| 264 | await this.ensure(io, "Docker container actions"); |
| 265 | if (image.startsWith("docker://")) { |
| 266 | const ref = image.slice("docker://".length); |
| 267 | await this.pull(io, ref); |
| 268 | return ref; |
| 269 | } |
| 270 | if (!actionDir) throw failure(`Invalid Docker image '${image}'`); |
| 271 | const dockerfile = path.resolve(actionDir, image); |
| 272 | if (!dockerfile.startsWith(`${path.resolve(actionDir)}/`) || !existsSync(dockerfile)) throw failure(`Dockerfile '${image}' not found in the action at ${actionDir}`); |
| 273 | const tag = `diffinery-action-${hash(dockerfile)}:latest`; |
| 274 | this.job.logger.line(io.logIndex, `##[command]docker build -t ${tag} -f ${dockerfile} ${actionDir}`); |
| 275 | const r = await this.docker(io, ["build", "-t", tag, "-f", dockerfile, actionDir]); |
| 276 | if (r.exitCode !== 0) throw failure(`Building the action's Dockerfile failed (exit code ${r.exitCode}).`); |
| 277 | return tag; |
| 278 | } |
| 279 | |
| 280 | /** Runs an action image as a sibling container on the job network. */ |
| 281 | async runWrap(io: StepIO, image: string, o: { entrypoint?: string; args: string[] }): Promise<Wrap> { |
| 282 | const network = await this.ensureNetwork(io); |
| 283 | const job = this.job; |
| 284 | const home = path.join(job.tempDir, "_github_home"); |
| 285 | await mkdir(home, { recursive: true }); |
| 286 | const tempDir = path.resolve(job.tempDir); |
| 287 | const name = `action_${hash(`${network}${++this.seq}`)}_${this.seq}`; |
| 288 | const mounts = ["-v", `${path.resolve(job.workspace)}:/github/workspace`, "-v", `${tempDir}:${tempDir}`, "-v", `${home}:${CONTAINER_HOME}`]; |
| 289 | if (existsSync(SOCKET)) mounts.push("-v", `${SOCKET}:${SOCKET}`); |
| 290 | return (_argv, env) => { |
| 291 | const inner: Record<string, string> = { ...env, GITHUB_WORKSPACE: "/github/workspace", HOME: CONTAINER_HOME }; |
| 292 | // The image keeps its own PATH unless the workflow set one. |
| 293 | if (env.PATH === undefined) delete inner.PATH; |
| 294 | const e = this.envArgs(inner); |
| 295 | const argv = [this.bin!, "run", "--rm", "--name", name, "--label", this.label, "--network", network, "--workdir", "/github/workspace", ...mounts, ...e.args]; |
| 296 | if (o.entrypoint) argv.push("--entrypoint", o.entrypoint); |
| 297 | argv.push(image, ...o.args); |
| 298 | return { argv, env: e.env, cwd: job.workspace }; |
| 299 | }; |
| 300 | } |
| 301 | |
| 302 | // ------------------------------------------------------------ cleanup |
| 303 | |
| 304 | async cleanup(io: StepIO): Promise<void> { |
| 305 | if (!this.bin) return; |
| 306 | const ids = (await this.docker(io, ["ps", "-aq", "--filter", `label=${this.label}`], { capture: true, silent: true })).stdout.split("\n").filter(Boolean); |
| 307 | if (ids.length) { |
| 308 | this.job.logger.line(io.logIndex, `Stopping ${ids.length} container${ids.length === 1 ? "" : "s"}`); |
| 309 | await this.docker(io, ["rm", "-f", "-v", ...ids], { capture: true, silent: true }); |
| 310 | } |
| 311 | if (this.network) await this.docker(io, ["network", "rm", this.network], { capture: true, silent: true }); |
| 312 | this.network = null; |
| 313 | this.container = null; |
| 314 | } |
| 315 | } |