| 1 | import { spawnSync } from "node:child_process"; |
| 2 | import { chmodSync, existsSync, readFileSync, rmSync, writeFileSync } from "node:fs"; |
| 3 | import { createServer } from "node:net"; |
| 4 | import path from "node:path"; |
| 5 | import { fileURLToPath } from "node:url"; |
| 6 | import { afterEach, beforeEach, describe, expect, it } from "vitest"; |
| 7 | import { apiBaseFor, runAgentJob } from "../src/agent.js"; |
| 8 | import { makeSpec, startFakeApi, tmp, type FakeApi } from "./helpers.js"; |
| 9 | |
| 10 | // The hosted runner image's docker wrapper (image/docker), run as it is apart from its fixed paths, |
| 11 | // against a stand-in daemon that takes a moment to come up and then keeps running, like dockerd. |
| 12 | const WRAPPER = path.join(path.dirname(fileURLToPath(import.meta.url)), "../image/docker"); |
| 13 | const tools = ["flock", "setsid", "openssl"].every((t) => spawnSync("sh", ["-c", `command -v ${t}`]).status === 0); |
| 14 | |
| 15 | let api: FakeApi; |
| 16 | let dir: string; |
| 17 | let relayPort: number; |
| 18 | |
| 19 | /** |
| 20 | * Writes the wrapper to `dir/bin/docker`, with its paths moved into `dir`, and the stand-ins next to it: |
| 21 | * a daemon that answers `daemonMs` after it starts, and a registry relay that listens `relayMs` after it |
| 22 | * starts (or never, when null). |
| 23 | */ |
| 24 | function install({ daemonMs = 1000, relayMs = 1000 as number | null } = {}): string { |
| 25 | const bin = path.join(dir, "bin"); |
| 26 | const sh = (name: string, body: string) => { |
| 27 | writeFileSync(path.join(bin, name), `#!/bin/bash\n${body}\n`); |
| 28 | chmodSync(path.join(bin, name), 0o755); |
| 29 | }; |
| 30 | spawnSync("mkdir", ["-p", bin, path.join(dir, "log")]); |
| 31 | let script = readFileSync(WRAPPER, "utf8"); |
| 32 | const moves: Array<[string, string]> = [ |
| 33 | ["real=/usr/bin/docker", `real=${bin}/docker-real`], |
| 34 | ["ready=/run/diffinery-dockerd.ready", `ready=${dir}/dockerd.ready`], |
| 35 | ["state=/run/diffinery\n", `state=${dir}/state\n`], |
| 36 | ["/run/diffinery-dockerd.lock", `${dir}/dockerd.lock`], |
| 37 | ["/var/log/", `${dir}/log/`], |
| 38 | ["sys_path=/usr/local/sbin:", `sys_path=${bin}:/usr/local/sbin:`], |
| 39 | ["relay_port=3129", `relay_port=${relayPort}`], |
| 40 | ["/etc/docker/certs.d/", `${dir}/certs.d/`], |
| 41 | ["/opt/diffinery/agent.mjs", `${dir}/relay.mjs`], |
| 42 | ['"$(id -u)" = 0', "0 = 0"], |
| 43 | ]; |
| 44 | for (const [from, to] of moves) { |
| 45 | expect(script, `the wrapper no longer has ${JSON.stringify(from)}`).toContain(from); |
| 46 | script = script.split(from).join(to); |
| 47 | } |
| 48 | writeFileSync(path.join(bin, "docker"), script); |
| 49 | chmodSync(path.join(bin, "docker"), 0o755); |
| 50 | // The daemon answers after a moment and keeps running (holding whatever it inherited). It notes |
| 51 | // the proxy it was started with. |
| 52 | sh("dockerd", `echo $$ >> ${dir}/dockerd.pids\necho "\${HTTPS_PROXY:-none}" >> ${dir}/dockerd.proxy\nsleep ${daemonMs / 1000}\ntouch ${dir}/dockerd.up\nexec sleep 60`); |
| 53 | sh("docker-real", `[ -e ${dir}/dockerd.up ] || exit 1\necho "docker $*"`); |
| 54 | sh("node", `exec ${process.execPath} "$@"`); |
| 55 | writeFileSync( |
| 56 | path.join(dir, "relay.mjs"), |
| 57 | relayMs === null |
| 58 | ? `process.exit(1);\n` |
| 59 | : `import { appendFileSync } from "node:fs";\nimport { createServer } from "node:net";\nappendFileSync(${JSON.stringify(`${dir}/dockerd.pids`)}, process.pid + "\\n");\nsetTimeout(() => createServer((s) => s.end()).listen(Number(process.env.DIFFINERY_RELAY_PORT), "127.0.0.1"), ${relayMs});\n`, |
| 60 | ); |
| 61 | return bin; |
| 62 | } |
| 63 | |
| 64 | /** Whether process `pid` is running: it exists and isn't a zombie. */ |
| 65 | function running(pid: string): boolean { |
| 66 | try { |
| 67 | return !/^\S+ \(.*\) Z /.test(readFileSync(`/proc/${pid}/stat`, "utf8")); |
| 68 | } catch { |
| 69 | return false; |
| 70 | } |
| 71 | } |
| 72 | |
| 73 | beforeEach(async () => { |
| 74 | api = await startFakeApi(); |
| 75 | dir = tmp("docker-wrapper-"); |
| 76 | relayPort = await new Promise<number>((resolve) => { |
| 77 | const srv = createServer().listen(0, "127.0.0.1", () => { |
| 78 | const port = (srv.address() as { port: number }).port; |
| 79 | srv.close(() => resolve(port)); |
| 80 | }); |
| 81 | }); |
| 82 | }); |
| 83 | afterEach(async () => { |
| 84 | await api.close(); |
| 85 | // Stand-in daemons started with setsid outlive the job, like the real one. |
| 86 | const pids = path.join(dir, "dockerd.pids"); |
| 87 | if (existsSync(pids)) spawnSync("kill", readFileSync(pids, "utf8").split("\n").filter(Boolean)); |
| 88 | rmSync(dir, { recursive: true, force: true }); |
| 89 | }); |
| 90 | |
| 91 | describe.skipIf(!tools)("docker wrapper", () => { |
| 92 | it("lets docker commands that arrive while the daemon is starting through, with the head start on", async () => { |
| 93 | const bin = install(); |
| 94 | // The head start runs `docker version` as the job begins; the step's commands arrive while it is |
| 95 | // still starting the daemon, and must not wait on its lock for good. |
| 96 | const spec = makeSpec(api, [{ run: "docker info & docker ps & wait\ndocker version" }], { timeoutMinutes: 1 }); |
| 97 | const started = Date.now(); |
| 98 | const done = await runAgentJob(spec, { |
| 99 | apiBase: apiBaseFor(spec.apiUrl, spec.jobId), |
| 100 | workRoot: path.join(dir, "work"), |
| 101 | toolCache: path.join(dir, "tool"), |
| 102 | logIntervalMs: 50, |
| 103 | echoLogs: false, |
| 104 | prewarmDocker: true, |
| 105 | baseEnv: { ...process.env, PATH: `${bin}:${process.env.PATH}` }, |
| 106 | }); |
| 107 | expect(done.conclusion, api.logText()).toBe("success"); |
| 108 | expect(Date.now() - started).toBeLessThan(15_000); |
| 109 | expect(api.logText()).toContain("docker info"); |
| 110 | expect(api.logText()).toContain("docker ps"); |
| 111 | expect(api.logText()).toContain("docker version"); |
| 112 | expect(readFileSync(path.join(dir, "log", "dockerd.log"), "utf8")).toBe(""); |
| 113 | }, 30_000); |
| 114 | |
| 115 | it("gives up with a message, not a hang, when the daemon never comes up", () => { |
| 116 | const bin = install(); |
| 117 | writeFileSync(path.join(bin, "dockerd"), `#!/bin/bash\necho $$ >> ${dir}/dockerd.pids\necho "failed to start daemon: no cgroups" >&2\nexec sleep 60\n`); |
| 118 | // Shorten the waits for the test: the start loop and the lock wait. |
| 119 | const script = readFileSync(path.join(bin, "docker"), "utf8").replace("seq 1 600", "seq 1 10").replace("flock -w 150 9", "flock -w 2 9"); |
| 120 | writeFileSync(path.join(bin, "docker"), script); |
| 121 | const env = { ...process.env, PATH: `${bin}:${process.env.PATH}` }; |
| 122 | const r = spawnSync("bash", ["-c", "docker version & sleep 0.2; docker info; echo \"second exited $?\"; wait"], { env, encoding: "utf8", timeout: 20_000 }); |
| 123 | expect(r.error).toBeUndefined(); |
| 124 | expect(r.stdout).toContain("second exited 1"); |
| 125 | expect(r.stderr).toContain("The Docker daemon did not start"); |
| 126 | expect(r.stderr).toContain("failed to start daemon: no cgroups"); |
| 127 | }, 30_000); |
| 128 | |
| 129 | it("starts the registry relay and the daemon together", () => { |
| 130 | // Two seconds each: one after the other they would take four. |
| 131 | const bin = install({ daemonMs: 2000, relayMs: 2000 }); |
| 132 | const env = { ...process.env, PATH: `${bin}:${process.env.PATH}`, DIFFINERY_API_URL: "https://diffinery.test/api" }; |
| 133 | const started = Date.now(); |
| 134 | const r = spawnSync("bash", ["-c", "docker version"], { env, encoding: "utf8", timeout: 20_000 }); |
| 135 | const took = Date.now() - started; |
| 136 | expect(r.status, r.stderr).toBe(0); |
| 137 | expect(took).toBeLessThan(3500); |
| 138 | expect(r.stderr).not.toContain("relay did not start"); |
| 139 | expect(readFileSync(path.join(dir, "dockerd.proxy"), "utf8")).toBe(`http://127.0.0.1:${relayPort}\n`); |
| 140 | expect(readFileSync(path.join(dir, "state", "relay.host"), "utf8")).toBe("diffinery.test\n"); |
| 141 | expect(existsSync(path.join(dir, "certs.d", "diffinery.test", "ca.crt"))).toBe(true); |
| 142 | }, 30_000); |
| 143 | |
| 144 | it("starts the daemon again without the relay when the relay never comes up", () => { |
| 145 | const bin = install({ relayMs: null }); |
| 146 | // Shorten the relay wait for the test. |
| 147 | const script = readFileSync(path.join(bin, "docker"), "utf8").replace("seq 1 400", "seq 1 10"); |
| 148 | writeFileSync(path.join(bin, "docker"), script); |
| 149 | const env = { ...process.env, PATH: `${bin}:${process.env.PATH}`, DIFFINERY_API_URL: "https://diffinery.test/api" }; |
| 150 | const r = spawnSync("bash", ["-c", "docker version"], { env, encoding: "utf8", timeout: 20_000 }); |
| 151 | expect(r.status, r.stderr).toBe(0); |
| 152 | expect(r.stdout).toContain("docker version"); |
| 153 | expect(r.stderr).toContain("The registry relay did not start"); |
| 154 | expect(readFileSync(path.join(dir, "dockerd.proxy"), "utf8")).toBe(`http://127.0.0.1:${relayPort}\nnone\n`); |
| 155 | const [first, second] = readFileSync(path.join(dir, "dockerd.pids"), "utf8").split("\n").filter(Boolean); |
| 156 | // Gone (or a zombie: nothing may reap it in a container), while the new one runs. |
| 157 | expect(running(first!)).toBe(false); |
| 158 | expect(running(second!)).toBe(true); |
| 159 | expect(existsSync(path.join(dir, "state", "relay.host"))).toBe(false); |
| 160 | }, 30_000); |
| 161 | }); |