| 1 | import { spawnSync } from "node:child_process"; |
| 2 | import { createHash } from "node:crypto"; |
| 3 | import { chmodSync, existsSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; |
| 4 | import { createServer, type Server } from "node:http"; |
| 5 | import path from "node:path"; |
| 6 | import type { StepSpec } from "@diffinery/shared"; |
| 7 | import { afterEach, beforeEach, describe, expect, it } from "vitest"; |
| 8 | import { apiBaseFor, runAgentJob } from "../src/agent.js"; |
| 9 | import type { JobRunner } from "../src/job.js"; |
| 10 | import { ToolPrefetch } from "../src/prefetch.js"; |
| 11 | import { makeSpec, startFakeApi, tmp, type FakeApi } from "./helpers.js"; |
| 12 | |
| 13 | // The toolchain head start (src/prefetch.ts), against a stand-in for go.dev and the Adoptium API. |
| 14 | |
| 15 | let api: FakeApi; |
| 16 | let dir: string; |
| 17 | let server: Server; |
| 18 | let base: string; |
| 19 | /** Paths the stand-in served, with when each was asked for. */ |
| 20 | let served: Array<{ path: string; at: number }>; |
| 21 | let files: Map<string, { body: Buffer; delayMs?: number }>; |
| 22 | const saved = { ...process.env }; |
| 23 | |
| 24 | /** A .tar.gz of `entries` (relative path -> shell script) under one top-level folder. */ |
| 25 | function tarball(top: string, entries: Record<string, string>): Buffer { |
| 26 | const src = path.join(dir, `src-${top}`); |
| 27 | for (const [rel, body] of Object.entries(entries)) { |
| 28 | const f = path.join(src, top, rel); |
| 29 | mkdirSync(path.dirname(f), { recursive: true }); |
| 30 | writeFileSync(f, body); |
| 31 | chmodSync(f, 0o755); |
| 32 | } |
| 33 | const out = path.join(dir, `${top}.tar.gz`); |
| 34 | expect(spawnSync("tar", ["-czf", out, "-C", src, top]).status).toBe(0); |
| 35 | return readFileSync(out); |
| 36 | } |
| 37 | |
| 38 | const sha = (b: Buffer) => createHash("sha256").update(b).digest("hex"); |
| 39 | |
| 40 | function goRelease(version: string, body: Buffer, sum = sha(body)) { |
| 41 | files.set(`/dl/go${version}.linux-amd64.tar.gz`, { body, delayMs: 1500 }); |
| 42 | return { version: `go${version}`, stable: true, files: [{ filename: `go${version}.linux-amd64.tar.gz`, os: "linux", arch: "amd64", kind: "archive", sha256: sum }] }; |
| 43 | } |
| 44 | |
| 45 | beforeEach(async () => { |
| 46 | api = await startFakeApi(); |
| 47 | dir = tmp("prefetch-"); |
| 48 | served = []; |
| 49 | files = new Map(); |
| 50 | server = createServer((req, res) => { |
| 51 | const p = (req.url ?? "").split("?")[0]!; |
| 52 | served.push({ path: p, at: Date.now() }); |
| 53 | const f = files.get(p); |
| 54 | if (!f) { |
| 55 | res.writeHead(404).end(); |
| 56 | return; |
| 57 | } |
| 58 | setTimeout(() => res.writeHead(200).end(f.body), f.delayMs ?? 0); |
| 59 | }); |
| 60 | await new Promise<void>((r) => server.listen(0, "127.0.0.1", r)); |
| 61 | base = `http://127.0.0.1:${(server.address() as { port: number }).port}`; |
| 62 | process.env.DIFFINERY_GO_INDEX = `${base}/dl/index.json`; |
| 63 | process.env.DIFFINERY_GO_DL = `${base}/dl/`; |
| 64 | process.env.DIFFINERY_ADOPTIUM_API = `${base}/adoptium`; |
| 65 | }); |
| 66 | afterEach(async () => { |
| 67 | process.env = { ...saved }; |
| 68 | await api.close(); |
| 69 | await new Promise((r) => server.close(r)); |
| 70 | rmSync(dir, { recursive: true, force: true }); |
| 71 | }); |
| 72 | |
| 73 | describe.skipIf(process.arch !== "x64")("toolchain head start", () => { |
| 74 | it("installs Go while the job's first steps run, and setup-go uses it", async () => { |
| 75 | const go = tarball("go", { "bin/go": `#!/bin/sh\ncase "$1" in version) echo "go version go1.99.1 linux/amd64" ;; env) echo "${dir}/gopath" ;; esac\n` }); |
| 76 | files.set("/dl/index.json", { body: Buffer.from(JSON.stringify([goRelease("1.99.1", go), goRelease("1.98.4", go)])) }); |
| 77 | const steps: StepSpec[] = [ |
| 78 | { name: "checkout stand-in", run: "sleep 1.5" }, |
| 79 | { uses: "actions/setup-go@v5", with: { "go-version": "${{ matrix.go }}", cache: "false" } }, |
| 80 | { run: "go version" }, |
| 81 | ]; |
| 82 | const spec = makeSpec(api, steps, { matrix: { go: "1.99" } }); |
| 83 | const started = Date.now(); |
| 84 | const done = await runAgentJob(spec, { |
| 85 | apiBase: apiBaseFor(spec.apiUrl, spec.jobId), |
| 86 | workRoot: path.join(dir, "work"), |
| 87 | toolCache: path.join(dir, "tool"), |
| 88 | logIntervalMs: 50, |
| 89 | echoLogs: false, |
| 90 | prefetchTools: true, |
| 91 | baseEnv: { ...process.env, PATH: "/usr/local/bin:/usr/bin:/bin" }, |
| 92 | }); |
| 93 | expect(done.conclusion, api.logText()).toBe("success"); |
| 94 | expect(api.logText()).toContain("go version go1.99.1 linux/amd64"); |
| 95 | expect(api.logText()).toContain("Found in cache @"); |
| 96 | // One download, asked for as the job began (before the first step's 1.5 s were up), not by the step. |
| 97 | const downloads = served.filter((s) => s.path.endsWith(".tar.gz")); |
| 98 | expect(downloads.map((s) => s.path)).toEqual(["/dl/go1.99.1.linux-amd64.tar.gz"]); |
| 99 | expect(downloads[0]!.at - started).toBeLessThan(1000); |
| 100 | expect(existsSync(path.join(dir, "tool", "go", "1.99.1", "amd64.complete"))).toBe(true); |
| 101 | }, 30_000); |
| 102 | |
| 103 | it("leaves nothing behind when a download doesn't match its published checksum", async () => { |
| 104 | const go = tarball("go", { "bin/go": "#!/bin/sh\necho go\n" }); |
| 105 | files.set("/dl/index.json", { body: Buffer.from(JSON.stringify([goRelease("1.99.1", go, "0".repeat(64))])) }); |
| 106 | const tool = path.join(dir, "tool"); |
| 107 | const p = new ToolPrefetch({ toolCache: tool, signal: new AbortController().signal } as unknown as JobRunner); |
| 108 | p.start(makeSpec(api, [{ uses: "actions/setup-go@v5", with: { "go-version": "1.99" } }])); |
| 109 | const lines: string[] = []; |
| 110 | await p.wait(0, (m) => lines.push(m)); |
| 111 | expect(lines).toEqual(["Waiting for Go 1.99, which the runner started installing as the job began"]); |
| 112 | // It was tried three times; none of it was kept. |
| 113 | expect(served.filter((s) => s.path.endsWith(".tar.gz"))).toHaveLength(3); |
| 114 | expect(readdirSync(path.join(tool, "go", "1.99.1"))).toEqual([]); |
| 115 | }, 30_000); |
| 116 | |
| 117 | it("lays a Temurin JDK out in the tool cache the way setup-java caches one", async () => { |
| 118 | const jdk = tarball("jdk-21.0.5+11", { "bin/java": "#!/bin/sh\necho java\n", release: 'JAVA_VERSION="21.0.5"\n' }); |
| 119 | files.set("/adoptium/v3/assets/latest/21/hotspot", { |
| 120 | body: Buffer.from(JSON.stringify([{ binary: { package: { link: `${base}/jdk.tar.gz`, checksum: sha(jdk) } }, version: { semver: "21.0.5+11.0.LTS" } }])), |
| 121 | }); |
| 122 | files.set("/jdk.tar.gz", { body: jdk, delayMs: 200 }); |
| 123 | const tool = path.join(dir, "tool"); |
| 124 | const p = new ToolPrefetch({ toolCache: tool, signal: new AbortController().signal } as unknown as JobRunner); |
| 125 | p.start(makeSpec(api, [{ run: "true" }, { uses: "actions/setup-java@v4", with: { distribution: "temurin", "java-version": "21", cache: "maven" } }])); |
| 126 | await p.wait(0, () => {}); |
| 127 | await p.wait(1, () => {}); |
| 128 | const home = path.join(tool, "Java_Temurin-Hotspot_jdk", "21.0.5-11.0.LTS", "x64"); |
| 129 | expect(readFileSync(path.join(home, "release"), "utf8")).toContain("21.0.5"); |
| 130 | expect(existsSync(path.join(home, "bin", "java"))).toBe(true); |
| 131 | expect(existsSync(`${home}.complete`)).toBe(true); |
| 132 | expect(readdirSync(path.dirname(home))).toEqual(["x64", "x64.complete"]); |
| 133 | |
| 134 | // A second job on the same machine finds it there and downloads nothing. |
| 135 | served = []; |
| 136 | const again = new ToolPrefetch({ toolCache: tool, signal: new AbortController().signal } as unknown as JobRunner); |
| 137 | again.start(makeSpec(api, [{ uses: "actions/setup-java@v4", with: { distribution: "temurin", "java-version": "21" } }])); |
| 138 | await again.wait(0, () => {}); |
| 139 | expect(served).toEqual([]); |
| 140 | }, 30_000); |
| 141 | |
| 142 | it("does nothing for jobs without a toolchain it can name", async () => { |
| 143 | const tool = path.join(dir, "tool"); |
| 144 | const steps: StepSpec[] = [ |
| 145 | { run: "go version" }, |
| 146 | { uses: "actions/setup-java@v4", with: { distribution: "zulu", "java-version": "21" } }, |
| 147 | { uses: "actions/setup-java@v4", with: { distribution: "temurin", "java-version": "${{ steps.v.outputs.java }}" } }, |
| 148 | { uses: "actions/setup-java@v4", with: { distribution: "temurin", "java-version": "21", "java-package": "jre" } }, |
| 149 | { uses: "actions/setup-go@v5", with: { "go-version-file": "go.mod" } }, |
| 150 | { uses: "actions/setup-go@v2", with: { "go-version": "1.22" } }, |
| 151 | { uses: "actions/setup-node@v4", with: { "node-version": "22" } }, |
| 152 | ]; |
| 153 | const p = new ToolPrefetch({ toolCache: tool, signal: new AbortController().signal } as unknown as JobRunner); |
| 154 | p.start(makeSpec(api, steps)); |
| 155 | p.start(makeSpec(api, [{ uses: "actions/setup-go@v5", with: { "go-version": "1.22" } }], { container: { image: "golang:1.22" } })); |
| 156 | for (let i = 0; i < steps.length; i++) await p.wait(i, () => {}); |
| 157 | await p.stop(); |
| 158 | expect(served).toEqual([]); |
| 159 | }); |
| 160 | }); |