| 1 | import { execFileSync } from "node:child_process"; |
| 2 | import { chmodSync, mkdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs"; |
| 3 | import { createServer } from "node:http"; |
| 4 | import type { AddressInfo } from "node:net"; |
| 5 | import path from "node:path"; |
| 6 | import { afterEach, describe, expect, it } from "vitest"; |
| 7 | import { getBuiltin } from "../src/builtins/index.js"; |
| 8 | import { backendIds, findFilesToUpload, mimeTypeOf, nameMatches } from "../src/builtins/artifacts.js"; |
| 9 | import { cacheVersion, isExactKeyMatch } from "../src/builtins/cache.js"; |
| 10 | import { RuntimeClient, RuntimeError } from "../src/runtime.js"; |
| 11 | import { crc32, extractZip, safeEntryPath, writeZip } from "../src/zip.js"; |
| 12 | import { tmp } from "./helpers.js"; |
| 13 | |
| 14 | const dirs: string[] = []; |
| 15 | const scratch = (prefix: string) => { |
| 16 | const d = tmp(prefix); |
| 17 | dirs.push(d); |
| 18 | return d; |
| 19 | }; |
| 20 | afterEach(() => { |
| 21 | for (const d of dirs.splice(0)) rmSync(d, { recursive: true, force: true }); |
| 22 | }); |
| 23 | |
| 24 | const write = (root: string, rel: string, body: string | Buffer = rel) => { |
| 25 | mkdirSync(path.dirname(path.join(root, rel)), { recursive: true }); |
| 26 | writeFileSync(path.join(root, rel), body); |
| 27 | }; |
| 28 | |
| 29 | describe("zip", () => { |
| 30 | it("computes CRC-32", () => { |
| 31 | expect(crc32(Buffer.from("123456789")).toString(16)).toBe("cbf43926"); |
| 32 | expect(crc32(Buffer.from("6789"), crc32(Buffer.from("12345")))).toBe(crc32(Buffer.from("123456789"))); |
| 33 | }); |
| 34 | |
| 35 | it("writes archives unzip reads, and reads archives zip writes", async () => { |
| 36 | const d = scratch("zip-"); |
| 37 | write(d, "src/a.txt", "alpha\n".repeat(1000)); |
| 38 | write(d, "src/sub/b.bin", Buffer.alloc(70_000, 7)); |
| 39 | write(d, "src/empty", ""); |
| 40 | write(d, "src/run.sh", "#!/bin/sh\necho hi\n"); |
| 41 | chmodSync(path.join(d, "src/run.sh"), 0o755); |
| 42 | const entries = ["a.txt", "sub/b.bin", "empty", "run.sh"].map((name) => ({ source: path.join(d, "src", name), name })); |
| 43 | for (const level of [0, 6]) { |
| 44 | const out = path.join(d, `l${level}.zip`); |
| 45 | const { size, sha256 } = await writeZip(out, entries, level); |
| 46 | expect(size).toBe(statSync(out).size); |
| 47 | expect(sha256).toMatch(/^[0-9a-f]{64}$/); |
| 48 | expect(execFileSync("unzip", ["-t", out]).toString()).toContain("No errors detected"); |
| 49 | const dest = path.join(d, `x${level}`); |
| 50 | expect(await extractZip(out, dest)).toBe(4); |
| 51 | expect(readFileSync(path.join(dest, "a.txt"), "utf8")).toBe("alpha\n".repeat(1000)); |
| 52 | expect(readFileSync(path.join(dest, "sub/b.bin")).equals(Buffer.alloc(70_000, 7))).toBe(true); |
| 53 | expect(readFileSync(path.join(dest, "empty")).length).toBe(0); |
| 54 | expect(statSync(path.join(dest, "run.sh")).mode & 0o111).not.toBe(0); |
| 55 | } |
| 56 | execFileSync("zip", ["-qr", path.join(d, "cli.zip"), "."], { cwd: path.join(d, "src") }); |
| 57 | const dest = path.join(d, "cli"); |
| 58 | await extractZip(path.join(d, "cli.zip"), dest); |
| 59 | expect(readFileSync(path.join(dest, "sub/b.bin")).length).toBe(70_000); |
| 60 | }); |
| 61 | |
| 62 | it("refuses entries that escape the destination", () => { |
| 63 | expect(() => safeEntryPath("/tmp/x", "../evil")).toThrow(); |
| 64 | expect(() => safeEntryPath("/tmp/x", "/etc/passwd")).toThrow(); |
| 65 | expect(safeEntryPath("/tmp/x", "a/b.txt")).toBe("/tmp/x/a/b.txt"); |
| 66 | }); |
| 67 | }); |
| 68 | |
| 69 | describe("built-in actions", () => { |
| 70 | it("mirror only the refs and inputs they understand", () => { |
| 71 | expect(getBuiltin("actions/checkout", "v4")).not.toBeNull(); |
| 72 | expect(getBuiltin("Actions/Checkout", "v4.2.1", { "fetch-depth": "0", LFS: "true" })).not.toBeNull(); |
| 73 | expect(getBuiltin("actions/checkout", "main")).toBeNull(); |
| 74 | expect(getBuiltin("actions/checkout", "11bd71901bbe5b1630ceea73d27597364c9af683")).toBeNull(); |
| 75 | expect(getBuiltin("actions/checkout", "v99")).toBeNull(); |
| 76 | expect(getBuiltin("actions/checkout", "v4", { "ssh-key": "k" })).toBeNull(); |
| 77 | expect(getBuiltin("actions/cache", "v4", { path: "x", key: "k", "new-input": "1" })).toBeNull(); |
| 78 | expect(getBuiltin("actions/setup-node", "v4", { cache: "bun" })).toBeNull(); |
| 79 | expect(getBuiltin("actions/setup-python", "v5", { "python-version": "pypy3.10" })).toBeNull(); |
| 80 | expect(getBuiltin("actions/setup-python", "v5", { cache: "poetry" })).toBeNull(); |
| 81 | expect(getBuiltin("actions/github-script", "v7")).toBeNull(); |
| 82 | }); |
| 83 | |
| 84 | it("step aside when the workflow or runner asks for the real action", () => { |
| 85 | expect(getBuiltin("actions/cache", "v4", {}, { DIFFINERY_BUILTINS: "false" })).toBeNull(); |
| 86 | expect(getBuiltin("actions/cache", "v4", {}, { DIFFINERY_BUILTINS: "true" })).not.toBeNull(); |
| 87 | process.env.DIFFINERY_DISABLE_BUILTINS = "1"; |
| 88 | try { |
| 89 | expect(getBuiltin("actions/cache", "v4")).toBeNull(); |
| 90 | } finally { |
| 91 | delete process.env.DIFFINERY_DISABLE_BUILTINS; |
| 92 | } |
| 93 | }); |
| 94 | }); |
| 95 | |
| 96 | describe("cache", () => { |
| 97 | it("versions entries like @actions/cache", () => { |
| 98 | // sha256("node_modules|zstd-without-long|1.0"), computed independently |
| 99 | const expected = execFileSync("sh", ["-c", "printf '%s' 'node_modules|zstd-without-long|1.0' | sha256sum"]).toString().split(" ")[0]; |
| 100 | expect(cacheVersion(["node_modules"], "zstd-without-long")).toBe(expected); |
| 101 | expect(cacheVersion(["a", "b"], "gzip")).not.toBe(cacheVersion(["b", "a"], "gzip")); |
| 102 | // gzip is part of the version too: @actions/cache's getCacheVersion pushes the compression |
| 103 | // method whatever it is (only enableCrossOsArchive adds anything else) |
| 104 | const gzip = execFileSync("sh", ["-c", "printf '%s' 'node_modules|gzip|1.0' | sha256sum"]).toString().split(" ")[0]; |
| 105 | expect(cacheVersion(["node_modules"], "gzip")).toBe(gzip); |
| 106 | }); |
| 107 | |
| 108 | it("matches keys case-insensitively, like actions/cache", () => { |
| 109 | expect(isExactKeyMatch("Linux-deps", "linux-DEPS")).toBe(true); |
| 110 | expect(isExactKeyMatch("linux-deps", "linux-deps-2")).toBe(false); |
| 111 | expect(isExactKeyMatch("linux-deps", undefined)).toBe(false); |
| 112 | }); |
| 113 | }); |
| 114 | |
| 115 | describe("artifacts", () => { |
| 116 | it("finds files and their root like upload-artifact", () => { |
| 117 | const ws = scratch("upload-"); |
| 118 | for (const f of ["dist/a.js", "dist/sub/b.js", "dist/.env", "dist/.cache/x", "logs/one.log", "logs/two.log", "README.md"]) write(ws, f); |
| 119 | const rel = (r: { files: string[]; root: string }) => ({ files: r.files.map((f) => path.relative(ws, f)).sort(), root: path.relative(ws, r.root) }); |
| 120 | expect(rel(findFilesToUpload(ws, "dist/", false))).toEqual({ files: ["dist/a.js", "dist/sub/b.js"], root: "dist" }); |
| 121 | expect(rel(findFilesToUpload(ws, "dist", true)).files).toEqual(["dist/.cache/x", "dist/.env", "dist/a.js", "dist/sub/b.js"]); |
| 122 | expect(rel(findFilesToUpload(ws, "README.md", false))).toEqual({ files: ["README.md"], root: "" }); |
| 123 | expect(rel(findFilesToUpload(ws, "dist/sub/b.js\nlogs/*.log\n!logs/two.log", false))).toEqual({ files: ["dist/sub/b.js", "logs/one.log"], root: "" }); |
| 124 | expect(rel(findFilesToUpload(ws, "dist/**/*.js", false))).toEqual({ files: ["dist/a.js", "dist/sub/b.js"], root: "dist" }); |
| 125 | expect(findFilesToUpload(ws, "missing/", false).files).toEqual([]); |
| 126 | }); |
| 127 | |
| 128 | it("matches artifact names like minimatch", () => { |
| 129 | expect(nameMatches("build-*", "build-linux")).toBe(true); |
| 130 | expect(nameMatches("build-*", "test-linux")).toBe(false); |
| 131 | expect(nameMatches("{a,b}-?", "b-1")).toBe(true); |
| 132 | expect(nameMatches("cov[0-9]", "cov7")).toBe(true); |
| 133 | }); |
| 134 | |
| 135 | it("knows the run and job from the runtime token, and guesses mime types", () => { |
| 136 | const b64 = (v: unknown) => Buffer.from(JSON.stringify(v)).toString("base64url"); |
| 137 | expect(backendIds(`${b64({})}.${b64({ scp: "Actions.GenericRead:12 Actions.Results:12:job-9" })}.sig`)).toEqual({ run: "12", job: "job-9" }); |
| 138 | expect(() => backendIds(`${b64({})}.${b64({ scp: "Actions.GenericRead:12" })}.sig`)).toThrow(); |
| 139 | expect(mimeTypeOf("report.HTML")).toBe("text/html"); |
| 140 | expect(mimeTypeOf("blob.weird")).toBe("application/octet-stream"); |
| 141 | }); |
| 142 | }); |
| 143 | |
| 144 | describe("runtime client", () => { |
| 145 | it("uploads large files as blocks, small ones in one request, and downloads them back", async () => { |
| 146 | const store = new Map<string, Buffer>(); |
| 147 | const calls: string[] = []; |
| 148 | let blocks = new Map<string, Buffer>(); |
| 149 | const server = createServer(async (req, res) => { |
| 150 | const chunks: Buffer[] = []; |
| 151 | for await (const c of req) chunks.push(c as Buffer); |
| 152 | const body = Buffer.concat(chunks); |
| 153 | const u = new URL(req.url!, "http://x"); |
| 154 | const comp = u.searchParams.get("comp"); |
| 155 | calls.push(`${req.method} ${comp ?? "-"}`); |
| 156 | if (req.method === "PUT" && comp === "block") blocks.set(u.searchParams.get("blockid")!, body); |
| 157 | else if (req.method === "PUT" && comp === "blocklist") { |
| 158 | const ids = [...body.toString().matchAll(/<Latest>([^<]+)<\/Latest>/g)].map((m) => m[1]!); |
| 159 | store.set(u.pathname, Buffer.concat(ids.map((id) => blocks.get(id)!))); |
| 160 | blocks = new Map(); |
| 161 | } else if (req.method === "PUT") store.set(u.pathname, body); |
| 162 | else { |
| 163 | const b = store.get(u.pathname); |
| 164 | res.writeHead(b ? 200 : 404, { "content-type": "application/zip", "content-disposition": "attachment; filename*=UTF-8''r%C3%A9sum%C3%A9.zip" }); |
| 165 | return res.end(b); |
| 166 | } |
| 167 | res.writeHead(201); |
| 168 | res.end(); |
| 169 | }); |
| 170 | await new Promise<void>((r) => server.listen(0, "127.0.0.1", () => r())); |
| 171 | const base = `http://127.0.0.1:${(server.address() as AddressInfo).port}`; |
| 172 | const d = scratch("blocks-"); |
| 173 | try { |
| 174 | const rt = new RuntimeClient("t", `${base}/`, { retries: 0 }); |
| 175 | const big = Buffer.alloc(33 * 1024 * 1024 + 5); |
| 176 | for (let i = 0; i < big.length; i += 4096) big[i] = i % 251; |
| 177 | writeFileSync(path.join(d, "big"), big); |
| 178 | writeFileSync(path.join(d, "small"), "hello"); |
| 179 | expect(await rt.upload(`${base}/big?sig=1`, path.join(d, "big"))).toBe(big.length); |
| 180 | expect(calls).toEqual(["PUT block", "PUT block", "PUT block", "PUT blocklist"]); // 16 MiB blocks |
| 181 | expect(await rt.upload(`${base}/small?sig=1`, path.join(d, "small"))).toBe(5); |
| 182 | expect(calls.at(-1)).toBe("PUT -"); |
| 183 | const meta = await rt.download(`${base}/big?sig=1`, path.join(d, "back")); |
| 184 | expect(meta).toEqual({ contentType: "application/zip", fileName: "résumé.zip" }); |
| 185 | expect(readFileSync(path.join(d, "back")).equals(big)).toBe(true); |
| 186 | await expect(rt.download(`${base}/nothing`, path.join(d, "none"))).rejects.toBeInstanceOf(RuntimeError); |
| 187 | } finally { |
| 188 | await new Promise<void>((r) => server.close(() => r())); |
| 189 | } |
| 190 | }); |
| 191 | |
| 192 | it("downloads in ranges, several at once, retrying a range on its own", async () => { |
| 193 | const blob = Buffer.alloc(8 * 1024 * 1024 * 3 + 12345); |
| 194 | for (let i = 0; i < blob.length; i += 997) blob[i] = (i / 997) % 256; |
| 195 | const seen: string[] = []; |
| 196 | let flaky = 1; |
| 197 | let inFlight = 0; |
| 198 | let most = 0; |
| 199 | // The ranges after the first are held until three of them are in flight at once (or 5 s pass), so the test |
| 200 | // doesn't depend on a busy host sending them within a few milliseconds of each other. |
| 201 | const held: Array<() => void> = []; |
| 202 | let released = false; |
| 203 | const server = createServer(async (req, res) => { |
| 204 | const b = req.url!.startsWith("/empty") ? Buffer.alloc(0) : blob; |
| 205 | const range = /^bytes=(\d+)-(\d+)$/.exec(req.headers.range ?? ""); |
| 206 | seen.push(req.headers.range ?? ""); |
| 207 | inFlight++; |
| 208 | most = Math.max(most, inFlight); |
| 209 | if (range && Number(range[1]) > 0 && !released) { |
| 210 | await new Promise<void>((r) => { |
| 211 | held.push(r); |
| 212 | if (held.length >= 3) { |
| 213 | released = true; |
| 214 | for (const go of held.splice(0)) go(); |
| 215 | } else setTimeout(r, 5000); |
| 216 | }); |
| 217 | } |
| 218 | await new Promise((r) => setTimeout(r, 20)); |
| 219 | inFlight--; |
| 220 | if (!range) return res.writeHead(200, { "content-length": b.length }).end(b); |
| 221 | if (!b.length) return res.writeHead(416).end(); |
| 222 | const start = Number(range[1]); |
| 223 | const end = Math.min(Number(range[2]), b.length - 1); |
| 224 | if (start > 0 && flaky-- > 0) return res.writeHead(503).end("busy"); |
| 225 | res.writeHead(206, { etag: '"x"', "content-range": `bytes ${start}-${end}/${b.length}` }).end(b.subarray(start, end + 1)); |
| 226 | }); |
| 227 | await new Promise<void>((r) => server.listen(0, "127.0.0.1", () => r())); |
| 228 | const base = `http://127.0.0.1:${(server.address() as AddressInfo).port}`; |
| 229 | const d = scratch("ranges-"); |
| 230 | try { |
| 231 | const rt = new RuntimeClient("t", `${base}/`, { retries: 2, minBackoffMs: 1 }); |
| 232 | await rt.download(`${base}/blob?sig=1`, path.join(d, "back")); |
| 233 | expect(readFileSync(path.join(d, "back")).equals(blob)).toBe(true); |
| 234 | // The first range, three more at once, and the one that failed again. |
| 235 | expect(seen[0]).toBe(`bytes=0-${8 * 1024 * 1024 - 1}`); |
| 236 | expect(seen).toHaveLength(5); |
| 237 | expect(most).toBeGreaterThanOrEqual(3); |
| 238 | // An empty blob: the range is refused (416), so it is fetched whole. |
| 239 | seen.length = 0; |
| 240 | await rt.download(`${base}/empty?sig=1`, path.join(d, "empty")); |
| 241 | expect(readFileSync(path.join(d, "empty")).length).toBe(0); |
| 242 | expect(seen).toEqual([`bytes=0-${8 * 1024 * 1024 - 1}`, ""]); |
| 243 | } finally { |
| 244 | await new Promise<void>((r) => server.close(() => r())); |
| 245 | } |
| 246 | }); |
| 247 | }); |