Diffinery

codician/diffinerypublic

Open-source, agent-first GitHub and GitHub Actions alternative that runs on your own Cloudflare account: git hosting, pull requests, code review and drop-in GitHub Actions YAML on Workers, Durable Objects, R2 and Containers.

Runner: start installing a job's Go or Java toolchain as the job begins (#239)

* Runner: start installing a job's Go or Java toolchain as the job begins
test-bot authored 1 parent 0c0b2f0commit 1c63ad9caf3da457ca2e0e9adff25ab82b554c36Browse files
8 files changed+467−32
Files (8)
modifiedapps/diffinery/docs/actions/runners.md+2−0View file
@@ -149,6 +149,8 @@
149149
150150 `actions/setup-node`, `actions/setup-python` and `actions/setup-go` use the preinstalled Node.js or Python when its version satisfies the one you ask for, and download the version otherwise. See [Compatibility with GitHub Actions](/docs/actions/compatibility) for their supported inputs.
151151
152+**Go and Java start installing as the job begins.** When a job has an `actions/setup-go` step (v4 to v6) or an `actions/setup-java` step (v2 to v5, `distribution: temurin`, a major version such as `java-version: 21`), and the version is written in the workflow or comes from `matrix`, `inputs` or `vars`, the runner starts downloading and extracting that toolchain as the job begins, while earlier steps such as checkout run. The setup step then finds it in the tool cache instead of downloading it. If it's still being installed when the step starts, the step logs `Waiting for Go 1.23, which the runner started installing as the job began` and continues once it's there. The runner installs the same release the action would pick, checks it against its published SHA-256, and puts it where the action looks (`setup-java` reports `Resolved Java ... from tool-cache`, as it does on GitHub's runners), so nothing else about the step changes. Jobs without these steps don't pay for any of this. Versions read from a file (`go-version-file`, `java-version-file`), other Java distributions, and jobs that run in a `container:` install the usual way.
153+
152154 > **Note:** Jobs run as `root`, and `sudo` runs commands without asking for a password, as on GitHub's runners. The image ships without apt package lists, so run `sudo apt-get update` before `sudo apt-get install ...`.
153155
154156 ### Docker on hosted runners
modifiedcontainers/runner/src/agent.ts+3−1View file
@@ -28,6 +28,8 @@
2828 measureUsage?: boolean | UsageSampler;
2929 /** Hosted runners: start the Docker daemon as the job begins when the job will use Docker (docker.ts prewarm). */
3030 prewarmDocker?: boolean;
31+ /** Hosted runners: start installing the job's Go or Java toolchain as the job begins (prefetch.ts). */
32+ prefetchTools?: boolean;
3133 }
3234
3335 export function apiBaseFor(apiUrl: string | undefined | null, jobId: string): string | null {
@@ -44,7 +46,7 @@
4446 onError: (e) => process.stderr.write(`[diffinery-runner] log upload failed: ${errMsg(e)}\n`),
4547 })
4648 : null;
47- const runner = new JobRunner(spec, api, null, { workRoot: o.workRoot, toolCache: o.toolCache, echoLogs: o.echoLogs ?? !api.online, baseEnv: o.baseEnv, prewarmDocker: o.prewarmDocker }, batcher);
49+ const runner = new JobRunner(spec, api, null, { workRoot: o.workRoot, toolCache: o.toolCache, echoLogs: o.echoLogs ?? !api.online, baseEnv: o.baseEnv, prewarmDocker: o.prewarmDocker, prefetchTools: o.prefetchTools }, batcher);
4850 o.onRunner?.(runner);
4951 const onSignal = (sig: string) => {
5052 runner.logger.line(-1, `##[error]Received ${sig}; cancelling the job.`);
modifiedcontainers/runner/src/builtins/setup-go.ts+40−16View file
@@ -7,9 +7,43 @@
77 interface GoRelease {
88 version: string; // "go1.22.3"
99 stable: boolean;
10- files: Array<{ filename: string; os: string; arch: string; kind: string }>;
10+ files: Array<{ filename: string; os: string; arch: string; kind: string; sha256?: string }>;
1111 }
1212
13+let index: { url: string; at: number; releases: Promise<GoRelease[]> } | null = null;
14+
15+/** Go's releases index, fetched at most every 10 minutes: the job's head start (prefetch.ts) and the step share it. */
16+function goReleases(): Promise<GoRelease[]> {
17+ const url = process.env.DIFFINERY_GO_INDEX ?? "https://go.dev/dl/?mode=json&include=all";
18+ if (!index || index.url !== url || Date.now() - index.at > 10 * 60_000) {
19+ const releases = fetchJson<GoRelease[]>(url);
20+ index = { url, at: Date.now(), releases };
21+ releases.catch(() => {
22+ if (index?.releases === releases) index = null;
23+ });
24+ }
25+ return index.releases;
26+}
27+
28+/** The stable Go release `spec` asks for ("1.22", "1.22.3", "stable", "oldstable", a range), as setup-go picks it. */
29+export async function resolveGo(spec: string): Promise<{ version: string; url: string; sha256?: string } | null> {
30+ const releases = await goReleases();
31+ const arch = toolArch("go");
32+ const stable = releases.filter((r) => r.stable && r.files.some((f) => f.os === "linux" && f.arch === arch && f.kind === "archive"));
33+ let v: string | null;
34+ if (spec === "stable") v = maxSatisfying(stable.map((r) => r.version.replace(/^go/, "")), "*");
35+ else if (spec === "oldstable") {
36+ const sorted = stable.map((r) => r.version.replace(/^go/, ""));
37+ const latest = maxSatisfying(sorted, "*");
38+ const latestMinor = latest ? latest.split(".").slice(0, 2).join(".") : "";
39+ v = maxSatisfying(sorted.filter((x) => !x.startsWith(latestMinor + ".") && x !== latestMinor), "*");
40+ } else v = maxSatisfying(stable.map((r) => r.version.replace(/^go/, "")), spec);
41+ if (!v) return null;
42+ const rel = stable.find((r) => r.version === `go${v}`)!;
43+ const f = rel.files.find((x) => x.os === "linux" && x.arch === arch && x.kind === "archive")!;
44+ return { version: v, url: `${process.env.DIFFINERY_GO_DL ?? "https://go.dev/dl/"}${f.filename}`, sha256: f.sha256 };
45+}
46+
1347 export const setupGo: Builtin = async (c) => {
1448 let spec = c.input("go-version");
1549 const file = c.input("go-version-file");
@@ -24,27 +58,17 @@
2458 if (useInstalled) {
2559 c.log(`Using pre-installed Go ${installed}`);
2660 } else {
27- let releases: GoRelease[];
61+ let rel: Awaited<ReturnType<typeof resolveGo>>;
2862 try {
29- releases = await fetchJson<GoRelease[]>(process.env.DIFFINERY_GO_INDEX ?? "https://go.dev/dl/?mode=json&include=all");
63+ rel = await resolveGo(spec);
3064 } catch (e) {
3165 throw failure(`Unable to fetch the Go versions index: ${errMsg(e)}`);
3266 }
3367 const arch = toolArch("go");
34- const stable = releases.filter((r) => r.stable && r.files.some((f) => f.os === "linux" && f.arch === arch && f.kind === "archive"));
35- let v: string | null;
36- if (spec === "stable") v = maxSatisfying(stable.map((r) => r.version.replace(/^go/, "")), "*");
37- else if (spec === "oldstable") {
38- const sorted = stable.map((r) => r.version.replace(/^go/, ""));
39- const latest = maxSatisfying(sorted, "*");
40- const latestMinor = latest ? latest.split(".").slice(0, 2).join(".") : "";
41- v = maxSatisfying(sorted.filter((x) => !x.startsWith(latestMinor + ".") && x !== latestMinor), "*");
42- } else v = maxSatisfying(stable.map((r) => r.version.replace(/^go/, "")), spec);
43- if (!v) throw failure(`Unable to find Go version '${spec}' for platform linux and architecture ${arch}.`);
44- const rel = stable.find((r) => r.version === `go${v}`)!;
45- const f = rel.files.find((x) => x.os === "linux" && x.arch === arch && x.kind === "archive")!;
68+ if (!rel) throw failure(`Unable to find Go version '${spec}' for platform linux and architecture ${arch}.`);
69+ const v = rel.version;
4670 const dir = toolDir(c, "go", v, arch);
47- if (!isCached(dir)) await installTarball(c, `https://go.dev/dl/${f.filename}`, dir);
71+ if (!isCached(dir)) await installTarball(c, rel.url, dir, 1, rel.sha256);
4872 else c.log(`Found in cache @ ${dir}`);
4973 c.addPath(path.join(dir, "bin"));
5074 version = v;
modifiedcontainers/runner/src/builtins/tools.ts+64−15View file
@@ -1,8 +1,9 @@
1-import { randomUUID } from "node:crypto";
1+import { spawn, type ChildProcess } from "node:child_process";
2+import { createHash, randomUUID } from "node:crypto";
23 import { createWriteStream, existsSync } from "node:fs";
3-import { mkdir, readFile, rm, writeFile } from "node:fs/promises";
4+import { mkdir, readFile, rename, rm, writeFile } from "node:fs/promises";
45 import path from "node:path";
5-import { Readable } from "node:stream";
6+import { Readable, Transform } from "node:stream";
67 import { pipeline } from "node:stream/promises";
78 import type { ReadableStream as WebReadableStream } from "node:stream/web";
89 import { failure } from "../job.js";
@@ -59,19 +60,67 @@
5960 }
6061
6162 /** Downloads a tarball and extracts it into `dir` (stripping the top-level folder). */
62-export async function installTarball(c: BuiltinContext, url: string, dir: string, strip = 1): Promise<void> {
63- c.log(`Downloading ${url}`);
64- const file = path.join(c.job.tempDir, `tool-${randomUUID()}${url.endsWith(".xz") ? ".tar.xz" : ".tar.gz"}`);
65- try {
66- await fetchToFile(url, file);
67- await rm(dir, { recursive: true, force: true });
68- await mkdir(dir, { recursive: true });
69- c.log(`Extracting to ${dir}`);
70- await c.run("tar", [url.endsWith(".xz") ? "-xJf" : "-xzf", file, "-C", dir, `--strip-components=${strip}`], { silent: true });
71- await writeFile(`${dir}.complete`, "");
72- } finally {
73- await rm(file, { force: true });
63+export async function installTarball(c: BuiltinContext, url: string, dir: string, strip = 1, sha256?: string): Promise<void> {
64+ c.log(`Downloading and extracting ${url} to ${dir}`);
65+ await extractUrl(url, dir, { strip, sha256, signal: c.job.signal });
66+}
67+
68+/**
69+ * Downloads a .tar.gz or .tar.xz and extracts it into `dir` as it arrives, stripping `strip` leading
70+ * folders and checking its SHA-256 when given, with a few retries. The tool appears at `dir` whole or
71+ * not at all: it is extracted next to it, renamed into place, then marked with `${dir}.complete`.
72+ */
73+export async function extractUrl(url: string, dir: string, o: { strip?: number; sha256?: string; signal?: AbortSignal } = {}): Promise<void> {
74+ let last: unknown;
75+ for (let i = 0; i < 3; i++) {
76+ if (i) await new Promise((r) => setTimeout(r, 1000 * 2 ** i));
77+ if (o.signal?.aborted) break;
78+ const part = `${dir}.part-${randomUUID().slice(0, 8)}`;
79+ let tar: ChildProcess | undefined;
80+ try {
81+ await mkdir(part, { recursive: true });
82+ const res = await fetch(url, { headers: { "user-agent": "diffinery-runner" }, redirect: "follow", signal: o.signal });
83+ if (res.status === 404) throw failure(`Not found: ${url}`);
84+ if (!res.ok || !res.body) throw new Error(`HTTP ${res.status} for ${url}`);
85+ const t = spawn("tar", [/\.(tar\.)?xz$/.test(new URL(url).pathname) ? "-xJf" : "-xzf", "-", "-C", part, `--strip-components=${o.strip ?? 1}`], { stdio: ["pipe", "ignore", "pipe"] });
86+ tar = t;
87+ let stderr = "";
88+ t.stderr!.on("data", (d: Buffer) => {
89+ if (stderr.length < 4000) stderr += d.toString();
90+ });
91+ const exited = new Promise<number>((resolve, reject) => {
92+ t.once("error", reject);
93+ t.once("close", (code) => resolve(code ?? -1));
94+ });
95+ const hash = createHash("sha256");
96+ const hashing = new Transform({
97+ transform(chunk: Buffer, _enc, cb) {
98+ hash.update(chunk);
99+ cb(null, chunk);
100+ },
101+ });
102+ const [piped, exit] = await Promise.allSettled([pipeline(Readable.fromWeb(res.body as unknown as WebReadableStream), hashing, t.stdin!), exited]);
103+ if (exit.status === "rejected") throw exit.reason;
104+ if (exit.value !== 0) throw new Error(`tar failed with exit code ${exit.value}: ${stderr.trim().split("\n").slice(-3).join(" ")}`);
105+ if (piped.status === "rejected") throw piped.reason;
106+ if (o.sha256 && hash.digest("hex") !== o.sha256.toLowerCase()) throw new Error(`${url} does not match its published SHA-256`);
107+ await rm(dir, { recursive: true, force: true });
108+ await rename(part, dir);
109+ await writeFile(`${dir}.complete`, "");
110+ return;
111+ } catch (e) {
112+ if (tar && tar.exitCode === null) {
113+ const gone = new Promise((r) => tar!.once("close", r));
114+ tar.kill("SIGKILL");
115+ await gone;
116+ }
117+ await rm(part, { recursive: true, force: true });
118+ if (e instanceof Error && e.message.startsWith("Not found:")) throw e;
119+ last = e;
120+ }
74121 }
122+ if (o.signal?.aborted) throw failure("cancelled");
123+ throw last instanceof Error ? last : new Error(String(last));
75124 }
76125
77126 /** Runs `cmd args` and returns trimmed stdout, or null if it fails. */
modifiedcontainers/runner/src/job.ts+14−0View file
@@ -11,6 +11,7 @@
1111 import { Docker, type Wrap } from "./docker.js";
1212 import { JobLogger, OutputProcessor, type Annotation, type OutputSink } from "./log.js";
1313 import { Masker } from "./mask.js";
14+import { ToolPrefetch } from "./prefetch.js";
1415 import { RuntimeClient } from "./runtime.js";
1516 import { runAction } from "./steps/actions.js";
1617 import { runScript } from "./steps/run.js";
@@ -64,6 +65,8 @@
6465 baseEnv?: NodeJS.ProcessEnv;
6566 /** Start the Docker daemon as the job begins when the job will use Docker (hosted runners). */
6667 prewarmDocker?: boolean;
68+ /** Hosted runners: start installing the job's Go or Java toolchain as the job begins (prefetch.ts). */
69+ prefetchTools?: boolean;
6770 }
6871
6972 export class StepFailure extends Error {}
@@ -99,10 +102,13 @@
99102 readonly summaries: string[] = [];
100103 readonly posts: PostStep[] = [];
101104 readonly children = new Set<ChildProcess>();
105+ /** Aborted when the job is cancelled or times out, for work the runner does itself (downloads). */
106+ private readonly aborter = new AbortController();
102107 readonly stepErrors = new Map<number, Annotation[]>();
103108 readonly stepReasons = new Map<number, string>();
104109 /** Job and service containers, and Docker actions (self-hosted runners with Docker). */
105110 readonly docker: Docker;
111+ readonly prefetch = new ToolPrefetch(this);
106112 /** The Actions runtime services client (null when the server sent none). */
107113 readonly runtime: RuntimeClient | null;
108114
@@ -155,6 +161,10 @@
155161
156162 // ------------------------------------------------------------ public
157163
164+ get signal(): AbortSignal {
165+ return this.aborter.signal;
166+ }
167+
158168 get isStopping(): boolean {
159169 return this.stopping !== null;
160170 }
@@ -163,6 +173,7 @@
163173 cancel(reason: "cancelled" | "timeout" = "cancelled"): void {
164174 if (this.stopping) return;
165175 this.stopping = reason;
176+ this.aborter.abort();
166177 for (const c of this.children) killTree(c);
167178 }
168179
@@ -178,6 +189,7 @@
178189 }
179190 const top: Scope = { steps: {}, inputs: spec.inputs ?? {}, failed: false, inheritedEnv: {}, composite: false, depth: 0 };
180191 if (this.opts.prewarmDocker && jobUsesDocker(spec)) this.docker.prewarm();
192+ if (this.opts.prefetchTools) this.prefetch.start(spec);
181193 try {
182194 await this.setup();
183195 await this.initContainers(top);
@@ -210,6 +222,7 @@
210222 this.firstFailure ??= { message: msg };
211223 } finally {
212224 if (jobTimer) clearTimeout(jobTimer);
225+ await this.prefetch.stop();
213226 try {
214227 await this.docker.cleanup({ logIndex: -1, outputs: {}, state: {} });
215228 } catch (e) {
@@ -528,6 +541,7 @@
528541 return { outcome: "skipped", conclusion: "skipped", outputs: {}, name };
529542 }
530543 await onStart?.(name);
544+ if (scope.depth === 0) await this.prefetch.wait(logIndex, (m) => this.logger.line(logIndex, m));
531545
532546 const io: StepIO = { logIndex, outputs: {}, state: {} };
533547 let outcome: Outcome = "success";
modifiedcontainers/runner/src/main.ts+1−0View file
@@ -152,6 +152,7 @@
152152 // A hosted runner is its own machine (Cloudflare sets the Durable Object id in its container).
153153 // test/docker-wrapper.test.ts runs the image's docker wrapper with the head start on.
154154 prewarmDocker: !!env.CLOUDFLARE_DURABLE_OBJECT_ID,
155+ prefetchTools: !!env.CLOUDFLARE_DURABLE_OBJECT_ID,
155156 measureUsage: !!env.CLOUDFLARE_DURABLE_OBJECT_ID || env.DIFFINERY_MEASURE_USAGE === "1" ? new UsageSampler({ cores: Number(env.DIFFINERY_RUNNER_VCPU) || undefined }) : false,
156157 });
157158 process.stdout.write(`[diffinery-runner] job finished: ${completion.conclusion}\n`);
addedcontainers/runner/src/prefetch.ts+183−0View file
@@ -1,0 +1,183 @@
1+import { existsSync, readdirSync } from "node:fs";
2+import path from "node:path";
3+import type { JobSpec, StepSpec } from "@diffinery/shared";
4+import { getBuiltin } from "./builtins/index.js";
5+import { resolveGo } from "./builtins/setup-go.js";
6+import { extractUrl, fetchJson, isCached, toolArch } from "./builtins/tools.js";
7+import type { JobRunner } from "./job.js";
8+import { parseUses } from "./steps/actions.js";
9+
10+/** How long a step waits for its head start before installing the tool itself. */
11+const WAIT_MS = 3 * 60_000;
12+
13+interface Tool {
14+ /** Same key, same download: two steps asking for the same tool share one. */
15+ key: string;
16+ what: string;
17+ fetch: (toolCache: string, signal: AbortSignal) => Promise<void>;
18+}
19+
20+interface Pending {
21+ what: string;
22+ done: Promise<void>;
23+ settled: boolean;
24+ abort: AbortController;
25+}
26+
27+/**
28+ * Hosted runners: the toolchain head start. For a job that sets up Go (actions/setup-go) or a Temurin
29+ * JDK (actions/setup-java) with a version the workflow names, the runner starts downloading and
30+ * extracting it into the tool cache as the job begins, while the job checks out its code, and the
31+ * setup step finds it there. It installs exactly what the step would (the same release, in the same
32+ * place), so the step behaves as it would without it. Jobs without such a step do nothing extra.
33+ */
34+export class ToolPrefetch {
35+ private readonly steps = new Map<number, Pending>();
36+ private readonly all: Pending[] = [];
37+
38+ constructor(private readonly job: JobRunner) {}
39+
40+ start(spec: JobSpec): void {
41+ // Steps in a job container run in another image, which may need another build of the tool.
42+ if (spec.container?.image) return;
43+ const byKey = new Map<string, Pending>();
44+ spec.steps.forEach((step, i) => {
45+ const tool = toolFor(step, spec);
46+ if (!tool) return;
47+ let p = byKey.get(tool.key);
48+ if (!p) {
49+ const abort = new AbortController();
50+ const signal = AbortSignal.any([abort.signal, this.job.signal]);
51+ const entry: Pending = { what: tool.what, settled: false, abort, done: Promise.resolve() };
52+ entry.done = tool
53+ .fetch(this.job.toolCache, signal)
54+ // On any failure the step installs the tool itself, as it would without a head start.
55+ .catch(() => {})
56+ .finally(() => {
57+ entry.settled = true;
58+ });
59+ p = entry;
60+ byKey.set(tool.key, p);
61+ this.all.push(p);
62+ }
63+ this.steps.set(i, p);
64+ });
65+ }
66+
67+ /** Before step `i` (of the job's own steps) runs: lets its head start finish, if it has one. */
68+ async wait(i: number, log: (msg: string) => void): Promise<void> {
69+ const p = this.steps.get(i);
70+ if (!p || p.settled) return;
71+ log(`Waiting for ${p.what}, which the runner started installing as the job began`);
72+ let timer: NodeJS.Timeout | undefined;
73+ const late = new Promise<void>((resolve) => {
74+ timer = setTimeout(() => {
75+ p.abort.abort();
76+ resolve();
77+ }, WAIT_MS);
78+ });
79+ await Promise.race([p.done, late]);
80+ clearTimeout(timer);
81+ // Aborted: let it clean up before the step installs the tool itself.
82+ await p.done;
83+ }
84+
85+ /** At the end of the job: stops head starts that no step waited for. */
86+ async stop(): Promise<void> {
87+ for (const p of this.all) if (!p.settled) p.abort.abort();
88+ await Promise.all(this.all.map((p) => p.done));
89+ }
90+}
91+
92+/** A `with:` value as the workflow names it: literal, or exactly `${{ matrix.x }}` / `${{ inputs.x }}` / `${{ vars.x }}`. */
93+function literal(v: string | undefined, spec: JobSpec): string | null {
94+ if (v === undefined) return "";
95+ if (!v.includes("${{")) return v.trim();
96+ const m = /^\$\{\{\s*(matrix|inputs|vars)\.([\w-]+)\s*\}\}$/.exec(v.trim());
97+ if (!m) return null;
98+ const ctx = (m[1] === "matrix" ? spec.matrix : m[1] === "inputs" ? spec.inputs : spec.vars) as Record<string, unknown> | undefined;
99+ const value = ctx?.[m[2]!];
100+ return typeof value === "string" || typeof value === "number" ? String(value).trim() : null;
101+}
102+
103+function toolFor(step: StepSpec, spec: JobSpec): Tool | null {
104+ if (!step.uses || step.run !== undefined) return null;
105+ let ref;
106+ try {
107+ ref = parseUses(step.uses);
108+ } catch {
109+ return null;
110+ }
111+ if (ref.kind !== "remote") return null;
112+ const name = ref.name.toLowerCase();
113+ const inputs: Record<string, string> = {};
114+ for (const [k, v] of Object.entries(step.with ?? {})) {
115+ const lit = literal(String(v), spec);
116+ if (lit === null) return null;
117+ inputs[k.toLowerCase()] = lit;
118+ }
119+ if (name === "actions/setup-go") return goTool(ref.ref, inputs, { ...spec.env, ...step.env });
120+ if (name === "actions/setup-java") return javaTool(ref.ref, inputs);
121+ return null;
122+}
123+
124+/** Go, for the runner's own setup-go (builtins/setup-go.ts), which installs it to `go/<version>/<arch>`. */
125+function goTool(version: string, inputs: Record<string, string>, env: Record<string, string>): Tool | null {
126+ if (!getBuiltin("actions/setup-go", version, inputs, env)) return null;
127+ const spec = (inputs["go-version"] ?? "").replace(/^go/, "");
128+ if (!spec) return null;
129+ return {
130+ key: `go ${spec}`,
131+ what: `Go ${spec}`,
132+ fetch: async (toolCache, signal) => {
133+ const rel = await resolveGo(spec);
134+ if (!rel) return;
135+ const dir = path.join(toolCache, "go", rel.version, toolArch("go"));
136+ if (!isCached(dir)) await extractUrl(rel.url, dir, { sha256: rel.sha256, signal });
137+ },
138+ };
139+}
140+
141+interface AdoptiumAsset {
142+ binary: { package: { link: string; checksum?: string } };
143+ version: { semver: string };
144+}
145+
146+/**
147+ * A Temurin JDK for the real actions/setup-java, laid out in the tool cache the way it caches one
148+ * (`Java_Temurin-Hotspot_jdk/<version, + as ->/<arch>`), so it finds the JDK there and uses it, as it
149+ * does with the JDKs preinstalled on GitHub's runners. Only a major version (`java-version: 21`),
150+ * which setup-java resolves to the same latest release.
151+ */
152+function javaTool(version: string, inputs: Record<string, string>): Tool | null {
153+ const m = /^v(\d+)(\.\d+){0,2}$/.exec(version);
154+ if (!m || Number(m[1]) < 2 || Number(m[1]) > 5) return null;
155+ const major = inputs["java-version"] ?? "";
156+ const arch = process.arch === "arm64" ? "arm64" : process.arch === "x64" ? "x64" : null;
157+ if (
158+ !arch ||
159+ !/^\d+$/.test(major) ||
160+ (inputs.distribution ?? "").toLowerCase() !== "temurin" ||
161+ !["", "jdk"].includes(inputs["java-package"] ?? "") ||
162+ !["", arch].includes((inputs.architecture ?? "").toLowerCase()) ||
163+ /^true$/i.test(inputs["check-latest"] ?? "") ||
164+ inputs.jdkfile
165+ ) {
166+ return null;
167+ }
168+ return {
169+ key: `java ${major}`,
170+ what: `Java ${major} (Temurin)`,
171+ fetch: async (toolCache, signal) => {
172+ const root = path.join(toolCache, "Java_Temurin-Hotspot_jdk");
173+ // Already there (a self-hosted runner's cache): setup-java will use it.
174+ if (existsSync(root) && readdirSync(root).some((v) => v.split(/[.+-]/)[0] === major && existsSync(path.join(root, v, `${arch}.complete`)))) return;
175+ const api = process.env.DIFFINERY_ADOPTIUM_API ?? "https://api.adoptium.net";
176+ const assets = await fetchJson<AdoptiumAsset[]>(`${api}/v3/assets/latest/${major}/hotspot?architecture=${arch === "arm64" ? "aarch64" : arch}&image_type=jdk&os=linux&vendor=eclipse`);
177+ const a = assets.find((x) => x.binary?.package?.link && x.version?.semver);
178+ if (!a) return;
179+ const dir = path.join(root, a.version.semver.replace("+", "-"), arch);
180+ if (!isCached(dir)) await extractUrl(a.binary.package.link, dir, { sha256: a.binary.package.checksum, signal });
181+ },
182+ };
183+}
addedcontainers/runner/test/prefetch.test.ts+160−0View file
@@ -1,0 +1,160 @@
1+import { spawnSync } from "node:child_process";
2+import { createHash } from "node:crypto";
3+import { chmodSync, existsSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
4+import { createServer, type Server } from "node:http";
5+import path from "node:path";
6+import type { StepSpec } from "@diffinery/shared";
7+import { afterEach, beforeEach, describe, expect, it } from "vitest";
8+import { apiBaseFor, runAgentJob } from "../src/agent.js";
9+import type { JobRunner } from "../src/job.js";
10+import { ToolPrefetch } from "../src/prefetch.js";
11+import { makeSpec, startFakeApi, tmp, type FakeApi } from "./helpers.js";
12+
13+// The toolchain head start (src/prefetch.ts), against a stand-in for go.dev and the Adoptium API.
14+
15+let api: FakeApi;
16+let dir: string;
17+let server: Server;
18+let base: string;
19+/** Paths the stand-in served, with when each was asked for. */
20+let served: Array<{ path: string; at: number }>;
21+let files: Map<string, { body: Buffer; delayMs?: number }>;
22+const saved = { ...process.env };
23+
24+/** A .tar.gz of `entries` (relative path -> shell script) under one top-level folder. */
25+function tarball(top: string, entries: Record<string, string>): Buffer {
26+ const src = path.join(dir, `src-${top}`);
27+ for (const [rel, body] of Object.entries(entries)) {
28+ const f = path.join(src, top, rel);
29+ mkdirSync(path.dirname(f), { recursive: true });
30+ writeFileSync(f, body);
31+ chmodSync(f, 0o755);
32+ }
33+ const out = path.join(dir, `${top}.tar.gz`);
34+ expect(spawnSync("tar", ["-czf", out, "-C", src, top]).status).toBe(0);
35+ return readFileSync(out);
36+}
37+
38+const sha = (b: Buffer) => createHash("sha256").update(b).digest("hex");
39+
40+function goRelease(version: string, body: Buffer, sum = sha(body)) {
41+ files.set(`/dl/go${version}.linux-amd64.tar.gz`, { body, delayMs: 1500 });
42+ return { version: `go${version}`, stable: true, files: [{ filename: `go${version}.linux-amd64.tar.gz`, os: "linux", arch: "amd64", kind: "archive", sha256: sum }] };
43+}
44+
45+beforeEach(async () => {
46+ api = await startFakeApi();
47+ dir = tmp("prefetch-");
48+ served = [];
49+ files = new Map();
50+ server = createServer((req, res) => {
51+ const p = (req.url ?? "").split("?")[0]!;
52+ served.push({ path: p, at: Date.now() });
53+ const f = files.get(p);
54+ if (!f) {
55+ res.writeHead(404).end();
56+ return;
57+ }
58+ setTimeout(() => res.writeHead(200).end(f.body), f.delayMs ?? 0);
59+ });
60+ await new Promise<void>((r) => server.listen(0, "127.0.0.1", r));
61+ base = `http://127.0.0.1:${(server.address() as { port: number }).port}`;
62+ process.env.DIFFINERY_GO_INDEX = `${base}/dl/index.json`;
63+ process.env.DIFFINERY_GO_DL = `${base}/dl/`;
64+ process.env.DIFFINERY_ADOPTIUM_API = `${base}/adoptium`;
65+});
66+afterEach(async () => {
67+ process.env = { ...saved };
68+ await api.close();
69+ await new Promise((r) => server.close(r));
70+ rmSync(dir, { recursive: true, force: true });
71+});
72+
73+describe.skipIf(process.arch !== "x64")("toolchain head start", () => {
74+ it("installs Go while the job's first steps run, and setup-go uses it", async () => {
75+ const go = tarball("go", { "bin/go": `#!/bin/sh\ncase "$1" in version) echo "go version go1.99.1 linux/amd64" ;; env) echo "${dir}/gopath" ;; esac\n` });
76+ files.set("/dl/index.json", { body: Buffer.from(JSON.stringify([goRelease("1.99.1", go), goRelease("1.98.4", go)])) });
77+ const steps: StepSpec[] = [
78+ { name: "checkout stand-in", run: "sleep 1.5" },
79+ { uses: "actions/setup-go@v5", with: { "go-version": "${{ matrix.go }}", cache: "false" } },
80+ { run: "go version" },
81+ ];
82+ const spec = makeSpec(api, steps, { matrix: { go: "1.99" } });
83+ const started = Date.now();
84+ const done = await runAgentJob(spec, {
85+ apiBase: apiBaseFor(spec.apiUrl, spec.jobId),
86+ workRoot: path.join(dir, "work"),
87+ toolCache: path.join(dir, "tool"),
88+ logIntervalMs: 50,
89+ echoLogs: false,
90+ prefetchTools: true,
91+ baseEnv: { ...process.env, PATH: "/usr/local/bin:/usr/bin:/bin" },
92+ });
93+ expect(done.conclusion, api.logText()).toBe("success");
94+ expect(api.logText()).toContain("go version go1.99.1 linux/amd64");
95+ expect(api.logText()).toContain("Found in cache @");
96+ // One download, asked for as the job began (before the first step's 1.5 s were up), not by the step.
97+ const downloads = served.filter((s) => s.path.endsWith(".tar.gz"));
98+ expect(downloads.map((s) => s.path)).toEqual(["/dl/go1.99.1.linux-amd64.tar.gz"]);
99+ expect(downloads[0]!.at - started).toBeLessThan(1000);
100+ expect(existsSync(path.join(dir, "tool", "go", "1.99.1", "amd64.complete"))).toBe(true);
101+ }, 30_000);
102+
103+ it("leaves nothing behind when a download doesn't match its published checksum", async () => {
104+ const go = tarball("go", { "bin/go": "#!/bin/sh\necho go\n" });
105+ files.set("/dl/index.json", { body: Buffer.from(JSON.stringify([goRelease("1.99.1", go, "0".repeat(64))])) });
106+ const tool = path.join(dir, "tool");
107+ const p = new ToolPrefetch({ toolCache: tool, signal: new AbortController().signal } as unknown as JobRunner);
108+ p.start(makeSpec(api, [{ uses: "actions/setup-go@v5", with: { "go-version": "1.99" } }]));
109+ const lines: string[] = [];
110+ await p.wait(0, (m) => lines.push(m));
111+ expect(lines).toEqual(["Waiting for Go 1.99, which the runner started installing as the job began"]);
112+ // It was tried three times; none of it was kept.
113+ expect(served.filter((s) => s.path.endsWith(".tar.gz"))).toHaveLength(3);
114+ expect(readdirSync(path.join(tool, "go", "1.99.1"))).toEqual([]);
115+ }, 30_000);
116+
117+ it("lays a Temurin JDK out in the tool cache the way setup-java caches one", async () => {
118+ const jdk = tarball("jdk-21.0.5+11", { "bin/java": "#!/bin/sh\necho java\n", release: 'JAVA_VERSION="21.0.5"\n' });
119+ files.set("/adoptium/v3/assets/latest/21/hotspot", {
120+ body: Buffer.from(JSON.stringify([{ binary: { package: { link: `${base}/jdk.tar.gz`, checksum: sha(jdk) } }, version: { semver: "21.0.5+11.0.LTS" } }])),
121+ });
122+ files.set("/jdk.tar.gz", { body: jdk, delayMs: 200 });
123+ const tool = path.join(dir, "tool");
124+ const p = new ToolPrefetch({ toolCache: tool, signal: new AbortController().signal } as unknown as JobRunner);
125+ p.start(makeSpec(api, [{ run: "true" }, { uses: "actions/setup-java@v4", with: { distribution: "temurin", "java-version": "21", cache: "maven" } }]));
126+ await p.wait(0, () => {});
127+ await p.wait(1, () => {});
128+ const home = path.join(tool, "Java_Temurin-Hotspot_jdk", "21.0.5-11.0.LTS", "x64");
129+ expect(readFileSync(path.join(home, "release"), "utf8")).toContain("21.0.5");
130+ expect(existsSync(path.join(home, "bin", "java"))).toBe(true);
131+ expect(existsSync(`${home}.complete`)).toBe(true);
132+ expect(readdirSync(path.dirname(home))).toEqual(["x64", "x64.complete"]);
133+
134+ // A second job on the same machine finds it there and downloads nothing.
135+ served = [];
136+ const again = new ToolPrefetch({ toolCache: tool, signal: new AbortController().signal } as unknown as JobRunner);
137+ again.start(makeSpec(api, [{ uses: "actions/setup-java@v4", with: { distribution: "temurin", "java-version": "21" } }]));
138+ await again.wait(0, () => {});
139+ expect(served).toEqual([]);
140+ }, 30_000);
141+
142+ it("does nothing for jobs without a toolchain it can name", async () => {
143+ const tool = path.join(dir, "tool");
144+ const steps: StepSpec[] = [
145+ { run: "go version" },
146+ { uses: "actions/setup-java@v4", with: { distribution: "zulu", "java-version": "21" } },
147+ { uses: "actions/setup-java@v4", with: { distribution: "temurin", "java-version": "${{ steps.v.outputs.java }}" } },
148+ { uses: "actions/setup-java@v4", with: { distribution: "temurin", "java-version": "21", "java-package": "jre" } },
149+ { uses: "actions/setup-go@v5", with: { "go-version-file": "go.mod" } },
150+ { uses: "actions/setup-go@v2", with: { "go-version": "1.22" } },
151+ { uses: "actions/setup-node@v4", with: { "node-version": "22" } },
152+ ];
153+ const p = new ToolPrefetch({ toolCache: tool, signal: new AbortController().signal } as unknown as JobRunner);
154+ p.start(makeSpec(api, steps));
155+ p.start(makeSpec(api, [{ uses: "actions/setup-go@v5", with: { "go-version": "1.22" } }], { container: { image: "golang:1.22" } }));
156+ for (let i = 0; i < steps.length; i++) await p.wait(i, () => {});
157+ await p.stop();
158+ expect(served).toEqual([]);
159+ });
160+});