Diffinery

codician/diffinerypublic

Open-source, agent-first GitHub and GitHub Actions alternative that runs on your own Cloudflare account: git hosting, pull requests, code review and drop-in GitHub Actions YAML on Workers, Durable Objects, R2 and Containers.

487 lines · 20.5 KBRaw History
1import { existsSync } from "node:fs";
2import { mkdir, readFile, rename, rm, writeFile } from "node:fs/promises";
3import path from "node:path";
4import type { StepSpec } from "@diffinery/shared";
5import { getBuiltin, makeBuiltinContext } from "../builtins/index.js";
6import { errMsg, failure, StepFailure, type JobRunner, type Scope, type StepIO } from "../job.js";
7import { exec, shellWords } from "../util.js";
8import { parseYaml } from "../yaml.js";
9import type { StepExec } from "./run.js";
10
11export type UsesRef =
12 | { kind: "docker"; image: string }
13 | { kind: "local"; path: string }
14 | { kind: "remote"; owner: string; repo: string; subpath: string; ref: string; name: string };
15
16export function parseUses(uses: string): UsesRef {
17 const u = uses.trim();
18 if (u.startsWith("docker://")) return { kind: "docker", image: u.slice("docker://".length) };
19 if (u.startsWith("./") || u.startsWith("../") || u === ".") return { kind: "local", path: u };
20 const at = u.lastIndexOf("@");
21 if (at <= 0) throw failure(`Expected format {owner}/{repo}[/path]@ref. Actual '${u}'`);
22 const ref = u.slice(at + 1);
23 const parts = u.slice(0, at).split("/");
24 if (parts.length < 2 || !parts[0] || !parts[1] || !ref) throw failure(`Expected format {owner}/{repo}[/path]@ref. Actual '${u}'`);
25 return { kind: "remote", owner: parts[0], repo: parts[1], subpath: parts.slice(2).join("/"), ref, name: parts.join("/") };
26}
27
28export interface ActionInput {
29 description?: string;
30 required?: boolean;
31 default?: unknown;
32 deprecationMessage?: string;
33}
34
35export interface ActionMeta {
36 name?: string;
37 inputs?: Record<string, ActionInput | null>;
38 outputs?: Record<string, { description?: string; value?: string } | null>;
39 runs: {
40 using: string;
41 main?: string;
42 pre?: string;
43 "pre-if"?: string;
44 post?: string;
45 "post-if"?: string;
46 steps?: StepSpec[];
47 image?: string;
48 env?: Record<string, string>;
49 entrypoint?: string;
50 args?: unknown[];
51 "pre-entrypoint"?: string;
52 "post-entrypoint"?: string;
53 };
54}
55
56export async function loadActionMeta(dir: string): Promise<ActionMeta> {
57 for (const f of ["action.yml", "action.yaml"]) {
58 const p = path.join(dir, f);
59 if (existsSync(p)) {
60 const meta = parseYaml(await readFile(p, "utf8")) as ActionMeta | null;
61 if (!meta || typeof meta !== "object" || !meta.runs || typeof meta.runs !== "object") {
62 throw failure(`Invalid action metadata in ${p}: missing 'runs'`);
63 }
64 return meta;
65 }
66 }
67 if (existsSync(path.join(dir, "Dockerfile"))) return { runs: { using: "docker", image: "Dockerfile" } };
68 throw failure(`Can't find 'action.yml' or 'action.yaml' under '${dir}'. Did you forget to run actions/checkout before running your local action?`);
69}
70
71/** Downloads `owner/repo@ref` into the action cache and returns its root dir. */
72export async function downloadAction(job: JobRunner, io: StepIO, owner: string, repo: string, ref: string): Promise<string> {
73 const dest = path.join(job.actionsDir, owner, repo, ref);
74 if (existsSync(path.join(dest, ".diffinery-complete"))) return dest;
75 const tpl = process.env.DIFFINERY_ACTIONS_DOWNLOAD_URL ?? "https://codeload.github.com/{owner}/{repo}/tar.gz/{ref}";
76 const url = tpl.replace("{owner}", owner).replace("{repo}", repo).replace("{ref}", encodeURIComponent(ref));
77 job.logger.line(io.logIndex, `Download action repository '${owner}/${repo}@${ref}'`);
78 const headers: Record<string, string> = { "user-agent": "diffinery-runner" };
79 const ghToken = GITHUB_TOKEN;
80 if (ghToken && url.startsWith("https://codeload.github.com/")) headers.authorization = `token ${ghToken}`;
81 let lastErr: unknown;
82 for (let attempt = 0; attempt < 4; attempt++) {
83 if (attempt) await new Promise((r) => setTimeout(r, 1000 * 2 ** attempt));
84 try {
85 const res = await fetch(url, { headers, redirect: "follow" });
86 if (res.status === 404) throw failure(`Unable to resolve action '${owner}/${repo}@${ref}', repository or version not found`);
87 if (res.status === 401 || res.status === 403) {
88 lastErr = new Error(`HTTP ${res.status} downloading ${url}`);
89 break;
90 }
91 if (!res.ok) throw new Error(`HTTP ${res.status} downloading ${url}`);
92 const buf = Buffer.from(await res.arrayBuffer());
93 const tmp = `${dest}.tmp-${process.pid}-${Date.now()}`;
94 await mkdir(tmp, { recursive: true });
95 const tarFile = `${tmp}.tar.gz`;
96 await writeFile(tarFile, buf);
97 const r = await exec("tar", ["-xzf", tarFile, "-C", tmp, "--strip-components=1"], { capture: true });
98 await rm(tarFile, { force: true });
99 if (r.exitCode !== 0) throw new Error(`Failed to extract action archive: ${r.stderr.trim()}`);
100 await rm(dest, { recursive: true, force: true });
101 await mkdir(path.dirname(dest), { recursive: true });
102 await rename(tmp, dest);
103 await writeFile(path.join(dest, ".diffinery-complete"), "");
104 return dest;
105 } catch (e) {
106 if (e instanceof StepFailure) throw e;
107 lastErr = e;
108 }
109 }
110 // The tarball endpoint can be unreachable (proxies, rate limits) while git still works.
111 if (!process.env.DIFFINERY_ACTIONS_DOWNLOAD_URL && (await cloneAction(dest, owner, repo, ref, ghToken))) return dest;
112 throw failure(`Failed to download action '${owner}/${repo}@${ref}': ${errMsg(lastErr)}`);
113}
114
115/** Fetches `owner/repo@ref` (branch, tag or commit SHA) from github.com with git; false when that fails. */
116async function cloneAction(dest: string, owner: string, repo: string, ref: string, token: string | undefined): Promise<boolean> {
117 if (!/^[\w.-]+$/.test(owner) || !/^[\w.-]+$/.test(repo) || ref.startsWith("-") || /[\s\\]|\.\.|[~^:?*[]/.test(ref)) return false;
118 const tmp = `${dest}.tmp-${process.pid}-${Date.now()}`;
119 // The header goes through the environment, not argv, where other processes could read it.
120 const n = Number(process.env.GIT_CONFIG_COUNT) || 0;
121 const auth: Record<string, string> = token
122 ? { GIT_CONFIG_COUNT: String(n + 1), [`GIT_CONFIG_KEY_${n}`]: "http.https://github.com/.extraheader", [`GIT_CONFIG_VALUE_${n}`]: `AUTHORIZATION: basic ${Buffer.from(`x-access-token:${token}`).toString("base64")}` }
123 : {};
124 const git = (args: string[]) => exec("git", args, { cwd: tmp, capture: true, env: { ...process.env, GIT_TERMINAL_PROMPT: "0", ...auth } });
125 try {
126 await mkdir(tmp, { recursive: true });
127 for (const args of [["init", "-q"], ["fetch", "-q", "--depth", "1", `https://github.com/${owner}/${repo}`, ref], ["checkout", "-q", "FETCH_HEAD"]]) {
128 if ((await git(args)).exitCode !== 0) throw new Error(args[0]);
129 }
130 await rm(path.join(tmp, ".git"), { recursive: true, force: true });
131 await rm(dest, { recursive: true, force: true });
132 await mkdir(path.dirname(dest), { recursive: true });
133 await rename(tmp, dest);
134 await writeFile(path.join(dest, ".diffinery-complete"), "");
135 return true;
136 } catch {
137 await rm(tmp, { recursive: true, force: true });
138 return false;
139 }
140}
141
142export function inputEnvName(name: string): string {
143 return `INPUT_${name.replace(/ /g, "_").toUpperCase()}`;
144}
145
146/** Evaluates `with:` plus defaults from action.yml. */
147export function computeInputs(job: JobRunner, x: StepExec, meta: ActionMeta | null): Record<string, string> {
148 const inputs: Record<string, string> = {};
149 const given = new Map<string, string>();
150 for (const [k, v] of Object.entries(x.step.with ?? {})) {
151 given.set(k.toLowerCase(), job.interp(v === null || v === undefined ? "" : String(v), x.ctx, x.scope));
152 inputs[k] = given.get(k.toLowerCase())!;
153 }
154 for (const [k, def] of Object.entries(meta?.inputs ?? {})) {
155 if (given.has(k.toLowerCase())) {
156 // normalise the key to the declared spelling
157 const actual = Object.keys(inputs).find((x) => x.toLowerCase() === k.toLowerCase())!;
158 if (actual !== k) {
159 inputs[k] = inputs[actual]!;
160 delete inputs[actual];
161 }
162 if (def?.deprecationMessage) job.logger.line(x.io.logIndex, `##[warning]Input '${k}' has been deprecated with message: ${def.deprecationMessage}`);
163 continue;
164 }
165 if (def && def.default !== undefined && def.default !== null) {
166 inputs[k] = job.interp(String(def.default), x.ctx, x.scope);
167 } else if (def?.required) {
168 job.logger.line(x.io.logIndex, `##[warning]Input required and not supplied: ${k}`);
169 }
170 }
171 return inputs;
172}
173
174export async function runAction(job: JobRunner, x: StepExec): Promise<void> {
175 const uses = job.interp(String(x.step.uses), x.ctx, x.scope);
176 const ref = parseUses(uses);
177 if (ref.kind === "docker") {
178 // `uses: docker://image`: `with.args` and `with.entrypoint` drive the container; the rest become INPUT_*.
179 const given = computeInputs(job, x, null);
180 const inputs: Record<string, string> = {};
181 let args: string[] = [];
182 let entrypoint: string | undefined;
183 for (const [k, v] of Object.entries(given)) {
184 if (k.toLowerCase() === "args") args = shellWords(v);
185 else if (k.toLowerCase() === "entrypoint") entrypoint = v || undefined;
186 else inputs[k] = v;
187 }
188 logActionHeader(job, x.io, uses, given);
189 await runDockerAction(job, x, { image: `docker://${ref.image}`, dir: null, args, entrypoint, env: {}, inputs, action: { path: "", repository: "", ref: "" } });
190 return;
191 }
192
193 if (ref.kind === "remote") {
194 const given = computeInputs(job, x, null);
195 const builtin = getBuiltin(ref.name, ref.ref, given, x.envCtx);
196 if (builtin) {
197 const inputs = given;
198 logActionHeader(job, x.io, uses, inputs);
199 const c = makeBuiltinContext(job, x, inputs, uses);
200 await builtin(c);
201 return;
202 }
203 }
204
205 let dir: string;
206 let repository: string;
207 let actionRef: string;
208 if (ref.kind === "local") {
209 dir = path.resolve(job.workspace, ref.path);
210 repository = "";
211 actionRef = "";
212 } else {
213 const root = await downloadAction(job, x.io, ref.owner, ref.repo, ref.ref);
214 dir = ref.subpath ? path.join(root, ref.subpath) : root;
215 repository = `${ref.owner}/${ref.repo}`;
216 actionRef = ref.ref;
217 }
218 const meta = await loadActionMeta(dir);
219 const using = String(meta.runs.using ?? "").toLowerCase();
220 const inputs = computeInputs(job, x, meta);
221 logActionHeader(job, x.io, uses, inputs);
222 const actionInfo = { path: dir, repository, ref: actionRef };
223
224 if (using === "docker" || meta.runs.image) {
225 const runs = meta.runs;
226 if (!runs.image) throw failure(`Action '${uses}' uses Docker but has no 'runs.image'`);
227 const actx = job.contexts({ ...x.scope, inputs }, x.envCtx);
228 const ev = (v: unknown) => job.interp(v === undefined || v === null ? "" : String(v), actx, x.scope);
229 const args = Array.isArray(runs.args) ? runs.args.map(ev) : [];
230 const env: Record<string, string> = {};
231 for (const [k, v] of Object.entries(runs.env ?? {})) env[k] = ev(v);
232 await runDockerAction(job, x, {
233 image: runs.image,
234 dir,
235 args,
236 entrypoint: runs.entrypoint ? ev(runs.entrypoint) : undefined,
237 pre: runs["pre-entrypoint"] ? ev(runs["pre-entrypoint"]) : undefined,
238 post: runs["post-entrypoint"] ? ev(runs["post-entrypoint"]) : undefined,
239 preIf: runs["pre-if"],
240 postIf: runs["post-if"],
241 env,
242 inputs,
243 action: actionInfo,
244 });
245 return;
246 }
247 if (using === "composite") {
248 await runComposite(job, x, meta, inputs, actionInfo);
249 return;
250 }
251 if (/^node\d+$/.test(using)) {
252 await runNodeAction(job, x, meta, inputs, actionInfo);
253 return;
254 }
255 throw failure(`Unsupported action runtime 'runs.using: ${meta.runs.using}' in ${uses}`);
256}
257
258function logActionHeader(job: JobRunner, io: StepIO, uses: string, inputs: Record<string, string>): void {
259 job.logger.line(io.logIndex, `##[group]Run ${uses}`);
260 const keys = Object.keys(inputs);
261 if (keys.length) {
262 job.logger.line(io.logIndex, "with:");
263 for (const k of keys) job.logger.line(io.logIndex, ` ${k}: ${inputs[k]!.split("\n").join("\n ")}`);
264 }
265 job.logger.line(io.logIndex, "##[endgroup]");
266}
267
268async function runNodeAction(
269 job: JobRunner,
270 x: StepExec,
271 meta: ActionMeta,
272 inputs: Record<string, string>,
273 action: { path: string; repository: string; ref: string },
274): Promise<void> {
275 const runs = meta.runs;
276 if (!runs.main) throw failure(`Action '${x.step.uses}' has no 'runs.main'`);
277 const inputEnv: Record<string, string> = {};
278 for (const [k, v] of Object.entries(inputs)) inputEnv[inputEnvName(k)] = v;
279 const baseExtra: Record<string, string> = {
280 ...job.runtimeEnv,
281 ...inputEnv,
282 GITHUB_ACTION: x.step.id ?? "__" + (action.repository || "local").replace(/[^A-Za-z0-9]/g, "_"),
283 GITHUB_ACTION_PATH: action.path,
284 GITHUB_ACTION_REPOSITORY: action.repository,
285 GITHUB_ACTION_REF: action.ref,
286 };
287 // @actions/artifact refuses to run on anything that looks like GitHub Enterprise Server
288 // (any GITHUB_SERVER_URL but github.com), so the artifact actions see github.com; their
289 // API calls still go to GITHUB_API_URL and the runtime services, and the URLs they
290 // output are pointed back at this server.
291 const serverUrl = job.processEnv(x.envCtx).GITHUB_SERVER_URL ?? "";
292 const artifactAction = ARTIFACT_ACTIONS.has(action.repository.toLowerCase());
293 if (artifactAction) baseExtra.GITHUB_SERVER_URL = "https://github.com";
294 const state: Record<string, string> = {};
295 const runScriptFile = async (file: string, io: StepIO, envCtx: Record<string, string>, extra: Record<string, string>, label: string) => {
296 const res = await job.runStepProcess([job.nodePath(), path.join(action.path, file)], {
297 cwd: job.workspace,
298 envCtx,
299 io,
300 timeoutMs: x.timeoutMs,
301 extraEnv: extra,
302 wrap: job.containerWrap(),
303 });
304 if (job.isStopping) return;
305 if (res.timedOut) throw failure(io.reason ?? "The step timed out.", true);
306 if (res.exitCode !== 0) {
307 const msg = `${label}: Process completed with exit code ${res.exitCode}.`;
308 job.logger.line(io.logIndex, `##[error]${msg}`);
309 job.addAnnotation(io.logIndex, { level: "error", message: msg });
310 io.reason = msg;
311 throw failure(msg, true);
312 }
313 };
314
315 const ctxScope = x.scope;
316 if (runs.pre) {
317 const cond = runs["pre-if"] ?? "always()";
318 let ok = true;
319 try {
320 ok = job.evalIf(cond, x.ctx, ctxScope);
321 } catch (e) {
322 throw failure(`Error evaluating pre-if: ${errMsg(e)}`);
323 }
324 if (ok) {
325 const preIo: StepIO = { logIndex: x.io.logIndex, outputs: {}, state: {} };
326 job.logger.line(x.io.logIndex, `Running pre: ${runs.pre}`);
327 await runScriptFile(runs.pre, preIo, x.envCtx, baseExtra, "pre");
328 Object.assign(state, preIo.state);
329 }
330 }
331 if (runs.post) {
332 const postFile = runs.post;
333 const envAtMain = { ...x.envCtx };
334 job.registerPost({
335 name: `Post ${x.name}`,
336 condition: runs["post-if"] ?? "always()",
337 run: async (logIndex) => {
338 const stateEnv: Record<string, string> = {};
339 for (const [k, v] of Object.entries(state)) stateEnv[`STATE_${k}`] = v;
340 const postIo: StepIO = { logIndex, outputs: {}, state: {} };
341 await runScriptFile(postFile, postIo, { ...envAtMain, ...job.envAccum }, { ...baseExtra, ...stateEnv }, "post");
342 },
343 });
344 }
345 const mainIo = x.io;
346 try {
347 await runScriptFile(runs.main, mainIo, x.envCtx, { ...baseExtra, ...stateEnvOf(state) }, "main");
348 } finally {
349 Object.assign(state, mainIo.state);
350 const url = mainIo.outputs["artifact-url"];
351 if (artifactAction && serverUrl && url?.startsWith("https://github.com/")) mainIo.outputs["artifact-url"] = `${serverUrl}/${url.slice("https://github.com/".length)}`;
352 }
353}
354
355const ARTIFACT_ACTIONS = new Set(["actions/upload-artifact", "actions/download-artifact"]);
356
357/**
358 * The operator's github.com token for downloading actions (rate limits, private
359 * actions), read once when the runner starts. Steps never see it: job.ts strips
360 * it from every child environment.
361 */
362const GITHUB_TOKEN = process.env.DIFFINERY_GITHUB_TOKEN || undefined;
363
364/** Docker container actions: `uses: docker://image`, or an action with `runs.using: docker`. */
365async function runDockerAction(
366 job: JobRunner,
367 x: StepExec,
368 d: {
369 image: string;
370 dir: string | null;
371 args: string[];
372 entrypoint?: string;
373 pre?: string;
374 post?: string;
375 preIf?: string;
376 postIf?: string;
377 env: Record<string, string>;
378 inputs: Record<string, string>;
379 action: { path: string; repository: string; ref: string };
380 },
381): Promise<void> {
382 const image = await job.docker.prepareAction(x.io, d.image, d.dir);
383 const extra: Record<string, string> = {
384 ...job.runtimeEnv,
385 GITHUB_ACTION: x.step.id ?? "__" + (d.action.repository || "docker").replace(/[^A-Za-z0-9]/g, "_"),
386 GITHUB_ACTION_REPOSITORY: d.action.repository,
387 GITHUB_ACTION_REF: d.action.ref,
388 };
389 for (const [k, v] of Object.entries(d.inputs)) extra[inputEnvName(k)] = v;
390 const runIn = async (io: StepIO, envCtx: Record<string, string>, o: { entrypoint?: string; args: string[] }, label: string) => {
391 const wrap = await job.docker.runWrap(io, image, o);
392 const res = await job.runStepProcess([], { cwd: job.workspace, envCtx: { ...envCtx, ...d.env }, io, timeoutMs: x.timeoutMs, extraEnv: extra, wrap });
393 if (job.isStopping) return;
394 if (res.timedOut) throw failure(io.reason ?? "The step timed out.", true);
395 if (res.exitCode !== 0) {
396 const msg = `${label}Docker run failed with exit code ${res.exitCode}`;
397 job.logger.line(io.logIndex, `##[error]${msg}`);
398 job.addAnnotation(io.logIndex, { level: "error", message: msg });
399 io.reason = msg;
400 throw failure(msg, true);
401 }
402 };
403 if (d.pre) {
404 let ok = true;
405 try {
406 ok = job.evalIf(d.preIf ?? "always()", x.ctx, x.scope);
407 } catch (e) {
408 throw failure(`Error evaluating pre-if: ${errMsg(e)}`);
409 }
410 if (ok) await runIn({ logIndex: x.io.logIndex, outputs: {}, state: {} }, x.envCtx, { entrypoint: d.pre, args: [] }, "pre: ");
411 }
412 if (d.post) {
413 const post = d.post;
414 const envAtMain = { ...x.envCtx };
415 job.registerPost({
416 name: `Post ${x.name}`,
417 condition: d.postIf ?? "always()",
418 run: async (logIndex) => runIn({ logIndex, outputs: {}, state: {} }, { ...envAtMain, ...job.envAccum }, { entrypoint: post, args: [] }, "post: "),
419 });
420 }
421 await runIn(x.io, x.envCtx, { entrypoint: d.entrypoint, args: d.args }, "");
422}
423
424function stateEnvOf(state: Record<string, string>): Record<string, string> {
425 const out: Record<string, string> = {};
426 for (const [k, v] of Object.entries(state)) out[`STATE_${k}`] = v;
427 return out;
428}
429
430async function runComposite(
431 job: JobRunner,
432 x: StepExec,
433 meta: ActionMeta,
434 inputs: Record<string, string>,
435 action: { path: string; repository: string; ref: string },
436): Promise<void> {
437 if (x.scope.depth > 9) throw failure("Composite action nesting is too deep (max 10).");
438 const steps = Array.isArray(meta.runs.steps) ? meta.runs.steps : [];
439 const scope: Scope = {
440 steps: {},
441 inputs,
442 failed: false,
443 inheritedEnv: { ...x.envCtx },
444 composite: true,
445 action,
446 depth: x.scope.depth + 1,
447 };
448 for (const s of steps) {
449 if (job.isStopping) break;
450 const sub = normaliseStep(s);
451 await job.executeStep(sub, scope, x.io.logIndex);
452 }
453 // outputs
454 const ctx = job.contexts(scope, job.baseEnvContext(scope));
455 for (const [k, o] of Object.entries(meta.outputs ?? {})) {
456 if (!o || o.value === undefined || o.value === null) continue;
457 try {
458 x.io.outputs[k] = job.interp(String(o.value), ctx, scope);
459 } catch (e) {
460 job.logger.line(x.io.logIndex, `##[warning]Failed to evaluate output '${k}': ${errMsg(e)}`);
461 }
462 }
463 if (job.isStopping) return;
464 if (scope.failed) throw failure(`Composite action '${x.step.uses}' failed.`, true);
465}
466
467function normaliseStep(s: unknown): StepSpec {
468 const o = (s ?? {}) as Record<string, unknown>;
469 const step: StepSpec = {};
470 if (o.id !== undefined) step.id = String(o.id);
471 if (o.name !== undefined) step.name = String(o.name);
472 if (o.if !== undefined && o.if !== null) step.if = typeof o.if === "boolean" ? String(o.if) : String(o.if);
473 if (o.run !== undefined) step.run = String(o.run);
474 if (o.uses !== undefined) step.uses = String(o.uses);
475 if (o.shell !== undefined) step.shell = String(o.shell);
476 if (o["working-directory"] !== undefined) step["working-directory"] = String(o["working-directory"]);
477 if (o["continue-on-error"] !== undefined) step["continue-on-error"] = o["continue-on-error"] as boolean | string;
478 if (o["timeout-minutes"] !== undefined) step["timeout-minutes"] = Number(o["timeout-minutes"]);
479 const strMap = (m: unknown) => {
480 const r: Record<string, string> = {};
481 if (m && typeof m === "object") for (const [k, v] of Object.entries(m)) r[k] = v === null || v === undefined ? "" : String(v);
482 return r;
483 };
484 if (o.with) step.with = strMap(o.with);
485 if (o.env) step.env = strMap(o.env);
486 return step;
487}