Diffinery

codician/diffinerypublic

Open-source, agent-first GitHub and GitHub Actions alternative that runs on your own Cloudflare account: git hosting, pull requests, code review and drop-in GitHub Actions YAML on Workers, Durable Objects, R2 and Containers.

433 lines · 17.4 KBRaw History
1import { existsSync } from "node:fs";
2import { mkdir, readFile, rename, rm, writeFile } from "node:fs/promises";
3import path from "node:path";
4import type { StepSpec } from "@diffinery/shared";
5import { getBuiltin, makeBuiltinContext } from "../builtins/index.js";
6import { errMsg, failure, StepFailure, type JobRunner, type Scope, type StepIO } from "../job.js";
7import { exec, shellWords } from "../util.js";
8import { parseYaml } from "../yaml.js";
9import type { StepExec } from "./run.js";
10
11export type UsesRef =
12 | { kind: "docker"; image: string }
13 | { kind: "local"; path: string }
14 | { kind: "remote"; owner: string; repo: string; subpath: string; ref: string; name: string };
15
16export function parseUses(uses: string): UsesRef {
17 const u = uses.trim();
18 if (u.startsWith("docker://")) return { kind: "docker", image: u.slice("docker://".length) };
19 if (u.startsWith("./") || u.startsWith("../") || u === ".") return { kind: "local", path: u };
20 const at = u.lastIndexOf("@");
21 if (at <= 0) throw failure(`Expected format {owner}/{repo}[/path]@ref. Actual '${u}'`);
22 const ref = u.slice(at + 1);
23 const parts = u.slice(0, at).split("/");
24 if (parts.length < 2 || !parts[0] || !parts[1] || !ref) throw failure(`Expected format {owner}/{repo}[/path]@ref. Actual '${u}'`);
25 return { kind: "remote", owner: parts[0], repo: parts[1], subpath: parts.slice(2).join("/"), ref, name: parts.join("/") };
26}
27
28export interface ActionInput {
29 description?: string;
30 required?: boolean;
31 default?: unknown;
32 deprecationMessage?: string;
33}
34
35export interface ActionMeta {
36 name?: string;
37 inputs?: Record<string, ActionInput | null>;
38 outputs?: Record<string, { description?: string; value?: string } | null>;
39 runs: {
40 using: string;
41 main?: string;
42 pre?: string;
43 "pre-if"?: string;
44 post?: string;
45 "post-if"?: string;
46 steps?: StepSpec[];
47 image?: string;
48 env?: Record<string, string>;
49 entrypoint?: string;
50 args?: unknown[];
51 "pre-entrypoint"?: string;
52 "post-entrypoint"?: string;
53 };
54}
55
56export async function loadActionMeta(dir: string): Promise<ActionMeta> {
57 for (const f of ["action.yml", "action.yaml"]) {
58 const p = path.join(dir, f);
59 if (existsSync(p)) {
60 const meta = parseYaml(await readFile(p, "utf8")) as ActionMeta | null;
61 if (!meta || typeof meta !== "object" || !meta.runs || typeof meta.runs !== "object") {
62 throw failure(`Invalid action metadata in ${p}: missing 'runs'`);
63 }
64 return meta;
65 }
66 }
67 if (existsSync(path.join(dir, "Dockerfile"))) return { runs: { using: "docker", image: "Dockerfile" } };
68 throw failure(`Can't find 'action.yml' or 'action.yaml' under '${dir}'. Did you forget to run actions/checkout before running your local action?`);
69}
70
71/** Downloads `owner/repo@ref` into the action cache and returns its root dir. */
72export async function downloadAction(job: JobRunner, io: StepIO, owner: string, repo: string, ref: string): Promise<string> {
73 const dest = path.join(job.actionsDir, owner, repo, ref);
74 if (existsSync(path.join(dest, ".diffinery-complete"))) return dest;
75 const tpl = process.env.DIFFINERY_ACTIONS_DOWNLOAD_URL ?? "https://codeload.github.com/{owner}/{repo}/tar.gz/{ref}";
76 const url = tpl.replace("{owner}", owner).replace("{repo}", repo).replace("{ref}", encodeURIComponent(ref));
77 job.logger.line(io.logIndex, `Download action repository '${owner}/${repo}@${ref}'`);
78 const headers: Record<string, string> = { "user-agent": "diffinery-runner" };
79 const ghToken = process.env.DIFFINERY_GITHUB_TOKEN;
80 if (ghToken && url.startsWith("https://codeload.github.com/")) headers.authorization = `token ${ghToken}`;
81 let lastErr: unknown;
82 for (let attempt = 0; attempt < 4; attempt++) {
83 if (attempt) await new Promise((r) => setTimeout(r, 1000 * 2 ** attempt));
84 try {
85 const res = await fetch(url, { headers, redirect: "follow" });
86 if (res.status === 404) throw failure(`Unable to resolve action '${owner}/${repo}@${ref}', repository or version not found`);
87 if (!res.ok) throw new Error(`HTTP ${res.status} downloading ${url}`);
88 const buf = Buffer.from(await res.arrayBuffer());
89 const tmp = `${dest}.tmp-${process.pid}-${Date.now()}`;
90 await mkdir(tmp, { recursive: true });
91 const tarFile = `${tmp}.tar.gz`;
92 await writeFile(tarFile, buf);
93 const r = await exec("tar", ["-xzf", tarFile, "-C", tmp, "--strip-components=1"], { capture: true });
94 await rm(tarFile, { force: true });
95 if (r.exitCode !== 0) throw new Error(`Failed to extract action archive: ${r.stderr.trim()}`);
96 await rm(dest, { recursive: true, force: true });
97 await mkdir(path.dirname(dest), { recursive: true });
98 await rename(tmp, dest);
99 await writeFile(path.join(dest, ".diffinery-complete"), "");
100 return dest;
101 } catch (e) {
102 if (e instanceof StepFailure) throw e;
103 lastErr = e;
104 }
105 }
106 throw failure(`Failed to download action '${owner}/${repo}@${ref}': ${errMsg(lastErr)}`);
107}
108
109export function inputEnvName(name: string): string {
110 return `INPUT_${name.replace(/ /g, "_").toUpperCase()}`;
111}
112
113/** Evaluates `with:` plus defaults from action.yml. */
114export function computeInputs(job: JobRunner, x: StepExec, meta: ActionMeta | null): Record<string, string> {
115 const inputs: Record<string, string> = {};
116 const given = new Map<string, string>();
117 for (const [k, v] of Object.entries(x.step.with ?? {})) {
118 given.set(k.toLowerCase(), job.interp(v === null || v === undefined ? "" : String(v), x.ctx, x.scope));
119 inputs[k] = given.get(k.toLowerCase())!;
120 }
121 for (const [k, def] of Object.entries(meta?.inputs ?? {})) {
122 if (given.has(k.toLowerCase())) {
123 // normalise the key to the declared spelling
124 const actual = Object.keys(inputs).find((x) => x.toLowerCase() === k.toLowerCase())!;
125 if (actual !== k) {
126 inputs[k] = inputs[actual]!;
127 delete inputs[actual];
128 }
129 if (def?.deprecationMessage) job.logger.line(x.io.logIndex, `##[warning]Input '${k}' has been deprecated with message: ${def.deprecationMessage}`);
130 continue;
131 }
132 if (def && def.default !== undefined && def.default !== null) {
133 inputs[k] = job.interp(String(def.default), x.ctx, x.scope);
134 } else if (def?.required) {
135 job.logger.line(x.io.logIndex, `##[warning]Input required and not supplied: ${k}`);
136 }
137 }
138 return inputs;
139}
140
141export async function runAction(job: JobRunner, x: StepExec): Promise<void> {
142 const uses = job.interp(String(x.step.uses), x.ctx, x.scope);
143 const ref = parseUses(uses);
144 if (ref.kind === "docker") {
145 // `uses: docker://image`: `with.args` and `with.entrypoint` drive the container; the rest become INPUT_*.
146 const given = computeInputs(job, x, null);
147 const inputs: Record<string, string> = {};
148 let args: string[] = [];
149 let entrypoint: string | undefined;
150 for (const [k, v] of Object.entries(given)) {
151 if (k.toLowerCase() === "args") args = shellWords(v);
152 else if (k.toLowerCase() === "entrypoint") entrypoint = v || undefined;
153 else inputs[k] = v;
154 }
155 logActionHeader(job, x.io, uses, given);
156 await runDockerAction(job, x, { image: `docker://${ref.image}`, dir: null, args, entrypoint, env: {}, inputs, action: { path: "", repository: "", ref: "" } });
157 return;
158 }
159
160 if (ref.kind === "remote") {
161 const builtin = getBuiltin(ref.name, ref.ref);
162 if (builtin) {
163 const inputs = computeInputs(job, x, null);
164 logActionHeader(job, x.io, uses, inputs);
165 const c = makeBuiltinContext(job, x, inputs, uses);
166 await builtin(c);
167 return;
168 }
169 }
170
171 let dir: string;
172 let repository: string;
173 let actionRef: string;
174 if (ref.kind === "local") {
175 dir = path.resolve(job.workspace, ref.path);
176 repository = "";
177 actionRef = "";
178 } else {
179 const root = await downloadAction(job, x.io, ref.owner, ref.repo, ref.ref);
180 dir = ref.subpath ? path.join(root, ref.subpath) : root;
181 repository = `${ref.owner}/${ref.repo}`;
182 actionRef = ref.ref;
183 }
184 const meta = await loadActionMeta(dir);
185 const using = String(meta.runs.using ?? "").toLowerCase();
186 const inputs = computeInputs(job, x, meta);
187 logActionHeader(job, x.io, uses, inputs);
188 const actionInfo = { path: dir, repository, ref: actionRef };
189
190 if (using === "docker" || meta.runs.image) {
191 const runs = meta.runs;
192 if (!runs.image) throw failure(`Action '${uses}' uses Docker but has no 'runs.image'`);
193 const actx = job.contexts({ ...x.scope, inputs }, x.envCtx);
194 const ev = (v: unknown) => job.interp(v === undefined || v === null ? "" : String(v), actx, x.scope);
195 const args = Array.isArray(runs.args) ? runs.args.map(ev) : [];
196 const env: Record<string, string> = {};
197 for (const [k, v] of Object.entries(runs.env ?? {})) env[k] = ev(v);
198 await runDockerAction(job, x, {
199 image: runs.image,
200 dir,
201 args,
202 entrypoint: runs.entrypoint ? ev(runs.entrypoint) : undefined,
203 pre: runs["pre-entrypoint"] ? ev(runs["pre-entrypoint"]) : undefined,
204 post: runs["post-entrypoint"] ? ev(runs["post-entrypoint"]) : undefined,
205 preIf: runs["pre-if"],
206 postIf: runs["post-if"],
207 env,
208 inputs,
209 action: actionInfo,
210 });
211 return;
212 }
213 if (using === "composite") {
214 await runComposite(job, x, meta, inputs, actionInfo);
215 return;
216 }
217 if (/^node\d+$/.test(using)) {
218 await runNodeAction(job, x, meta, inputs, actionInfo);
219 return;
220 }
221 throw failure(`Unsupported action runtime 'runs.using: ${meta.runs.using}' in ${uses}`);
222}
223
224function logActionHeader(job: JobRunner, io: StepIO, uses: string, inputs: Record<string, string>): void {
225 job.logger.line(io.logIndex, `##[group]Run ${uses}`);
226 const keys = Object.keys(inputs);
227 if (keys.length) {
228 job.logger.line(io.logIndex, "with:");
229 for (const k of keys) job.logger.line(io.logIndex, ` ${k}: ${inputs[k]!.split("\n").join("\n ")}`);
230 }
231 job.logger.line(io.logIndex, "##[endgroup]");
232}
233
234async function runNodeAction(
235 job: JobRunner,
236 x: StepExec,
237 meta: ActionMeta,
238 inputs: Record<string, string>,
239 action: { path: string; repository: string; ref: string },
240): Promise<void> {
241 const runs = meta.runs;
242 if (!runs.main) throw failure(`Action '${x.step.uses}' has no 'runs.main'`);
243 const inputEnv: Record<string, string> = {};
244 for (const [k, v] of Object.entries(inputs)) inputEnv[inputEnvName(k)] = v;
245 const baseExtra: Record<string, string> = {
246 ...inputEnv,
247 GITHUB_ACTION: x.step.id ?? "__" + (action.repository || "local").replace(/[^A-Za-z0-9]/g, "_"),
248 GITHUB_ACTION_PATH: action.path,
249 GITHUB_ACTION_REPOSITORY: action.repository,
250 GITHUB_ACTION_REF: action.ref,
251 };
252 const state: Record<string, string> = {};
253 const runScriptFile = async (file: string, io: StepIO, envCtx: Record<string, string>, extra: Record<string, string>, label: string) => {
254 const res = await job.runStepProcess([job.nodePath(), path.join(action.path, file)], {
255 cwd: job.workspace,
256 envCtx,
257 io,
258 timeoutMs: x.timeoutMs,
259 extraEnv: extra,
260 wrap: job.containerWrap(),
261 });
262 if (job.isStopping) return;
263 if (res.timedOut) throw failure(io.reason ?? "The step timed out.", true);
264 if (res.exitCode !== 0) {
265 const msg = `${label}: Process completed with exit code ${res.exitCode}.`;
266 job.logger.line(io.logIndex, `##[error]${msg}`);
267 job.addAnnotation(io.logIndex, { level: "error", message: msg });
268 io.reason = msg;
269 throw failure(msg, true);
270 }
271 };
272
273 const ctxScope = x.scope;
274 if (runs.pre) {
275 const cond = runs["pre-if"] ?? "always()";
276 let ok = true;
277 try {
278 ok = job.evalIf(cond, x.ctx, ctxScope);
279 } catch (e) {
280 throw failure(`Error evaluating pre-if: ${errMsg(e)}`);
281 }
282 if (ok) {
283 const preIo: StepIO = { logIndex: x.io.logIndex, outputs: {}, state: {} };
284 job.logger.line(x.io.logIndex, `Running pre: ${runs.pre}`);
285 await runScriptFile(runs.pre, preIo, x.envCtx, baseExtra, "pre");
286 Object.assign(state, preIo.state);
287 }
288 }
289 if (runs.post) {
290 const postFile = runs.post;
291 const envAtMain = { ...x.envCtx };
292 job.registerPost({
293 name: `Post ${x.name}`,
294 condition: runs["post-if"] ?? "always()",
295 run: async (logIndex) => {
296 const stateEnv: Record<string, string> = {};
297 for (const [k, v] of Object.entries(state)) stateEnv[`STATE_${k}`] = v;
298 const postIo: StepIO = { logIndex, outputs: {}, state: {} };
299 await runScriptFile(postFile, postIo, { ...envAtMain, ...job.envAccum }, { ...baseExtra, ...stateEnv }, "post");
300 },
301 });
302 }
303 const mainIo = x.io;
304 try {
305 await runScriptFile(runs.main, mainIo, x.envCtx, { ...baseExtra, ...stateEnvOf(state) }, "main");
306 } finally {
307 Object.assign(state, mainIo.state);
308 }
309}
310
311/** Docker container actions: `uses: docker://image`, or an action with `runs.using: docker`. */
312async function runDockerAction(
313 job: JobRunner,
314 x: StepExec,
315 d: {
316 image: string;
317 dir: string | null;
318 args: string[];
319 entrypoint?: string;
320 pre?: string;
321 post?: string;
322 preIf?: string;
323 postIf?: string;
324 env: Record<string, string>;
325 inputs: Record<string, string>;
326 action: { path: string; repository: string; ref: string };
327 },
328): Promise<void> {
329 const image = await job.docker.prepareAction(x.io, d.image, d.dir);
330 const extra: Record<string, string> = {
331 GITHUB_ACTION: x.step.id ?? "__" + (d.action.repository || "docker").replace(/[^A-Za-z0-9]/g, "_"),
332 GITHUB_ACTION_REPOSITORY: d.action.repository,
333 GITHUB_ACTION_REF: d.action.ref,
334 };
335 for (const [k, v] of Object.entries(d.inputs)) extra[inputEnvName(k)] = v;
336 const runIn = async (io: StepIO, envCtx: Record<string, string>, o: { entrypoint?: string; args: string[] }, label: string) => {
337 const wrap = await job.docker.runWrap(io, image, o);
338 const res = await job.runStepProcess([], { cwd: job.workspace, envCtx: { ...envCtx, ...d.env }, io, timeoutMs: x.timeoutMs, extraEnv: extra, wrap });
339 if (job.isStopping) return;
340 if (res.timedOut) throw failure(io.reason ?? "The step timed out.", true);
341 if (res.exitCode !== 0) {
342 const msg = `${label}Docker run failed with exit code ${res.exitCode}`;
343 job.logger.line(io.logIndex, `##[error]${msg}`);
344 job.addAnnotation(io.logIndex, { level: "error", message: msg });
345 io.reason = msg;
346 throw failure(msg, true);
347 }
348 };
349 if (d.pre) {
350 let ok = true;
351 try {
352 ok = job.evalIf(d.preIf ?? "always()", x.ctx, x.scope);
353 } catch (e) {
354 throw failure(`Error evaluating pre-if: ${errMsg(e)}`);
355 }
356 if (ok) await runIn({ logIndex: x.io.logIndex, outputs: {}, state: {} }, x.envCtx, { entrypoint: d.pre, args: [] }, "pre: ");
357 }
358 if (d.post) {
359 const post = d.post;
360 const envAtMain = { ...x.envCtx };
361 job.registerPost({
362 name: `Post ${x.name}`,
363 condition: d.postIf ?? "always()",
364 run: async (logIndex) => runIn({ logIndex, outputs: {}, state: {} }, { ...envAtMain, ...job.envAccum }, { entrypoint: post, args: [] }, "post: "),
365 });
366 }
367 await runIn(x.io, x.envCtx, { entrypoint: d.entrypoint, args: d.args }, "");
368}
369
370function stateEnvOf(state: Record<string, string>): Record<string, string> {
371 const out: Record<string, string> = {};
372 for (const [k, v] of Object.entries(state)) out[`STATE_${k}`] = v;
373 return out;
374}
375
376async function runComposite(
377 job: JobRunner,
378 x: StepExec,
379 meta: ActionMeta,
380 inputs: Record<string, string>,
381 action: { path: string; repository: string; ref: string },
382): Promise<void> {
383 if (x.scope.depth > 9) throw failure("Composite action nesting is too deep (max 10).");
384 const steps = Array.isArray(meta.runs.steps) ? meta.runs.steps : [];
385 const scope: Scope = {
386 steps: {},
387 inputs,
388 failed: false,
389 inheritedEnv: { ...x.envCtx },
390 composite: true,
391 action,
392 depth: x.scope.depth + 1,
393 };
394 for (const s of steps) {
395 if (job.isStopping) break;
396 const sub = normaliseStep(s);
397 await job.executeStep(sub, scope, x.io.logIndex);
398 }
399 // outputs
400 const ctx = job.contexts(scope, job.baseEnvContext(scope));
401 for (const [k, o] of Object.entries(meta.outputs ?? {})) {
402 if (!o || o.value === undefined || o.value === null) continue;
403 try {
404 x.io.outputs[k] = job.interp(String(o.value), ctx, scope);
405 } catch (e) {
406 job.logger.line(x.io.logIndex, `##[warning]Failed to evaluate output '${k}': ${errMsg(e)}`);
407 }
408 }
409 if (job.isStopping) return;
410 if (scope.failed) throw failure(`Composite action '${x.step.uses}' failed.`, true);
411}
412
413function normaliseStep(s: unknown): StepSpec {
414 const o = (s ?? {}) as Record<string, unknown>;
415 const step: StepSpec = {};
416 if (o.id !== undefined) step.id = String(o.id);
417 if (o.name !== undefined) step.name = String(o.name);
418 if (o.if !== undefined && o.if !== null) step.if = typeof o.if === "boolean" ? String(o.if) : String(o.if);
419 if (o.run !== undefined) step.run = String(o.run);
420 if (o.uses !== undefined) step.uses = String(o.uses);
421 if (o.shell !== undefined) step.shell = String(o.shell);
422 if (o["working-directory"] !== undefined) step["working-directory"] = String(o["working-directory"]);
423 if (o["continue-on-error"] !== undefined) step["continue-on-error"] = o["continue-on-error"] as boolean | string;
424 if (o["timeout-minutes"] !== undefined) step["timeout-minutes"] = Number(o["timeout-minutes"]);
425 const strMap = (m: unknown) => {
426 const r: Record<string, string> = {};
427 if (m && typeof m === "object") for (const [k, v] of Object.entries(m)) r[k] = v === null || v === undefined ? "" : String(v);
428 return r;
429 };
430 if (o.with) step.with = strMap(o.with);
431 if (o.env) step.env = strMap(o.env);
432 return step;
433}