Diffinery

codician/diffinerypublic

Open-source, agent-first GitHub and GitHub Actions alternative that runs on your own Cloudflare account: git hosting, pull requests, code review and drop-in GitHub Actions YAML on Workers, Durable Objects, R2 and Containers.

421 lines · 20.3 KBRaw History
1import { randomUUID } from "node:crypto";
2import { copyFile, mkdir, rm } from "node:fs/promises";
3import { homedir } from "node:os";
4import path from "node:path";
5import { errMsg, failure, StepFailure } from "../job.js";
6import { RuntimeError, type RuntimeClient } from "../runtime.js";
7import { globPaths, truthyInput } from "../util.js";
8import { extractZip, sha256File, writeZip } from "../zip.js";
9import type { Builtin, BuiltinContext } from "./index.js";
10
11/**
12 * actions/upload-artifact and actions/download-artifact (v4 and later), on the
13 * artifact service the official actions use: the same zip layout, names, ids,
14 * digests and options, so artifacts move freely between built-in and real
15 * actions, jobs and runs.
16 */
17
18const MIME: Record<string, string> = {
19 ".txt": "text/plain", ".html": "text/html", ".htm": "text/html", ".css": "text/css", ".csv": "text/csv", ".xml": "text/xml", ".md": "text/markdown",
20 ".js": "application/javascript", ".mjs": "application/javascript", ".json": "application/json",
21 ".png": "image/png", ".jpg": "image/jpeg", ".jpeg": "image/jpeg", ".gif": "image/gif", ".svg": "image/svg+xml", ".webp": "image/webp", ".ico": "image/x-icon", ".bmp": "image/bmp", ".tiff": "image/tiff", ".tif": "image/tiff",
22 ".mp3": "audio/mpeg", ".wav": "audio/wav", ".ogg": "audio/ogg", ".flac": "audio/flac", ".mp4": "video/mp4", ".webm": "video/webm", ".avi": "video/x-msvideo", ".mov": "video/quicktime",
23 ".pdf": "application/pdf", ".zip": "application/zip", ".tar": "application/x-tar", ".gz": "application/gzip", ".rar": "application/vnd.rar", ".7z": "application/x-7z-compressed",
24 ".wasm": "application/wasm", ".yaml": "application/x-yaml", ".yml": "application/x-yaml", ".woff": "font/woff", ".woff2": "font/woff2", ".ttf": "font/ttf", ".otf": "font/otf",
25};
26
27export const mimeTypeOf = (file: string) => MIME[path.extname(file).toLowerCase()] ?? "application/octet-stream";
28
29const INVALID_NAME_CHARS = ['"', ":", "<", ">", "|", "*", "?", "\r", "\n", "\\", "/"];
30const INVALID_PATH_CHARS = ['"', ":", "<", ">", "|", "*", "?", "\r", "\n"];
31
32/** The action's major version (`@v4` -> 4); builtins only run for known majors. */
33function major(c: BuiltinContext): number {
34 return Number(/@v?(\d+)/.exec(c.uses)?.[1] ?? 99);
35}
36
37/** The run and job ids the artifact service expects, from the runtime token's `scp` claim. */
38export function backendIds(token: string): { run: string; job: string } {
39 const payload = JSON.parse(Buffer.from(token.split(".")[1] ?? "", "base64url").toString("utf8")) as { scp?: string };
40 for (const scope of (payload.scp ?? "").split(" ")) {
41 const parts = scope.split(":");
42 if (parts[0] === "Actions.Results" && parts.length === 3) return { run: parts[1]!, job: parts[2]! };
43 }
44 throw new Error("the runtime token has no Actions.Results scope");
45}
46
47function runtimeFor(c: BuiltinContext): RuntimeClient {
48 if (!c.job.runtime) throw failure("The artifact service is not available to this job.");
49 return c.job.runtime;
50}
51
52// ---------------------------------------------------------------- search (like upload-artifact's findFilesToUpload)
53
54const hasGlob = (s: string) => /[*?[]/.test(s);
55
56/** @actions/glob's search path of a pattern: its longest literal directory prefix. */
57function searchPathOf(abs: string): string {
58 const norm = path.normalize(abs).replace(/\/+$/, "") || "/";
59 if (!hasGlob(norm)) return norm;
60 const parts = norm.split("/");
61 const lit: string[] = [];
62 for (const p of parts) {
63 if (hasGlob(p)) break;
64 lit.push(p);
65 }
66 return lit.join("/") || "/";
67}
68
69function lca(paths: string[]): string {
70 let parts = paths[0]!.split("/");
71 for (const p of paths.slice(1)) {
72 const q = p.split("/");
73 let i = 0;
74 while (i < parts.length && i < q.length && parts[i] === q[i]) i++;
75 parts = parts.slice(0, i);
76 }
77 return parts.join("/") || "/";
78}
79
80export function findFilesToUpload(workspace: string, searchPath: string, includeHidden: boolean): { files: string[]; root: string } {
81 const lines = searchPath
82 .split("\n")
83 .map((l) => l.trim())
84 .filter((l) => l && !l.startsWith("#"));
85 const expand = (p: string) => (p === "~" || p.startsWith("~/") ? path.join(homedir(), p.slice(1)) : p);
86 const negations = lines.filter((l) => l.startsWith("!")).map((l) => "!" + path.resolve(workspace, expand(l.slice(1).trim())));
87 const includes = lines.filter((l) => !l.startsWith("!")).map((l) => path.resolve(workspace, expand(l)));
88 // Search paths, minus those inside another one (as @actions/glob does).
89 const candidates = includes.map(searchPathOf);
90 const roots: string[] = [];
91 for (const s of candidates) {
92 if (roots.includes(s)) continue;
93 if (candidates.some((o) => o !== s && (s.startsWith(o + "/") || o === "/"))) continue;
94 roots.push(s);
95 }
96 const files: string[] = [];
97 const seen = new Set<string>();
98 for (const inc of includes) {
99 const root = searchPathOf(inc);
100 for (const f of globPaths(workspace, [inc, ...negations], { filesOnly: true })) {
101 if (seen.has(f)) continue;
102 // Hidden: the search path's own name or anything below it starts with a dot.
103 const rel = path.relative(path.dirname(root), f).split("/");
104 if (!includeHidden && rel.some((s) => s.startsWith(".") && s !== ".." && s !== ".")) continue;
105 seen.add(f);
106 files.push(f);
107 }
108 }
109 let root: string;
110 if (roots.length > 1) root = lca(roots);
111 else if (files.length === 1 && roots[0] === files[0]) root = path.dirname(files[0]!);
112 else root = roots[0] ?? workspace;
113 return { files, root };
114}
115
116// ---------------------------------------------------------------- upload
117
118export const artifactUpload: Builtin = async (c) => {
119 const searchPath = c.input("path");
120 if (!searchPath) throw failure("Input required and not supplied: path");
121 const ifNone = c.input("if-no-files-found", "warn").toLowerCase();
122 if (!["warn", "error", "ignore"].includes(ifNone)) throw failure(`Unrecognized if-no-files-found input. Provided: ${ifNone}. Available options: warn,error,ignore`);
123 const archive = truthyInput(c.input("archive"), true);
124 const retentionRaw = c.input("retention-days");
125 const retentionDays = retentionRaw ? parseInt(retentionRaw, 10) : undefined;
126 if (retentionDays !== undefined && Number.isNaN(retentionDays)) throw failure("Invalid retention-days");
127 const levelRaw = c.input("compression-level");
128 const level = levelRaw ? parseInt(levelRaw, 10) : 6;
129 if (Number.isNaN(level)) throw failure("Invalid compression-level");
130 if (level < 0 || level > 9) throw failure("Invalid compression-level. Valid values are 0-9");
131
132 const { files, root } = findFilesToUpload(c.job.workspace, searchPath, truthyInput(c.input("include-hidden-files"), false));
133 if (files.length === 0) {
134 const msg = `No files were found with the provided path: ${searchPath}. No artifacts will be uploaded.`;
135 if (ifNone === "error") throw failure(msg);
136 if (ifNone === "warn") c.warn(msg);
137 else c.log(msg);
138 return;
139 }
140 c.log(`With the provided path, there will be ${files.length} file${files.length === 1 ? "" : "s"} uploaded`);
141 if (!archive && files.length > 1) throw failure(`When 'archive' is set to false, only a single file can be uploaded. Found ${files.length} files to upload.`);
142 const rt = runtimeFor(c);
143 const name = archive ? c.input("name", "artifact") : path.basename(files[0]!);
144 const bad = INVALID_NAME_CHARS.find((ch) => name.includes(ch));
145 if (!name || bad) throw failure(`The artifact name is not valid: ${name}. Contains the following character: ${JSON.stringify(bad ?? "")}`);
146 const ids = backendIds(rt.token);
147
148 if (truthyInput(c.input("overwrite"), false)) {
149 try {
150 await rt.twirp("ArtifactService", "DeleteArtifact", { workflow_run_backend_id: ids.run, workflow_job_run_backend_id: ids.job, name });
151 } catch (e) {
152 if (!(e instanceof RuntimeError && e.status === 404)) c.log(`Unable to delete artifact: ${errMsg(e)}`);
153 }
154 }
155
156 const entries = files.map((f) => {
157 const name = path.relative(root, f);
158 const badChar = INVALID_PATH_CHARS.find((ch) => name.includes(ch));
159 if (badChar) throw failure(`The path for one of the files in artifact is not valid: ${name}. Contains the following character: ${JSON.stringify(badChar)}`);
160 return { source: f, name };
161 });
162
163 const req: Record<string, unknown> = {
164 workflow_run_backend_id: ids.run,
165 workflow_job_run_backend_id: ids.job,
166 name,
167 version: 7,
168 mime_type: archive ? "application/zip" : mimeTypeOf(files[0]!),
169 };
170 if (retentionDays) {
171 const max = Number(process.env.GITHUB_RETENTION_DAYS ?? c.envCtx.GITHUB_RETENTION_DAYS ?? 90) || 90;
172 let days = retentionDays;
173 if (days > max) {
174 c.warn(`Retention days cannot be greater than the maximum allowed retention set within the repository. Using ${max} instead.`);
175 days = max;
176 }
177 req.expires_at = new Date(Date.now() + days * 86400_000).toISOString();
178 }
179
180 const work = path.join(c.job.tempDir, `artifact-${randomUUID()}`);
181 try {
182 let created: { ok: boolean; signed_upload_url?: string };
183 try {
184 created = await rt.twirp("ArtifactService", "CreateArtifact", req);
185 } catch (e) {
186 if (e instanceof RuntimeError && e.status === 409) throw failure(`Failed to CreateArtifact: an artifact with the name '${name}' already exists on the workflow run. Set 'overwrite: true' to replace it.`);
187 throw e;
188 }
189 if (!created.ok || !created.signed_upload_url) throw failure("CreateArtifact: response from backend was not ok");
190 await mkdir(work, { recursive: true });
191 let file: string;
192 let digest: string;
193 if (archive) {
194 file = path.join(work, `${name}.zip`);
195 digest = (await writeZip(file, entries, level)).sha256;
196 } else {
197 file = files[0]!;
198 digest = await sha256File(file);
199 }
200 c.log(`Uploading artifact: ${archive ? `${name}.zip` : name}`);
201 const size = await rt.upload(created.signed_upload_url, file, req.mime_type as string);
202 c.log("Finalizing artifact upload");
203 const done = await rt.twirp<{ ok: boolean; artifact_id?: string }>("ArtifactService", "FinalizeArtifact", {
204 workflow_run_backend_id: ids.run,
205 workflow_job_run_backend_id: ids.job,
206 name,
207 size: String(size),
208 hash: `sha256:${digest}`,
209 });
210 if (!done.ok || !done.artifact_id) throw failure("FinalizeArtifact: response from backend was not ok");
211 c.log(`Artifact ${name} has been successfully uploaded! Final size is ${size} bytes. Artifact ID is ${done.artifact_id}`);
212 c.setOutput("artifact-id", done.artifact_id);
213 c.setOutput("artifact-digest", digest);
214 const env = c.toolEnv();
215 const url = `${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}/artifacts/${done.artifact_id}`;
216 c.log(`Artifact download URL: ${url}`);
217 c.setOutput("artifact-url", url);
218 } catch (e) {
219 if (e instanceof StepFailure) throw e;
220 throw failure(`Failed to upload artifact '${name}': ${errMsg(e)}`);
221 } finally {
222 await rm(work, { recursive: true, force: true });
223 }
224};
225
226// ---------------------------------------------------------------- download
227
228interface Listed {
229 id: number;
230 name: string;
231 size: number;
232 digest?: string;
233 createdAt?: string;
234}
235
236/** Keeps the newest artifact of each name (the toolkit's `latest: true`). */
237function latest(list: Listed[]): Listed[] {
238 const out = new Map<string, Listed>();
239 for (const a of [...list].sort((x, y) => y.id - x.id)) if (!out.has(a.name)) out.set(a.name, a);
240 return [...out.values()];
241}
242
243/** Minimatch-style match of an artifact name: `*`, `**`, `?`, `[...]` and `{a,b}`. */
244export function nameMatches(pattern: string, name: string): boolean {
245 let re = "";
246 for (let i = 0; i < pattern.length; i++) {
247 const ch = pattern[i]!;
248 if (ch === "*") {
249 if (pattern[i + 1] === "*") {
250 re += ".*";
251 i++;
252 } else re += "[^/]*";
253 } else if (ch === "?") re += "[^/]";
254 else if (ch === "[") {
255 const end = pattern.indexOf("]", i + 1);
256 if (end < 0) re += "\\[";
257 else {
258 re += `[${pattern.slice(i + 1, end).replace(/^!/, "^").replace(/\\/g, "\\\\")}]`;
259 i = end;
260 }
261 } else if (ch === "{") {
262 const end = pattern.indexOf("}", i + 1);
263 if (end < 0) re += "\\{";
264 else {
265 re += `(?:${pattern
266 .slice(i + 1, end)
267 .split(",")
268 .map((s) => s.replace(/[.+^$()|\\]/g, "\\$&").replace(/\*/g, "[^/]*").replace(/\?/g, "[^/]"))
269 .join("|")})`;
270 i = end;
271 }
272 } else re += ch.replace(/[.+^$()|\\]/g, "\\$&");
273 }
274 return new RegExp(`^${re}$`).test(name);
275}
276
277interface Source {
278 list(): Promise<Listed[]>;
279 /** Downloads artifact `a` to `file`; returns the response's content type and file name. */
280 fetch(a: Listed, file: string): Promise<{ contentType: string; fileName: string | null; url: string }>;
281}
282
283/** Artifacts of the job's own run, through the artifact service. */
284function ownRun(c: BuiltinContext): Source {
285 const rt = runtimeFor(c);
286 const ids = backendIds(rt.token);
287 const base = { workflow_run_backend_id: ids.run, workflow_job_run_backend_id: ids.job };
288 return {
289 async list() {
290 const r = await rt.twirp<{ artifacts?: Array<{ database_id: string; name: string; size: string; created_at?: string; digest?: string }> }>("ArtifactService", "ListArtifacts", base);
291 return (r.artifacts ?? []).map((a) => ({ id: Number(a.database_id), name: a.name, size: Number(a.size), createdAt: a.created_at, digest: a.digest }));
292 },
293 async fetch(a, file) {
294 const { signed_url } = await rt.twirp<{ signed_url: string }>("ArtifactService", "GetSignedArtifactURL", { ...base, name: a.name });
295 return { ...(await rt.download(signed_url, file)), url: signed_url };
296 },
297 };
298}
299
300/** Another run's (or repository's) artifacts, through the REST API with `github-token`. */
301function otherRun(c: BuiltinContext, token: string, repository: string, runId: string): Source {
302 const env = c.toolEnv();
303 const api = (env.GITHUB_API_URL ?? `${c.job.spec.apiUrl.replace(/\/$/, "")}/api/v3`).replace(/\/$/, "");
304 const [owner, repo] = repository.split("/");
305 if (!owner || !repo) throw failure(`Invalid repository: '${repository}'. Must be in format owner/repo`);
306 if (!/^\d+$/.test(runId)) throw failure(`Invalid run-id: '${runId}'`);
307 c.job.masker.add(token);
308 const headers = { authorization: `Bearer ${token}`, accept: "application/vnd.github+json", "user-agent": "diffinery-runner" };
309 const base = `${api}/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}/actions`;
310 return {
311 async list() {
312 const out: Listed[] = [];
313 for (let page = 1; page <= 50; page++) {
314 const res = await fetch(`${base}/runs/${runId}/artifacts?per_page=100&page=${page}`, { headers });
315 if (!res.ok) throw new Error(`Listing artifacts of run ${runId} in ${repository} failed: HTTP ${res.status}`);
316 const j = (await res.json()) as { total_count: number; artifacts: Array<{ id: number; name: string; size_in_bytes: number; digest?: string | null; created_at?: string }> };
317 for (const a of j.artifacts) out.push({ id: a.id, name: a.name, size: a.size_in_bytes, digest: a.digest ?? undefined, createdAt: a.created_at });
318 if (out.length >= j.total_count || j.artifacts.length === 0) break;
319 }
320 return out;
321 },
322 async fetch(a, file) {
323 const res = await fetch(`${base}/artifacts/${a.id}/zip`, { headers, redirect: "manual" });
324 const location = res.headers.get("location");
325 if (res.status !== 302 || !location) throw new Error(`Unable to download artifact ${a.id}: HTTP ${res.status}`);
326 return { ...(await runtimeFor(c).download(location, file)), url: location };
327 },
328 };
329}
330
331export const artifactDownload: Builtin = async (c) => {
332 const v = major(c);
333 const name = c.input("name");
334 const pattern = c.input("pattern");
335 const idsInput = c.input("artifact-ids");
336 const merge = truthyInput(c.input("merge-multiple"), false);
337 const skipDecompress = truthyInput(c.input("skip-decompress"), false);
338 const onMismatch = (c.input("digest-mismatch") || (v >= 8 ? "error" : "warn")).toLowerCase();
339 if (!["ignore", "info", "warn", "error"].includes(onMismatch)) throw failure(`Invalid value for 'digest-mismatch': '${onMismatch}'. Valid options are: ignore, info, warn, error`);
340 if (name && idsInput) throw failure("Inputs 'name' and 'artifact-ids' cannot be used together. Please specify only one.");
341 let dest = c.input("path") || c.job.workspace;
342 if (dest.startsWith("~")) dest = dest.replace("~", homedir());
343 const resolved = path.resolve(c.job.workspace, dest);
344 const token = c.input("github-token");
345 const source = token ? otherRun(c, token, c.input("repository", `${c.job.spec.repo.owner}/${c.job.spec.repo.name}`), c.input("run-id")) : ownRun(c);
346
347 let artifacts: Listed[];
348 try {
349 const all = latest(await source.list());
350 if (name) {
351 const a = all.find((x) => x.name === name);
352 if (!a) throw failure(`Unable to download artifact(s): Artifact not found for name: ${name}
353 Please ensure that your artifact is not expired and the artifact was uploaded using a compatible version of toolkit/upload-artifact.
354 For more information, visit the GitHub Artifacts FAQ: https://github.com/actions/toolkit/blob/main/packages/artifact/docs/faq.md`);
355 artifacts = [a];
356 } else if (idsInput) {
357 const wanted = idsInput.split(",").map((s) => s.trim()).filter(Boolean).map((s) => {
358 const n = parseInt(s, 10);
359 if (Number.isNaN(n)) throw failure(`Invalid artifact ID: '${s}'. Must be a number.`);
360 return n;
361 });
362 if (!wanted.length) throw failure("No valid artifact IDs provided in 'artifact-ids' input");
363 artifacts = all.filter((a) => wanted.includes(a.id));
364 if (!artifacts.length) throw failure("None of the provided artifact IDs were found");
365 const missing = wanted.filter((id) => !artifacts.some((a) => a.id === id));
366 if (missing.length) c.warn(`Could not find the following artifact IDs: ${missing.join(", ")}`);
367 } else {
368 artifacts = pattern ? all.filter((a) => nameMatches(pattern, a.name)) : all;
369 if (pattern) c.log(`Filtering artifacts by pattern '${pattern}'`);
370 else c.log("No input name, artifact-ids or pattern filtered specified, downloading all artifacts");
371 }
372 } catch (e) {
373 if (e instanceof StepFailure) throw e;
374 throw failure(`Unable to download artifact(s): ${errMsg(e)}`);
375 }
376 if (artifacts.length) {
377 c.log("Preparing to download the following artifacts:");
378 for (const a of artifacts) c.log(`- ${a.name} (ID: ${a.id}, Size: ${a.size}, Expected Digest: ${a.digest})`);
379 }
380
381 const single = !!name || merge || (v >= 5 && artifacts.length === 1);
382 const mismatches: string[] = [];
383 const work = path.join(c.job.tempDir, `download-${randomUUID()}`);
384 try {
385 await mkdir(work, { recursive: true });
386 for (const a of artifacts) {
387 const into = single ? resolved : path.join(resolved, a.name);
388 await mkdir(into, { recursive: true });
389 const file = path.join(work, String(a.id));
390 let got: { contentType: string; fileName: string | null; url: string };
391 try {
392 got = await source.fetch(a, file);
393 } catch (e) {
394 throw failure(`Unable to download artifact(s): ${errMsg(e)}`);
395 }
396 const digest = `sha256:${await sha256File(file)}`;
397 if (a.digest && a.digest !== digest) {
398 mismatches.push(a.name);
399 const msg = `Artifact '${a.name}' digest validation failed. Please verify the integrity of the artifact.`;
400 if (onMismatch === "warn") c.warn(msg);
401 else if (onMismatch === "info") c.log(msg);
402 }
403 const isZip = ["application/zip", "application/x-zip-compressed", "application/zip-compressed"].includes(got.contentType) || new URL(got.url).pathname.toLowerCase().endsWith(".zip");
404 if (isZip && !skipDecompress) await extractZip(file, into);
405 else {
406 const fileName = path.basename(got.fileName || (isZip ? `${a.name}.zip` : a.name) || "artifact");
407 c.log(`Downloading raw file (non-zip) to: ${path.join(into, fileName)}`);
408 await copyFile(file, path.join(into, fileName));
409 }
410 await rm(file, { force: true });
411 c.log(`Artifact '${a.name}' downloaded to ${into}`);
412 }
413 } finally {
414 await rm(work, { recursive: true, force: true });
415 }
416 if (mismatches.length && onMismatch === "error") throw failure(`Digest validation failed for artifact(s): ${mismatches.join(", ")}. Use 'digest-mismatch: warn' to continue on mismatch.`);
417 c.log(`Total of ${artifacts.length} artifact(s) downloaded`);
418 c.setOutput("download-path", resolved);
419 c.log("Download artifact has finished successfully");
420};
421