| 1 | import { createHash, randomUUID } from "node:crypto"; |
| 2 | import { lstat, mkdir, readFile, rm, rmdir, stat, writeFile } from "node:fs/promises"; |
| 3 | import path from "node:path"; |
| 4 | import type { Readable } from "node:stream"; |
| 5 | import { errMsg, failure, type JobRunner } from "../job.js"; |
| 6 | import { RuntimeError } from "../runtime.js"; |
| 7 | import { globPaths, inputList, truthyInput } from "../util.js"; |
| 8 | import type { Builtin, BuiltinContext } from "./index.js"; |
| 9 | |
| 10 | /** |
| 11 | * actions/cache (and the `cache:` of setup-*), speaking the same cache service |
| 12 | * and archive format as the official toolkit: a `tar --posix` archive of |
| 13 | * workspace-relative paths, zstd-compressed when zstd is installed, keyed by |
| 14 | * key + version (a hash of the path list and compression). Entries saved here |
| 15 | * restore with the real action and vice versa. |
| 16 | */ |
| 17 | |
| 18 | type Compression = "zstd-without-long" | "gzip"; |
| 19 | |
| 20 | const compressionOf = new WeakMap<JobRunner, Promise<Compression>>(); |
| 21 | |
| 22 | /** Same rule as @actions/cache: zstd when `zstd` runs, else gzip. */ |
| 23 | export function cacheCompression(c: BuiltinContext): Promise<Compression> { |
| 24 | let p = compressionOf.get(c.job); |
| 25 | if (!p) { |
| 26 | p = c.exec("zstd", ["--quiet", "--version"], { capture: true, silent: true }).then( |
| 27 | (r) => (r.exitCode === 0 && (r.stdout + r.stderr).trim() ? "zstd-without-long" : "gzip"), |
| 28 | () => "gzip" as const, |
| 29 | ); |
| 30 | compressionOf.set(c.job, p); |
| 31 | } |
| 32 | return p; |
| 33 | } |
| 34 | |
| 35 | /** @actions/cache's version: the path patterns as given, the compression, and its salt. */ |
| 36 | export function cacheVersion(paths: string[], compression: Compression): string { |
| 37 | return createHash("sha256").update([...paths, compression, "1.0"].join("|")).digest("hex"); |
| 38 | } |
| 39 | |
| 40 | const archiveName = (m: Compression) => (m === "gzip" ? "cache.tgz" : "cache.tzst"); |
| 41 | const compressArgs = (m: Compression, mode: "create" | "extract") => (m === "gzip" ? ["-z"] : ["--use-compress-program", mode === "create" ? "zstdmt" : "unzstd"]); |
| 42 | |
| 43 | /** Case-insensitive like actions/cache's isExactKeyMatch. */ |
| 44 | export const isExactKeyMatch = (key: string, matched?: string) => !!matched && key.localeCompare(matched, undefined, { sensitivity: "accent" }) === 0; |
| 45 | |
| 46 | export interface RestoreResult { |
| 47 | matchedKey?: string; |
| 48 | exact: boolean; |
| 49 | } |
| 50 | |
| 51 | function runtimeOf(c: BuiltinContext) { |
| 52 | const rt = c.job.runtime; |
| 53 | if (!rt) c.warn("The cache service is not available to this job; skipping."); |
| 54 | return rt; |
| 55 | } |
| 56 | |
| 57 | /** Restores a cache entry; never throws for service errors (warns instead). */ |
| 58 | export async function restoreCache(c: BuiltinContext, paths: string[], key: string, restoreKeys: string[], lookupOnly = false): Promise<RestoreResult> { |
| 59 | if (!key) throw failure("Input required and not supplied: key"); |
| 60 | if (paths.length === 0) throw failure("Input required and not supplied: path"); |
| 61 | for (const k of [key, ...restoreKeys]) { |
| 62 | if (k.length > 512) throw failure(`Key Validation Error: ${k} cannot be larger than 512 characters.`); |
| 63 | if (k.includes(",")) throw failure(`Key Validation Error: ${k} cannot contain commas.`); |
| 64 | } |
| 65 | const rt = runtimeOf(c); |
| 66 | if (!rt) return { exact: false }; |
| 67 | const compression = await cacheCompression(c); |
| 68 | let hit: { ok: boolean; signed_download_url?: string; matched_key?: string }; |
| 69 | try { |
| 70 | hit = await rt.twirp("CacheService", "GetCacheEntryDownloadURL", { key, restore_keys: restoreKeys, version: cacheVersion(paths, compression) }); |
| 71 | } catch (e) { |
| 72 | c.warn(`Failed to restore: ${errMsg(e)}`); |
| 73 | return { exact: false }; |
| 74 | } |
| 75 | if (!hit.ok || !hit.matched_key || !hit.signed_download_url) return { exact: false }; |
| 76 | const exact = isExactKeyMatch(key, hit.matched_key); |
| 77 | if (lookupOnly) return { matchedKey: hit.matched_key, exact }; |
| 78 | const dir = path.join(c.job.tempDir, `cache-${randomUUID()}`); |
| 79 | const ws = c.job.workspace; |
| 80 | const indexes = [path.join(dir, "extracted-1"), path.join(dir, "extracted-2")]; |
| 81 | const since = Date.now(); |
| 82 | const extract = (from: string, index: string, input?: Readable) => |
| 83 | c.exec("tar", ["-xvf", from, "--index-file", index, "-P", "-C", ws, ...compressArgs(compression, "extract")], { silent: true, capture: true, input }); |
| 84 | const tarFailed = (r: { exitCode: number; stderr: string }) => new Error(`tar failed with exit code ${r.exitCode}: ${r.stderr.trim().split("\n").slice(-3).join(" ")}`); |
| 85 | try { |
| 86 | await mkdir(dir, { recursive: true }); |
| 87 | await mkdir(ws, { recursive: true }); |
| 88 | // Downloaded in ranges, several at once, straight into tar: the archive extracts while it arrives. |
| 89 | let streamError: unknown; |
| 90 | try { |
| 91 | const dl = rt.downloadStream(hit.signed_download_url, { onSize: (size) => c.log(`Cache Size: ~${Math.round(size / 1024 / 1024)} MB (${size} B)`) }); |
| 92 | dl.once("error", (e) => (streamError = e)); |
| 93 | const r = await extract("-", indexes[0]!, dl); |
| 94 | if (streamError) throw streamError; |
| 95 | if (r.exitCode !== 0) throw tarFailed(r); |
| 96 | return { matchedKey: hit.matched_key, exact }; |
| 97 | } catch (e) { |
| 98 | if (c.job.isStopping) throw e; |
| 99 | c.log(`Streaming the cache entry failed (${errMsg(streamError ?? e)}); downloading it in full and extracting it again.`); |
| 100 | } |
| 101 | // Once more from a file, like @actions/cache: extracting the whole archive again replaces anything the stream left half done. |
| 102 | const file = path.join(dir, archiveName(compression)); |
| 103 | await rt.download(hit.signed_download_url, file); |
| 104 | const r = await extract(file, indexes[1]!); |
| 105 | if (r.exitCode !== 0) throw tarFailed(r); |
| 106 | return { matchedKey: hit.matched_key, exact }; |
| 107 | } catch (e) { |
| 108 | // Nothing half restored stays behind: a job that misses the cache installs from scratch, one with half a cache may not notice. |
| 109 | await undoExtraction(ws, indexes, since).catch((u: unknown) => c.warn(`Could not remove a partly restored cache: ${errMsg(u)}`)); |
| 110 | c.warn(`Failed to restore: ${errMsg(e)}`); |
| 111 | return { exact: false }; |
| 112 | } finally { |
| 113 | await rm(dir, { recursive: true, force: true }); |
| 114 | } |
| 115 | } |
| 116 | |
| 117 | /** |
| 118 | * Removes what a failed restore extracted: the entries tar listed (its --index-file) that were |
| 119 | * created or replaced since the restore began (their ctime, which tar can't set). Directories go |
| 120 | * only when empty, deepest first; the workspace itself stays. |
| 121 | */ |
| 122 | export async function undoExtraction(ws: string, indexes: string[], since: number): Promise<void> { |
| 123 | const dirs = new Set<string>(); |
| 124 | for (const index of indexes) { |
| 125 | let text: string; |
| 126 | try { |
| 127 | text = await readFile(index, "utf8"); |
| 128 | } catch { |
| 129 | continue; |
| 130 | } |
| 131 | for (const name of text.split("\n")) { |
| 132 | if (!name) continue; |
| 133 | const p = path.resolve(ws, name); |
| 134 | if (p === ws || p === path.parse(p).root) continue; |
| 135 | const st = await lstat(p).catch(() => null); |
| 136 | if (!st || st.ctimeMs < since - 1) continue; |
| 137 | if (st.isDirectory()) dirs.add(p); |
| 138 | else await rm(p, { force: true }); |
| 139 | } |
| 140 | } |
| 141 | for (const d of [...dirs].sort((a, b) => b.length - a.length)) await rmdir(d).catch(() => {}); |
| 142 | } |
| 143 | |
| 144 | /** Archives `paths` and saves it under `key`. Never throws for service errors. */ |
| 145 | export async function saveCache(c: BuiltinContext, paths: string[], key: string): Promise<boolean> { |
| 146 | const rt = runtimeOf(c); |
| 147 | if (!rt) return false; |
| 148 | const ws = c.job.workspace; |
| 149 | const resolved = globPaths(ws, paths).map((f) => path.relative(ws, f) || "."); |
| 150 | if (resolved.length === 0) { |
| 151 | c.warn("Path Validation Error: Path(s) specified in the action for caching do(es) not exist, hence no cache is being saved."); |
| 152 | return false; |
| 153 | } |
| 154 | const compression = await cacheCompression(c); |
| 155 | const dir = path.join(c.job.tempDir, `cache-${randomUUID()}`); |
| 156 | const name = archiveName(compression); |
| 157 | try { |
| 158 | await mkdir(dir, { recursive: true }); |
| 159 | await writeFile(path.join(dir, "manifest.txt"), resolved.join("\n")); |
| 160 | await c.run("tar", ["--posix", "-cf", name, "--exclude", name, "-P", "-C", ws, "--files-from", "manifest.txt", ...compressArgs(compression, "create")], { cwd: dir, silent: true }); |
| 161 | const file = path.join(dir, name); |
| 162 | const size = (await stat(file)).size; |
| 163 | c.log(`Cache Size: ~${Math.round(size / 1024 / 1024)} MB (${size} B)`); |
| 164 | if (size > 10 * 1024 ** 3) { |
| 165 | c.warn(`Cache size of ~${Math.round(size / 1024 / 1024)} MB (${size} B) is over the 10GB limit, not saving cache.`); |
| 166 | return false; |
| 167 | } |
| 168 | const version = cacheVersion(paths, compression); |
| 169 | let created: { ok: boolean; signed_upload_url?: string; message?: string }; |
| 170 | try { |
| 171 | created = await rt.twirp("CacheService", "CreateCacheEntry", { key, version }); |
| 172 | } catch (e) { |
| 173 | if (e instanceof RuntimeError && e.status === 409) { |
| 174 | c.warn(`Failed to save: Unable to reserve cache with key ${key}, another job may be creating this cache.`); |
| 175 | return false; |
| 176 | } |
| 177 | throw e; |
| 178 | } |
| 179 | if (!created.ok || !created.signed_upload_url) throw new Error(created.message || `Unable to reserve cache with key ${key}`); |
| 180 | await rt.upload(created.signed_upload_url, file); |
| 181 | const done = await rt.twirp<{ ok: boolean; message?: string }>("CacheService", "FinalizeCacheEntryUpload", { key, version, size_bytes: String(size) }); |
| 182 | if (!done.ok) throw new Error(done.message || `Unable to finalize cache with key ${key}`); |
| 183 | c.log(`Cache saved with key: ${key}`); |
| 184 | return true; |
| 185 | } catch (e) { |
| 186 | c.warn(`Failed to save: ${errMsg(e)}`); |
| 187 | return false; |
| 188 | } finally { |
| 189 | await rm(dir, { recursive: true, force: true }); |
| 190 | } |
| 191 | } |
| 192 | |
| 193 | /** actions/cache's path list: one per line, `! pattern` normalised to `!pattern`. */ |
| 194 | const pathInput = (c: BuiltinContext) => inputList(c.input("path")).map((p) => p.replace(/^!\s+/, "!")); |
| 195 | |
| 196 | function restoreInputs(c: BuiltinContext) { |
| 197 | const key = c.input("key"); |
| 198 | if (!key) throw failure("Input required and not supplied: key"); |
| 199 | const paths = pathInput(c); |
| 200 | if (!paths.length) throw failure("Input required and not supplied: path"); |
| 201 | return { key, paths, restoreKeys: inputList(c.input("restore-keys")), lookupOnly: truthyInput(c.input("lookup-only"), false), failOnMiss: truthyInput(c.input("fail-on-cache-miss"), false) }; |
| 202 | } |
| 203 | |
| 204 | async function restoreStep(c: BuiltinContext, o: ReturnType<typeof restoreInputs>): Promise<RestoreResult> { |
| 205 | const r = await restoreCache(c, o.paths, o.key, o.restoreKeys, o.lookupOnly); |
| 206 | if (!r.matchedKey) { |
| 207 | if (o.failOnMiss) throw failure(`Failed to restore cache entry. Exiting as fail-on-cache-miss is set. Input key: ${o.key}`); |
| 208 | c.log(`Cache not found for input keys: ${[o.key, ...o.restoreKeys].join(", ")}`); |
| 209 | return r; |
| 210 | } |
| 211 | c.setOutput("cache-hit", r.exact ? "true" : "false"); |
| 212 | c.log(o.lookupOnly ? `Cache found and can be restored from key: ${r.matchedKey}` : `Cache restored from key: ${r.matchedKey}`); |
| 213 | return r; |
| 214 | } |
| 215 | |
| 216 | export const cacheAction: Builtin = async (c) => { |
| 217 | const o = restoreInputs(c); |
| 218 | const r = await restoreStep(c, o); |
| 219 | const saveAlways = truthyInput(c.input("save-always"), false); |
| 220 | c.registerPost(`Post ${c.name}`, saveAlways ? "always()" : "success()", async (pc) => { |
| 221 | if (isExactKeyMatch(o.key, r.matchedKey)) { |
| 222 | pc.log(`Cache hit occurred on the primary key ${o.key}, not saving cache.`); |
| 223 | return; |
| 224 | } |
| 225 | await saveCache(pc, o.paths, o.key); |
| 226 | }); |
| 227 | }; |
| 228 | |
| 229 | export const cacheRestoreAction: Builtin = async (c) => { |
| 230 | const o = restoreInputs(c); |
| 231 | c.setOutput("cache-primary-key", o.key); |
| 232 | const r = await restoreStep(c, o); |
| 233 | if (r.matchedKey) c.setOutput("cache-matched-key", r.matchedKey); |
| 234 | }; |
| 235 | |
| 236 | export const cacheSaveAction: Builtin = async (c) => { |
| 237 | const key = c.input("key"); |
| 238 | if (!key) throw failure("Input required and not supplied: key"); |
| 239 | const paths = pathInput(c); |
| 240 | if (!paths.length) throw failure("Input required and not supplied: path"); |
| 241 | await saveCache(c, paths, key); |
| 242 | }; |