| 1 | import { createHash, randomUUID } from "node:crypto"; |
| 2 | import { mkdir, rm, stat, writeFile } from "node:fs/promises"; |
| 3 | import path from "node:path"; |
| 4 | import { errMsg, failure, type JobRunner } from "../job.js"; |
| 5 | import { RuntimeError } from "../runtime.js"; |
| 6 | import { globPaths, inputList, truthyInput } from "../util.js"; |
| 7 | import type { Builtin, BuiltinContext } from "./index.js"; |
| 8 | |
| 9 | /** |
| 10 | * actions/cache (and the `cache:` of setup-*), speaking the same cache service |
| 11 | * and archive format as the official toolkit: a `tar --posix` archive of |
| 12 | * workspace-relative paths, zstd-compressed when zstd is installed, keyed by |
| 13 | * key + version (a hash of the path list and compression). Entries saved here |
| 14 | * restore with the real action and vice versa. |
| 15 | */ |
| 16 | |
| 17 | type Compression = "zstd-without-long" | "gzip"; |
| 18 | |
| 19 | const compressionOf = new WeakMap<JobRunner, Promise<Compression>>(); |
| 20 | |
| 21 | /** Same rule as @actions/cache: zstd when `zstd` runs, else gzip. */ |
| 22 | export function cacheCompression(c: BuiltinContext): Promise<Compression> { |
| 23 | let p = compressionOf.get(c.job); |
| 24 | if (!p) { |
| 25 | p = c.exec("zstd", ["--quiet", "--version"], { capture: true, silent: true }).then( |
| 26 | (r) => (r.exitCode === 0 && (r.stdout + r.stderr).trim() ? "zstd-without-long" : "gzip"), |
| 27 | () => "gzip" as const, |
| 28 | ); |
| 29 | compressionOf.set(c.job, p); |
| 30 | } |
| 31 | return p; |
| 32 | } |
| 33 | |
| 34 | /** @actions/cache's version: the path patterns as given, the compression, and its salt. */ |
| 35 | export function cacheVersion(paths: string[], compression: Compression): string { |
| 36 | return createHash("sha256").update([...paths, compression, "1.0"].join("|")).digest("hex"); |
| 37 | } |
| 38 | |
| 39 | const archiveName = (m: Compression) => (m === "gzip" ? "cache.tgz" : "cache.tzst"); |
| 40 | const compressArgs = (m: Compression, mode: "create" | "extract") => (m === "gzip" ? ["-z"] : ["--use-compress-program", mode === "create" ? "zstdmt" : "unzstd"]); |
| 41 | |
| 42 | /** Case-insensitive like actions/cache's isExactKeyMatch. */ |
| 43 | export const isExactKeyMatch = (key: string, matched?: string) => !!matched && key.localeCompare(matched, undefined, { sensitivity: "accent" }) === 0; |
| 44 | |
| 45 | export interface RestoreResult { |
| 46 | matchedKey?: string; |
| 47 | exact: boolean; |
| 48 | } |
| 49 | |
| 50 | function runtimeOf(c: BuiltinContext) { |
| 51 | const rt = c.job.runtime; |
| 52 | if (!rt) c.warn("The cache service is not available to this job; skipping."); |
| 53 | return rt; |
| 54 | } |
| 55 | |
| 56 | /** Restores a cache entry; never throws for service errors (warns instead). */ |
| 57 | export async function restoreCache(c: BuiltinContext, paths: string[], key: string, restoreKeys: string[], lookupOnly = false): Promise<RestoreResult> { |
| 58 | if (!key) throw failure("Input required and not supplied: key"); |
| 59 | if (paths.length === 0) throw failure("Input required and not supplied: path"); |
| 60 | for (const k of [key, ...restoreKeys]) { |
| 61 | if (k.length > 512) throw failure(`Key Validation Error: ${k} cannot be larger than 512 characters.`); |
| 62 | if (k.includes(",")) throw failure(`Key Validation Error: ${k} cannot contain commas.`); |
| 63 | } |
| 64 | const rt = runtimeOf(c); |
| 65 | if (!rt) return { exact: false }; |
| 66 | const compression = await cacheCompression(c); |
| 67 | let hit: { ok: boolean; signed_download_url?: string; matched_key?: string }; |
| 68 | try { |
| 69 | hit = await rt.twirp("CacheService", "GetCacheEntryDownloadURL", { key, restore_keys: restoreKeys, version: cacheVersion(paths, compression) }); |
| 70 | } catch (e) { |
| 71 | c.warn(`Failed to restore: ${errMsg(e)}`); |
| 72 | return { exact: false }; |
| 73 | } |
| 74 | if (!hit.ok || !hit.matched_key || !hit.signed_download_url) return { exact: false }; |
| 75 | const exact = isExactKeyMatch(key, hit.matched_key); |
| 76 | if (lookupOnly) return { matchedKey: hit.matched_key, exact }; |
| 77 | const dir = path.join(c.job.tempDir, `cache-${randomUUID()}`); |
| 78 | const file = path.join(dir, archiveName(compression)); |
| 79 | try { |
| 80 | await mkdir(dir, { recursive: true }); |
| 81 | await rt.download(hit.signed_download_url, file); |
| 82 | const size = (await stat(file)).size; |
| 83 | c.log(`Cache Size: ~${Math.round(size / 1024 / 1024)} MB (${size} B)`); |
| 84 | await mkdir(c.job.workspace, { recursive: true }); |
| 85 | await c.run("tar", ["-xf", file, "-P", "-C", c.job.workspace, ...compressArgs(compression, "extract")], { silent: true }); |
| 86 | return { matchedKey: hit.matched_key, exact }; |
| 87 | } catch (e) { |
| 88 | c.warn(`Failed to restore: ${errMsg(e)}`); |
| 89 | return { exact: false }; |
| 90 | } finally { |
| 91 | await rm(dir, { recursive: true, force: true }); |
| 92 | } |
| 93 | } |
| 94 | |
| 95 | /** Archives `paths` and saves it under `key`. Never throws for service errors. */ |
| 96 | export async function saveCache(c: BuiltinContext, paths: string[], key: string): Promise<boolean> { |
| 97 | const rt = runtimeOf(c); |
| 98 | if (!rt) return false; |
| 99 | const ws = c.job.workspace; |
| 100 | const resolved = globPaths(ws, paths).map((f) => path.relative(ws, f) || "."); |
| 101 | if (resolved.length === 0) { |
| 102 | c.warn("Path Validation Error: Path(s) specified in the action for caching do(es) not exist, hence no cache is being saved."); |
| 103 | return false; |
| 104 | } |
| 105 | const compression = await cacheCompression(c); |
| 106 | const dir = path.join(c.job.tempDir, `cache-${randomUUID()}`); |
| 107 | const name = archiveName(compression); |
| 108 | try { |
| 109 | await mkdir(dir, { recursive: true }); |
| 110 | await writeFile(path.join(dir, "manifest.txt"), resolved.join("\n")); |
| 111 | await c.run("tar", ["--posix", "-cf", name, "--exclude", name, "-P", "-C", ws, "--files-from", "manifest.txt", ...compressArgs(compression, "create")], { cwd: dir, silent: true }); |
| 112 | const file = path.join(dir, name); |
| 113 | const size = (await stat(file)).size; |
| 114 | c.log(`Cache Size: ~${Math.round(size / 1024 / 1024)} MB (${size} B)`); |
| 115 | if (size > 10 * 1024 ** 3) { |
| 116 | c.warn(`Cache size of ~${Math.round(size / 1024 / 1024)} MB (${size} B) is over the 10GB limit, not saving cache.`); |
| 117 | return false; |
| 118 | } |
| 119 | const version = cacheVersion(paths, compression); |
| 120 | let created: { ok: boolean; signed_upload_url?: string; message?: string }; |
| 121 | try { |
| 122 | created = await rt.twirp("CacheService", "CreateCacheEntry", { key, version }); |
| 123 | } catch (e) { |
| 124 | if (e instanceof RuntimeError && e.status === 409) { |
| 125 | c.warn(`Failed to save: Unable to reserve cache with key ${key}, another job may be creating this cache.`); |
| 126 | return false; |
| 127 | } |
| 128 | throw e; |
| 129 | } |
| 130 | if (!created.ok || !created.signed_upload_url) throw new Error(created.message || `Unable to reserve cache with key ${key}`); |
| 131 | await rt.upload(created.signed_upload_url, file); |
| 132 | const done = await rt.twirp<{ ok: boolean; message?: string }>("CacheService", "FinalizeCacheEntryUpload", { key, version, size_bytes: String(size) }); |
| 133 | if (!done.ok) throw new Error(done.message || `Unable to finalize cache with key ${key}`); |
| 134 | c.log(`Cache saved with key: ${key}`); |
| 135 | return true; |
| 136 | } catch (e) { |
| 137 | c.warn(`Failed to save: ${errMsg(e)}`); |
| 138 | return false; |
| 139 | } finally { |
| 140 | await rm(dir, { recursive: true, force: true }); |
| 141 | } |
| 142 | } |
| 143 | |
| 144 | /** actions/cache's path list: one per line, `! pattern` normalised to `!pattern`. */ |
| 145 | const pathInput = (c: BuiltinContext) => inputList(c.input("path")).map((p) => p.replace(/^!\s+/, "!")); |
| 146 | |
| 147 | function restoreInputs(c: BuiltinContext) { |
| 148 | const key = c.input("key"); |
| 149 | if (!key) throw failure("Input required and not supplied: key"); |
| 150 | const paths = pathInput(c); |
| 151 | if (!paths.length) throw failure("Input required and not supplied: path"); |
| 152 | return { key, paths, restoreKeys: inputList(c.input("restore-keys")), lookupOnly: truthyInput(c.input("lookup-only"), false), failOnMiss: truthyInput(c.input("fail-on-cache-miss"), false) }; |
| 153 | } |
| 154 | |
| 155 | async function restoreStep(c: BuiltinContext, o: ReturnType<typeof restoreInputs>): Promise<RestoreResult> { |
| 156 | const r = await restoreCache(c, o.paths, o.key, o.restoreKeys, o.lookupOnly); |
| 157 | if (!r.matchedKey) { |
| 158 | if (o.failOnMiss) throw failure(`Failed to restore cache entry. Exiting as fail-on-cache-miss is set. Input key: ${o.key}`); |
| 159 | c.log(`Cache not found for input keys: ${[o.key, ...o.restoreKeys].join(", ")}`); |
| 160 | return r; |
| 161 | } |
| 162 | c.setOutput("cache-hit", r.exact ? "true" : "false"); |
| 163 | c.log(o.lookupOnly ? `Cache found and can be restored from key: ${r.matchedKey}` : `Cache restored from key: ${r.matchedKey}`); |
| 164 | return r; |
| 165 | } |
| 166 | |
| 167 | export const cacheAction: Builtin = async (c) => { |
| 168 | const o = restoreInputs(c); |
| 169 | const r = await restoreStep(c, o); |
| 170 | const saveAlways = truthyInput(c.input("save-always"), false); |
| 171 | c.registerPost(`Post ${c.name}`, saveAlways ? "always()" : "success()", async (pc) => { |
| 172 | if (isExactKeyMatch(o.key, r.matchedKey)) { |
| 173 | pc.log(`Cache hit occurred on the primary key ${o.key}, not saving cache.`); |
| 174 | return; |
| 175 | } |
| 176 | await saveCache(pc, o.paths, o.key); |
| 177 | }); |
| 178 | }; |
| 179 | |
| 180 | export const cacheRestoreAction: Builtin = async (c) => { |
| 181 | const o = restoreInputs(c); |
| 182 | c.setOutput("cache-primary-key", o.key); |
| 183 | const r = await restoreStep(c, o); |
| 184 | if (r.matchedKey) c.setOutput("cache-matched-key", r.matchedKey); |
| 185 | }; |
| 186 | |
| 187 | export const cacheSaveAction: Builtin = async (c) => { |
| 188 | const key = c.input("key"); |
| 189 | if (!key) throw failure("Input required and not supplied: key"); |
| 190 | const paths = pathInput(c); |
| 191 | if (!paths.length) throw failure("Input required and not supplied: path"); |
| 192 | await saveCache(c, paths, key); |
| 193 | }; |