| 1 | import { createWriteStream } from "node:fs"; |
| 2 | import { open, stat } from "node:fs/promises"; |
| 3 | import { Readable } from "node:stream"; |
| 4 | import { pipeline } from "node:stream/promises"; |
| 5 | import type { ReadableStream as WebReadableStream } from "node:stream/web"; |
| 6 | import type { JobSpec } from "@diffinery/shared"; |
| 7 | |
| 8 | /** |
| 9 | * Client for the Actions runtime services (ACTIONS_RESULTS_URL): the Twirp |
| 10 | * cache and artifact services and their signed blob URLs. The built-in cache |
| 11 | * and artifact actions use it, so they share storage (and archive formats) |
| 12 | * with the official actions. |
| 13 | */ |
| 14 | |
| 15 | export class RuntimeError extends Error { |
| 16 | constructor( |
| 17 | readonly status: number, |
| 18 | readonly code: string, |
| 19 | message: string, |
| 20 | ) { |
| 21 | super(message); |
| 22 | } |
| 23 | } |
| 24 | |
| 25 | const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); |
| 26 | const retryable = (status: number) => status === 408 || status === 429 || status >= 500; |
| 27 | |
| 28 | /** Uploads at most this much in one request; larger files go up as blocks. */ |
| 29 | const SINGLE_SHOT = 32 * 1024 * 1024; |
| 30 | const BLOCK_SIZE = 32 * 1024 * 1024; |
| 31 | const UPLOAD_CONCURRENCY = 4; |
| 32 | |
| 33 | export class RuntimeClient { |
| 34 | constructor( |
| 35 | readonly token: string, |
| 36 | readonly resultsUrl: string, |
| 37 | private readonly opts: { retries?: number; minBackoffMs?: number } = {}, |
| 38 | ) {} |
| 39 | |
| 40 | static fromSpec(spec: JobSpec): RuntimeClient | null { |
| 41 | return spec.runtime ? new RuntimeClient(spec.runtime.token, spec.runtime.resultsUrl) : null; |
| 42 | } |
| 43 | |
| 44 | private async withRetries<T>(what: string, fn: () => Promise<T>): Promise<T> { |
| 45 | const retries = this.opts.retries ?? 5; |
| 46 | let last: unknown; |
| 47 | for (let attempt = 0; attempt <= retries; attempt++) { |
| 48 | if (attempt) await sleep((this.opts.minBackoffMs ?? 1000) * 2 ** (attempt - 1)); |
| 49 | try { |
| 50 | return await fn(); |
| 51 | } catch (e) { |
| 52 | if (e instanceof RuntimeError && !retryable(e.status)) throw e; |
| 53 | last = e; |
| 54 | } |
| 55 | } |
| 56 | throw last instanceof Error ? new Error(`${what}: ${last.message}`) : new Error(`${what} failed`); |
| 57 | } |
| 58 | |
| 59 | /** Calls a Twirp method (JSON). Errors become RuntimeError with the Twirp code. */ |
| 60 | async twirp<T>(service: "CacheService" | "ArtifactService", method: string, body: Record<string, unknown>): Promise<T> { |
| 61 | const url = `${new URL(this.resultsUrl).origin}/twirp/github.actions.results.api.v1.${service}/${method}`; |
| 62 | return this.withRetries(method, async () => { |
| 63 | const res = await fetch(url, { |
| 64 | method: "POST", |
| 65 | headers: { authorization: `Bearer ${this.token}`, "content-type": "application/json", "user-agent": "diffinery-runner" }, |
| 66 | body: JSON.stringify(body), |
| 67 | }); |
| 68 | const text = await res.text(); |
| 69 | if (!res.ok) { |
| 70 | let code = "unknown"; |
| 71 | let msg = text.slice(0, 300); |
| 72 | try { |
| 73 | const j = JSON.parse(text) as { code?: string; msg?: string }; |
| 74 | code = j.code ?? code; |
| 75 | msg = j.msg ?? msg; |
| 76 | } catch { |
| 77 | /* not Twirp JSON */ |
| 78 | } |
| 79 | throw new RuntimeError(res.status, code, `${method} failed (${res.status} ${code}): ${msg}`); |
| 80 | } |
| 81 | return JSON.parse(text) as T; |
| 82 | }); |
| 83 | } |
| 84 | |
| 85 | /** Uploads a file to a signed URL (Put Blob, or Put Block + Put Block List). */ |
| 86 | async upload(url: string, file: string, contentType = "application/octet-stream"): Promise<number> { |
| 87 | const size = (await stat(file)).size; |
| 88 | const fh = await open(file, "r"); |
| 89 | try { |
| 90 | const read = async (offset: number, length: number) => { |
| 91 | const buf = Buffer.alloc(length); |
| 92 | let got = 0; |
| 93 | while (got < length) { |
| 94 | const { bytesRead } = await fh.read(buf, got, length - got, offset + got); |
| 95 | if (bytesRead === 0) break; |
| 96 | got += bytesRead; |
| 97 | } |
| 98 | return buf.subarray(0, got); |
| 99 | }; |
| 100 | const put = (target: string, body: Buffer, headers: Record<string, string>) => |
| 101 | this.withRetries("upload", async () => { |
| 102 | const res = await fetch(target, { method: "PUT", headers: { "x-ms-version": "2025-01-05", ...headers }, body }); |
| 103 | if (res.status !== 201) { |
| 104 | const text = await res.text().catch(() => ""); |
| 105 | throw new RuntimeError(res.status, "upload", `upload failed (${res.status}): ${text.slice(0, 300)}`); |
| 106 | } |
| 107 | }); |
| 108 | if (size <= SINGLE_SHOT) { |
| 109 | await put(url, await read(0, size), { "x-ms-blob-type": "BlockBlob", "x-ms-blob-content-type": contentType, "content-type": "application/octet-stream" }); |
| 110 | return size; |
| 111 | } |
| 112 | const blocks: string[] = []; |
| 113 | for (let off = 0, i = 0; off < size; off += BLOCK_SIZE, i++) blocks.push(Buffer.from(`block-${String(i).padStart(8, "0")}`).toString("base64")); |
| 114 | let next = 0; |
| 115 | const worker = async () => { |
| 116 | for (let i = next++; i < blocks.length; i = next++) { |
| 117 | const body = await read(i * BLOCK_SIZE, Math.min(BLOCK_SIZE, size - i * BLOCK_SIZE)); |
| 118 | await put(`${url}&comp=block&blockid=${encodeURIComponent(blocks[i]!)}`, body, { "content-type": "application/octet-stream" }); |
| 119 | } |
| 120 | }; |
| 121 | await Promise.all(Array.from({ length: Math.min(UPLOAD_CONCURRENCY, blocks.length) }, worker)); |
| 122 | const xml = `<?xml version="1.0" encoding="utf-8"?><BlockList>${blocks.map((b) => `<Latest>${b}</Latest>`).join("")}</BlockList>`; |
| 123 | await put(`${url}&comp=blocklist`, Buffer.from(xml), { "x-ms-blob-content-type": contentType, "content-type": "application/xml" }); |
| 124 | return size; |
| 125 | } finally { |
| 126 | await fh.close(); |
| 127 | } |
| 128 | } |
| 129 | |
| 130 | /** Downloads a signed URL to a file. Returns the response's content type and file name. */ |
| 131 | async download(url: string, file: string): Promise<{ contentType: string; fileName: string | null }> { |
| 132 | return this.withRetries("download", async () => { |
| 133 | const res = await fetch(url, { headers: { "user-agent": "diffinery-runner" } }); |
| 134 | if (!res.ok || !res.body) { |
| 135 | const text = await res.text().catch(() => ""); |
| 136 | throw new RuntimeError(res.status, "download", `download failed (${res.status}): ${text.slice(0, 200)}`); |
| 137 | } |
| 138 | await pipeline(Readable.fromWeb(res.body as unknown as WebReadableStream), createWriteStream(file)); |
| 139 | const cd = res.headers.get("content-disposition") ?? ""; |
| 140 | const star = /filename\*\s*=\s*UTF-8''([^;\r\n]*)/i.exec(cd)?.[1]; |
| 141 | const plain = /(?<!\*)filename\s*=\s*"?([^;\r\n"]*)"?/i.exec(cd)?.[1]; |
| 142 | let fileName: string | null = null; |
| 143 | try { |
| 144 | fileName = star ? decodeURIComponent(star) : (plain ?? null); |
| 145 | } catch { |
| 146 | fileName = plain ?? null; |
| 147 | } |
| 148 | return { contentType: (res.headers.get("content-type") ?? "").split(";")[0]!.trim().toLowerCase(), fileName }; |
| 149 | }); |
| 150 | } |
| 151 | } |