| 1 | import { BUSY_BUCKETS, type JobUsage } from "@diffinery/shared"; |
| 2 | |
| 3 | /** |
| 4 | * Hosted runner sizes. Each is its own container class (wrangler.jsonc `containers`) on a Cloudflare |
| 5 | * Containers instance type. Cloudflare bills a running container's memory and disk for what the |
| 6 | * instance type provisions, used or not, and its CPU for what it actually uses; so a job that waits |
| 7 | * on the network, or uses one core of four, costs most of a large runner's price in memory alone. |
| 8 | * |
| 9 | * A job runs on `large` unless it asks for a size (`runs-on: diffinery-medium`, ...) or its own |
| 10 | * earlier runs show that a smaller one would serve it as well (chooseSize). Smaller is never meant to |
| 11 | * be slower or less reliable: anything that could make it so keeps the job on `large`. |
| 12 | */ |
| 13 | export type RunnerSize = "large" | "medium" | "small" | "tiny"; |
| 14 | export type RunnerBinding = "RUNNER" | "RUNNER_MEDIUM" | "RUNNER_SMALL" | "RUNNER_TINY"; |
| 15 | |
| 16 | export const RUNNER_SIZES: Record<RunnerSize, { binding: RunnerBinding; instanceType: string; vcpu: number; memoryGiB: number; diskGB: number }> = { |
| 17 | large: { binding: "RUNNER", instanceType: "standard-4", vcpu: 4, memoryGiB: 12, diskGB: 20 }, |
| 18 | medium: { binding: "RUNNER_MEDIUM", instanceType: "standard-3", vcpu: 2, memoryGiB: 8, diskGB: 16 }, |
| 19 | small: { binding: "RUNNER_SMALL", instanceType: "standard-1", vcpu: 0.5, memoryGiB: 4, diskGB: 8 }, |
| 20 | tiny: { binding: "RUNNER_TINY", instanceType: "basic", vcpu: 0.25, memoryGiB: 1, diskGB: 4 }, |
| 21 | }; |
| 22 | /** Smallest first: the order chooseSize tries them in. */ |
| 23 | const ASCENDING: RunnerSize[] = ["tiny", "small", "medium", "large"]; |
| 24 | |
| 25 | /** The size a `runs-on` label asks for (`diffinery-small`, ...), the largest if several; null for none. */ |
| 26 | export function sizeLabel(labels: readonly string[]): RunnerSize | null { |
| 27 | let out: RunnerSize | null = null; |
| 28 | for (const l of labels) { |
| 29 | const m = /^diffinery-(large|medium|small|tiny)$/i.exec(l.trim()); |
| 30 | if (m && (!out || ASCENDING.indexOf(m[1]!.toLowerCase() as RunnerSize) > ASCENDING.indexOf(out))) out = m[1]!.toLowerCase() as RunnerSize; |
| 31 | } |
| 32 | return out; |
| 33 | } |
| 34 | |
| 35 | /** |
| 36 | * The container name of a hosted job: `job-<id>` (and `-r<n>` once restarted) on a large runner, |
| 37 | * with `.<size>` after it on the others. The name says which container class runs it. |
| 38 | */ |
| 39 | export function containerName(jobId: string, restarts: number, size: RunnerSize): string { |
| 40 | return `job-${jobId}${restarts ? `-r${restarts}` : ""}${size === "large" ? "" : `.${size}`}`; |
| 41 | } |
| 42 | |
| 43 | export function sizeOfContainer(name: string): RunnerSize { |
| 44 | const m = /\.(medium|small|tiny)$/.exec(name); |
| 45 | return (m?.[1] as RunnerSize | undefined) ?? "large"; |
| 46 | } |
| 47 | |
| 48 | /** One earlier run of the same job: how it ended, on which size, and what it used. */ |
| 49 | export type SizedRun = { conclusion: string | null; size: RunnerSize; usage: JobUsage | null }; |
| 50 | |
| 51 | /** Headroom a smaller runner must leave over what the job used: memory and disk are sampled, so a peak can be missed. */ |
| 52 | export const MEMORY_HEADROOM = { factor: 1.5, plusGiB: 0.5 }; |
| 53 | export const DISK_HEADROOM = { factor: 1.25, plusGB: 1 }; |
| 54 | /** |
| 55 | * CPU headroom: a smaller runner must have this many times the cores the job kept busy at its busiest, so a |
| 56 | * burst shorter than a sample (or a little busier than the last runs) still has room and doesn't queue. |
| 57 | */ |
| 58 | export const CPU_HEADROOM = 1.5; |
| 59 | /** |
| 60 | * A run on a smaller runner that took longer than this over the job's typical time on large (the median of |
| 61 | * its measured large runs) was slowed by it, or may have been: the job goes back to large. |
| 62 | */ |
| 63 | export const SLOWER = { factor: 1.2, plusSeconds: 10 }; |
| 64 | /** A run busy at its runner's limit this share of its time may have wanted more: no smaller runner than that one. */ |
| 65 | export const SATURATED = 0.05; |
| 66 | /** Earlier runs looked at, and how many measured ones a job needs before it can get a smaller runner. */ |
| 67 | export const HISTORY = 5; |
| 68 | export const MIN_MEASURED = 3; |
| 69 | |
| 70 | /** |
| 71 | * The most cores a run kept busy in any 2-second window, counted at its level's upper bound (never below |
| 72 | * what was measured). A smaller runner must have at least this many vCPUs: its average says little about a |
| 73 | * test suite's bursts, and a job that is short of CPU even for seconds can miss its own timeouts. |
| 74 | */ |
| 75 | export function peakCores(u: JobUsage): number { |
| 76 | let top = 0; |
| 77 | u.busySeconds.forEach((secs, i) => { |
| 78 | if (secs > 0) top = Math.max(top, Math.min(BUSY_BUCKETS[i] ?? Infinity, u.cores)); |
| 79 | }); |
| 80 | return Math.max(top, u.coresMax); |
| 81 | } |
| 82 | |
| 83 | /** Why a size would not do for a run, or null when it would. */ |
| 84 | function unfit(size: RunnerSize, r: SizedRun & { usage: JobUsage }): string | null { |
| 85 | const s = RUNNER_SIZES[size]; |
| 86 | const u = r.usage; |
| 87 | if (u.memoryPeak / 1024 ** 3 * MEMORY_HEADROOM.factor + MEMORY_HEADROOM.plusGiB > s.memoryGiB) return "memory"; |
| 88 | if (u.diskPeak / 1e9 * DISK_HEADROOM.factor + DISK_HEADROOM.plusGB > s.diskGB) return "disk"; |
| 89 | // Held back by its own runner's CPU: it may need more than it showed, so nothing smaller than that runner. |
| 90 | if (u.saturated > SATURATED && s.vcpu <= RUNNER_SIZES[r.size].vcpu) return "cpu"; |
| 91 | if (peakCores(u) * CPU_HEADROOM > s.vcpu) return "cpu"; |
| 92 | return null; |
| 93 | } |
| 94 | |
| 95 | export type SizeChoice = { size: RunnerSize; why: string }; |
| 96 | |
| 97 | /** |
| 98 | * The size for a hosted job from its earlier runs (the same workflow and job name, newest first). It is |
| 99 | * `large` unless each of its last few runs (at least MIN_MEASURED, all measured) fits a smaller one with headroom: peak memory |
| 100 | * and disk well inside it, and half as many cores again as it kept busy at its busiest. A run that ran out |
| 101 | * of memory, failed on a smaller runner, took longer there than the job takes on large, or wasn't measured |
| 102 | * keeps the job on `large`, so a smaller runner is never what makes a job fail or slow down twice. A re-run |
| 103 | * of a failed job always gets `large`. Runs past the last few only give the job's typical time on large. |
| 104 | */ |
| 105 | export function chooseSize(history: readonly SizedRun[], o: { rerun?: boolean } = {}): SizeChoice { |
| 106 | if (o.rerun) return { size: "large", why: "a re-run runs on the large runner" }; |
| 107 | const runs = history.filter((r) => r.conclusion !== "cancelled" && r.conclusion !== "skipped").slice(0, HISTORY); |
| 108 | if (!runs.length) return { size: "large", why: "no earlier run of this job to size it by" }; |
| 109 | for (const r of runs) { |
| 110 | if (!r.usage) return { size: "large", why: "an earlier run of this job wasn't measured" }; |
| 111 | if (r.usage.oomKills) return { size: "large", why: "an earlier run of this job ran out of memory" }; |
| 112 | if (r.size !== "large" && r.conclusion !== "success") return { size: "large", why: `an earlier run of this job failed on a ${r.size} runner` }; |
| 113 | } |
| 114 | const onLarge = history |
| 115 | .filter((r) => r.size === "large" && r.conclusion === "success" && r.usage) |
| 116 | .map((r) => r.usage!.seconds) |
| 117 | .sort((a, b) => a - b); |
| 118 | if (onLarge.length) { |
| 119 | const typical = onLarge[Math.floor((onLarge.length - 1) / 2)]!; |
| 120 | const slow = runs.find((r) => r.size !== "large" && r.usage!.seconds > typical * SLOWER.factor + SLOWER.plusSeconds); |
| 121 | if (slow) return { size: "large", why: `an earlier run of this job took longer on a ${slow.size} runner than it does on large` }; |
| 122 | } |
| 123 | if (runs.length < MIN_MEASURED) return { size: "large", why: `fewer than ${MIN_MEASURED} measured runs of this job to size it by` }; |
| 124 | const measured = runs as Array<SizedRun & { usage: JobUsage }>; |
| 125 | for (const size of ASCENDING) { |
| 126 | if (size === "large") break; |
| 127 | if (measured.every((r) => !unfit(size, r))) { |
| 128 | const peak = Math.max(...measured.map((r) => r.usage.memoryPeak)) / 1024 ** 3; |
| 129 | return { size, why: `sized by this job's last ${measured.length} runs (peak memory ${peak.toFixed(1)} GiB)` }; |
| 130 | } |
| 131 | } |
| 132 | return { size: "large", why: "this job uses what a smaller runner doesn't have" }; |
| 133 | } |
| 134 | |
| 135 | /** The log line naming the job's runner size and why it got it. */ |
| 136 | export function describeSize(size: RunnerSize, why: string, pinned: boolean): string { |
| 137 | const s = RUNNER_SIZES[size]; |
| 138 | const cores = s.vcpu >= 1 ? `${s.vcpu} vCPU` : `${s.vcpu} of a vCPU`; |
| 139 | return `Runner size: ${size} (${cores}, ${s.memoryGiB} GiB of memory, ${s.diskGB} GB of disk), ${pinned ? "asked for by runs-on" : `${why}; runs-on: diffinery-large (or -medium, -small, -tiny) sets a size`}`; |
| 140 | } |
| 141 | |
| 142 | /** |
| 143 | * A runner's usage report as stored (JSON), or null when it isn't one. The runner holding the job's token |
| 144 | * writes it, so only known fields are kept, as finite numbers in range: at worst a job misreports its own |
| 145 | * use and is sized for that, and a run that then fails on a smaller runner goes back to large. |
| 146 | */ |
| 147 | export function cleanUsage(raw: unknown): string | null { |
| 148 | if (!raw || typeof raw !== "object") return null; |
| 149 | const u = raw as Record<string, unknown>; |
| 150 | const num = (k: string, max: number) => { |
| 151 | const v = u[k]; |
| 152 | return typeof v === "number" && Number.isFinite(v) && v >= 0 ? Math.min(v, max) : null; |
| 153 | }; |
| 154 | const out: Record<string, unknown> = {}; |
| 155 | const limits: Record<string, number> = { seconds: 1e6, cpuSeconds: 1e8, coresP95: 1024, coresMax: 1024, saturated: 1, memoryPeak: 2 ** 50, diskPeak: 2 ** 50, cores: 1024, memory: 2 ** 50, disk: 2 ** 50 }; |
| 156 | for (const [k, max] of Object.entries(limits)) { |
| 157 | const v = num(k, max); |
| 158 | if (v === null) return null; |
| 159 | out[k] = v; |
| 160 | } |
| 161 | out.oomKills = u.oomKills === null ? null : num("oomKills", 1e6); |
| 162 | const busy = u.busySeconds; |
| 163 | if (!Array.isArray(busy) || busy.length !== BUSY_BUCKETS.length || !busy.every((v) => typeof v === "number" && Number.isFinite(v) && v >= 0)) return null; |
| 164 | out.busySeconds = busy.map((v: number) => Math.min(v, 1e6)); |
| 165 | return JSON.stringify(out); |
| 166 | } |