| 1 | import { BUSY_BUCKETS, type JobUsage } from "@diffinery/shared"; |
| 2 | |
| 3 | /** |
| 4 | * Hosted runner sizes. Each is its own container class (wrangler.jsonc `containers`) on a Cloudflare |
| 5 | * Containers instance type. Cloudflare bills a running container's memory and disk for what the |
| 6 | * instance type provisions, used or not, and its CPU for what it actually uses; so a job that waits |
| 7 | * on the network, or uses one core of four, costs most of a large runner's price in memory alone. |
| 8 | * |
| 9 | * A job runs on `large` unless it asks for a size (`runs-on: diffinery-medium`, ...) or its own |
| 10 | * earlier runs show that a smaller one would serve it as well (chooseSize). Smaller is never meant to |
| 11 | * be slower or less reliable: anything that could make it so keeps the job on `large`. |
| 12 | */ |
| 13 | export type RunnerSize = "large" | "medium" | "small" | "tiny"; |
| 14 | export type RunnerBinding = "RUNNER" | "RUNNER_MEDIUM" | "RUNNER_SMALL" | "RUNNER_TINY"; |
| 15 | |
| 16 | export const RUNNER_SIZES: Record<RunnerSize, { binding: RunnerBinding; instanceType: string; vcpu: number; memoryGiB: number; diskGB: number }> = { |
| 17 | large: { binding: "RUNNER", instanceType: "standard-4", vcpu: 4, memoryGiB: 12, diskGB: 20 }, |
| 18 | medium: { binding: "RUNNER_MEDIUM", instanceType: "standard-3", vcpu: 2, memoryGiB: 8, diskGB: 16 }, |
| 19 | small: { binding: "RUNNER_SMALL", instanceType: "standard-1", vcpu: 0.5, memoryGiB: 4, diskGB: 8 }, |
| 20 | tiny: { binding: "RUNNER_TINY", instanceType: "basic", vcpu: 0.25, memoryGiB: 1, diskGB: 4 }, |
| 21 | }; |
| 22 | /** Smallest first: the order chooseSize tries them in. */ |
| 23 | const ASCENDING: RunnerSize[] = ["tiny", "small", "medium", "large"]; |
| 24 | |
| 25 | /** The size a `runs-on` label asks for (`diffinery-small`, ...), the largest if several; null for none. */ |
| 26 | export function sizeLabel(labels: readonly string[]): RunnerSize | null { |
| 27 | let out: RunnerSize | null = null; |
| 28 | for (const l of labels) { |
| 29 | const m = /^diffinery-(large|medium|small|tiny)$/i.exec(l.trim()); |
| 30 | if (m && (!out || ASCENDING.indexOf(m[1]!.toLowerCase() as RunnerSize) > ASCENDING.indexOf(out))) out = m[1]!.toLowerCase() as RunnerSize; |
| 31 | } |
| 32 | return out; |
| 33 | } |
| 34 | |
| 35 | /** |
| 36 | * The container name of a hosted job: `job-<id>` (and `-r<n>` once restarted) on a large runner, |
| 37 | * with `.<size>` after it on the others. The name says which container class runs it. |
| 38 | */ |
| 39 | export function containerName(jobId: string, restarts: number, size: RunnerSize): string { |
| 40 | return `job-${jobId}${restarts ? `-r${restarts}` : ""}${size === "large" ? "" : `.${size}`}`; |
| 41 | } |
| 42 | |
| 43 | export function sizeOfContainer(name: string): RunnerSize { |
| 44 | const m = /\.(medium|small|tiny)$/.exec(name); |
| 45 | return (m?.[1] as RunnerSize | undefined) ?? "large"; |
| 46 | } |
| 47 | |
| 48 | /** |
| 49 | * One earlier run of the same job: how it ended, on which size, what it used, and the host processor it |
| 50 | * ran on as the runner reported it (`Host:` in the log, e.g. "AMD EPYC 9654 96-Core Processor (family 25, model 17), ..."). |
| 51 | */ |
| 52 | export type SizedRun = { conclusion: string | null; size: RunnerSize; usage: JobUsage | null; cpu?: string | null }; |
| 53 | |
| 54 | /** |
| 55 | * How long a fixed single-thread loop took on each host processor Cloudflare's runners landed on (cpu |
| 56 | * family/model; scalability probe, 2026-10-16): Zen 4 623 ms, Zen 3 837 ms at best, Zen 2 1190 ms; the slowest |
| 57 | * host seen took SLOWEST_HOST_MS. A core on a slow host does about half the work of one on a fast host. |
| 58 | */ |
| 59 | const HOST_LOOP_MS: Record<string, number> = { "23/49": 1190, "25/1": 837, "25/17": 623 }; |
| 60 | export const SLOWEST_HOST_MS = 1362; |
| 61 | |
| 62 | /** A host processor's family and model ("25/17"), from the runner's description of it; null when it doesn't say. */ |
| 63 | export function hostModel(cpu: string | null | undefined): string | null { |
| 64 | const m = /\(family (\d+), model (\d+)\)/.exec(cpu ?? ""); |
| 65 | return m ? `${m[1]}/${m[2]}` : null; |
| 66 | } |
| 67 | |
| 68 | /** |
| 69 | * How many times the cores a run kept busy it would keep busy for the same work on the slowest host. Where |
| 70 | * a job lands isn't ours to pick, so a smaller runner must serve it there too; an unknown processor |
| 71 | * counts as the fastest one, which gives the largest factor. |
| 72 | */ |
| 73 | export function hostFactor(cpu: string | null | undefined): number { |
| 74 | const ms = HOST_LOOP_MS[hostModel(cpu) ?? ""] ?? Math.min(...Object.values(HOST_LOOP_MS)); |
| 75 | return Math.max(1, SLOWEST_HOST_MS / ms); |
| 76 | } |
| 77 | |
| 78 | /** Headroom a smaller runner must leave over what the job used: memory and disk are sampled, so a peak can be missed. */ |
| 79 | export const MEMORY_HEADROOM = { factor: 1.5, plusGiB: 0.5 }; |
| 80 | export const DISK_HEADROOM = { factor: 1.25, plusGB: 1 }; |
| 81 | /** |
| 82 | * CPU headroom: a smaller runner must have this many times the cores the job kept busy at its busiest, so a |
| 83 | * burst shorter than a sample (or a little busier than the last runs) still has room and doesn't queue. |
| 84 | */ |
| 85 | export const CPU_HEADROOM = 1.5; |
| 86 | /** |
| 87 | * A run on a smaller runner was slowed by it, or may have been, when it took longer than the job takes on |
| 88 | * large plus its own run-to-run noise there: over a tenth and 2 seconds above the median of its measured |
| 89 | * large runs (on the same host processor when there are any), and above the slowest but one of the last |
| 90 | * ten of them (see slowLimit). The job then goes back to large. |
| 91 | */ |
| 92 | export const SLOWER = { factor: 1.1, plusSeconds: 2 }; |
| 93 | /** |
| 94 | * Busy cores are measured over windows of at most this many seconds (the runner's own sampling). Shorter |
| 95 | * windows catch a short burst at its height: averaged over 2 seconds, a third of a second on one core reads |
| 96 | * as a sixth of a core, and a runner sized by that would run each such burst at half speed or less. |
| 97 | * Runs measured with longer windows (runners from before 2026-10-16) count as not measured. |
| 98 | */ |
| 99 | export const MAX_WINDOW_SECONDS = 0.25; |
| 100 | /** A run busy at its runner's limit this share of its time may have wanted more: no smaller runner than that one. */ |
| 101 | export const SATURATED = 0.05; |
| 102 | /** Earlier runs looked at, and how many measured ones a job needs before it can get a smaller runner. */ |
| 103 | export const HISTORY = 5; |
| 104 | export const MIN_MEASURED = 3; |
| 105 | /** |
| 106 | * Once a smaller runner ran a job out of memory, failed it or slowed it, the job stays on large while that |
| 107 | * run is among its last LOOKBACK, so it doesn't move back and forth between sizes. |
| 108 | */ |
| 109 | export const LOOKBACK = 20; |
| 110 | |
| 111 | /** |
| 112 | * The most cores a run kept busy in any quarter-second window, counted at its level's upper bound (never below |
| 113 | * what was measured). A smaller runner must have at least this many vCPUs: its average says little about a |
| 114 | * test suite's bursts, and a job that is short of CPU even for seconds can miss its own timeouts. |
| 115 | */ |
| 116 | export function peakCores(u: JobUsage): number { |
| 117 | let top = 0; |
| 118 | u.busySeconds.forEach((secs, i) => { |
| 119 | if (secs > 0) top = Math.max(top, Math.min(BUSY_BUCKETS[i] ?? Infinity, u.cores)); |
| 120 | }); |
| 121 | return Math.max(top, u.coresMax); |
| 122 | } |
| 123 | |
| 124 | /** Why a size would not do for a run, or null when it would. */ |
| 125 | function unfit(size: RunnerSize, r: SizedRun & { usage: JobUsage }): string | null { |
| 126 | const s = RUNNER_SIZES[size]; |
| 127 | const u = r.usage; |
| 128 | if (u.memoryPeak / 1024 ** 3 * MEMORY_HEADROOM.factor + MEMORY_HEADROOM.plusGiB > s.memoryGiB) return "memory"; |
| 129 | if (u.diskPeak / 1e9 * DISK_HEADROOM.factor + DISK_HEADROOM.plusGB > s.diskGB) return "disk"; |
| 130 | // Held back by its own runner's CPU: it may need more than it showed, so nothing smaller than that runner. |
| 131 | if (u.saturated > SATURATED && s.vcpu <= RUNNER_SIZES[r.size].vcpu) return "cpu"; |
| 132 | // Counted as on the slowest host, with headroom for bursts shorter than a sample. |
| 133 | if (peakCores(u) * hostFactor(r.cpu) * CPU_HEADROOM > s.vcpu) return "cpu"; |
| 134 | return null; |
| 135 | } |
| 136 | |
| 137 | export type SizeChoice = { size: RunnerSize; why: string }; |
| 138 | |
| 139 | /** |
| 140 | * The size for a hosted job from its earlier runs (the same workflow and job name, newest first). It is |
| 141 | * `large` unless each of its last few runs (at least MIN_MEASURED, all measured) fits a smaller one with |
| 142 | * headroom: peak memory and disk well inside it, and half as many cores again as it kept busy at its |
| 143 | * busiest, counted as on the slowest host. A run that ran out of memory, failed on a smaller runner, or |
| 144 | * took longer there than the job takes on large keeps the job on `large` for its next LOOKBACK runs, so a |
| 145 | * smaller runner is never what makes a job fail or slow down twice. A re-run of a failed job always gets |
| 146 | * `large`. `large` gives the job's measured runs on large when they go back further than `history`. |
| 147 | */ |
| 148 | export function chooseSize(history: readonly SizedRun[], o: { rerun?: boolean; large?: readonly SizedRun[] } = {}): SizeChoice { |
| 149 | if (o.rerun) return { size: "large", why: "a re-run runs on the large runner" }; |
| 150 | const counted = history.filter((r) => r.conclusion !== "cancelled" && r.conclusion !== "skipped").slice(0, LOOKBACK); |
| 151 | const runs = counted.slice(0, HISTORY); |
| 152 | if (!runs.length) return { size: "large", why: "no earlier run of this job to size it by" }; |
| 153 | for (const r of runs) { |
| 154 | if (!r.usage) return { size: "large", why: "an earlier run of this job wasn't measured" }; |
| 155 | if (!(r.usage.windowSeconds! <= MAX_WINDOW_SECONDS)) return { size: "large", why: "an earlier run of this job wasn't measured closely enough (busy cores every quarter second)" }; |
| 156 | } |
| 157 | const onLarge = (o.large ?? history).filter((r) => r.size === "large" && r.conclusion === "success" && r.usage); |
| 158 | for (const r of counted) { |
| 159 | if (r.usage?.oomKills) return { size: "large", why: "an earlier run of this job ran out of memory" }; |
| 160 | if (r.size === "large") continue; |
| 161 | if (r.conclusion !== "success") return { size: "large", why: `an earlier run of this job failed on a ${r.size} runner` }; |
| 162 | const limit = slowLimit(onLarge, r.cpu); |
| 163 | if (r.usage && limit !== null && r.usage.seconds > limit) { |
| 164 | return { size: "large", why: `an earlier run of this job took longer on a ${r.size} runner than it does on large` }; |
| 165 | } |
| 166 | } |
| 167 | if (runs.length < MIN_MEASURED) return { size: "large", why: `fewer than ${MIN_MEASURED} measured runs of this job to size it by` }; |
| 168 | const measured = runs as Array<SizedRun & { usage: JobUsage }>; |
| 169 | for (const size of ASCENDING) { |
| 170 | if (size === "large") break; |
| 171 | if (measured.every((r) => !unfit(size, r))) { |
| 172 | const peak = Math.max(...measured.map((r) => r.usage.memoryPeak)) / 1024 ** 3; |
| 173 | return { size, why: `sized by this job's last ${measured.length} runs (peak memory ${peak.toFixed(1)} GiB)` }; |
| 174 | } |
| 175 | } |
| 176 | return { size: "large", why: "this job uses what a smaller runner doesn't have" }; |
| 177 | } |
| 178 | |
| 179 | /** |
| 180 | * The longest a run on a smaller runner may take before it counts as slowed: the job's time on large plus its |
| 181 | * own noise there. Its measured large runs on the same host processor as `cpu` are compared (cores differ |
| 182 | * about 2x between hosts), or all of them when none ran on that one; null with none. The limit is a tenth and |
| 183 | * 2 seconds over their median, or, with 5 or more of them, their slowest but one when that is higher: a job |
| 184 | * whose own large runs vary that much can't be told to be slower by less. |
| 185 | */ |
| 186 | export function slowLimit(onLarge: readonly SizedRun[], cpu: string | null | undefined): number | null { |
| 187 | const model = hostModel(cpu); |
| 188 | const same = model ? onLarge.filter((r) => hostModel(r.cpu) === model) : []; |
| 189 | const secs = (same.length ? same : onLarge).slice(0, 10).map((r) => r.usage!.seconds).sort((a, b) => a - b); |
| 190 | if (!secs.length) return null; |
| 191 | const median = secs[Math.floor((secs.length - 1) / 2)]!; |
| 192 | const spread = secs.length >= 5 ? secs[secs.length - 2]! : 0; |
| 193 | return Math.max(median * SLOWER.factor + SLOWER.plusSeconds, spread); |
| 194 | } |
| 195 | |
| 196 | /** The log line naming the job's runner size and why it got it. */ |
| 197 | export function describeSize(size: RunnerSize, why: string, pinned: boolean): string { |
| 198 | const s = RUNNER_SIZES[size]; |
| 199 | const cores = s.vcpu >= 1 ? `${s.vcpu} vCPU` : `${s.vcpu} of a vCPU`; |
| 200 | return `Runner size: ${size} (${cores}, ${s.memoryGiB} GiB of memory, ${s.diskGB} GB of disk), ${pinned ? "asked for by runs-on" : `${why}; runs-on: diffinery-large (or -medium, -small, -tiny) sets a size`}`; |
| 201 | } |
| 202 | |
| 203 | /** |
| 204 | * A runner's usage report as stored (JSON), or null when it isn't one. The runner holding the job's token |
| 205 | * writes it, so only known fields are kept, as finite numbers in range: at worst a job misreports its own |
| 206 | * use and is sized for that, and a run that then fails on a smaller runner goes back to large. |
| 207 | */ |
| 208 | export function cleanUsage(raw: unknown): string | null { |
| 209 | if (!raw || typeof raw !== "object") return null; |
| 210 | const u = raw as Record<string, unknown>; |
| 211 | const num = (k: string, max: number) => { |
| 212 | const v = u[k]; |
| 213 | return typeof v === "number" && Number.isFinite(v) && v >= 0 ? Math.min(v, max) : null; |
| 214 | }; |
| 215 | const out: Record<string, unknown> = {}; |
| 216 | const limits: Record<string, number> = { seconds: 1e6, cpuSeconds: 1e8, coresP95: 1024, coresMax: 1024, saturated: 1, memoryPeak: 2 ** 50, diskPeak: 2 ** 50, cores: 1024, memory: 2 ** 50, disk: 2 ** 50 }; |
| 217 | for (const [k, max] of Object.entries(limits)) { |
| 218 | const v = num(k, max); |
| 219 | if (v === null) return null; |
| 220 | out[k] = v; |
| 221 | } |
| 222 | out.oomKills = u.oomKills === null ? null : num("oomKills", 1e6); |
| 223 | const busy = u.busySeconds; |
| 224 | if (!Array.isArray(busy) || busy.length !== BUSY_BUCKETS.length || !busy.every((v) => typeof v === "number" && Number.isFinite(v) && v >= 0)) return null; |
| 225 | out.busySeconds = busy.map((v: number) => Math.min(v, 1e6)); |
| 226 | // Optional: absent from runners before 2026-10-16 (such a run counts as not measured closely enough). |
| 227 | const w = u.windowSeconds; |
| 228 | if (typeof w === "number" && Number.isFinite(w) && w > 0) out.windowSeconds = Math.min(w, 60); |
| 229 | return JSON.stringify(out); |
| 230 | } |