| 1 | import { readFileSync, statfsSync } from "node:fs"; |
| 2 | import os from "node:os"; |
| 3 | import path from "node:path"; |
| 4 | import { BUSY_BUCKETS, type JobUsage } from "@diffinery/shared"; |
| 5 | |
| 6 | /** Reads of the machine at one moment: CPU time (busy and all, in ticks), memory and disk in use. */ |
| 7 | type Reading = { busy: number; total: number; memUsed: number; memTotal: number; diskUsed: number; diskTotal: number; oom: number | null }; |
| 8 | |
| 9 | export interface UsageSamplerOptions { |
| 10 | /** Where /proc is (tests point it at a fake one). */ |
| 11 | procRoot?: string; |
| 12 | /** The filesystem whose use is measured (the root filesystem holds the image, the workspace and the tool cache). */ |
| 13 | diskPath?: string; |
| 14 | /** How often memory and disk are read; CPU is read on the same tick. */ |
| 15 | sampleMs?: number; |
| 16 | /** Busy cores are measured over windows this long. */ |
| 17 | windowMs?: number; |
| 18 | /** The machine's share of CPU in vCPUs (a quarter or half for the smallest hosted runners); by default the CPUs it sees. */ |
| 19 | cores?: number; |
| 20 | /** CPUs the machine shows in /proc/stat (busy time is a share of all of them). */ |
| 21 | visible?: number; |
| 22 | } |
| 23 | |
| 24 | /** |
| 25 | * Measures what a job uses of its machine while it runs: busy cores (from /proc/stat), memory in use |
| 26 | * (MemTotal less MemAvailable from /proc/meminfo, which leaves out cache the kernel can drop), disk in |
| 27 | * use, and out-of-memory kills (/proc/vmstat). A hosted container is its own virtual machine, so these |
| 28 | * are the job's own numbers. Diffinery sizes the job's later runs by them (apps/diffinery/src/runner/sizes.ts). |
| 29 | * Off Linux, or where /proc can't be read, it measures nothing and reports null. |
| 30 | */ |
| 31 | export class UsageSampler { |
| 32 | private readonly proc: string; |
| 33 | private readonly diskPath: string; |
| 34 | private readonly sampleMs: number; |
| 35 | private readonly windowMs: number; |
| 36 | private readonly cores: number; |
| 37 | private readonly visible: number; |
| 38 | private timer: NodeJS.Timeout | null = null; |
| 39 | private first: Reading | null = null; |
| 40 | private windowStart: Reading | null = null; |
| 41 | private windowStartedAt = 0; |
| 42 | private startedAt = 0; |
| 43 | private busyWindows: number[] = []; |
| 44 | private busySeconds = BUSY_BUCKETS.map(() => 0); |
| 45 | private cpuSeconds = 0; |
| 46 | private memPeak = 0; |
| 47 | private diskPeak = 0; |
| 48 | private last: Reading | null = null; |
| 49 | |
| 50 | constructor(o: UsageSamplerOptions = {}) { |
| 51 | this.proc = o.procRoot ?? "/proc"; |
| 52 | this.diskPath = o.diskPath ?? "/"; |
| 53 | // A quarter second: a short burst of CPU is measured at its height, not averaged away |
| 54 | // (apps/diffinery/src/runner/sizes.ts sizes later runs by the busiest window). |
| 55 | this.sampleMs = o.sampleMs ?? 250; |
| 56 | this.windowMs = o.windowMs ?? 250; |
| 57 | this.visible = o.visible ?? os.availableParallelism(); |
| 58 | this.cores = o.cores && o.cores > 0 ? o.cores : this.visible; |
| 59 | } |
| 60 | |
| 61 | start(): void { |
| 62 | const r = this.read(); |
| 63 | if (!r) return; |
| 64 | this.first = this.windowStart = this.last = r; |
| 65 | this.startedAt = this.windowStartedAt = Date.now(); |
| 66 | this.note(r); |
| 67 | this.timer = setInterval(() => this.tick(), this.sampleMs); |
| 68 | this.timer.unref(); |
| 69 | } |
| 70 | |
| 71 | /** Stops sampling and returns what the job used, or null if nothing could be measured. */ |
| 72 | stop(): JobUsage | null { |
| 73 | if (this.timer) clearInterval(this.timer); |
| 74 | this.timer = null; |
| 75 | if (!this.first) return null; |
| 76 | this.tick(true); |
| 77 | const first = this.first; |
| 78 | const last = this.last ?? first; |
| 79 | this.first = null; |
| 80 | const seconds = (Date.now() - this.startedAt) / 1000; |
| 81 | const sorted = [...this.busyWindows].sort((a, b) => a - b); |
| 82 | const pct = (p: number) => (sorted.length ? sorted[Math.min(sorted.length - 1, Math.floor(p * sorted.length))]! : 0); |
| 83 | const round = (n: number, d = 2) => Math.round(n * 10 ** d) / 10 ** d; |
| 84 | return { |
| 85 | seconds: round(seconds, 1), |
| 86 | cpuSeconds: round(this.cpuSeconds, 1), |
| 87 | coresP95: round(pct(0.95)), |
| 88 | coresMax: round(sorted.length ? sorted[sorted.length - 1]! : 0), |
| 89 | saturated: round(sorted.length ? sorted.filter((b) => b >= saturatedAt(this.cores)).length / sorted.length : 0, 3), |
| 90 | busySeconds: this.busySeconds.map((s) => round(s, 1)), |
| 91 | windowSeconds: this.windowMs / 1000, |
| 92 | memoryPeak: this.memPeak, |
| 93 | diskPeak: this.diskPeak, |
| 94 | cores: this.cores, |
| 95 | memory: last.memTotal, |
| 96 | disk: last.diskTotal, |
| 97 | oomKills: first.oom !== null && last.oom !== null ? Math.max(0, last.oom - first.oom) : null, |
| 98 | }; |
| 99 | } |
| 100 | |
| 101 | private tick(final = false): void { |
| 102 | const r = this.read(); |
| 103 | if (!r) return; |
| 104 | this.last = r; |
| 105 | this.note(r); |
| 106 | const now = Date.now(); |
| 107 | if (this.windowStart && (final || now - this.windowStartedAt >= this.windowMs)) { |
| 108 | const dTotal = r.total - this.windowStart.total; |
| 109 | const dBusy = r.busy - this.windowStart.busy; |
| 110 | const secs = (now - this.windowStartedAt) / 1000; |
| 111 | if (dTotal > 0 && secs > 0) { |
| 112 | const busyCores = Math.min(this.cores, Math.max(0, (dBusy / dTotal) * this.visible)); |
| 113 | // A last window much shorter than the rest says little about how busy the job was. |
| 114 | if (!final || secs >= this.windowMs / 2000) this.busyWindows.push(busyCores); |
| 115 | this.cpuSeconds += busyCores * secs; |
| 116 | const i = BUSY_BUCKETS.findIndex((b) => busyCores <= b); |
| 117 | this.busySeconds[i < 0 ? BUSY_BUCKETS.length - 1 : i]! += secs; |
| 118 | } |
| 119 | this.windowStart = r; |
| 120 | this.windowStartedAt = now; |
| 121 | } |
| 122 | } |
| 123 | |
| 124 | private note(r: Reading): void { |
| 125 | this.memPeak = Math.max(this.memPeak, r.memUsed); |
| 126 | this.diskPeak = Math.max(this.diskPeak, r.diskUsed); |
| 127 | } |
| 128 | |
| 129 | private read(): Reading | null { |
| 130 | try { |
| 131 | const stat = readFileSync(path.join(this.proc, "stat"), "utf8"); |
| 132 | const cpu = stat.split("\n", 1)[0]!.trim().split(/\s+/); |
| 133 | if (cpu[0] !== "cpu") return null; |
| 134 | // user nice system idle iowait irq softirq steal: busy is what ran here, not idle, waiting or stolen time. |
| 135 | const [user, nice, system, idle, iowait, irq, softirq, steal] = cpu.slice(1, 9).map((v) => Number(v) || 0); |
| 136 | const busy = user! + nice! + system! + irq! + softirq!; |
| 137 | const total = busy + idle! + iowait! + steal!; |
| 138 | const mem = readFileSync(path.join(this.proc, "meminfo"), "utf8"); |
| 139 | const kb = (name: string) => Number(new RegExp(`^${name}:\\s+(\\d+)`, "m").exec(mem)?.[1] ?? NaN) * 1024; |
| 140 | const memTotal = kb("MemTotal"); |
| 141 | const memAvailable = kb("MemAvailable"); |
| 142 | if (!Number.isFinite(memTotal) || !Number.isFinite(memAvailable)) return null; |
| 143 | let diskUsed = 0; |
| 144 | let diskTotal = 0; |
| 145 | try { |
| 146 | const fs = statfsSync(this.diskPath); |
| 147 | diskTotal = fs.blocks * fs.bsize; |
| 148 | diskUsed = (fs.blocks - fs.bfree) * fs.bsize; |
| 149 | } catch { |
| 150 | /* no disk numbers */ |
| 151 | } |
| 152 | let oom: number | null = null; |
| 153 | try { |
| 154 | const m = /^oom_kill\s+(\d+)/m.exec(readFileSync(path.join(this.proc, "vmstat"), "utf8")); |
| 155 | if (m) oom = Number(m[1]); |
| 156 | } catch { |
| 157 | /* an older kernel doesn't count them */ |
| 158 | } |
| 159 | return { busy, total, memUsed: memTotal - memAvailable, memTotal, diskUsed, diskTotal, oom }; |
| 160 | } catch { |
| 161 | return null; |
| 162 | } |
| 163 | } |
| 164 | } |
| 165 | |
| 166 | /** Busy cores at which a machine of `cores` vCPUs counts as held back by its CPU. */ |
| 167 | export const saturatedAt = (cores: number) => Math.max(cores * 0.9, cores - 0.25); |
| 168 | |
| 169 | const gib = (b: number) => `${(b / 1024 ** 3).toFixed(1)} GiB`; |
| 170 | const gb = (b: number) => `${(b / 1e9).toFixed(1)} GB`; |
| 171 | |
| 172 | /** One log line saying what the job used of its machine. */ |
| 173 | export function describeUsage(u: JobUsage): string { |
| 174 | const avg = u.seconds > 0 ? u.cpuSeconds / u.seconds : 0; |
| 175 | return `Resources: ${avg.toFixed(1)} of ${u.cores} vCPUs busy on average (${u.coresP95.toFixed(1)} at the 95th percentile), peak memory ${gib(u.memoryPeak)} of ${gib(u.memory)}, peak disk ${gb(u.diskPeak)} of ${gb(u.disk)}${u.oomKills ? `, ${u.oomKills} process(es) killed for want of memory` : ""}`; |
| 176 | } |