containers/runner/src/usage.ts at 26c09b152037265bc1cbd6dfcb1905b028365c4d · codician/diffinery · Diffinery

1 import { readFileSync, statfsSync } from "node:fs";
2 import os from "node:os";
3 import path from "node:path";
4 import { BUSY_BUCKETS, type JobUsage } from "@diffinery/shared";
5
6 /** Reads of the machine at one moment: CPU time (busy and all, in ticks), memory and disk in use. */
7 type Reading = { busy: number; total: number; memUsed: number; memTotal: number; diskUsed: number; diskTotal: number; oom: number | null };
8
9 export interface UsageSamplerOptions {
10   /** Where /proc is (tests point it at a fake one). */
11   procRoot?: string;
12   /** The filesystem whose use is measured (the root filesystem holds the image, the workspace and the tool cache). */
13   diskPath?: string;
14   /** How often memory and disk are read; CPU is read on the same tick. */
15   sampleMs?: number;
16   /** Busy cores are measured over windows this long. */
17   windowMs?: number;
18   /** The machine's share of CPU in vCPUs (a quarter or half for the smallest hosted runners); by default the CPUs it sees. */
19   cores?: number;
20   /** CPUs the machine shows in /proc/stat (busy time is a share of all of them). */
21   visible?: number;
22 }
23
24 /**
25  * Measures what a job uses of its machine while it runs: busy cores (from /proc/stat), memory in use
26  * (MemTotal less MemAvailable from /proc/meminfo, which leaves out cache the kernel can drop), disk in
27  * use, and out-of-memory kills (/proc/vmstat). A hosted container is its own virtual machine, so these
28  * are the job's own numbers. Diffinery sizes the job's later runs by them (apps/diffinery/src/runner/sizes.ts).
29  * Off Linux, or where /proc can't be read, it measures nothing and reports null.
30  */
31 export class UsageSampler {
32   private readonly proc: string;
33   private readonly diskPath: string;
34   private readonly sampleMs: number;
35   private readonly windowMs: number;
36   private readonly cores: number;
37   private readonly visible: number;
38   private timer: NodeJS.Timeout | null = null;
39   private first: Reading | null = null;
40   private windowStart: Reading | null = null;
41   private windowStartedAt = 0;
42   private startedAt = 0;
43   private busyWindows: number[] = [];
44   private busySeconds = BUSY_BUCKETS.map(() => 0);
45   private cpuSeconds = 0;
46   private memPeak = 0;
47   private diskPeak = 0;
48   private last: Reading | null = null;
49
50   constructor(o: UsageSamplerOptions = {}) {
51     this.proc = o.procRoot ?? "/proc";
52     this.diskPath = o.diskPath ?? "/";
53     this.sampleMs = o.sampleMs ?? 500;
54     this.windowMs = o.windowMs ?? 2000;
55     this.visible = o.visible ?? os.availableParallelism();
56     this.cores = o.cores && o.cores > 0 ? o.cores : this.visible;
57   }
58
59   start(): void {
60     const r = this.read();
61     if (!r) return;
62     this.first = this.windowStart = this.last = r;
63     this.startedAt = this.windowStartedAt = Date.now();
64     this.note(r);
65     this.timer = setInterval(() => this.tick(), this.sampleMs);
66     this.timer.unref();
67   }
68
69   /** Stops sampling and returns what the job used, or null if nothing could be measured. */
70   stop(): JobUsage | null {
71     if (this.timer) clearInterval(this.timer);
72     this.timer = null;
73     if (!this.first) return null;
74     this.tick(true);
75     const first = this.first;
76     const last = this.last ?? first;
77     this.first = null;
78     const seconds = (Date.now() - this.startedAt) / 1000;
79     const sorted = [...this.busyWindows].sort((a, b) => a - b);
80     const pct = (p: number) => (sorted.length ? sorted[Math.min(sorted.length - 1, Math.floor(p * sorted.length))]! : 0);
81     const round = (n: number, d = 2) => Math.round(n * 10 ** d) / 10 ** d;
82     return {
83       seconds: round(seconds, 1),
84       cpuSeconds: round(this.cpuSeconds, 1),
85       coresP95: round(pct(0.95)),
86       coresMax: round(sorted.length ? sorted[sorted.length - 1]! : 0),
87       saturated: round(sorted.length ? sorted.filter((b) => b >= saturatedAt(this.cores)).length / sorted.length : 0, 3),
88       busySeconds: this.busySeconds.map((s) => round(s, 1)),
89       memoryPeak: this.memPeak,
90       diskPeak: this.diskPeak,
91       cores: this.cores,
92       memory: last.memTotal,
93       disk: last.diskTotal,
94       oomKills: first.oom !== null && last.oom !== null ? Math.max(0, last.oom - first.oom) : null,
95     };
96   }
97
98   private tick(final = false): void {
99     const r = this.read();
100     if (!r) return;
101     this.last = r;
102     this.note(r);
103     const now = Date.now();
104     if (this.windowStart && (final || now - this.windowStartedAt >= this.windowMs)) {
105       const dTotal = r.total - this.windowStart.total;
106       const dBusy = r.busy - this.windowStart.busy;
107       const secs = (now - this.windowStartedAt) / 1000;
108       if (dTotal > 0 && secs > 0) {
109         const busyCores = Math.min(this.cores, Math.max(0, (dBusy / dTotal) * this.visible));
110         // A last window much shorter than the rest says little about how busy the job was.
111         if (!final || secs >= this.windowMs / 2000) this.busyWindows.push(busyCores);
112         this.cpuSeconds += busyCores * secs;
113         const i = BUSY_BUCKETS.findIndex((b) => busyCores <= b);
114         this.busySeconds[i < 0 ? BUSY_BUCKETS.length - 1 : i]! += secs;
115       }
116       this.windowStart = r;
117       this.windowStartedAt = now;
118     }
119   }
120
121   private note(r: Reading): void {
122     this.memPeak = Math.max(this.memPeak, r.memUsed);
123     this.diskPeak = Math.max(this.diskPeak, r.diskUsed);
124   }
125
126   private read(): Reading | null {
127     try {
128       const stat = readFileSync(path.join(this.proc, "stat"), "utf8");
129       const cpu = stat.split("\n", 1)[0]!.trim().split(/\s+/);
130       if (cpu[0] !== "cpu") return null;
131       // user nice system idle iowait irq softirq steal: busy is what ran here, not idle, waiting or stolen time.
132       const [user, nice, system, idle, iowait, irq, softirq, steal] = cpu.slice(1, 9).map((v) => Number(v) || 0);
133       const busy = user! + nice! + system! + irq! + softirq!;
134       const total = busy + idle! + iowait! + steal!;
135       const mem = readFileSync(path.join(this.proc, "meminfo"), "utf8");
136       const kb = (name: string) => Number(new RegExp(`^${name}:\\s+(\\d+)`, "m").exec(mem)?.[1] ?? NaN) * 1024;
137       const memTotal = kb("MemTotal");
138       const memAvailable = kb("MemAvailable");
139       if (!Number.isFinite(memTotal) || !Number.isFinite(memAvailable)) return null;
140       let diskUsed = 0;
141       let diskTotal = 0;
142       try {
143         const fs = statfsSync(this.diskPath);
144         diskTotal = fs.blocks * fs.bsize;
145         diskUsed = (fs.blocks - fs.bfree) * fs.bsize;
146       } catch {
147         /* no disk numbers */
148       }
149       let oom: number | null = null;
150       try {
151         const m = /^oom_kill\s+(\d+)/m.exec(readFileSync(path.join(this.proc, "vmstat"), "utf8"));
152         if (m) oom = Number(m[1]);
153       } catch {
154         /* an older kernel doesn't count them */
155       }
156       return { busy, total, memUsed: memTotal - memAvailable, memTotal, diskUsed, diskTotal, oom };
157     } catch {
158       return null;
159     }
160   }
161 }
162
163 /** Busy cores at which a machine of `cores` vCPUs counts as held back by its CPU. */
164 export const saturatedAt = (cores: number) => Math.max(cores * 0.9, cores - 0.25);
165
166 const gib = (b: number) => `${(b / 1024 ** 3).toFixed(1)} GiB`;
167 const gb = (b: number) => `${(b / 1e9).toFixed(1)} GB`;
168
169 /** One log line saying what the job used of its machine. */
170 export function describeUsage(u: JobUsage): string {
171   const avg = u.seconds > 0 ? u.cpuSeconds / u.seconds : 0;
172   return `Resources: ${avg.toFixed(1)} of ${u.cores} vCPUs busy on average (${u.coresP95.toFixed(1)} at the 95th percentile), peak memory ${gib(u.memoryPeak)} of ${gib(u.memory)}, peak disk ${gb(u.diskPeak)} of ${gb(u.disk)}${u.oomKills ? `, ${u.oomKills} process(es) killed for want of memory` : ""}`;
173 }
1	import { readFileSync, statfsSync } from "node:fs";
2	import os from "node:os";
3	import path from "node:path";
4	import { BUSY_BUCKETS, type JobUsage } from "@diffinery/shared";
5
6	/** Reads of the machine at one moment: CPU time (busy and all, in ticks), memory and disk in use. */
7	type Reading = { busy: number; total: number; memUsed: number; memTotal: number; diskUsed: number; diskTotal: number; oom: number \| null };
8
9	export interface UsageSamplerOptions {
10	/** Where /proc is (tests point it at a fake one). */
11	procRoot?: string;
12	/** The filesystem whose use is measured (the root filesystem holds the image, the workspace and the tool cache). */
13	diskPath?: string;
14	/** How often memory and disk are read; CPU is read on the same tick. */
15	sampleMs?: number;
16	/** Busy cores are measured over windows this long. */
17	windowMs?: number;
18	/** The machine's share of CPU in vCPUs (a quarter or half for the smallest hosted runners); by default the CPUs it sees. */
19	cores?: number;
20	/** CPUs the machine shows in /proc/stat (busy time is a share of all of them). */
21	visible?: number;
22	}
23
24	/**
25	* Measures what a job uses of its machine while it runs: busy cores (from /proc/stat), memory in use
26	* (MemTotal less MemAvailable from /proc/meminfo, which leaves out cache the kernel can drop), disk in
27	* use, and out-of-memory kills (/proc/vmstat). A hosted container is its own virtual machine, so these
28	* are the job's own numbers. Diffinery sizes the job's later runs by them (apps/diffinery/src/runner/sizes.ts).
29	* Off Linux, or where /proc can't be read, it measures nothing and reports null.
30	*/
31	export class UsageSampler {
32	private readonly proc: string;
33	private readonly diskPath: string;
34	private readonly sampleMs: number;
35	private readonly windowMs: number;
36	private readonly cores: number;
37	private readonly visible: number;
38	private timer: NodeJS.Timeout \| null = null;
39	private first: Reading \| null = null;
40	private windowStart: Reading \| null = null;
41	private windowStartedAt = 0;
42	private startedAt = 0;
43	private busyWindows: number[] = [];
44	private busySeconds = BUSY_BUCKETS.map(() => 0);
45	private cpuSeconds = 0;
46	private memPeak = 0;
47	private diskPeak = 0;
48	private last: Reading \| null = null;
49
50	constructor(o: UsageSamplerOptions = {}) {
51	this.proc = o.procRoot ?? "/proc";
52	this.diskPath = o.diskPath ?? "/";
53	this.sampleMs = o.sampleMs ?? 500;
54	this.windowMs = o.windowMs ?? 2000;
55	this.visible = o.visible ?? os.availableParallelism();
56	this.cores = o.cores && o.cores > 0 ? o.cores : this.visible;
57	}
58
59	start(): void {
60	const r = this.read();
61	if (!r) return;
62	this.first = this.windowStart = this.last = r;
63	this.startedAt = this.windowStartedAt = Date.now();
64	this.note(r);
65	this.timer = setInterval(() => this.tick(), this.sampleMs);
66	this.timer.unref();
67	}
68
69	/** Stops sampling and returns what the job used, or null if nothing could be measured. */
70	stop(): JobUsage \| null {
71	if (this.timer) clearInterval(this.timer);
72	this.timer = null;
73	if (!this.first) return null;
74	this.tick(true);
75	const first = this.first;
76	const last = this.last ?? first;
77	this.first = null;
78	const seconds = (Date.now() - this.startedAt) / 1000;
79	const sorted = [...this.busyWindows].sort((a, b) => a - b);
80	const pct = (p: number) => (sorted.length ? sorted[Math.min(sorted.length - 1, Math.floor(p * sorted.length))]! : 0);
81	const round = (n: number, d = 2) => Math.round(n * 10 d) / 10 d;
82	return {
83	seconds: round(seconds, 1),
84	cpuSeconds: round(this.cpuSeconds, 1),
85	coresP95: round(pct(0.95)),
86	coresMax: round(sorted.length ? sorted[sorted.length - 1]! : 0),
87	saturated: round(sorted.length ? sorted.filter((b) => b >= saturatedAt(this.cores)).length / sorted.length : 0, 3),
88	busySeconds: this.busySeconds.map((s) => round(s, 1)),
89	memoryPeak: this.memPeak,
90	diskPeak: this.diskPeak,
91	cores: this.cores,
92	memory: last.memTotal,
93	disk: last.diskTotal,
94	oomKills: first.oom !== null && last.oom !== null ? Math.max(0, last.oom - first.oom) : null,
95	};
96	}
97
98	private tick(final = false): void {
99	const r = this.read();
100	if (!r) return;
101	this.last = r;
102	this.note(r);
103	const now = Date.now();
104	if (this.windowStart && (final \|\| now - this.windowStartedAt >= this.windowMs)) {
105	const dTotal = r.total - this.windowStart.total;
106	const dBusy = r.busy - this.windowStart.busy;
107	const secs = (now - this.windowStartedAt) / 1000;
108	if (dTotal > 0 && secs > 0) {
109	const busyCores = Math.min(this.cores, Math.max(0, (dBusy / dTotal) * this.visible));
110	// A last window much shorter than the rest says little about how busy the job was.
111	if (!final \|\| secs >= this.windowMs / 2000) this.busyWindows.push(busyCores);
112	this.cpuSeconds += busyCores * secs;
113	const i = BUSY_BUCKETS.findIndex((b) => busyCores <= b);
114	this.busySeconds[i < 0 ? BUSY_BUCKETS.length - 1 : i]! += secs;
115	}
116	this.windowStart = r;
117	this.windowStartedAt = now;
118	}
119	}
120
121	private note(r: Reading): void {
122	this.memPeak = Math.max(this.memPeak, r.memUsed);
123	this.diskPeak = Math.max(this.diskPeak, r.diskUsed);
124	}
125
126	private read(): Reading \| null {
127	try {
128	const stat = readFileSync(path.join(this.proc, "stat"), "utf8");
129	const cpu = stat.split("\n", 1)[0]!.trim().split(/\s+/);
130	if (cpu[0] !== "cpu") return null;
131	// user nice system idle iowait irq softirq steal: busy is what ran here, not idle, waiting or stolen time.
132	const [user, nice, system, idle, iowait, irq, softirq, steal] = cpu.slice(1, 9).map((v) => Number(v) \|\| 0);
133	const busy = user! + nice! + system! + irq! + softirq!;
134	const total = busy + idle! + iowait! + steal!;
135	const mem = readFileSync(path.join(this.proc, "meminfo"), "utf8");
136	const kb = (name: string) => Number(new RegExp(`^${name}:\\s+(\\d+)`, "m").exec(mem)?.[1] ?? NaN) * 1024;
137	const memTotal = kb("MemTotal");
138	const memAvailable = kb("MemAvailable");
139	if (!Number.isFinite(memTotal) \|\| !Number.isFinite(memAvailable)) return null;
140	let diskUsed = 0;
141	let diskTotal = 0;
142	try {
143	const fs = statfsSync(this.diskPath);
144	diskTotal = fs.blocks * fs.bsize;
145	diskUsed = (fs.blocks - fs.bfree) * fs.bsize;
146	} catch {
147	/* no disk numbers */
148	}
149	let oom: number \| null = null;
150	try {
151	const m = /^oom_kill\s+(\d+)/m.exec(readFileSync(path.join(this.proc, "vmstat"), "utf8"));
152	if (m) oom = Number(m[1]);
153	} catch {
154	/* an older kernel doesn't count them */
155	}
156	return { busy, total, memUsed: memTotal - memAvailable, memTotal, diskUsed, diskTotal, oom };
157	} catch {
158	return null;
159	}
160	}
161	}
162
163	/** Busy cores at which a machine of `cores` vCPUs counts as held back by its CPU. */
164	export const saturatedAt = (cores: number) => Math.max(cores * 0.9, cores - 0.25);
165
166	const gib = (b: number) => `${(b / 1024 ** 3).toFixed(1)} GiB`;
167	const gb = (b: number) => `${(b / 1e9).toFixed(1)} GB`;
168
169	/** One log line saying what the job used of its machine. */
170	export function describeUsage(u: JobUsage): string {
171	const avg = u.seconds > 0 ? u.cpuSeconds / u.seconds : 0;
172	return `Resources: ${avg.toFixed(1)} of ${u.cores} vCPUs busy on average (${u.coresP95.toFixed(1)} at the 95th percentile), peak memory ${gib(u.memoryPeak)} of ${gib(u.memory)}, peak disk ${gb(u.diskPeak)} of ${gb(u.disk)}${u.oomKills ? `, ${u.oomKills} process(es) killed for want of memory` : ""}`;
173	}