| 1 | import type { Env } from "../env.js"; |
| 2 | import { HttpError, now } from "./util.js"; |
| 3 | |
| 4 | /** |
| 5 | * Per-account quotas on what Diffinery runs and keeps for an account. The account that pays is the |
| 6 | * owner of the repository (a person or an organization), whoever started the work. |
| 7 | * |
| 8 | * - hosted_minutes: minutes of hosted Actions jobs and agent sandboxes per calendar month (UTC). |
| 9 | * - hosted_jobs: hosted Actions jobs running at once across the account's repositories; more wait. |
| 10 | * - sandboxes: agent sandboxes open at once across the account's repositories. |
| 11 | * - repositories: repositories the account owns. |
| 12 | * - storage_bytes: bytes the account stores: its repositories' git objects, Git LFS objects and release |
| 13 | * assets, and its packages' blobs and manifests. Actions artifacts and caches don't count: each |
| 14 | * repository has its own caps on them, and they expire. |
| 15 | * |
| 16 | * Defaults come from the environment (QUOTA_*) and otherwise DEFAULT_QUOTAS; a site administrator |
| 17 | * can override them per account (account_quotas). -1 means unlimited. Accounts run by a site |
| 18 | * administrator (one, or an organization with one as owner) have no quotas, so the instance's own |
| 19 | * work never stops on them. |
| 20 | * |
| 21 | * Over every account, the instance runs at most fleetLimits().total hosted jobs at once, and one account |
| 22 | * at most half of them, exempt accounts included: the fleet stays under the runner container classes' |
| 23 | * max_instances, and no account can keep the others' jobs from starting. |
| 24 | */ |
| 25 | export type QuotaKey = "hosted_minutes" | "hosted_jobs" | "sandboxes" | "repositories" | "storage_bytes"; |
| 26 | export const QUOTA_KEYS: readonly QuotaKey[] = ["hosted_minutes", "hosted_jobs", "sandboxes", "repositories", "storage_bytes"]; |
| 27 | export type Quotas = Record<QuotaKey, number>; |
| 28 | |
| 29 | export const DEFAULT_QUOTAS: Quotas = { hosted_minutes: 3000, hosted_jobs: 20, sandboxes: 5, repositories: 500, storage_bytes: 10 * 1024 ** 3 }; |
| 30 | const ENV_KEYS: Record<QuotaKey, string> = { hosted_minutes: "QUOTA_HOSTED_MINUTES", hosted_jobs: "QUOTA_HOSTED_JOBS", sandboxes: "QUOTA_SANDBOXES", repositories: "QUOTA_REPOSITORIES", storage_bytes: "QUOTA_STORAGE_BYTES" }; |
| 31 | /** The largest quota a site administrator can set: storage is in bytes, the rest are counts. */ |
| 32 | const MAX_QUOTA: Record<QuotaKey, number> = { hosted_minutes: 1e9, hosted_jobs: 1e9, sandboxes: 1e9, repositories: 1e9, storage_bytes: 2 ** 50 }; |
| 33 | |
| 34 | /** |
| 35 | * The longest a job or sandbox can run (a job's 6 hours plus start-up, a sandbox's 24-hour token): |
| 36 | * a row still open after that is counted as ended then, so a lost end can't hold a slot forever. |
| 37 | */ |
| 38 | export const MAX_RUN_MS: Record<"job" | "sandbox", number> = { job: 7 * 3600_000, sandbox: 24 * 3600_000 }; |
| 39 | |
| 40 | const UNLIMITED: Quotas = { hosted_minutes: -1, hosted_jobs: -1, sandboxes: -1, repositories: -1, storage_bytes: -1 }; |
| 41 | |
| 42 | function envDefault(env: Env, k: QuotaKey): number { |
| 43 | const raw = (env as unknown as Record<string, string | undefined>)[ENV_KEYS[k]]; |
| 44 | const n = raw === undefined || raw === "" ? NaN : Math.trunc(Number(raw)); |
| 45 | return Number.isFinite(n) && n >= -1 ? n : DEFAULT_QUOTAS[k]; |
| 46 | } |
| 47 | |
| 48 | /** Whether the account is run by a site administrator (itself, or an organization with one as owner). */ |
| 49 | async function adminRun(env: Env, accountId: string): Promise<boolean> { |
| 50 | const r = await env.DB.prepare( |
| 51 | "SELECT 1 FROM accounts a WHERE a.id = ? AND (a.site_admin = 1 OR EXISTS (SELECT 1 FROM org_members m JOIN accounts o ON o.id = m.user_id WHERE m.org_id = a.id AND m.role = 'owner' AND o.site_admin = 1 AND o.status = 'active'))", |
| 52 | ) |
| 53 | .bind(accountId) |
| 54 | .first(); |
| 55 | return !!r; |
| 56 | } |
| 57 | |
| 58 | /** The quotas that apply to an account, and whether they're its own (overridden) or the defaults. */ |
| 59 | export async function quotasFor(env: Env, accountId: string): Promise<Quotas & { exempt: boolean }> { |
| 60 | if (await adminRun(env, accountId)) return { ...UNLIMITED, exempt: true }; |
| 61 | const o = await env.DB.prepare("SELECT * FROM account_quotas WHERE account_id = ?").bind(accountId).first<Record<QuotaKey, number | null>>(); |
| 62 | const q = {} as Quotas; |
| 63 | for (const k of QUOTA_KEYS) q[k] = o?.[k] ?? envDefault(env, k); |
| 64 | return { ...q, exempt: false }; |
| 65 | } |
| 66 | |
| 67 | /** Start of the current calendar month (UTC). */ |
| 68 | export function monthStart(at = new Date()): string { |
| 69 | return new Date(Date.UTC(at.getUTCFullYear(), at.getUTCMonth(), 1)).toISOString(); |
| 70 | } |
| 71 | |
| 72 | export interface Usage { |
| 73 | /** Whole minutes of hosted jobs and sandboxes this month, rounded up per job or sandbox. */ |
| 74 | hosted_minutes: number; |
| 75 | hosted_jobs: number; |
| 76 | sandboxes: number; |
| 77 | repositories: number; |
| 78 | storage_bytes: number; |
| 79 | } |
| 80 | |
| 81 | /** What an account uses now. */ |
| 82 | export async function usageOf(env: Env, accountId: string, at = new Date()): Promise<Usage> { |
| 83 | const [compute, repos, storage] = await Promise.all([computeUsage(env, accountId, at), env.DB.prepare("SELECT count(*) AS n FROM repos WHERE owner_id = ?").bind(accountId).first<{ n: number }>(), storedBytes(env, accountId)]); |
| 84 | return { ...compute, repositories: repos?.n ?? 0, storage_bytes: storage }; |
| 85 | } |
| 86 | |
| 87 | /** The hosted minutes, jobs and sandboxes an account uses now: all a job or sandbox start needs to know. */ |
| 88 | export async function computeUsage(env: Env, accountId: string, at = new Date()): Promise<Pick<Usage, "hosted_minutes" | "hosted_jobs" | "sandboxes">> { |
| 89 | const t = at.getTime(); |
| 90 | const month = monthStart(at); |
| 91 | // Anything that started earlier ended before the month began (or is counted as ended by then). |
| 92 | const rows = await env.DB.prepare("SELECT kind, started_at, ended_at FROM compute_usage WHERE account_id = ? AND started_at >= ?") |
| 93 | .bind(accountId, new Date(Math.min(Date.parse(month), t) - MAX_RUN_MS.sandbox).toISOString()) |
| 94 | .all<{ kind: "job" | "sandbox"; started_at: string; ended_at: string | null }>(); |
| 95 | let minutes = 0; |
| 96 | let jobs = 0; |
| 97 | let sandboxes = 0; |
| 98 | for (const r of rows.results) { |
| 99 | const start = Date.parse(r.started_at); |
| 100 | const cap = start + MAX_RUN_MS[r.kind]; |
| 101 | const end = Math.min(r.ended_at ? Date.parse(r.ended_at) : t, cap); |
| 102 | const from = Math.max(start, Date.parse(month)); |
| 103 | if (end > from) minutes += Math.ceil((end - from) / 60_000); |
| 104 | if (!r.ended_at && t < cap) { |
| 105 | if (r.kind === "job") jobs++; |
| 106 | else sandboxes++; |
| 107 | } |
| 108 | } |
| 109 | return { hosted_minutes: minutes, hosted_jobs: jobs, sandboxes }; |
| 110 | } |
| 111 | |
| 112 | /** Repositories not measured yet (made before storage was counted) measured by one check. */ |
| 113 | const MEASURE_AT_ONCE = 25; |
| 114 | |
| 115 | /** |
| 116 | * Bytes the account stores: what its repositories last reported (repo_storage) and its packages' blobs |
| 117 | * and manifests (a blob several of its packages use counts once). `live` is a repository counting its |
| 118 | * own bytes, which are fresher than its report. Repositories that never reported are asked first, a |
| 119 | * few per call, so storage made before it was counted is found without listing R2. |
| 120 | */ |
| 121 | export async function storedBytes(env: Env, accountId: string, live?: { repoId: string; bytes: number }): Promise<number> { |
| 122 | const unmeasured = await env.DB.prepare("SELECT r.id FROM repos r LEFT JOIN repo_storage s ON s.repo_id = r.id WHERE r.owner_id = ? AND s.repo_id IS NULL AND r.id != ? LIMIT ?") |
| 123 | .bind(accountId, live?.repoId ?? "", MEASURE_AT_ONCE) |
| 124 | .all<{ id: string }>(); |
| 125 | await Promise.all(unmeasured.results.map((r) => env.REPO.get(env.REPO.idFromName(r.id)).reportStorage().catch((e: unknown) => console.error("could not measure a repository's storage", r.id, e)))); |
| 126 | const row = await env.DB.prepare( |
| 127 | `SELECT (SELECT coalesce(sum(s.bytes), 0) FROM repo_storage s JOIN repos r ON r.id = s.repo_id WHERE r.owner_id = ?1 AND r.id != ?2) |
| 128 | + (SELECT coalesce(sum(b.size), 0) FROM registry_blobs b WHERE b.digest IN (SELECT pb.digest FROM package_blobs pb JOIN packages p ON p.id = pb.package_id WHERE p.owner_id = ?1)) |
| 129 | + (SELECT coalesce(sum(m.size), 0) FROM package_manifests m JOIN packages p ON p.id = m.package_id WHERE p.owner_id = ?1) AS n`, |
| 130 | ) |
| 131 | .bind(accountId, live?.repoId ?? "") |
| 132 | .first<{ n: number }>(); |
| 133 | return (row?.n ?? 0) + (live?.bytes ?? 0); |
| 134 | } |
| 135 | |
| 136 | /** |
| 137 | * Why a write of `adding` more bytes (0 when not known in advance) must be refused, or null: it would |
| 138 | * take the account past its storage quota. Reads and deletes never ask, so an account past its quota |
| 139 | * can always get back under it. |
| 140 | */ |
| 141 | export async function storageRefusal(env: Env, accountId: string, login: string, adding = 0, live?: { repoId: string; bytes: number }): Promise<string | null> { |
| 142 | const q = await quotasFor(env, accountId); |
| 143 | if (q.exempt || q.storage_bytes < 0) return null; |
| 144 | if ((await storedBytes(env, accountId, live)) + adding <= q.storage_bytes) return null; |
| 145 | return `${login} has reached its storage limit of ${q.storage_bytes} bytes (git objects, Git LFS objects, release assets and packages); delete some or ask a site administrator to raise it`; |
| 146 | } |
| 147 | |
| 148 | /** Bytes the account may still store: Infinity when it has no limit. */ |
| 149 | export async function storageHeadroom(env: Env, accountId: string): Promise<number> { |
| 150 | const q = await quotasFor(env, accountId); |
| 151 | if (q.exempt || q.storage_bytes < 0) return Infinity; |
| 152 | return Math.max(0, q.storage_bytes - (await storedBytes(env, accountId))); |
| 153 | } |
| 154 | |
| 155 | /** Refuse a write past the account's storage quota (see storageRefusal). */ |
| 156 | export async function checkStorageQuota(env: Env, accountId: string, login: string, adding = 0): Promise<void> { |
| 157 | const refused = await storageRefusal(env, accountId, login, adding); |
| 158 | if (refused) throw new HttpError(403, refused); |
| 159 | } |
| 160 | |
| 161 | const within = (limit: number, used: number) => limit < 0 || used < limit; |
| 162 | |
| 163 | /** |
| 164 | * Hosted jobs the instance runs at once by default. Under the runner container classes' max_instances |
| 165 | * (100) with room to spare: repositories start jobs side by side, each counting before the others' starts |
| 166 | * are recorded, so a burst can pass the limit by a few. |
| 167 | */ |
| 168 | export const HOSTED_JOBS_LIMIT = 80; |
| 169 | |
| 170 | /** Hosted jobs at once: `total` over the whole instance (HOSTED_JOBS_LIMIT), `share` for one account (half of it). */ |
| 171 | export function fleetLimits(env: Env): { total: number; share: number } { |
| 172 | const n = Math.trunc(Number(env.HOSTED_JOBS_LIMIT)); |
| 173 | const total = Number.isFinite(n) && n >= 2 ? n : HOSTED_JOBS_LIMIT; |
| 174 | return { total, share: Math.floor(total / 2) }; |
| 175 | } |
| 176 | |
| 177 | /** Hosted jobs running now, the instance's or one account's (a row left open past MAX_RUN_MS no longer counts, as in computeUsage). */ |
| 178 | async function runningJobs(env: Env, accountId?: string): Promise<number> { |
| 179 | const since = new Date(Date.now() - MAX_RUN_MS.job).toISOString(); |
| 180 | const q = accountId |
| 181 | ? env.DB.prepare("SELECT count(*) AS n FROM compute_usage WHERE account_id = ? AND kind = 'job' AND ended_at IS NULL AND started_at > ?").bind(accountId, since) |
| 182 | : env.DB.prepare("SELECT count(*) AS n FROM compute_usage WHERE ended_at IS NULL AND started_at > ? AND kind = 'job'").bind(since); |
| 183 | return (await q.first<{ n: number }>())?.n ?? 0; |
| 184 | } |
| 185 | |
| 186 | /** |
| 187 | * Whether the account may start a hosted job or a sandbox now: "ok", "wait" (a job waits for one of |
| 188 | * the account's running jobs to end), or an error message (minutes used up, or too many sandboxes). |
| 189 | */ |
| 190 | export async function mayStart(env: Env, accountId: string, kind: "job" | "sandbox"): Promise<{ ok: true } | { ok: false; wait: boolean; message: string }> { |
| 191 | const q = await quotasFor(env, accountId); |
| 192 | let running: number | null = null; |
| 193 | if (!q.exempt) { |
| 194 | // Not usageOf: a start doesn't need the repository count or the storage total (up to 25 repositories asked). |
| 195 | const u = await computeUsage(env, accountId); |
| 196 | if (!within(q.hosted_minutes, u.hosted_minutes)) |
| 197 | return { ok: false, wait: false, message: `this account has used its ${q.hosted_minutes} hosted minutes for this month (Actions jobs on hosted runners and agent sandboxes); they renew on the 1st (UTC). Self-hosted runners still work` }; |
| 198 | if (kind === "job" && !within(q.hosted_jobs, u.hosted_jobs)) return { ok: false, wait: true, message: `waiting for one of this account's ${q.hosted_jobs} hosted jobs to finish` }; |
| 199 | if (kind === "sandbox" && !within(q.sandboxes, u.sandboxes)) return { ok: false, wait: false, message: `this account already has ${q.sandboxes} agent sandboxes open; close one or wait for one to finish` }; |
| 200 | running = u.hosted_jobs; |
| 201 | } |
| 202 | if (kind === "job") { |
| 203 | // The fleet's limits hold for every account, exempt ones too. |
| 204 | const fleet = fleetLimits(env); |
| 205 | if ((running ?? (await runningJobs(env, accountId))) >= fleet.share) return { ok: false, wait: true, message: `waiting for one of this account's hosted jobs to finish: one account runs at most ${fleet.share} at once, half of this Diffinery's hosted runners` }; |
| 206 | if ((await runningJobs(env)) >= fleet.total) return { ok: false, wait: true, message: `waiting for a hosted runner: all ${fleet.total} of this Diffinery's hosted runners are busy` }; |
| 207 | } |
| 208 | return { ok: true }; |
| 209 | } |
| 210 | |
| 211 | /** Record that a hosted job or sandbox started; `id` is the job or sandbox id. */ |
| 212 | export async function startCompute(env: Env, row: { id: string; accountId: string; repoId: string; kind: "job" | "sandbox" }): Promise<void> { |
| 213 | await env.DB.prepare("INSERT OR REPLACE INTO compute_usage (id, account_id, repo_id, kind, started_at) VALUES (?, ?, ?, ?, ?)").bind(row.id, row.accountId, row.repoId, row.kind, now()).run(); |
| 214 | } |
| 215 | |
| 216 | /** Record that a hosted job or sandbox ended (idempotent). */ |
| 217 | export async function endCompute(env: Env, id: string): Promise<void> { |
| 218 | await env.DB.prepare("UPDATE compute_usage SET ended_at = ? WHERE id = ? AND ended_at IS NULL").bind(now(), id).run(); |
| 219 | } |
| 220 | |
| 221 | /** Refuse a new repository past the owner's quota. */ |
| 222 | export async function checkRepositoryQuota(env: Env, ownerId: string, ownerLogin: string): Promise<void> { |
| 223 | const q = await quotasFor(env, ownerId); |
| 224 | if (q.exempt || q.repositories < 0) return; |
| 225 | const n = (await env.DB.prepare("SELECT count(*) AS n FROM repos WHERE owner_id = ?").bind(ownerId).first<{ n: number }>())?.n ?? 0; |
| 226 | if (n >= q.repositories) throw new HttpError(403, `${ownerLogin} has reached its limit of ${q.repositories} repositories; delete one or ask a site administrator to raise it`); |
| 227 | } |
| 228 | |
| 229 | /** A site administrator's overrides for an account; null clears one back to the default. */ |
| 230 | export async function setQuotas(env: Env, accountId: string, patch: Partial<Record<QuotaKey, number | null>>, by: string): Promise<void> { |
| 231 | for (const [k, v] of Object.entries(patch)) { |
| 232 | if (!QUOTA_KEYS.includes(k as QuotaKey)) throw new HttpError(422, `unknown quota ${k}; use ${QUOTA_KEYS.join(", ")}`); |
| 233 | if (v !== null && (typeof v !== "number" || !Number.isInteger(v) || v < -1 || v > MAX_QUOTA[k as QuotaKey])) throw new HttpError(422, `${k} must be a whole number, -1 for unlimited, or null for the default`); |
| 234 | } |
| 235 | const cur = (await env.DB.prepare("SELECT * FROM account_quotas WHERE account_id = ?").bind(accountId).first<Record<QuotaKey, number | null>>()) ?? null; |
| 236 | const next = Object.fromEntries(QUOTA_KEYS.map((k) => [k, k in patch ? (patch[k] ?? null) : (cur?.[k] ?? null)])) as Record<QuotaKey, number | null>; |
| 237 | await env.DB.prepare( |
| 238 | "INSERT INTO account_quotas (account_id, hosted_minutes, hosted_jobs, sandboxes, repositories, storage_bytes, updated_at, updated_by) VALUES (?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(account_id) DO UPDATE SET hosted_minutes = excluded.hosted_minutes, hosted_jobs = excluded.hosted_jobs, sandboxes = excluded.sandboxes, repositories = excluded.repositories, storage_bytes = excluded.storage_bytes, updated_at = excluded.updated_at, updated_by = excluded.updated_by", |
| 239 | ) |
| 240 | .bind(accountId, next.hosted_minutes, next.hosted_jobs, next.sandboxes, next.repositories, next.storage_bytes, now(), by) |
| 241 | .run(); |
| 242 | } |