| 1 | import { ZERO_OID, diffTrees, fromUtf8, isTreeMode, readCommit, readTree, resolvePath } from "@diffinery/git-core"; |
| 2 | import type { Actor, Conclusion, Json, JobCompletion, JobSpec, LogLine, NeedsResult, StepReport } from "@diffinery/shared"; |
| 3 | import { |
| 4 | evaluateIf, |
| 5 | evaluateValue, |
| 6 | expandMatrix, |
| 7 | interpolate, |
| 8 | isWarning, |
| 9 | jobOrder, |
| 10 | matchesTrigger, |
| 11 | parseWorkflow, |
| 12 | type ExprContext, |
| 13 | type Job, |
| 14 | type StatusFns, |
| 15 | type TriggerEvent, |
| 16 | type Workflow, |
| 17 | } from "@diffinery/workflow-yaml"; |
| 18 | import { decryptSecret, encryptSecret } from "../lib/crypto.js"; |
| 19 | import { Directory } from "../lib/directory.js"; |
| 20 | import { getFirst, headFirst, legacyArtifactKeys, legacyCacheKeys, logKeys } from "../lib/storage-keys.js"; |
| 21 | import { permissionsForJob, type JobPermissions } from "../lib/job-permissions.js"; |
| 22 | import { signRuntimeToken } from "../lib/runtime-auth.js"; |
| 23 | import { ghRepo, numericId } from "../http/gh.js"; |
| 24 | import { HttpError, newId, now, randomToken, sha256 } from "../lib/util.js"; |
| 25 | import { localOnly } from "../auth/dev.js"; |
| 26 | import { endCompute, mayStart, startCompute } from "../lib/quotas.js"; |
| 27 | import { blockedBy, killSwitches, type Work } from "../lib/killswitch.js"; |
| 28 | import type { Env } from "../env.js"; |
| 29 | import type { RepoDO } from "./RepoDO.js"; |
| 30 | import { completePayload } from "./event-payloads.js"; |
| 31 | import { CACHE_IDLE_MS, CACHE_LIMIT_BYTES, artifactFileName, artifactObjectKeys } from "./runtime.js"; |
| 32 | import { atLeast, type Caller } from "./types.js"; |
| 33 | import { isJobCaller } from "./token-policy.js"; |
| 34 | import { ReusableError, expandReusable, localKey, parseCallRef, type Expanded, type Source } from "./reusable.js"; |
| 35 | import { ancestors, combineResults, jobStatus, runnerLabels, type JobNode } from "./job-graph.js"; |
| 36 | import { schedulerStub } from "../lib/schedules.js"; |
| 37 | import { APPROVAL_DAYS, type Deployment, type DeploymentReview, type Reviewer } from "./deployments.js"; |
| 38 | import { referencedSecrets, RUNNER_SECRETS, secretsFor } from "./job-secrets.js"; |
| 39 | |
| 40 | /** How long a sandbox token works after the sandbox starts. */ |
| 41 | export const SANDBOX_TOKEN_TTL_MS = 24 * 3600_000; |
| 42 | /** Sandboxes one person or agent may have open in a repository at once: their containers are shared by the whole instance. */ |
| 43 | export const MAX_OPEN_SANDBOXES = 5; |
| 44 | |
| 45 | const WORKFLOW_DIRS = [".github/workflows", ".diffinery/workflows"]; |
| 46 | /** Characters of text a job's log keeps; the rest is dropped after a line saying so. */ |
| 47 | export const MAX_JOB_LOG = 16 * 1048576; |
| 48 | /** Every push reads and parses a commit's workflow files: at most this many, each at most this large. */ |
| 49 | export const MAX_WORKFLOW_FILES = 256; |
| 50 | export const MAX_WORKFLOW_BYTES = 512 * 1024; |
| 51 | /** Hosted jobs one repository runs at once (GitHub's free plan allows 20 per account); more wait their turn. */ |
| 52 | export const MAX_HOSTED_JOBS = 20; |
| 53 | /** |
| 54 | * A hosted job whose container never got a machine (the platform had none free, usually when many |
| 55 | * jobs start at once or right after a deploy) goes back in line and starts again later, this many |
| 56 | * times, before it fails. Only jobs whose runner never reported in: nothing of theirs has run. |
| 57 | */ |
| 58 | export const MAX_PLACEMENT_TRIES = 8; |
| 59 | /** Seconds before the next start of a job whose container got no machine: 30 s, doubling to 5 minutes. */ |
| 60 | export const placementDelay = (tries: number): number => Math.min(30 * 2 ** (tries - 1), 300); |
| 61 | const NO_INSTANCE = "no container instance"; |
| 62 | /** Runs one repository may have unfinished at once; past it, new ones aren't started. */ |
| 63 | export const MAX_ACTIVE_RUNS = 500; |
| 64 | /** Self-hosted runners one repository may have registered. */ |
| 65 | export const MAX_RUNNERS = 100; |
| 66 | /** How long a runner registration token works, as on GitHub. */ |
| 67 | const REGISTRATION_TTL_MS = 3600_000; |
| 68 | /** A runner that polled this recently shows as online. */ |
| 69 | const RUNNER_ONLINE_MS = 90_000; |
| 70 | |
| 71 | type RunnerRow = { id: number; name: string; labels: string; token_hash: string | null; ephemeral: number; created_by: string; created_at: string; last_seen_at: string | null }; |
| 72 | |
| 73 | export interface RunnerInfo { |
| 74 | id: number; |
| 75 | name: string; |
| 76 | os: string; |
| 77 | status: "online" | "offline"; |
| 78 | busy: boolean; |
| 79 | ephemeral: boolean; |
| 80 | labels: Array<{ id: number; name: string; type: "read-only" | "custom" }>; |
| 81 | } |
| 82 | const SUPPORTED_LABEL = /^(ubuntu-.*|linux|diffinery(-.*)?|self-hosted|x64|amd64)$/i; |
| 83 | |
| 84 | export type RunRow = { |
| 85 | id: string; |
| 86 | run_number: number; |
| 87 | workflow_path: string; |
| 88 | workflow_name: string; |
| 89 | event: string; |
| 90 | ref: string; |
| 91 | sha: string; |
| 92 | actor: string; |
| 93 | status: "queued" | "in_progress" | "completed" | "waiting"; |
| 94 | conclusion: Conclusion | "timed_out" | null; |
| 95 | created_at: string; |
| 96 | updated_at: string; |
| 97 | pr_number: number | null; |
| 98 | attempt: number; |
| 99 | event_payload: string; |
| 100 | inputs: string; |
| 101 | concurrency_group: string | null; |
| 102 | display_title: string; |
| 103 | /** Commit the run reports checks on, when it differs from `sha` (pull request runs). */ |
| 104 | head_sha: string | null; |
| 105 | /** 1 when the code the run executes came from an agent; its jobs then act as an agent. */ |
| 106 | agent_origin?: number; |
| 107 | /** Account ids that started earlier attempts of this run (JSON list), for prevent_self_review. */ |
| 108 | prior_actors?: string | null; |
| 109 | }; |
| 110 | |
| 111 | export type JobRow = { |
| 112 | id: string; |
| 113 | run_id: string; |
| 114 | job_key: string; |
| 115 | name: string; |
| 116 | /** |
| 117 | * pending: held back by its concurrency group until the group's running job finishes. |
| 118 | * approval: waiting on its environment's required reviewers or wait timer; nothing has started. |
| 119 | */ |
| 120 | status: "queued" | "pending" | "approval" | "waiting" | "in_progress" | "completed"; |
| 121 | conclusion: Conclusion | "timed_out" | null; |
| 122 | matrix: string; |
| 123 | strategy: string; |
| 124 | started_at: string | null; |
| 125 | completed_at: string | null; |
| 126 | outputs: string; |
| 127 | steps: string; |
| 128 | failure: string | null; |
| 129 | token_hash: string | null; |
| 130 | summary: string | null; |
| 131 | container: string | null; |
| 132 | timeout_minutes: number; |
| 133 | /** runs-on labels for self-hosted jobs (JSON array); null for Diffinery-hosted containers. */ |
| 134 | labels: string | null; |
| 135 | /** `continue-on-error` as evaluated for this matrix leg (0 or 1). */ |
| 136 | continue_on_error: number; |
| 137 | /** Job-level `concurrency` group as evaluated for this leg, and its cancel-in-progress (0 or 1). */ |
| 138 | concurrency_group: string | null; |
| 139 | concurrency_cancel: number; |
| 140 | log_chars: number; |
| 141 | fleet_held: number; |
| 142 | runner_id: number | null; |
| 143 | /** `environment:` as evaluated at launch (null for jobs without one, and for jobs launched before it was stored). */ |
| 144 | environment: string | null; |
| 145 | /** When the job started waiting on its environment's protection rules, and "released" once they passed. */ |
| 146 | gate_since: string | null; |
| 147 | gate_state: string | null; |
| 148 | placement_tries: number; |
| 149 | }; |
| 150 | |
| 151 | /** An environment a run's jobs wait on (GitHub's pending deployment). */ |
| 152 | export type PendingDeployment = { |
| 153 | environment: { id: number; name: string }; |
| 154 | waitTimer: number; |
| 155 | waitTimerStartedAt: string; |
| 156 | reviewers: Reviewer[]; |
| 157 | preventSelfReview: boolean; |
| 158 | /** Still waiting for a required reviewer (false once approved, or when only a timer applies). */ |
| 159 | needsReview: boolean; |
| 160 | currentUserCanApprove: boolean; |
| 161 | /** The viewer is listed but started the run, and the environment prevents self-review. */ |
| 162 | selfBlocked: boolean; |
| 163 | }; |
| 164 | |
| 165 | export type RunInfo = { |
| 166 | id: string; |
| 167 | number: number; |
| 168 | attempt: number; |
| 169 | workflowPath: string; |
| 170 | workflowName: string; |
| 171 | title: string; |
| 172 | event: string; |
| 173 | ref: string; |
| 174 | sha: string; |
| 175 | /** The commit checks are reported on: the pull request head for pull_request runs. */ |
| 176 | headSha: string; |
| 177 | /** Branch the run is about: the pull request's head branch for pull request runs. */ |
| 178 | headBranch: string; |
| 179 | actor: Actor; |
| 180 | status: RunRow["status"]; |
| 181 | conclusion: RunRow["conclusion"]; |
| 182 | createdAt: string; |
| 183 | updatedAt: string; |
| 184 | prNumber: number | null; |
| 185 | }; |
| 186 | |
| 187 | export type JobInfo = { |
| 188 | id: string; |
| 189 | runId: string; |
| 190 | key: string; |
| 191 | name: string; |
| 192 | status: JobRow["status"]; |
| 193 | conclusion: JobRow["conclusion"]; |
| 194 | matrix: Record<string, Json>; |
| 195 | startedAt: string | null; |
| 196 | completedAt: string | null; |
| 197 | outputs: Record<string, string>; |
| 198 | steps: Array<StepReport & { index: number }>; |
| 199 | failure: JobCompletion["failure"] | null; |
| 200 | summary: string | null; |
| 201 | }; |
| 202 | |
| 203 | const toRun = (r: RunRow): RunInfo => ({ |
| 204 | id: r.id, |
| 205 | number: r.run_number, |
| 206 | attempt: r.attempt, |
| 207 | workflowPath: r.workflow_path, |
| 208 | workflowName: r.workflow_name, |
| 209 | title: r.display_title, |
| 210 | event: r.event, |
| 211 | ref: r.ref, |
| 212 | sha: r.sha, |
| 213 | headSha: r.head_sha ?? r.sha, |
| 214 | headBranch: (r.pr_number ? (JSON.parse(r.event_payload) as { pull_request?: { head?: { ref?: string } } }).pull_request?.head?.ref : undefined) ?? r.ref.replace(/^refs\/(heads|tags)\//, ""), |
| 215 | actor: JSON.parse(r.actor), |
| 216 | status: r.status, |
| 217 | conclusion: r.conclusion, |
| 218 | createdAt: r.created_at, |
| 219 | updatedAt: r.updated_at, |
| 220 | prNumber: r.pr_number, |
| 221 | }); |
| 222 | |
| 223 | const toJob = (j: JobRow): JobInfo => ({ |
| 224 | id: j.id, |
| 225 | runId: j.run_id, |
| 226 | key: j.job_key, |
| 227 | name: j.name, |
| 228 | status: j.status, |
| 229 | conclusion: j.conclusion, |
| 230 | matrix: JSON.parse(j.matrix), |
| 231 | startedAt: j.started_at, |
| 232 | completedAt: j.completed_at, |
| 233 | outputs: JSON.parse(j.outputs), |
| 234 | steps: JSON.parse(j.steps), |
| 235 | failure: j.failure ? JSON.parse(j.failure) : null, |
| 236 | summary: j.summary, |
| 237 | }); |
| 238 | |
| 239 | /** |
| 240 | * GitHub Actions compatible run orchestration state. The Cloudflare Workflow |
| 241 | * (`RunWorkflow`) drives a run through these methods; the runner agent inside |
| 242 | * each job's container reports back through the runner* methods. |
| 243 | */ |
| 244 | export class Actions { |
| 245 | constructor( |
| 246 | private repo: RepoDO, |
| 247 | private env: Env, |
| 248 | ) {} |
| 249 | |
| 250 | private get sql() { |
| 251 | return this.repo.sql; |
| 252 | } |
| 253 | |
| 254 | // ---------------------------------------------------------------- workflow discovery |
| 255 | |
| 256 | async loadWorkflows(sha: string): Promise<Array<{ path: string; workflow?: Workflow; errors: string[] }>> { |
| 257 | const commit = await readCommit(this.repo.store, sha); |
| 258 | const out: Array<{ path: string; workflow?: Workflow; errors: string[] }> = []; |
| 259 | for (const dir of WORKFLOW_DIRS) { |
| 260 | const entry = await resolvePath(this.repo.store, commit.tree, dir); |
| 261 | if (!entry || !isTreeMode(entry.mode)) continue; |
| 262 | for (const f of await readTree(this.repo.store, entry.oid)) { |
| 263 | if (!/\.ya?ml$/.test(f.name) || isTreeMode(f.mode)) continue; |
| 264 | if (out.length >= MAX_WORKFLOW_FILES) break; |
| 265 | const path = `${dir}/${f.name}`; |
| 266 | if ((this.repo.store.size(f.oid) ?? 0) > MAX_WORKFLOW_BYTES) { |
| 267 | out.push({ path, errors: [`: the workflow file is larger than ${MAX_WORKFLOW_BYTES / 1024} KiB`] }); |
| 268 | continue; |
| 269 | } |
| 270 | const o = await this.repo.store.read(f.oid); |
| 271 | if (!o) continue; |
| 272 | const { workflow, errors } = parseWorkflow(fromUtf8(o.content), path); |
| 273 | out.push({ path, workflow, errors: errors.filter((e) => !isWarning(e)).map((e) => `${e.path}${e.line ? `:${e.line}` : ""}: ${e.message}`) }); |
| 274 | } |
| 275 | } |
| 276 | return out; |
| 277 | } |
| 278 | |
| 279 | async listWorkflows(): Promise<Array<{ path: string; name: string; errors: string[]; triggers: string[] }>> { |
| 280 | const sha = await this.repo.resolve("HEAD"); |
| 281 | if (!sha) return []; |
| 282 | return (await this.loadWorkflows(sha)).map((w) => ({ path: w.path, name: w.workflow?.name ?? w.path.split("/").pop()!, errors: w.errors, triggers: w.workflow ? Object.keys(w.workflow.on) : [] })); |
| 283 | } |
| 284 | |
| 285 | /** The `repository` object of event payloads: GitHub's shape, with the repository's real visibility. */ |
| 286 | async repositoryPayload(): Promise<Record<string, unknown>> { |
| 287 | const m = this.repo.meta; |
| 288 | const row = await new Directory(this.env).repoById(m.repoId); |
| 289 | if (row) return ghRepo(this.env.PUBLIC_URL, row); |
| 290 | return { id: numericId(m.repoId), name: m.name, full_name: `${m.owner}/${m.name}`, owner: { login: m.owner }, default_branch: m.defaultBranch, private: true, visibility: "private", html_url: `${this.env.PUBLIC_URL}/${m.owner}/${m.name}`, clone_url: `${this.env.PUBLIC_URL}/${m.owner}/${m.name}.git` }; |
| 291 | } |
| 292 | |
| 293 | // ---------------------------------------------------------------- triggers |
| 294 | |
| 295 | async onPush(ev: { ref: string; before: string; after: string; forced: boolean; created: boolean; deleted: boolean }, caller: Caller): Promise<void> { |
| 296 | if (ev.deleted) return; |
| 297 | const workflows = await this.loadWorkflows(ev.after); |
| 298 | let changedFiles: string[] | undefined; |
| 299 | if (ev.before !== ZERO_OID && this.repo.store.has(ev.before)) { |
| 300 | const [a, b] = await Promise.all([readCommit(this.repo.store, ev.before), readCommit(this.repo.store, ev.after)]); |
| 301 | changedFiles = (await diffTrees(this.repo.store, a.tree, b.tree)).map((c) => c.path); |
| 302 | } |
| 303 | const head = await readCommit(this.repo.store, ev.after); |
| 304 | const payload = { |
| 305 | ref: ev.ref, |
| 306 | before: ev.before, |
| 307 | after: ev.after, |
| 308 | created: ev.created, |
| 309 | deleted: ev.deleted, |
| 310 | forced: ev.forced, |
| 311 | repository: await this.repositoryPayload(), |
| 312 | pusher: { name: caller.login }, |
| 313 | sender: { login: caller.login, type: caller.kind === "agent" ? "Bot" : "User" }, |
| 314 | head_commit: { id: ev.after, message: head.message, timestamp: new Date(head.committer.timestamp * 1000).toISOString(), author: { name: head.author.name, email: head.author.email } }, |
| 315 | commits: [{ id: ev.after, message: head.message }], |
| 316 | }; |
| 317 | if (ev.ref === `refs/heads/${this.repo.meta.defaultBranch}`) await this.syncSchedules(workflows); |
| 318 | for (const w of workflows) { |
| 319 | if (!w.workflow) { |
| 320 | if (w.errors.length) await this.createFailedRun(w.path, "push", ev.ref, ev.after, caller, w.errors); |
| 321 | continue; |
| 322 | } |
| 323 | if (!matchesTrigger(w.workflow, { name: "push", ref: ev.ref, changedFiles })) continue; |
| 324 | await this.createRun(w.workflow, { event: "push", ref: ev.ref, sha: ev.after, payload, caller, title: head.message.split("\n")[0] ?? "" }); |
| 325 | } |
| 326 | } |
| 327 | |
| 328 | /** |
| 329 | * Pull request workflows, like GitHub's: `pull_request` runs the workflow from the test |
| 330 | * merge commit (`refs/pull/<n>/merge`, the head merged into the base; nothing runs while |
| 331 | * they conflict), and `pull_request_target` runs the base branch's own workflow on the |
| 332 | * base commit, so a pull request can never change what runs with the base's secrets. |
| 333 | * Both report their checks on the pull request's head commit. |
| 334 | */ |
| 335 | async onPullRequest(action: string, number: number, caller: Caller): Promise<void> { |
| 336 | const { issue } = this.repo.collab.get(number); |
| 337 | const pull = issue.pull!; |
| 338 | // Once merged, the base already holds the changes: keep the refs as they were and let path filters pass. |
| 339 | const { baseSha, mergeSha } = pull.merged ? { baseSha: this.repo.getRef(`refs/heads/${pull.base}`), mergeSha: null } : await this.repo.collab.refreshPullRefs(number); |
| 340 | const changedFiles = pull.merged ? undefined : (await this.repo.collab.pullFiles(number)).files.map((f) => f.path); |
| 341 | const m = this.repo.meta; |
| 342 | const repository = await this.repositoryPayload(); |
| 343 | const payload = { |
| 344 | action, |
| 345 | number, |
| 346 | pull_request: { |
| 347 | number, |
| 348 | id: number, |
| 349 | state: issue.state, |
| 350 | title: issue.title, |
| 351 | body: issue.body, |
| 352 | draft: pull.draft, |
| 353 | merged: pull.merged, |
| 354 | merged_at: pull.mergedAt, |
| 355 | merged_by: pull.mergedBy ? { login: pull.mergedBy } : null, |
| 356 | merge_commit_sha: pull.merged ? pull.mergeCommit : mergeSha, |
| 357 | user: { login: issue.author.login }, |
| 358 | labels: issue.labels.map((name) => ({ name })), |
| 359 | assignees: issue.assignees.map((login) => ({ login })), |
| 360 | head: { ref: pull.head, sha: pull.headSha, repo: repository }, |
| 361 | base: { ref: pull.base, sha: baseSha, repo: repository }, |
| 362 | html_url: `${this.env.PUBLIC_URL}/${m.owner}/${m.name}/pull/${number}`, |
| 363 | url: `${this.env.PUBLIC_URL}/api/v3/repos/${m.owner}/${m.name}/pulls/${number}`, |
| 364 | }, |
| 365 | repository, |
| 366 | sender: { login: caller.login }, |
| 367 | }; |
| 368 | const common = { payload, caller, prNumber: number, headSha: pull.headSha, headRef: pull.head, baseRef: pull.base, title: issue.title }; |
| 369 | // A merged pull request's runs use its merge commit, as on GitHub. |
| 370 | const prSha = pull.merged ? pull.mergeCommit : mergeSha; |
| 371 | if (prSha) { |
| 372 | for (const w of await this.loadWorkflows(prSha)) { |
| 373 | if (!w.workflow || !matchesTrigger(w.workflow, { name: "pull_request", action, baseRef: pull.base, changedFiles })) continue; |
| 374 | await this.createRun(w.workflow, { ...common, event: "pull_request", ref: `refs/pull/${number}/merge`, sha: prSha }); |
| 375 | } |
| 376 | } |
| 377 | if (baseSha) { |
| 378 | for (const w of await this.loadWorkflows(baseSha)) { |
| 379 | if (!w.workflow || !matchesTrigger(w.workflow, { name: "pull_request_target", action, baseRef: pull.base, changedFiles })) continue; |
| 380 | await this.createRun(w.workflow, { ...common, event: "pull_request_target", ref: `refs/heads/${pull.base}`, sha: baseSha }); |
| 381 | } |
| 382 | } |
| 383 | } |
| 384 | |
| 385 | /** `merge_group` workflows for a merge queue group, run from the group's commit. */ |
| 386 | async onMergeGroup(g: { number: number; base: string; ref: string; sha: string; parent: string; title: string; sender: Actor }): Promise<void> { |
| 387 | const head = await readCommit(this.repo.store, g.sha); |
| 388 | const payload = { |
| 389 | action: "checks_requested", |
| 390 | merge_group: { |
| 391 | head_sha: g.sha, |
| 392 | head_ref: g.ref, |
| 393 | base_sha: g.parent, |
| 394 | base_ref: `refs/heads/${g.base}`, |
| 395 | head_commit: { |
| 396 | id: g.sha, |
| 397 | tree_id: head.tree, |
| 398 | message: head.message, |
| 399 | timestamp: new Date(head.committer.timestamp * 1000).toISOString(), |
| 400 | author: { name: head.author.name, email: head.author.email }, |
| 401 | committer: { name: head.committer.name, email: head.committer.email }, |
| 402 | }, |
| 403 | }, |
| 404 | repository: await this.repositoryPayload(), |
| 405 | sender: { login: g.sender.login }, |
| 406 | }; |
| 407 | for (const w of await this.loadWorkflows(g.sha)) { |
| 408 | if (!w.workflow) { |
| 409 | if (w.errors.length) await this.createFailedRun(w.path, "merge_group", g.ref, g.sha, g.sender, w.errors); |
| 410 | continue; |
| 411 | } |
| 412 | if (!matchesTrigger(w.workflow, { name: "merge_group", action: "checks_requested", baseRef: g.base })) continue; |
| 413 | await this.createRun(w.workflow, { event: "merge_group", ref: g.ref, sha: g.sha, payload, caller: g.sender, title: `Merge group for #${g.number}: ${g.title}`, prNumbers: [g.number] }); |
| 414 | } |
| 415 | } |
| 416 | |
| 417 | /** Cancel unfinished runs on a ref (a merge group branch that is going away). */ |
| 418 | async cancelRunsOnRef(ref: string, reason: string): Promise<void> { |
| 419 | for (const r of this.sql.exec<{ id: string }>("SELECT id FROM runs WHERE ref = ? AND status != 'completed'", ref).toArray()) await this.cancelInternal(r.id, reason); |
| 420 | } |
| 421 | |
| 422 | async dispatch(workflowFile: string, ref: string, inputs: Record<string, unknown>, caller: Caller): Promise<RunInfo> { |
| 423 | if (!atLeast(caller.role, "write")) throw new HttpError(403, "write access is required to run workflows"); |
| 424 | await this.refuseIfBlocked("run"); |
| 425 | const sha = await this.repo.resolve(ref); |
| 426 | if (!sha) throw new HttpError(422, `unknown ref ${ref}`); |
| 427 | const all = await this.loadWorkflows(sha); |
| 428 | const w = all.find((x) => x.path === workflowFile || x.path.endsWith(`/${workflowFile}`)); |
| 429 | if (!w?.workflow) throw new HttpError(404, `workflow ${workflowFile} not found at ${ref}${w?.errors.length ? `: ${w.errors.join("; ")}` : ""}`); |
| 430 | if (!w.workflow.on.workflow_dispatch) throw new HttpError(422, "workflow does not have a workflow_dispatch trigger"); |
| 431 | const defs = w.workflow.on.workflow_dispatch.inputs ?? {}; |
| 432 | const resolved: Record<string, unknown> = {}; |
| 433 | for (const [k, d] of Object.entries(defs)) { |
| 434 | const v = inputs[k] ?? d.default; |
| 435 | if (d.required && (v === undefined || v === "")) throw new HttpError(422, `input '${k}' is required`); |
| 436 | if (d.type === "choice" && d.options && v !== undefined && !d.options.includes(String(v))) throw new HttpError(422, `input '${k}' must be one of ${d.options.join(", ")}`); |
| 437 | resolved[k] = d.type === "boolean" ? v === true || v === "true" : d.type === "number" ? Number(v) : v; |
| 438 | } |
| 439 | const fullRef = ref.startsWith("refs/") ? ref : this.repo.getRef(`refs/heads/${ref}`) ? `refs/heads/${ref}` : `refs/tags/${ref}`; |
| 440 | const payload = { inputs: resolved, ref: fullRef, repository: await this.repositoryPayload(), sender: { login: caller.login }, workflow: w.path }; |
| 441 | const m = this.repo.meta; |
| 442 | this.repo.emit({ type: "workflow_dispatch", repoId: m.repoId, owner: m.owner, repo: m.name, ref: fullRef, workflow: w.path, inputs: resolved as Record<string, Json>, actor: { id: caller.id, login: caller.login, kind: caller.kind, onBehalfOf: caller.onBehalfOf }, at: now() }); |
| 443 | return (await this.createRun(w.workflow, { event: "workflow_dispatch", ref: fullRef, sha, payload, caller, inputs: resolved, title: w.workflow.name ?? w.path }))!; |
| 444 | } |
| 445 | |
| 446 | async schedule(workflowPath: string, cron: string): Promise<void> { |
| 447 | const sha = await this.repo.resolve("HEAD"); |
| 448 | if (!sha) return; |
| 449 | const w = (await this.loadWorkflows(sha)).find((x) => x.path === workflowPath); |
| 450 | if (!w?.workflow || !matchesTrigger(w.workflow, { name: "schedule", cron })) return; |
| 451 | const system: Caller = { id: "system", login: "diffinery-scheduler", kind: "system", role: "write" }; |
| 452 | await this.createRun(w.workflow, { event: "schedule", ref: this.repo.headRef, sha, payload: { schedule: cron, repository: await this.repositoryPayload() }, caller: system, title: w.workflow.name ?? w.path }); |
| 453 | } |
| 454 | |
| 455 | private async syncSchedules(workflows: Array<{ path: string; workflow?: Workflow }>): Promise<void> { |
| 456 | const repoId = this.repo.meta.repoId; |
| 457 | const stmts = [this.env.DB.prepare("DELETE FROM schedules WHERE repo_id = ?").bind(repoId)]; |
| 458 | for (const w of workflows) for (const s of w.workflow?.on.schedule ?? []) stmts.push(this.env.DB.prepare("INSERT OR IGNORE INTO schedules (repo_id, workflow_path, cron) VALUES (?, ?, ?)").bind(repoId, w.path, s.cron)); |
| 459 | await this.env.DB.batch(stmts); |
| 460 | // The scheduler's alarm starts with the first schedule, without waiting for the cron trigger. |
| 461 | if (stmts.length > 1) await schedulerStub(this.env).ensure().catch((e) => console.error("scheduler alarm check failed", e)); |
| 462 | } |
| 463 | |
| 464 | // ---------------------------------------------------------------- runs |
| 465 | |
| 466 | private nextRunNumber(): number { |
| 467 | this.sql.exec("INSERT INTO counters (name, value) VALUES ('run', 1) ON CONFLICT(name) DO UPDATE SET value = value + 1"); |
| 468 | return this.sql.exec<{ value: number }>("SELECT value FROM counters WHERE name = 'run'").one().value; |
| 469 | } |
| 470 | |
| 471 | private githubContext(run: RunRow, jobKey: string, token: string): Record<string, unknown> { |
| 472 | const m = this.repo.meta; |
| 473 | const payload = JSON.parse(run.event_payload) as Record<string, unknown>; |
| 474 | const actor = JSON.parse(run.actor) as Actor; |
| 475 | const refName = run.ref.replace(/^refs\/(heads|tags)\//, "").replace(/^refs\/pull\/(\d+)\/(head|merge)$/, "$1/merge"); |
| 476 | const pr = payload.pull_request as { head?: { ref?: string }; base?: { ref?: string } } | undefined; |
| 477 | return { |
| 478 | event_name: run.event, |
| 479 | event: payload, |
| 480 | sha: run.sha, |
| 481 | ref: run.ref, |
| 482 | ref_name: refName, |
| 483 | ref_type: run.ref.startsWith("refs/tags/") ? "tag" : "branch", |
| 484 | ref_protected: false, |
| 485 | head_ref: pr?.head?.ref ?? "", |
| 486 | base_ref: pr?.base?.ref ?? "", |
| 487 | repository: `${m.owner}/${m.name}`, |
| 488 | repository_id: String(numericId(m.repoId)), |
| 489 | repository_owner: m.owner, |
| 490 | repositoryUrl: `${this.env.PUBLIC_URL}/${m.owner}/${m.name}.git`, |
| 491 | actor: actor.login, |
| 492 | triggering_actor: actor.login, |
| 493 | // Numeric like GitHub's (actions parse it, e.g. download-artifact's run-id); the API accepts it as a run id. |
| 494 | run_id: String(run.run_number), |
| 495 | run_number: String(run.run_number), |
| 496 | run_attempt: String(run.attempt), |
| 497 | workflow: run.workflow_name, |
| 498 | workflow_ref: `${m.owner}/${m.name}/${run.workflow_path}@${run.ref}`, |
| 499 | job: jobKey, |
| 500 | server_url: this.env.PUBLIC_URL, |
| 501 | api_url: `${this.env.PUBLIC_URL}/api/v3`, |
| 502 | graphql_url: `${this.env.PUBLIC_URL}/api/graphql`, |
| 503 | token, |
| 504 | retention_days: "90", |
| 505 | secret_source: "Actions", |
| 506 | }; |
| 507 | } |
| 508 | |
| 509 | private async createFailedRun(path: string, event: string, ref: string, sha: string, caller: Actor, errors: string[]): Promise<void> { |
| 510 | // While Actions are suspended no run is recorded, not even a failed one. |
| 511 | if (await this.blocked("run")) return; |
| 512 | const id = newId(); |
| 513 | const t = now(); |
| 514 | this.sql.exec( |
| 515 | "INSERT INTO runs (id, run_number, workflow_path, workflow_name, event, ref, sha, actor, status, conclusion, created_at, updated_at, event_payload, display_title) VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'completed', 'failure', ?, ?, ?, ?)", |
| 516 | id, |
| 517 | this.nextRunNumber(), |
| 518 | path, |
| 519 | path, |
| 520 | event, |
| 521 | ref, |
| 522 | sha, |
| 523 | JSON.stringify({ id: caller.id, login: caller.login, kind: caller.kind }), |
| 524 | t, |
| 525 | t, |
| 526 | "{}", |
| 527 | `Invalid workflow file: ${errors[0] ?? path}`, |
| 528 | ); |
| 529 | const jobId = newId(); |
| 530 | this.sql.exec( |
| 531 | "INSERT INTO jobs (id, run_id, job_key, name, status, conclusion, started_at, completed_at, failure) VALUES (?, ?, 'invalid', 'Invalid workflow file', 'completed', 'failure', ?, ?, ?)", |
| 532 | jobId, |
| 533 | id, |
| 534 | t, |
| 535 | t, |
| 536 | JSON.stringify({ message: errors.join("\n") }), |
| 537 | ); |
| 538 | } |
| 539 | |
| 540 | async createRun( |
| 541 | wf: Workflow, |
| 542 | o: { |
| 543 | event: string; |
| 544 | ref: string; |
| 545 | sha: string; |
| 546 | headSha?: string; |
| 547 | payload: Record<string, unknown>; |
| 548 | caller: Caller | { id: string; login: string; kind: Actor["kind"] }; |
| 549 | prNumber?: number; |
| 550 | inputs?: Record<string, unknown>; |
| 551 | headRef?: string; |
| 552 | baseRef?: string; |
| 553 | title: string; |
| 554 | /** Started by a run whose code came from an agent (a re-run, or a workflow_run follower). */ |
| 555 | agentOrigin?: boolean; |
| 556 | /** Pull requests whose code the run includes (a merge group's). */ |
| 557 | prNumbers?: number[]; |
| 558 | }, |
| 559 | ): Promise<RunInfo | null> { |
| 560 | // Like GitHub, what a job does with its GITHUB_TOKEN (pushes, pull requests) starts no new |
| 561 | // runs, so workflows cannot trigger themselves in a loop; workflow_dispatch and |
| 562 | // repository_dispatch are the exceptions. |
| 563 | if (isJobCaller(o.caller) && o.event !== "workflow_dispatch" && o.event !== "repository_dispatch") return null; |
| 564 | // A site administrator suspended Actions here (lib/killswitch.ts): events start nothing, and a |
| 565 | // dispatch, which someone asked for, is refused with the reason. |
| 566 | const block = await this.blocked("run"); |
| 567 | if (block) { |
| 568 | if (o.event === "workflow_dispatch" || o.event === "repository_dispatch") throw new HttpError(403, block); |
| 569 | return null; |
| 570 | } |
| 571 | // A dispatch loop (or a flood of pushes) stops growing here. |
| 572 | const active = this.sql.exec<{ n: number }>("SELECT count(*) AS n FROM runs WHERE status != 'completed'").one().n; |
| 573 | if (active >= MAX_ACTIVE_RUNS) { |
| 574 | if (o.event === "workflow_dispatch" || o.event === "repository_dispatch") throw new HttpError(429, `this repository has ${active} unfinished workflow runs, the most allowed; wait for some to finish`); |
| 575 | console.warn(`not starting ${wf.path}: ${active} unfinished runs`); |
| 576 | return null; |
| 577 | } |
| 578 | const id = newId(); |
| 579 | const t = now(); |
| 580 | const run: RunRow = { |
| 581 | id, |
| 582 | run_number: this.nextRunNumber(), |
| 583 | workflow_path: wf.path, |
| 584 | workflow_name: wf.name ?? wf.path, |
| 585 | event: o.event, |
| 586 | ref: o.ref, |
| 587 | sha: o.sha, |
| 588 | actor: JSON.stringify({ id: o.caller.id, login: o.caller.login, kind: o.caller.kind }), |
| 589 | status: "queued", |
| 590 | conclusion: null, |
| 591 | created_at: t, |
| 592 | updated_at: t, |
| 593 | pr_number: o.prNumber ?? null, |
| 594 | attempt: 1, |
| 595 | event_payload: JSON.stringify(completePayload({ publicUrl: this.env.PUBLIC_URL, meta: this.repo.meta, visibility: (o.payload.repository as { visibility?: string } | undefined)?.visibility === "public" ? "public" : "private" }, o.payload, o.caller)), |
| 596 | inputs: JSON.stringify(o.inputs ?? {}), |
| 597 | concurrency_group: null, |
| 598 | display_title: o.title, |
| 599 | head_sha: o.headSha && o.headSha !== o.sha ? o.headSha : null, |
| 600 | agent_origin: this.repo.policy.agentOrigin({ |
| 601 | caller: o.caller, |
| 602 | ref: o.ref, |
| 603 | prNumbers: [...(o.prNumber !== undefined ? [o.prNumber] : []), ...(o.prNumbers ?? [])], |
| 604 | inherited: o.agentOrigin, |
| 605 | }) |
| 606 | ? 1 |
| 607 | : 0, |
| 608 | }; |
| 609 | const ctx: ExprContext = { github: this.githubContext(run, "", ""), inputs: o.inputs ?? {}, vars: this.variables() }; |
| 610 | try { |
| 611 | if (wf.runName) run.display_title = interpolate(wf.runName, ctx); |
| 612 | if (wf.concurrency) run.concurrency_group = interpolate(wf.concurrency.group, ctx); |
| 613 | } catch (e) { |
| 614 | console.warn("run-name/concurrency evaluation failed", e); |
| 615 | } |
| 616 | this.sql.exec( |
| 617 | "INSERT INTO runs (id, run_number, workflow_path, workflow_name, event, ref, sha, actor, status, created_at, updated_at, pr_number, attempt, event_payload, inputs, concurrency_group, display_title, head_sha, agent_origin) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", |
| 618 | run.id, |
| 619 | run.run_number, |
| 620 | run.workflow_path, |
| 621 | run.workflow_name, |
| 622 | run.event, |
| 623 | run.ref, |
| 624 | run.sha, |
| 625 | run.actor, |
| 626 | run.status, |
| 627 | run.created_at, |
| 628 | run.updated_at, |
| 629 | run.pr_number, |
| 630 | run.attempt, |
| 631 | run.event_payload, |
| 632 | run.inputs, |
| 633 | run.concurrency_group, |
| 634 | run.display_title, |
| 635 | run.head_sha, |
| 636 | run.agent_origin ?? 0, |
| 637 | ); |
| 638 | if (run.concurrency_group && wf.concurrency) { |
| 639 | const cancelInProgress = evaluateValue(wf.concurrency.cancelInProgress, ctx); |
| 640 | const others = this.sql.exec<{ id: string }>("SELECT id FROM runs WHERE concurrency_group = ? AND status != 'completed' AND id != ?", run.concurrency_group, id).toArray(); |
| 641 | for (const other of others) if (cancelInProgress === true || cancelInProgress === "true") await this.cancelInternal(other.id, "cancelled by a newer run in the same concurrency group"); |
| 642 | } |
| 643 | await this.env.RUN_WORKFLOW.create({ id, params: { repoId: this.repo.meta.repoId, runId: id } }); |
| 644 | const m = this.repo.meta; |
| 645 | this.repo.emit({ type: "workflow_run", action: "requested", repoId: m.repoId, owner: m.owner, repo: m.name, runId: id, actor: JSON.parse(run.actor), at: t }); |
| 646 | this.repo.broadcast("runs", { action: "created", runId: id }); |
| 647 | await this.onWorkflowRun(run, "requested"); |
| 648 | return toRun(run); |
| 649 | } |
| 650 | |
| 651 | /** A run by id, or by its number (github.run_id; the latest attempt). */ |
| 652 | private runRow(id: string): RunRow { |
| 653 | const r = /^\d+$/.test(id) |
| 654 | ? this.sql.exec<RunRow>("SELECT * FROM runs WHERE run_number = ? ORDER BY attempt DESC, created_at DESC LIMIT 1", Number(id)).toArray()[0] |
| 655 | : this.sql.exec<RunRow>("SELECT * FROM runs WHERE id = ?", id).toArray()[0]; |
| 656 | if (!r) throw new HttpError(404, "run not found"); |
| 657 | return r; |
| 658 | } |
| 659 | |
| 660 | private jobRow(id: string): JobRow { |
| 661 | const j = this.sql.exec<JobRow>("SELECT * FROM jobs WHERE id = ?", id).toArray()[0]; |
| 662 | if (!j) throw new HttpError(404, "job not found"); |
| 663 | return j; |
| 664 | } |
| 665 | |
| 666 | listRuns(opts: { workflow?: string; branch?: string; status?: string; event?: string; sha?: string; prNumber?: number; limit?: number; offset?: number } = {}): { runs: RunInfo[]; total: number } { |
| 667 | const where: string[] = []; |
| 668 | const args: unknown[] = []; |
| 669 | if (opts.workflow) { |
| 670 | where.push("(workflow_path = ? OR workflow_path LIKE ?)"); |
| 671 | args.push(opts.workflow, `%/${opts.workflow}`); |
| 672 | } |
| 673 | if (opts.branch) { |
| 674 | where.push("ref = ?"); |
| 675 | args.push(`refs/heads/${opts.branch}`); |
| 676 | } |
| 677 | if (opts.status) { |
| 678 | where.push(opts.status === "completed" || opts.status === "queued" || opts.status === "in_progress" ? "status = ?" : "conclusion = ?"); |
| 679 | args.push(opts.status); |
| 680 | } |
| 681 | if (opts.event) { |
| 682 | where.push("event = ?"); |
| 683 | args.push(opts.event); |
| 684 | } |
| 685 | if (opts.sha) { |
| 686 | where.push("(sha = ? OR head_sha = ?)"); |
| 687 | args.push(opts.sha, opts.sha); |
| 688 | } |
| 689 | if (opts.prNumber) { |
| 690 | where.push("pr_number = ?"); |
| 691 | args.push(opts.prNumber); |
| 692 | } |
| 693 | const w = where.length ? `WHERE ${where.join(" AND ")}` : ""; |
| 694 | const total = this.sql.exec<{ n: number }>(`SELECT count(*) AS n FROM runs ${w}`, ...args).one().n; |
| 695 | const rows = this.sql.exec<RunRow>(`SELECT * FROM runs ${w} ORDER BY created_at DESC LIMIT ? OFFSET ?`, ...args, Math.min(opts.limit ?? 25, 100), opts.offset ?? 0).toArray(); |
| 696 | return { runs: rows.map(toRun), total }; |
| 697 | } |
| 698 | |
| 699 | getRun(id: string): RunInfo & { jobs: JobInfo[] } { |
| 700 | const run = this.runRow(id); |
| 701 | return { ...toRun(run), jobs: this.jobs(run.id) }; |
| 702 | } |
| 703 | |
| 704 | jobs(runId: string): JobInfo[] { |
| 705 | if (/^\d+$/.test(runId)) runId = this.runRow(runId).id; |
| 706 | return this.sql.exec<JobRow>("SELECT * FROM jobs WHERE run_id = ? ORDER BY rowid", runId).toArray().map(toJob); |
| 707 | } |
| 708 | |
| 709 | getJob(id: string): JobInfo { |
| 710 | return toJob(this.jobRow(id)); |
| 711 | } |
| 712 | |
| 713 | async logs(jobId: string, afterSeq = -1): Promise<{ lines: Array<LogLine & { seq: number }>; complete: boolean }> { |
| 714 | const j = this.jobRow(jobId); |
| 715 | const rows = this.sql.exec<LogLine & { seq: number }>("SELECT seq, t, step, text FROM logs WHERE job_id = ? AND seq > ? ORDER BY seq LIMIT 5000", jobId, afterSeq).toArray(); |
| 716 | if (rows.length === 0 && j.status === "completed") { |
| 717 | const archived = await getFirst(this.env.BUCKET, logKeys(this.repo.meta.repoId, jobId)); |
| 718 | if (archived && afterSeq < 0) return { lines: (await archived.json()) as Array<LogLine & { seq: number }>, complete: true }; |
| 719 | } |
| 720 | return { lines: rows, complete: j.status === "completed" }; |
| 721 | } |
| 722 | |
| 723 | private logKey(jobId: string) { |
| 724 | return logKeys(this.repo.meta.repoId, jobId)[0]; |
| 725 | } |
| 726 | |
| 727 | // ---------------------------------------------------------------- orchestration (called by RunWorkflow) |
| 728 | |
| 729 | /** |
| 730 | * The run's job graph: each job with the jobs it waits for (reusable workflow calls |
| 731 | * expanded). RunWorkflow starts a job as soon as its own needs have finished. |
| 732 | */ |
| 733 | async plan(runId: string): Promise<{ jobs: JobNode[]; error?: string; concurrencyBlocked: boolean }> { |
| 734 | const run = this.runRow(runId); |
| 735 | if (run.status === "completed") return { jobs: [], error: "run already completed", concurrencyBlocked: false }; |
| 736 | let exp: Expanded; |
| 737 | try { |
| 738 | exp = await this.expanded(run); |
| 739 | } catch (e) { |
| 740 | return { jobs: [], error: (e as Error).message, concurrencyBlocked: false }; |
| 741 | } |
| 742 | const { order, errors } = jobOrder(exp.workflow); |
| 743 | if (errors.length) return { jobs: [], error: errors.map((e) => e.message).join("; "), concurrencyBlocked: false }; |
| 744 | const blocked = |
| 745 | !!run.concurrency_group && |
| 746 | this.sql.exec<{ n: number }>("SELECT count(*) AS n FROM runs WHERE concurrency_group = ? AND status = 'in_progress' AND id != ?", run.concurrency_group, runId).one().n > 0; |
| 747 | if (!blocked) this.sql.exec("UPDATE runs SET status = 'in_progress', updated_at = ? WHERE id = ? AND status = 'queued'", now(), runId); |
| 748 | return { jobs: order.flat().map((key) => ({ key, needs: exp.workflow.jobs[key]!.needs })), concurrencyBlocked: blocked }; |
| 749 | } |
| 750 | |
| 751 | /** The run's workflow with reusable workflow calls expanded (see ./reusable.ts). */ |
| 752 | private async expanded(run: RunRow): Promise<Expanded> { |
| 753 | const wf = (await this.loadWorkflows(run.sha)).find((w) => w.path === run.workflow_path); |
| 754 | if (!wf?.workflow) throw new ReusableError(`workflow file not found or invalid: ${wf?.errors.join("; ") || run.workflow_path}`); |
| 755 | return expandReusable(wf.workflow, { repoId: null, sha: run.sha }, (uses, from) => this.loadCalled(uses, from)); |
| 756 | } |
| 757 | |
| 758 | /** Load a called workflow: `./path` from the same commit, or `owner/repo/path@ref` from a repository on this instance. */ |
| 759 | private async loadCalled(uses: string, from: Source): Promise<{ workflow: Workflow; source: Source }> { |
| 760 | const ref = parseCallRef(uses); |
| 761 | const m = this.repo.meta; |
| 762 | let source: Source; |
| 763 | let text: string | null; |
| 764 | if (ref.kind === "local" && from.repoId === null) { |
| 765 | source = from; |
| 766 | const b = await this.repo.blob(from.sha, ref.path); |
| 767 | text = b && !b.binary ? fromUtf8(b.content) : null; |
| 768 | } else { |
| 769 | let repoId: string; |
| 770 | if (ref.kind === "local") repoId = from.repoId!; |
| 771 | else if (ref.owner.toLowerCase() === m.owner.toLowerCase() && ref.repo.toLowerCase() === m.name.toLowerCase()) repoId = m.repoId; |
| 772 | else { |
| 773 | const dir = new Directory(this.env); |
| 774 | // A transferred or renamed repository's workflows are found under the old name too. |
| 775 | const target = (await dir.repo(ref.owner, ref.repo)) ?? (await dir.movedRepo(ref.owner, ref.repo)); |
| 776 | // Like GitHub: public workflows can be shared; private ones only within the same owner. |
| 777 | if (!target || (target.visibility === "private" && target.owner_id !== m.ownerId)) throw new Error(`repository ${ref.owner}/${ref.repo} not found, or its workflows are not shared with ${m.owner}/${m.name}`); |
| 778 | repoId = target.id; |
| 779 | } |
| 780 | if (repoId === m.repoId) { |
| 781 | const sha = ref.kind === "local" ? from.sha : await this.repo.resolve(ref.ref); |
| 782 | if (!sha) throw new Error(`unknown ref ${ref.kind === "remote" ? ref.ref : ""}`); |
| 783 | source = { repoId: null, sha }; |
| 784 | const b = await this.repo.blob(sha, ref.path); |
| 785 | text = b && !b.binary ? fromUtf8(b.content) : null; |
| 786 | } else { |
| 787 | const stub = this.env.REPO.get(this.env.REPO.idFromName(repoId)); |
| 788 | const sha = ref.kind === "local" ? from.sha : await stub.resolve(ref.ref); |
| 789 | if (!sha) throw new Error(`unknown ref ${ref.kind === "remote" ? ref.ref : ""} in ${ref.kind === "remote" ? `${ref.owner}/${ref.repo}` : "the called repository"}`); |
| 790 | source = { repoId, sha }; |
| 791 | const b = await stub.blob(sha, ref.path); |
| 792 | text = b && !b.binary ? fromUtf8(b.content) : null; |
| 793 | } |
| 794 | } |
| 795 | if (text === null) throw new Error(`${uses} not found`); |
| 796 | const { workflow, errors } = parseWorkflow(text, ref.path); |
| 797 | const fatal = errors.filter((e) => !isWarning(e)); |
| 798 | if (!workflow || fatal.length) throw new Error(`${uses} is invalid: ${fatal.map((e) => `${e.path}: ${e.message}`).join("; ")}`); |
| 799 | return { workflow, source }; |
| 800 | } |
| 801 | |
| 802 | /** `needs` context for a job or call: keys are as written inside its own workflow. */ |
| 803 | private async needsResults(run: RunRow, exp: Expanded, keys: string[], scope: string | null): Promise<Record<string, NeedsResult>> { |
| 804 | const out: Record<string, NeedsResult> = {}; |
| 805 | for (const key of keys) out[localKey(key, scope)] = await this.resultOf(run, exp, key); |
| 806 | return out; |
| 807 | } |
| 808 | |
| 809 | /** Status functions for the `if:` of a job or call: from the results of all its ancestors, like GitHub. */ |
| 810 | private statusOf(run: RunRow, exp: Expanded, key: string): StatusFns { |
| 811 | const results = ancestors(exp.needs, exp.needs.get(key) ?? []).map((k) => combineResults(this.rowsOf(run, exp, k))); |
| 812 | return jobStatus(results, run.conclusion === "cancelled"); |
| 813 | } |
| 814 | |
| 815 | /** The job rows of a job, or of every job inside a reusable workflow call. */ |
| 816 | private rowsOf(run: RunRow, exp: Expanded, key: string): Array<JobRow & { continueOnError: boolean }> { |
| 817 | const rows = exp.calls.has(key) |
| 818 | ? this.sql.exec<JobRow>("SELECT * FROM jobs WHERE run_id = ? AND substr(job_key, 1, ?) = ?", run.id, key.length + 1, `${key}/`).toArray() |
| 819 | : this.sql.exec<JobRow>("SELECT * FROM jobs WHERE run_id = ? AND job_key = ?", run.id, key).toArray(); |
| 820 | return rows.map((r) => ({ ...r, continueOnError: !!r.continue_on_error })); |
| 821 | } |
| 822 | |
| 823 | /** Result and outputs of a job, or of a reusable workflow call (from its `on.workflow_call.outputs`). */ |
| 824 | private async resultOf(run: RunRow, exp: Expanded, key: string): Promise<NeedsResult> { |
| 825 | const call = exp.calls.get(key); |
| 826 | const rows = this.rowsOf(run, exp, key); |
| 827 | const result = combineResults(rows); |
| 828 | let outputs: Record<string, string> = {}; |
| 829 | if (!call) for (const r of rows) Object.assign(outputs, JSON.parse(r.outputs)); |
| 830 | if (call && Object.keys(call.workflow.on.workflow_call?.outputs ?? {}).length) { |
| 831 | const jobs: Record<string, NeedsResult> = {}; |
| 832 | for (const k of Object.keys(call.workflow.jobs)) jobs[k] = await this.resultOf(run, exp, `${key}/${k}`); |
| 833 | const { inputs } = await this.callContext(run, exp, key, false); |
| 834 | const ctx: ExprContext = { github: this.githubContext(run, localKey(key, call.parent), ""), jobs, inputs, vars: this.variables() }; |
| 835 | outputs = {}; |
| 836 | for (const [name, d] of Object.entries(call.workflow.on.workflow_call!.outputs!)) { |
| 837 | try { |
| 838 | const v = evaluateValue(d.value, ctx); |
| 839 | outputs[name] = v === undefined || v === null ? "" : typeof v === "object" ? JSON.stringify(v) : String(v); |
| 840 | } catch { |
| 841 | outputs[name] = ""; |
| 842 | } |
| 843 | } |
| 844 | } |
| 845 | return { result, outputs }; |
| 846 | } |
| 847 | |
| 848 | /** |
| 849 | * `inputs` and `secrets` inside a reusable workflow call (or at the top level when |
| 850 | * `callKey` is null): inputs from the caller's `with` and the declared defaults, |
| 851 | * secrets from `secrets: inherit` or the caller's mapping. |
| 852 | */ |
| 853 | private async callContext(run: RunRow, exp: Expanded, callKey: string | null, withSecrets: boolean): Promise<{ inputs: Record<string, unknown>; secrets: Record<string, string> }> { |
| 854 | if (callKey === null) return { inputs: JSON.parse(run.inputs), secrets: withSecrets ? { ...(await this.orgSecrets()), ...(await this.repoSecrets()) } : {} }; |
| 855 | const call = exp.calls.get(callKey)!; |
| 856 | const parent = await this.callContext(run, exp, call.parent, withSecrets); |
| 857 | const ctx: ExprContext = { github: this.githubContext(run, localKey(callKey, call.parent), ""), needs: await this.needsResults(run, exp, call.needs, call.parent), inputs: parent.inputs, vars: this.variables() }; |
| 858 | const inputs: Record<string, unknown> = {}; |
| 859 | for (const [k, d] of Object.entries(call.workflow.on.workflow_call?.inputs ?? {})) { |
| 860 | let v = call.job.with && k in call.job.with ? evaluateValue(call.job.with[k], ctx) : evaluateValue(d.default, ctx); |
| 861 | if (v === undefined || v === null || v === "") { |
| 862 | if (d.required) throw new ReusableError(`jobs.${callKey}: input '${k}' is required by ${call.job.uses}`); |
| 863 | v = d.type === "boolean" ? false : d.type === "number" ? 0 : ""; |
| 864 | } |
| 865 | inputs[k] = d.type === "boolean" ? v === true || v === "true" : d.type === "number" ? Number(v) : v; |
| 866 | } |
| 867 | const secrets: Record<string, string> = {}; |
| 868 | if (withSecrets) { |
| 869 | if (call.job.secrets === "inherit") Object.assign(secrets, parent.secrets); |
| 870 | else for (const [name, expr] of Object.entries(call.job.secrets ?? {})) secrets[name.toUpperCase()] = String(evaluateValue(expr, { ...ctx, secrets: parent.secrets }) ?? ""); |
| 871 | for (const [name, d] of Object.entries(call.workflow.on.workflow_call?.secrets ?? {})) { |
| 872 | if (d.required && !secrets[name.toUpperCase()]) throw new ReusableError(`jobs.${callKey}: secret '${name}' is required by ${call.job.uses}`); |
| 873 | } |
| 874 | } |
| 875 | return { inputs, secrets }; |
| 876 | } |
| 877 | |
| 878 | /** |
| 879 | * Start every matrix instance of a job. Returns the ids the workflow must wait |
| 880 | * for (instances beyond max-parallel are queued and started as others finish). |
| 881 | */ |
| 882 | async startJobKey(runId: string, jobKey: string): Promise<{ jobIds: string[] }> { |
| 883 | const run = this.runRow(runId); |
| 884 | const exp = await this.expanded(run); |
| 885 | const job = exp.workflow.jobs[jobKey]!; |
| 886 | const owner = exp.workflowOf.get(jobKey)!; |
| 887 | const parent = exp.parentOf.get(jobKey) ?? null; |
| 888 | const chain: string[] = []; |
| 889 | for (let c = parent; c; c = exp.calls.get(c)!.parent) chain.unshift(c); |
| 890 | const prefix = chain.map((c) => exp.calls.get(c)!.job.name ?? localKey(c, exp.calls.get(c)!.parent)).join(" / "); |
| 891 | const label = (name: string) => (prefix ? `${prefix} / ${name}` : name); |
| 892 | const fail = (name: string, message: string, conclusion: Conclusion = "failure") => { |
| 893 | const id = newId(); |
| 894 | const t = now(); |
| 895 | this.sql.exec("INSERT INTO jobs (id, run_id, job_key, name, status, conclusion, started_at, completed_at, failure) VALUES (?, ?, ?, ?, 'completed', ?, ?, ?, ?)", id, runId, jobKey, name, conclusion, t, t, conclusion === "failure" ? JSON.stringify({ message }) : null); |
| 896 | this.repo.broadcast(`run:${runId}`, { action: "job", jobId: id }); |
| 897 | return { jobIds: [] as string[] }; |
| 898 | }; |
| 899 | const short = localKey(jobKey, parent); |
| 900 | let needs: Record<string, NeedsResult>; |
| 901 | let baseCtx: ExprContext; |
| 902 | let shouldRun: boolean; |
| 903 | try { |
| 904 | // The callers decide first: a skipped call, or one whose ancestors failed or were skipped, skips everything inside it. |
| 905 | for (const c of chain) { |
| 906 | const call = exp.calls.get(c)!; |
| 907 | const callNeeds = await this.needsResults(run, exp, call.needs, call.parent); |
| 908 | const outer = await this.callContext(run, exp, call.parent, false); |
| 909 | const callCtx: ExprContext = { github: this.githubContext(run, localKey(c, call.parent), ""), needs: callNeeds, inputs: outer.inputs, vars: this.variables() }; |
| 910 | if (!evaluateIf(call.job.if, callCtx, this.statusOf(run, exp, c))) return fail(label(job.name ?? short), "", "skipped"); |
| 911 | } |
| 912 | // Resolve the call's inputs and secrets now, so a missing required one fails here rather than in the runner. |
| 913 | if (parent) await this.callContext(run, exp, parent, true); |
| 914 | needs = await this.needsResults(run, exp, exp.needs.get(jobKey) ?? [], parent); |
| 915 | baseCtx = { github: this.githubContext(run, short, ""), needs, inputs: (await this.callContext(run, exp, parent, false)).inputs, vars: this.variables(), env: owner.env }; |
| 916 | shouldRun = evaluateIf(job.if, baseCtx, this.statusOf(run, exp, jobKey)); |
| 917 | } catch (e) { |
| 918 | return fail(label(job.name ?? short), e instanceof ReusableError ? e.message : `Error evaluating 'if': ${(e as Error).message}`); |
| 919 | } |
| 920 | if (!shouldRun) return fail(label(job.name ?? short), "", "skipped"); |
| 921 | |
| 922 | let combos: Array<Record<string, unknown>>; |
| 923 | try { |
| 924 | combos = expandMatrix(job.strategy?.matrix, baseCtx); |
| 925 | } catch (e) { |
| 926 | return fail(label(job.name ?? short), `Error expanding matrix: ${(e as Error).message}`); |
| 927 | } |
| 928 | if (combos.length > 256) return fail(label(job.name ?? short), "A matrix may produce at most 256 jobs."); |
| 929 | const failFast = evaluateValue(job.strategy?.failFast ?? true, baseCtx) !== false; |
| 930 | const maxParallel = Number(evaluateValue(job.strategy?.maxParallel ?? combos.length, baseCtx)) || combos.length; |
| 931 | const ids: string[] = []; |
| 932 | combos.forEach((matrix, i) => { |
| 933 | // Like GitHub, name, runs-on, timeout-minutes, continue-on-error and concurrency are evaluated for each matrix leg. |
| 934 | const ctx = { ...baseCtx, matrix, strategy: { "fail-fast": failFast, "job-index": i, "job-total": combos.length, "max-parallel": maxParallel } }; |
| 935 | let name = short; |
| 936 | if (!job.name && Object.keys(matrix).length) name = `${short} (${Object.values(matrix).map((v) => (typeof v === "object" ? JSON.stringify(v) : String(v))).join(", ")})`; |
| 937 | let leg: { labels: string[]; timeout: number; continueOnError: boolean; group: string | null; cancel: boolean; environment: string | null }; |
| 938 | try { |
| 939 | if (job.name) name = interpolate(job.name, ctx); |
| 940 | const continueOnError = evaluateValue(job.continueOnError, ctx); |
| 941 | const cancel = job.concurrency ? evaluateValue(job.concurrency.cancelInProgress, ctx) : false; |
| 942 | leg = { |
| 943 | labels: runnerLabels(job.runsOn, ctx), |
| 944 | timeout: Number(evaluateValue(job.timeoutMinutes, ctx)) || 360, |
| 945 | continueOnError: continueOnError === true || continueOnError === "true", |
| 946 | group: (job.concurrency ? interpolate(job.concurrency.group, ctx).trim() : "") || null, |
| 947 | cancel: cancel === true || cancel === "true", |
| 948 | // Evaluated once, here: its protection rules gate the launch, and runnerSpec deploys to this same name. |
| 949 | environment: job.environment ? interpolate(job.environment.name, ctx).trim() : null, |
| 950 | }; |
| 951 | } catch (e) { |
| 952 | fail(label(name), `Error evaluating the job for this matrix combination: ${(e as Error).message}`); |
| 953 | return; |
| 954 | } |
| 955 | const unsupported = leg.labels.filter((l) => !SUPPORTED_LABEL.test(l)); |
| 956 | const selfHosted = leg.labels.some((l) => l.toLowerCase() === "self-hosted"); |
| 957 | if (!leg.labels.length || (unsupported.length && !selfHosted)) { |
| 958 | fail(label(name), `No Diffinery runner matches runs-on: ${unsupported.join(", ") || "(no labels)"}. Linux (ubuntu-*) runners are hosted; anything else needs runs-on: [self-hosted, ...] and a registered runner.`); |
| 959 | return; |
| 960 | } |
| 961 | const id = newId(); |
| 962 | this.sql.exec( |
| 963 | "INSERT INTO jobs (id, run_id, job_key, name, status, matrix, strategy, timeout_minutes, labels, continue_on_error, concurrency_group, concurrency_cancel, environment) VALUES (?, ?, ?, ?, 'queued', ?, ?, ?, ?, ?, ?, ?, ?)", |
| 964 | id, |
| 965 | runId, |
| 966 | jobKey, |
| 967 | label(name), |
| 968 | JSON.stringify(matrix), |
| 969 | JSON.stringify(ctx.strategy), |
| 970 | Math.min(leg.timeout, 360), |
| 971 | selfHosted ? JSON.stringify(leg.labels.map((l) => l.toLowerCase())) : null, |
| 972 | leg.continueOnError ? 1 : 0, |
| 973 | leg.group, |
| 974 | leg.cancel ? 1 : 0, |
| 975 | leg.environment, |
| 976 | ); |
| 977 | ids.push(id); |
| 978 | }); |
| 979 | for (const id of ids.slice(0, maxParallel)) await this.launch(id); |
| 980 | this.repo.broadcast(`run:${runId}`, { action: "jobs" }); |
| 981 | return { jobIds: ids }; |
| 982 | } |
| 983 | |
| 984 | private checkName(run: RunRow, job: JobRow): string { |
| 985 | return `${run.workflow_name} / ${job.name}`; |
| 986 | } |
| 987 | |
| 988 | private async launch(jobId: string): Promise<void> { |
| 989 | const job = this.jobRow(jobId); |
| 990 | if (job.status === "completed") return; |
| 991 | // Kill switches (lib/killswitch.ts): a suspended repository or owner starts no job; a paused instance no hosted one. |
| 992 | const block = await this.blocked(job.labels ? "self-hosted-job" : "hosted-job"); |
| 993 | if (block) return this.completeJob(jobId, { conclusion: "failure", outputs: {}, failure: { message: `The job was not started: ${block}.` } }); |
| 994 | if (!(await this.gate(job)) || !(await this.concurrencySlot(job))) return; |
| 995 | const run = this.runRow(job.run_id); |
| 996 | const m = this.repo.meta; |
| 997 | if (job.labels) { |
| 998 | // Self-hosted: park the job until a registered runner with matching labels claims it. |
| 999 | this.sql.exec("UPDATE jobs SET status = 'waiting', container = NULL WHERE id = ?", jobId); |
| 1000 | this.repo.upsertCheck({ sha: run.head_sha ?? run.sha, name: this.checkName(run, job), status: "queued", runId: run.id, jobId, detailsUrl: `${this.env.PUBLIC_URL}/${m.owner}/${m.name}/actions/runs/${run.id}/job/${jobId}` }); |
| 1001 | return; |
| 1002 | } |
| 1003 | // The account that owns the repository pays (lib/quotas.ts): past its monthly minutes the job |
| 1004 | // fails; past its hosted jobs at once it waits (fleet_held = 2, retried by the alarm) for one to end. |
| 1005 | const quota = await mayStart(this.env, m.ownerId, "job").catch((e: unknown) => { |
| 1006 | console.error("quota check failed", e); |
| 1007 | return { ok: true } as const; |
| 1008 | }); |
| 1009 | if (!quota.ok) { |
| 1010 | if (!quota.wait) return this.completeJob(jobId, { conclusion: "failure", outputs: {}, failure: { message: `The job was not started: ${quota.message}.` } }); |
| 1011 | this.sql.exec("UPDATE jobs SET status = 'queued', fleet_held = 2 WHERE id = ?", jobId); |
| 1012 | this.repo.upsertCheck({ sha: run.head_sha ?? run.sha, name: this.checkName(run, job), status: "queued", runId: run.id, jobId, detailsUrl: `${this.env.PUBLIC_URL}/${m.owner}/${m.name}/actions/runs/${run.id}/job/${jobId}` }); |
| 1013 | this.repo.armAlarm(null); |
| 1014 | return; |
| 1015 | } |
| 1016 | // One repository can't take the whole fleet: past its share, the job waits for one of its own to finish. |
| 1017 | const busy = this.sql.exec<{ n: number }>("SELECT count(*) AS n FROM jobs WHERE labels IS NULL AND container IS NOT NULL AND status IN ('waiting', 'in_progress') AND id != ?", jobId).one().n; |
| 1018 | if (busy >= MAX_HOSTED_JOBS) { |
| 1019 | this.sql.exec("UPDATE jobs SET status = 'queued', fleet_held = 1 WHERE id = ?", jobId); |
| 1020 | this.repo.upsertCheck({ sha: run.head_sha ?? run.sha, name: this.checkName(run, job), status: "queued", runId: run.id, jobId, detailsUrl: `${this.env.PUBLIC_URL}/${m.owner}/${m.name}/actions/runs/${run.id}/job/${jobId}` }); |
| 1021 | return; |
| 1022 | } |
| 1023 | this.sql.exec("UPDATE jobs SET fleet_held = 0 WHERE id = ?", jobId); |
| 1024 | const token = `djt_${m.repoId}_${jobId}_${randomToken(24)}`; |
| 1025 | const container = `job-${jobId}`; |
| 1026 | this.sql.exec("UPDATE jobs SET status = 'waiting', token_hash = ?, container = ? WHERE id = ?", await sha256(token), container, jobId); |
| 1027 | await startCompute(this.env, { id: jobId, accountId: m.ownerId, repoId: m.repoId, kind: "job" }).catch((e: unknown) => console.error("compute usage failed", e)); |
| 1028 | this.repo.upsertCheck({ sha: run.head_sha ?? run.sha, name: this.checkName(run, job), status: "queued", runId: run.id, jobId, detailsUrl: `${this.env.PUBLIC_URL}/${m.owner}/${m.name}/actions/runs/${run.id}/job/${jobId}` }); |
| 1029 | try { |
| 1030 | const stub = this.env.RUNNER.get(this.env.RUNNER.idFromName(container)); |
| 1031 | await stub.startJob({ repoId: m.repoId, jobId, token, apiUrl: this.env.PUBLIC_URL }); |
| 1032 | } catch (e) { |
| 1033 | const message = (e as Error).message; |
| 1034 | if (message.includes(NO_INSTANCE)) return this.placementFailed(jobId, message); |
| 1035 | await this.completeJob(jobId, { conclusion: "failure", outputs: {}, failure: { message: `Could not start a runner: ${message}` } }); |
| 1036 | } |
| 1037 | } |
| 1038 | |
| 1039 | |
| 1040 | // ---------------------------------------------------------------- environment protection rules |
| 1041 | |
| 1042 | private jobUrl(run: RunRow, jobId: string): string { |
| 1043 | const m = this.repo.meta; |
| 1044 | return `${this.env.PUBLIC_URL}/${m.owner}/${m.name}/actions/runs/${run.id}/job/${jobId}`; |
| 1045 | } |
| 1046 | |
| 1047 | /** |
| 1048 | * Whose run this is, for prevent_self_review, as account ids (logins can be renamed): who started |
| 1049 | * it and its earlier attempts, and for each of them that is an agent, the account that owns it. |
| 1050 | */ |
| 1051 | async selfIds(run: RunRow): Promise<Set<string>> { |
| 1052 | const actor = JSON.parse(run.actor) as Actor; |
| 1053 | const ids = new Set([actor.id, ...((run.prior_actors ? JSON.parse(run.prior_actors) : []) as string[])].filter(Boolean)); |
| 1054 | const dir = new Directory(this.env); |
| 1055 | for (const id of [...ids]) { |
| 1056 | const a = await dir.account(id).catch(() => null); |
| 1057 | if (a?.kind === "agent" && a.owner_id) ids.add(a.owner_id); |
| 1058 | } |
| 1059 | return ids; |
| 1060 | } |
| 1061 | |
| 1062 | /** |
| 1063 | * A job whose environment has required reviewers or a wait timer waits here, before a token is |
| 1064 | * minted or a runner started, in the status `approval`: no concurrency slot, no hosted slot. |
| 1065 | * Returns whether the job may go on to start. The branch policy is checked first, so a ref that |
| 1066 | * may never deploy fails at once instead of asking anyone. |
| 1067 | */ |
| 1068 | private async gate(job: JobRow): Promise<boolean> { |
| 1069 | if (!job.environment || job.gate_state === "released") return true; |
| 1070 | const rules = this.repo.deploys.protection(job.environment); |
| 1071 | if (!rules) { |
| 1072 | if (job.status === "approval") this.release(job); |
| 1073 | return true; |
| 1074 | } |
| 1075 | const run = this.runRow(job.run_id); |
| 1076 | const fail = async (message: string) => { |
| 1077 | await this.completeJob(job.id, { conclusion: "failure", outputs: {}, failure: { message } }); |
| 1078 | return false; |
| 1079 | }; |
| 1080 | if (!this.repo.deploys.refAllowed(job.environment, run.ref)) return fail(`${run.ref.replace(/^refs\/(heads|tags)\//, "")} is not allowed to deploy to ${job.environment} due to environment protection rules.`); |
| 1081 | const decision = rules.reviewers.length ? this.repo.deploys.decision(run.id, job.environment) : null; |
| 1082 | if (decision === "rejected") { |
| 1083 | const r = this.repo.deploys.reviews(run.id).find((x) => x.environment.toLowerCase() === job.environment!.toLowerCase() && x.state === "rejected")!; |
| 1084 | return fail(`The deployment to ${job.environment} was rejected by ${r.user.login}${r.comment ? `: ${r.comment}` : "."}`); |
| 1085 | } |
| 1086 | let since = job.gate_since; |
| 1087 | if (job.status !== "approval") { |
| 1088 | since = now(); |
| 1089 | this.sql.exec("UPDATE jobs SET status = 'approval', gate_since = ? WHERE id = ?", since, job.id); |
| 1090 | this.sql.exec("UPDATE runs SET status = 'waiting', updated_at = ? WHERE id = ? AND status != 'completed'", since, run.id); |
| 1091 | const actor = JSON.parse(run.actor) as Actor; |
| 1092 | this.repo.deploys.jobWaiting({ id: job.id, ref: run.ref, sha: run.sha, logUrl: this.jobUrl(run, job.id), actor: run.agent_origin ? { ...actor, kind: "agent" } : actor }, job.environment); |
| 1093 | this.repo.upsertCheck({ sha: run.head_sha ?? run.sha, name: this.checkName(run, job), status: "queued", runId: run.id, jobId: job.id, detailsUrl: this.jobUrl(run, job.id) }); |
| 1094 | // Reviewers hear about it once per run and environment, from its first job to wait. |
| 1095 | const first = !this.sql.exec("SELECT 1 FROM jobs WHERE run_id = ? AND environment = ? COLLATE NOCASE AND gate_since IS NOT NULL AND id != ?", run.id, job.environment, job.id).toArray().length; |
| 1096 | if (first && rules.reviewers.length && !decision) this.emitReview("requested", run, job.environment, rules.reviewers, actor, null, ""); |
| 1097 | this.repo.broadcast(`run:${run.id}`, { action: "job", jobId: job.id }); |
| 1098 | this.repo.broadcast("runs", { action: "waiting", runId: run.id }); |
| 1099 | } |
| 1100 | const timerEnds = Date.parse(since!) + rules.waitTimer * timerMinute(this.env); |
| 1101 | if ((rules.reviewers.length && decision !== "approved") || Date.now() < timerEnds) return false; |
| 1102 | this.release({ ...job, status: "approval" }); |
| 1103 | return true; |
| 1104 | } |
| 1105 | |
| 1106 | /** The job's protection rules passed: it goes on to start, and its run stops waiting once no other job does. */ |
| 1107 | private release(job: JobRow): void { |
| 1108 | this.sql.exec("UPDATE jobs SET gate_state = 'released', status = 'queued' WHERE id = ?", job.id); |
| 1109 | this.syncWaiting(job.run_id); |
| 1110 | } |
| 1111 | |
| 1112 | private syncWaiting(runId: string): void { |
| 1113 | if (this.sql.exec("SELECT 1 FROM jobs WHERE run_id = ? AND status = 'approval'", runId).toArray().length) return; |
| 1114 | this.sql.exec("UPDATE runs SET status = 'in_progress', updated_at = ? WHERE id = ? AND status = 'waiting'", now(), runId); |
| 1115 | } |
| 1116 | |
| 1117 | private async wakeGate(runId: string, jobId: string): Promise<void> { |
| 1118 | try { |
| 1119 | await (await this.env.RUN_WORKFLOW.get(runId)).sendEvent({ type: `gate-${jobId}`, payload: {} }); |
| 1120 | } catch (e) { |
| 1121 | console.error("failed to wake a job waiting for approval", e); |
| 1122 | } |
| 1123 | } |
| 1124 | |
| 1125 | private emitReview(action: "requested" | "approved" | "rejected", run: RunRow, environment: string, reviewers: Reviewer[], by: Actor, state: string | null, comment: string): void { |
| 1126 | const m = this.repo.meta; |
| 1127 | const url = `${this.env.PUBLIC_URL}/${m.owner}/${m.name}/actions/runs/${run.id}`; |
| 1128 | const ids = Object.fromEntries(this.repo.deploys.environments().map((e) => [e.name.toLowerCase(), e.id])); |
| 1129 | this.repo.emit({ |
| 1130 | type: "deployment_review", |
| 1131 | action, |
| 1132 | repoId: m.repoId, |
| 1133 | owner: m.owner, |
| 1134 | repo: m.name, |
| 1135 | payload: { |
| 1136 | since: now(), |
| 1137 | environment: environment, |
| 1138 | environment_id: ids[environment.toLowerCase()] ?? null, |
| 1139 | comment, |
| 1140 | ...(state ? { state } : {}), |
| 1141 | workflow_run: { id: run.id, run_number: run.run_number, name: run.workflow_name, display_title: run.display_title, head_sha: run.head_sha ?? run.sha, head_branch: run.ref.replace(/^refs\/(heads|tags)\//, ""), html_url: url }, |
| 1142 | reviewers: reviewers.map((r) => ({ type: "User", reviewer: { login: r.login, id: r.id } })), |
| 1143 | ...(action === "requested" ? { requestor: { login: by.login } } : { approver: { login: by.login } }), |
| 1144 | }, |
| 1145 | actor: by, |
| 1146 | at: now(), |
| 1147 | }); |
| 1148 | } |
| 1149 | |
| 1150 | /** |
| 1151 | * Where a job waiting on its environment stands, for the run's Workflow: what it waits for and |
| 1152 | * until when. Re-checks the rules first, so a job whose time came starts here. |
| 1153 | */ |
| 1154 | async jobGate(jobId: string): Promise<{ status: JobRow["status"]; wait: "review" | "timer" | null; until: string | null }> { |
| 1155 | let job = this.jobRow(jobId); |
| 1156 | if (job.status === "approval") await this.launch(jobId); |
| 1157 | job = this.jobRow(jobId); |
| 1158 | if (job.status !== "approval") return { status: job.status, wait: null, until: null }; |
| 1159 | const rules = this.repo.deploys.protection(job.environment!) ?? { reviewers: [], waitTimer: 0, preventSelfReview: false }; |
| 1160 | const since = Date.parse(job.gate_since!); |
| 1161 | const deadline = since + APPROVAL_DAYS * 86400_000; |
| 1162 | if (Date.now() >= deadline) { |
| 1163 | await this.completeJob(jobId, { conclusion: "failure", outputs: {}, failure: { message: `The deployment to ${job.environment} wasn't approved within ${APPROVAL_DAYS} days.` } }); |
| 1164 | return { status: "completed", wait: null, until: null }; |
| 1165 | } |
| 1166 | const decided = this.repo.deploys.decision(job.run_id, job.environment!) === "approved"; |
| 1167 | if (rules.reviewers.length && !decided) return { status: "approval", wait: "review", until: new Date(deadline).toISOString() }; |
| 1168 | return { status: "approval", wait: "timer", until: new Date(since + rules.waitTimer * timerMinute(this.env)).toISOString() }; |
| 1169 | } |
| 1170 | |
| 1171 | /** The environments a run's jobs wait on, with who may approve them (GitHub's pending deployments). */ |
| 1172 | async pendingDeployments(runId: string, viewer: { id: string; login: string; mayReview?: boolean } | null): Promise<PendingDeployment[]> { |
| 1173 | const run = this.runRow(runId); |
| 1174 | const waiting = this.sql.exec<{ environment: string; since: string }>("SELECT environment, MIN(gate_since) AS since FROM jobs WHERE run_id = ? AND status = 'approval' GROUP BY environment COLLATE NOCASE", run.id).toArray(); |
| 1175 | const self = viewer ? await this.selfIds(run) : new Set<string>(); |
| 1176 | return waiting.flatMap((w) => { |
| 1177 | const env = this.repo.deploys.environments().find((e) => e.name.toLowerCase() === w.environment.toLowerCase()); |
| 1178 | if (!env) return []; |
| 1179 | const p = env.protection; |
| 1180 | const decided = this.repo.deploys.decision(run.id, env.name); |
| 1181 | // A listed reviewer who lost access to the repository no longer decides (the caller says so in mayReview). |
| 1182 | const listed = !!viewer && viewer.mayReview !== false && p.reviewers.some((r) => r.id === viewer.id); |
| 1183 | const selfBlocked = !!viewer && p.preventSelfReview && self.has(viewer.id); |
| 1184 | return [ |
| 1185 | { |
| 1186 | environment: { id: env.id, name: env.name }, |
| 1187 | waitTimer: p.waitTimer, |
| 1188 | waitTimerStartedAt: w.since, |
| 1189 | reviewers: p.reviewers, |
| 1190 | preventSelfReview: p.preventSelfReview, |
| 1191 | needsReview: p.reviewers.length > 0 && decided !== "approved", |
| 1192 | currentUserCanApprove: listed && !selfBlocked && p.reviewers.length > 0 && !decided, |
| 1193 | selfBlocked, |
| 1194 | }, |
| 1195 | ]; |
| 1196 | }); |
| 1197 | } |
| 1198 | |
| 1199 | /** |
| 1200 | * A required reviewer approves or rejects environments a run waits on. The caller checked that |
| 1201 | * `reviewer` is a person signed in or using a write token (never an agent or a job); here the |
| 1202 | * reviewer must be listed on every environment, and not whoever started the run when the |
| 1203 | * environment prevents self-review. One approval is enough. Approved jobs start once their |
| 1204 | * wait timer has also passed; rejected ones fail. |
| 1205 | */ |
| 1206 | async reviewDeployments(runId: string, input: { environmentIds: number[]; state: "approved" | "rejected"; comment: string }, reviewer: { id: string; login: string; mayReview: boolean }): Promise<{ reviews: DeploymentReview[]; deployments: Deployment[] }> { |
| 1207 | const run = this.runRow(runId); |
| 1208 | if (!reviewer.mayReview) throw new HttpError(403, "only people with access to this repository review its deployments"); |
| 1209 | if (!input.environmentIds.length) throw new HttpError(422, "environment_ids must name at least one environment"); |
| 1210 | const pending = await this.pendingDeployments(run.id, reviewer); |
| 1211 | const chosen = input.environmentIds.map((id) => { |
| 1212 | const p = pending.find((x) => x.environment.id === id); |
| 1213 | if (!p || !p.needsReview) throw new HttpError(422, `environment ${id} isn't waiting for a review in this run`); |
| 1214 | if (p.selfBlocked) throw new HttpError(403, `${p.environment.name} doesn't let whoever started a run approve it`); |
| 1215 | if (!p.currentUserCanApprove) throw new HttpError(403, `you aren't a required reviewer for ${p.environment.name}`); |
| 1216 | return p; |
| 1217 | }); |
| 1218 | const by: Actor = { id: reviewer.id, login: reviewer.login, kind: "user" }; |
| 1219 | const out: DeploymentReview[] = []; |
| 1220 | const jobIds: string[] = []; |
| 1221 | for (const p of chosen) { |
| 1222 | out.push(this.repo.deploys.addReview(run.id, p.environment.name, input.state, input.comment, reviewer)); |
| 1223 | this.emitReview(input.state, run, p.environment.name, p.reviewers, by, input.state, input.comment); |
| 1224 | const jobs = this.sql.exec<JobRow>("SELECT * FROM jobs WHERE run_id = ? AND status = 'approval' AND environment = ? COLLATE NOCASE", run.id, p.environment.name).toArray(); |
| 1225 | for (const j of jobs) { |
| 1226 | jobIds.push(j.id); |
| 1227 | await this.launch(j.id); |
| 1228 | await this.wakeGate(run.id, j.id); |
| 1229 | } |
| 1230 | } |
| 1231 | this.repo.broadcast(`run:${run.id}`, { action: "reviewed" }); |
| 1232 | return { reviews: out, deployments: this.repo.deploys.jobDeployments(jobIds) }; |
| 1233 | } |
| 1234 | |
| 1235 | deploymentReviews(runId: string): DeploymentReview[] { |
| 1236 | return this.repo.deploys.reviews(this.runRow(runId).id); |
| 1237 | } |
| 1238 | |
| 1239 | /** |
| 1240 | * The job's container got no machine (its start failed, or it stopped before the runner reported |
| 1241 | * in). Put the job back in line and have the run's workflow start it again after a pause, or fail it |
| 1242 | * once it has waited MAX_PLACEMENT_TRIES times. |
| 1243 | */ |
| 1244 | private async placementFailed(jobId: string, why: string): Promise<void> { |
| 1245 | const job = this.jobRow(jobId); |
| 1246 | if (job.status === "completed") return; |
| 1247 | const tries = job.placement_tries + 1; |
| 1248 | if (tries > MAX_PLACEMENT_TRIES) { |
| 1249 | await this.completeJob(jobId, { conclusion: "failure", outputs: {}, failure: { message: `No hosted runner could be started: Diffinery tried ${MAX_PLACEMENT_TRIES} times over about half an hour (${why}). Re-run the job to try again.` } }); |
| 1250 | return; |
| 1251 | } |
| 1252 | this.sql.exec("UPDATE jobs SET status = 'queued', container = NULL, token_hash = NULL, placement_tries = ? WHERE id = ?", tries, jobId); |
| 1253 | // Not running while it waits: it holds none of the account's hosted jobs and uses no minutes. |
| 1254 | await endCompute(this.env, jobId).catch((e: unknown) => console.error("compute usage failed", e)); |
| 1255 | this.repo.broadcast(`run:${job.run_id}`, { action: "job", jobId }); |
| 1256 | try { |
| 1257 | const instance = await this.env.RUN_WORKFLOW.get(job.run_id); |
| 1258 | await instance.sendEvent({ type: `job-${jobId}`, payload: { retryInSeconds: placementDelay(tries) } }); |
| 1259 | } catch (e) { |
| 1260 | console.error("could not ask the run's workflow to start the job again", e); |
| 1261 | await this.completeJob(jobId, { conclusion: "failure", outputs: {}, failure: { message: `Could not start a runner: ${why}` } }); |
| 1262 | } |
| 1263 | } |
| 1264 | |
| 1265 | /** The run's workflow starts a job again after placementFailed put it back in line. */ |
| 1266 | async relaunchJob(jobId: string): Promise<void> { |
| 1267 | if (this.jobRow(jobId).status === "queued") await this.launch(jobId); |
| 1268 | } |
| 1269 | |
| 1270 | /** |
| 1271 | * Job-level `concurrency`, like GitHub: at most one job of a group runs at a time in |
| 1272 | * the repository. With cancel-in-progress the newcomer cancels the group's other jobs; |
| 1273 | * without it the newcomer waits as `pending` and replaces any older pending job. |
| 1274 | * Returns whether the job may start now. |
| 1275 | */ |
| 1276 | private async concurrencySlot(job: JobRow): Promise<boolean> { |
| 1277 | if (!job.concurrency_group) return true; |
| 1278 | const others = this.sql.exec<JobRow>("SELECT * FROM jobs WHERE concurrency_group = ? AND id != ? AND status IN ('pending', 'waiting', 'in_progress') ORDER BY rowid", job.concurrency_group, job.id).toArray(); |
| 1279 | const pending = others.filter((o) => o.status === "pending"); |
| 1280 | const running = others.filter((o) => o.status !== "pending"); |
| 1281 | const failure = { message: `Canceling since a higher priority waiting request for '${job.concurrency_group}' exists` }; |
| 1282 | if (running.length && !job.concurrency_cancel) { |
| 1283 | this.sql.exec("UPDATE jobs SET status = 'pending' WHERE id = ?", job.id); |
| 1284 | this.repo.broadcast(`run:${job.run_id}`, { action: "job", jobId: job.id }); |
| 1285 | for (const o of pending) await this.completeJob(o.id, { conclusion: "cancelled", outputs: {}, failure }); |
| 1286 | return false; |
| 1287 | } |
| 1288 | // Take the group before cancelling anything, so a job that starts meanwhile waits for this one. |
| 1289 | this.sql.exec("UPDATE jobs SET status = 'waiting' WHERE id = ?", job.id); |
| 1290 | for (const o of [...pending, ...running]) { |
| 1291 | await this.stopContainer(o); |
| 1292 | await this.completeJob(o.id, { conclusion: "cancelled", outputs: {}, failure }); |
| 1293 | } |
| 1294 | return true; |
| 1295 | } |
| 1296 | |
| 1297 | /** Whether jobs wait on the account's hosted-job quota (RepoDO retries them on its alarm). */ |
| 1298 | hasAccountHeldJobs(): boolean { |
| 1299 | return this.sql.exec<{ n: number }>("SELECT count(*) AS n FROM jobs WHERE fleet_held = 2 AND status = 'queued'").one().n > 0; |
| 1300 | } |
| 1301 | |
| 1302 | /** Try again to start jobs waiting on the account's hosted-job quota, oldest first, until one waits again. */ |
| 1303 | async releaseAccountHeld(): Promise<void> { |
| 1304 | for (const { id } of this.sql.exec<{ id: string }>("SELECT id FROM jobs WHERE fleet_held = 2 AND status = 'queued' ORDER BY rowid").toArray()) { |
| 1305 | await this.launch(id); |
| 1306 | if (this.jobRow(id).fleet_held === 2) return; |
| 1307 | } |
| 1308 | } |
| 1309 | |
| 1310 | async jobTimedOut(jobId: string): Promise<void> { |
| 1311 | const j = this.jobRow(jobId); |
| 1312 | if (j.status === "completed") return; |
| 1313 | if (j.status === "queued" && j.fleet_held === 2) { |
| 1314 | await this.completeJob(jobId, { conclusion: "failure", outputs: {}, failure: { message: "The job waited 6 hours for one of this account's hosted jobs to finish, and was not started." } }, "timed_out"); |
| 1315 | return; |
| 1316 | } |
| 1317 | await this.stopContainer(j); |
| 1318 | await this.completeJob(jobId, { conclusion: "failure", outputs: {}, failure: { message: `The job exceeded its timeout of ${j.timeout_minutes} minutes.` } }, "timed_out"); |
| 1319 | } |
| 1320 | |
| 1321 | async containerStopped(jobId: string, exitCode: number | null): Promise<void> { |
| 1322 | const j = this.sql.exec<JobRow>("SELECT * FROM jobs WHERE id = ?", jobId).toArray()[0]; |
| 1323 | if (!j || j.status === "completed") return; |
| 1324 | // Stopped while still waiting for its runner, without an exit code: the container never got a machine. |
| 1325 | if (j.status === "waiting" && !j.labels && !exitCode) return this.placementFailed(jobId, "the container stopped before the runner started"); |
| 1326 | await this.completeJob(jobId, { conclusion: "failure", outputs: {}, failure: { message: exitCode ? `The runner stopped unexpectedly (exit code ${exitCode}).` : "The runner's container stopped before the job finished, without an exit code (the platform restarted or replaced it)." } }); |
| 1327 | } |
| 1328 | |
| 1329 | /** The oldest waiting job whose runs-on labels the runner all offers, handed to it with a fresh job token. */ |
| 1330 | private async assignJob(runnerName: string, runnerLabels: string[], runnerId: number | null): Promise<{ jobId: string; token: string; runId: string; name: string } | null> { |
| 1331 | const offered = new Set(["self-hosted", ...runnerLabels.map((l) => l.toLowerCase())]); |
| 1332 | const waiting = this.sql |
| 1333 | .exec<JobRow>("SELECT j.* FROM jobs j JOIN runs r ON r.id = j.run_id WHERE j.status = 'waiting' AND j.labels IS NOT NULL AND j.container IS NULL AND r.status != 'completed' ORDER BY r.created_at, j.rowid") |
| 1334 | .toArray(); |
| 1335 | const job = waiting.find((j) => (JSON.parse(j.labels!) as string[]).every((l) => offered.has(l))); |
| 1336 | if (!job) return null; |
| 1337 | // A job parked before Actions were suspended here isn't handed out. |
| 1338 | const block = await this.blocked("self-hosted-job"); |
| 1339 | if (block) { |
| 1340 | await this.completeJob(job.id, { conclusion: "failure", outputs: {}, failure: { message: `The job was not started: ${block}.` } }); |
| 1341 | return null; |
| 1342 | } |
| 1343 | const m = this.repo.meta; |
| 1344 | const token = `djt_${m.repoId}_${job.id}_${randomToken(24)}`; |
| 1345 | const container = `self-hosted:${runnerName.slice(0, 64)}`; |
| 1346 | this.sql.exec("UPDATE jobs SET token_hash = ?, container = ?, runner_id = ? WHERE id = ? AND container IS NULL", await sha256(token), container, runnerId, job.id); |
| 1347 | this.repo.broadcast(`run:${job.run_id}`, { action: "job", jobId: job.id }); |
| 1348 | return { jobId: job.id, token, runId: job.run_id, name: job.name }; |
| 1349 | } |
| 1350 | |
| 1351 | // ------------------------------------------------------------ registered self-hosted runners |
| 1352 | |
| 1353 | /** |
| 1354 | * GitHub's runner registration token: a repository admin's credential, good for an hour, that a |
| 1355 | * machine trades for its own runner token. So the machine never holds a person's token, and the |
| 1356 | * jobs it runs (as the same OS user) can't read one. |
| 1357 | */ |
| 1358 | async runnerRegistrationToken(caller: Caller): Promise<{ token: string; expires_at: string }> { |
| 1359 | if (caller.kind !== "user" || caller.permissions || !atLeast(caller.role, "admin")) throw new HttpError(403, "only repository admins can add self-hosted runners"); |
| 1360 | const token = `drg_${randomToken(32)}`; |
| 1361 | const expires = new Date(Date.now() + REGISTRATION_TTL_MS).toISOString(); |
| 1362 | const hash = await sha256(token); |
| 1363 | this.sql.exec("DELETE FROM runner_registrations WHERE expires_at < ?", now()); |
| 1364 | this.sql.exec("INSERT INTO runner_registrations (token_hash, created_by, expires_at) VALUES (?, ?, ?)", hash, caller.login, expires); |
| 1365 | return { token, expires_at: expires }; |
| 1366 | } |
| 1367 | |
| 1368 | /** Register a runner with a registration token; returns its own token, shown once. */ |
| 1369 | async registerRunner(registration: string, name: string, labels: string[], ephemeral: boolean): Promise<{ id: number; name: string; labels: string[]; ephemeral: boolean; token: string; registeredBy: string }> { |
| 1370 | const hash = await sha256(registration); |
| 1371 | const reg = this.sql.exec<{ created_by: string }>("SELECT created_by FROM runner_registrations WHERE token_hash = ? AND expires_at > ?", hash, now()).toArray()[0]; |
| 1372 | if (!reg) throw new HttpError(401, "invalid or expired registration token"); |
| 1373 | const clean = [...new Set(labels.map((l) => String(l).trim().toLowerCase()).filter((l) => l && l !== "self-hosted"))]; |
| 1374 | if (clean.length > 20) throw new HttpError(422, "a runner can offer at most 20 labels"); |
| 1375 | const bad = clean.find((l) => !/^[a-z0-9][a-z0-9._-]{0,63}$/.test(l)); |
| 1376 | if (bad) throw new HttpError(422, `invalid label ${JSON.stringify(bad.slice(0, 70))}: use letters, digits, '.', '_' and '-' (up to 64)`); |
| 1377 | const runnerName = name.trim().slice(0, 64) || "runner"; |
| 1378 | const count = this.sql.exec<{ n: number }>("SELECT count(*) AS n FROM runners").one().n; |
| 1379 | if (count >= MAX_RUNNERS) throw new HttpError(429, `this repository has ${count} self-hosted runners, the most allowed; remove one first`); |
| 1380 | const id = this.sql.exec<{ id: number }>("INSERT INTO runners (name, labels, ephemeral, created_by, created_at) VALUES (?, ?, ?, ?, ?) RETURNING id", runnerName, JSON.stringify(clean), ephemeral ? 1 : 0, reg.created_by, now()).one().id; |
| 1381 | const token = `drn_${this.repo.meta.repoId}_${id}_${randomToken(32)}`; |
| 1382 | this.sql.exec("UPDATE runners SET token_hash = ? WHERE id = ?", await sha256(token), id); |
| 1383 | return { id, name: runnerName, labels: clean, ephemeral, token, registeredBy: reg.created_by }; |
| 1384 | } |
| 1385 | |
| 1386 | /** A registered runner asks for work with its own token. An ephemeral runner's token ends with the job it takes. */ |
| 1387 | async claimJobAsRunner(token: string): Promise<{ jobId: string; token: string; runId: string; name: string } | null> { |
| 1388 | const id = Number(token.split("_")[2]); |
| 1389 | const r = Number.isSafeInteger(id) ? this.sql.exec<RunnerRow>("SELECT * FROM runners WHERE id = ?", id).toArray()[0] : undefined; |
| 1390 | if (!r || !r.token_hash || r.token_hash !== (await sha256(token))) throw new HttpError(401, "this runner was removed, or its token is invalid; register it again"); |
| 1391 | this.sql.exec("UPDATE runners SET last_seen_at = ? WHERE id = ?", now(), r.id); |
| 1392 | const job = await this.assignJob(r.name, JSON.parse(r.labels) as string[], r.id); |
| 1393 | if (job && r.ephemeral) this.sql.exec("UPDATE runners SET token_hash = NULL WHERE id = ?", r.id); |
| 1394 | return job; |
| 1395 | } |
| 1396 | |
| 1397 | runners(caller: Caller): RunnerInfo[] { |
| 1398 | if (!atLeast(caller.role, "admin") || caller.permissions) throw new HttpError(403, "only repository admins can list self-hosted runners"); |
| 1399 | const busy = new Set(this.sql.exec<{ runner_id: number }>("SELECT DISTINCT runner_id FROM jobs WHERE runner_id IS NOT NULL AND status != 'completed'").toArray().map((x) => x.runner_id)); |
| 1400 | return this.sql.exec<RunnerRow>("SELECT * FROM runners ORDER BY id").toArray().map((r) => ({ |
| 1401 | id: r.id, |
| 1402 | name: r.name, |
| 1403 | os: "unknown", |
| 1404 | status: r.last_seen_at && Date.now() - Date.parse(r.last_seen_at) < RUNNER_ONLINE_MS ? "online" : "offline", |
| 1405 | busy: busy.has(r.id), |
| 1406 | ephemeral: !!r.ephemeral, |
| 1407 | labels: ["self-hosted", ...(JSON.parse(r.labels) as string[])].map((name, i) => ({ id: i + 1, name, type: i === 0 ? "read-only" : "custom" })), |
| 1408 | })); |
| 1409 | } |
| 1410 | |
| 1411 | /** Remove a runner: its token stops working at once. A job it is running carries on with its own job token. */ |
| 1412 | deleteRunner(id: number, caller: Caller): void { |
| 1413 | if (caller.kind !== "user" || caller.permissions || !atLeast(caller.role, "admin")) throw new HttpError(403, "only repository admins can remove self-hosted runners"); |
| 1414 | if (!this.sql.exec("DELETE FROM runners WHERE id = ? RETURNING id", id).toArray().length) throw new HttpError(404, "runner not found"); |
| 1415 | } |
| 1416 | |
| 1417 | private async stopContainer(j: JobRow): Promise<void> { |
| 1418 | if (!j.container || j.container.startsWith("self-hosted:")) return; |
| 1419 | try { |
| 1420 | await this.env.RUNNER.get(this.env.RUNNER.idFromName(j.container)).stopJob(); |
| 1421 | } catch (e) { |
| 1422 | console.warn("stop container failed", e); |
| 1423 | } |
| 1424 | } |
| 1425 | |
| 1426 | async finishRun(runId: string): Promise<RunInfo> { |
| 1427 | const run = this.runRow(runId); |
| 1428 | const jobs = this.sql.exec<JobRow>("SELECT * FROM jobs WHERE run_id = ?", runId).toArray(); |
| 1429 | let conclusion: RunRow["conclusion"] = run.conclusion === "cancelled" ? "cancelled" : "success"; |
| 1430 | if (conclusion !== "cancelled") { |
| 1431 | // A job with continue-on-error may fail without failing the run. |
| 1432 | if (jobs.some((j) => (j.conclusion === "failure" || j.conclusion === "timed_out") && !j.continue_on_error)) conclusion = "failure"; |
| 1433 | else if (jobs.some((j) => j.conclusion === "cancelled")) conclusion = "cancelled"; |
| 1434 | // As on GitHub, a run whose jobs were all skipped is skipped, not a success, so a |
| 1435 | // workflow_run follower that checks for success (a live test after a deploy) doesn't run. |
| 1436 | else if (jobs.length && jobs.every((j) => j.conclusion === "skipped")) conclusion = "skipped"; |
| 1437 | } |
| 1438 | const t = now(); |
| 1439 | this.sql.exec("UPDATE runs SET status = 'completed', conclusion = ?, updated_at = ? WHERE id = ?", conclusion, t, runId); |
| 1440 | const m = this.repo.meta; |
| 1441 | this.repo.emit({ type: "workflow_run", action: "completed", repoId: m.repoId, owner: m.owner, repo: m.name, runId, conclusion: conclusion ?? undefined, actor: JSON.parse(run.actor), at: t }); |
| 1442 | this.repo.broadcast(`run:${runId}`, { action: "completed", conclusion }); |
| 1443 | this.repo.broadcast("runs", { action: "completed", runId }); |
| 1444 | await this.onWorkflowRun(this.runRow(runId), "completed"); |
| 1445 | return toRun(this.runRow(runId)); |
| 1446 | } |
| 1447 | |
| 1448 | /** |
| 1449 | * Starts the default branch's `on: workflow_run` workflows that follow `run`. Like |
| 1450 | * GitHub, they run on the default branch, must name the workflows they follow, and |
| 1451 | * chains stop after three levels (A -> B -> C -> D; D's runs trigger nothing). |
| 1452 | */ |
| 1453 | private async onWorkflowRun(run: RunRow, action: "requested" | "completed"): Promise<void> { |
| 1454 | if (this.chainDepth(run) >= 3) return; |
| 1455 | try { |
| 1456 | const sha = this.repo.getRef(this.repo.headRef); |
| 1457 | if (!sha) return; |
| 1458 | const followers = (await this.loadWorkflows(sha)).filter((w) => w.workflow?.on.workflow_run?.workflows?.length); |
| 1459 | if (!followers.length) return; |
| 1460 | const m = this.repo.meta; |
| 1461 | const payload = JSON.parse(run.event_payload) as { pull_request?: { head?: { ref?: string } } }; |
| 1462 | const branch = payload.pull_request?.head?.ref ?? run.ref.replace(/^refs\/(heads|tags)\//, ""); |
| 1463 | const actor = JSON.parse(run.actor) as Actor; |
| 1464 | const workflowRun = { |
| 1465 | id: run.run_number, |
| 1466 | name: run.workflow_name, |
| 1467 | display_title: run.display_title, |
| 1468 | path: run.workflow_path, |
| 1469 | run_number: run.run_number, |
| 1470 | run_attempt: run.attempt, |
| 1471 | event: run.event, |
| 1472 | status: run.status, |
| 1473 | conclusion: run.conclusion, |
| 1474 | head_branch: branch, |
| 1475 | head_sha: run.head_sha ?? run.sha, |
| 1476 | actor: { login: actor.login }, |
| 1477 | triggering_actor: { login: actor.login }, |
| 1478 | pull_requests: run.pr_number ? [{ number: run.pr_number }] : [], |
| 1479 | created_at: run.created_at, |
| 1480 | updated_at: run.updated_at, |
| 1481 | html_url: `${this.env.PUBLIC_URL}/${m.owner}/${m.name}/actions/runs/${run.id}`, |
| 1482 | }; |
| 1483 | for (const w of followers) { |
| 1484 | if (!matchesTrigger(w.workflow!, { name: "workflow_run", action, workflow: run.workflow_name, branch })) continue; |
| 1485 | await this.createRun(w.workflow!, { |
| 1486 | event: "workflow_run", |
| 1487 | ref: this.repo.headRef, |
| 1488 | sha, |
| 1489 | payload: { action, workflow_run: workflowRun, workflow: { name: run.workflow_name, path: run.workflow_path }, repository: await this.repositoryPayload(), sender: { login: actor.login } }, |
| 1490 | caller: actor, |
| 1491 | title: w.workflow!.name ?? w.path, |
| 1492 | // A follower acts on what the run it follows produced (artifacts, outputs), so it stays an agent's. |
| 1493 | agentOrigin: !!run.agent_origin, |
| 1494 | }); |
| 1495 | } |
| 1496 | } catch (e) { |
| 1497 | console.warn("workflow_run trigger failed", e); |
| 1498 | } |
| 1499 | } |
| 1500 | |
| 1501 | /** How many workflow_run links led to `run` (0 for a run started by any other event). */ |
| 1502 | private chainDepth(run: RunRow): number { |
| 1503 | let depth = 0; |
| 1504 | for (let r: RunRow | undefined = run; r?.event === "workflow_run" && depth < 3; depth++) { |
| 1505 | const parent: string | number | undefined = (JSON.parse(r.event_payload) as { workflow_run?: { id?: string | number } }).workflow_run?.id; |
| 1506 | r = parent !== undefined ? this.sql.exec<RunRow>("SELECT * FROM runs WHERE id = ? OR run_number = ? ORDER BY attempt DESC LIMIT 1", String(parent), Number(parent) || -1).toArray()[0] : undefined; |
| 1507 | } |
| 1508 | return depth; |
| 1509 | } |
| 1510 | |
| 1511 | async plannedError(runId: string, message: string): Promise<void> { |
| 1512 | const t = now(); |
| 1513 | this.sql.exec("INSERT INTO jobs (id, run_id, job_key, name, status, conclusion, started_at, completed_at, failure) VALUES (?, ?, 'setup', 'Set up workflow', 'completed', 'failure', ?, ?, ?)", newId(), runId, t, t, JSON.stringify({ message })); |
| 1514 | } |
| 1515 | |
| 1516 | // ---------------------------------------------------------------- cancellation and re-runs |
| 1517 | |
| 1518 | private async cancelInternal(runId: string, reason: string): Promise<void> { |
| 1519 | const run = this.runRow(runId); |
| 1520 | if (run.status === "completed") return; |
| 1521 | this.sql.exec("UPDATE runs SET conclusion = 'cancelled', updated_at = ? WHERE id = ?", now(), run.id); |
| 1522 | for (const j of this.sql.exec<JobRow>("SELECT * FROM jobs WHERE run_id = ? AND status != 'completed'", run.id).toArray()) { |
| 1523 | await this.stopContainer(j); |
| 1524 | await this.completeJob(j.id, { conclusion: "cancelled", outputs: {}, failure: { message: reason } }); |
| 1525 | } |
| 1526 | } |
| 1527 | |
| 1528 | async cancel(runId: string, caller: Caller): Promise<void> { |
| 1529 | if (!atLeast(caller.role, "write")) throw new HttpError(403, "permission denied"); |
| 1530 | await this.cancelInternal(runId, `Cancelled by ${caller.login}`); |
| 1531 | } |
| 1532 | |
| 1533 | async rerun(runId: string, caller: Caller): Promise<RunInfo> { |
| 1534 | if (!atLeast(caller.role, "write")) throw new HttpError(403, "permission denied"); |
| 1535 | if (caller.id.startsWith("job:")) throw new HttpError(403, "a workflow's GITHUB_TOKEN cannot re-run workflows"); |
| 1536 | await this.refuseIfBlocked("run"); |
| 1537 | const old = this.runRow(runId); |
| 1538 | const wf = (await this.loadWorkflows(old.sha)).find((w) => w.path === old.workflow_path)?.workflow; |
| 1539 | if (!wf) throw new HttpError(422, "workflow file no longer valid at this commit"); |
| 1540 | const run = await this.createRun(wf, { |
| 1541 | event: old.event, |
| 1542 | ref: old.ref, |
| 1543 | sha: old.sha, |
| 1544 | headSha: old.head_sha ?? undefined, |
| 1545 | payload: JSON.parse(old.event_payload), |
| 1546 | caller, |
| 1547 | prNumber: old.pr_number ?? undefined, |
| 1548 | inputs: JSON.parse(old.inputs), |
| 1549 | title: old.display_title, |
| 1550 | // Re-running code an agent wrote keeps it acting as the agent, whoever re-runs it. |
| 1551 | agentOrigin: !!old.agent_origin, |
| 1552 | }); |
| 1553 | // Not started after all (Actions suspended meanwhile, or too many unfinished runs). |
| 1554 | if (!run) throw new HttpError(409, "the run was not started; try again later"); |
| 1555 | // Whoever started an earlier attempt still can't approve this one where an environment prevents self-review. |
| 1556 | const prior = [...new Set([...((old.prior_actors ? JSON.parse(old.prior_actors) : []) as string[]), (JSON.parse(old.actor) as Actor).id])]; |
| 1557 | this.sql.exec("UPDATE runs SET attempt = ?, run_number = ?, prior_actors = ? WHERE id = ?", old.attempt + 1, old.run_number, JSON.stringify(prior), run.id); |
| 1558 | return toRun(this.runRow(run.id)); |
| 1559 | } |
| 1560 | |
| 1561 | // ---------------------------------------------------------------- runner protocol |
| 1562 | |
| 1563 | /** Validate a job or sandbox token. Returns what it grants. */ |
| 1564 | async runnerAuth( |
| 1565 | token: string, |
| 1566 | ): Promise<{ kind: "job"; jobId: string; runId: string; role: Caller["role"]; permissions: JobPermissions; actorKind: Actor["kind"] } | { kind: "sandbox"; sandboxId: string; caller: Caller; taskPerson?: string } | null> { |
| 1567 | const parts = token.split("_"); |
| 1568 | if (parts.length < 4) return null; |
| 1569 | const hash = await sha256(token); |
| 1570 | if (parts[0] === "djt") { |
| 1571 | const j = this.sql.exec<JobRow>("SELECT * FROM jobs WHERE id = ?", parts[2]!).toArray()[0]; |
| 1572 | if (!j || j.token_hash !== hash || j.status === "completed") return null; |
| 1573 | const grant = this.repo.runtime.grantFor(j.id); |
| 1574 | // Code an agent wrote, pushed or opened a pull request with runs as that agent (repo/token-policy.ts). |
| 1575 | const run = this.sql.exec<{ agent_origin: number }>("SELECT agent_origin FROM runs WHERE id = ?", j.run_id).toArray()[0]; |
| 1576 | const actorKind: Actor["kind"] = run?.agent_origin ? "agent" : "system"; |
| 1577 | return { kind: "job", jobId: j.id, runId: j.run_id, role: grant.role, permissions: grant.permissions, actorKind }; |
| 1578 | } |
| 1579 | if (parts[0] === "dst") { |
| 1580 | const s = this.sql.exec<{ id: string; secret: string; created_by: string; status: string; created_at: string; task_person: string | null }>("SELECT * FROM sandboxes WHERE id = ?", parts[2]!).toArray()[0]; |
| 1581 | if (!s || s.secret !== hash || s.status !== "running") return null; |
| 1582 | // A sandbox's token lasts a day at most, even if nobody closes the sandbox. |
| 1583 | if (Date.now() - Date.parse(s.created_at) > SANDBOX_TOKEN_TTL_MS) return null; |
| 1584 | return { kind: "sandbox", sandboxId: s.id, caller: JSON.parse(s.created_by) as Caller, ...(s.task_person ? { taskPerson: s.task_person } : {}) }; |
| 1585 | } |
| 1586 | return null; |
| 1587 | } |
| 1588 | |
| 1589 | private async authJob(jobId: string, token: string): Promise<JobRow> { |
| 1590 | const auth = await this.runnerAuth(token); |
| 1591 | if (!auth || auth.kind !== "job" || auth.jobId !== jobId) throw new HttpError(401, "invalid job token"); |
| 1592 | return this.jobRow(jobId); |
| 1593 | } |
| 1594 | |
| 1595 | /** Claims for an OIDC id token; side-effect free (unlike runnerSpec). */ |
| 1596 | async oidcClaims(jobId: string, token: string): Promise<{ allowed: boolean; github: Record<string, string> }> { |
| 1597 | const job = await this.authJob(jobId, token); |
| 1598 | if (job.status !== "in_progress") throw new HttpError(409, "job is not running"); |
| 1599 | const run = this.runRow(job.run_id); |
| 1600 | const exp = await this.expanded(run); |
| 1601 | const def: Job = exp.workflow.jobs[job.job_key]!; |
| 1602 | const wf = exp.workflowOf.get(job.job_key)!; |
| 1603 | const allowed = this.repo.runtime.grantFor(jobId).permissions["id-token"] === "write"; |
| 1604 | const ctx = this.githubContext(run, localKey(job.job_key, exp.parentOf.get(job.job_key) ?? null), token) as Record<string, Json>; |
| 1605 | const github: Record<string, string> = {}; |
| 1606 | for (const [k, v] of Object.entries(ctx)) if (typeof v === "string" || typeof v === "number") github[k] = String(v); |
| 1607 | // Ids don't change when a name is reused, so a cloud trust policy can pin them; the environment is the one |
| 1608 | // whose protection rules this job passed (runnerSpec records its deployment). |
| 1609 | github.repository_owner_id = String(numericId(this.repo.meta.ownerId)); |
| 1610 | const actor = JSON.parse(run.actor) as Actor; |
| 1611 | if (actor.id) github.actor_id = String(numericId(actor.id)); |
| 1612 | const env = this.sql.exec<{ environment: string }>("SELECT environment FROM deployments WHERE job_id = ?", jobId).toArray()[0]; |
| 1613 | if (env) github.environment = env.environment; |
| 1614 | return { allowed, github }; |
| 1615 | } |
| 1616 | |
| 1617 | /** |
| 1618 | * What a job token may do with container packages (the `packages` permission). |
| 1619 | * A workflow without a permissions block gets the repository's default: read in |
| 1620 | * repositories with read-only tokens (the default for new ones), write in those that |
| 1621 | * keep GitHub's old permissive default. A permissions map that leaves `packages` out gets none. |
| 1622 | */ |
| 1623 | async packagesPermission(token: string): Promise<"none" | "read" | "write"> { |
| 1624 | const auth = await this.runnerAuth(token); |
| 1625 | return auth?.kind === "job" ? auth.permissions.packages : "none"; |
| 1626 | } |
| 1627 | |
| 1628 | /** |
| 1629 | * A pull_request run for a pull request from another repository (a fork): its jobs |
| 1630 | * never get write or secrets. Decided from the pull request's row, not the event payload. |
| 1631 | */ |
| 1632 | private untrusted(run: RunRow): boolean { |
| 1633 | return run.event === "pull_request" && this.repo.policy.isForkPull(run.pr_number); |
| 1634 | } |
| 1635 | |
| 1636 | async runnerSpec(jobId: string, token: string): Promise<JobSpec> { |
| 1637 | const job = await this.authJob(jobId, token); |
| 1638 | const run = this.runRow(job.run_id); |
| 1639 | const exp = await this.expanded(run); |
| 1640 | const def: Job = exp.workflow.jobs[job.job_key]!; |
| 1641 | const wf = exp.workflowOf.get(job.job_key)!; |
| 1642 | const parent = exp.parentOf.get(job.job_key) ?? null; |
| 1643 | const short = localKey(job.job_key, parent); |
| 1644 | let scope: { inputs: Record<string, unknown>; secrets: Record<string, string> }; |
| 1645 | try { |
| 1646 | // A fork's code never sees the repository's secrets, as on GitHub, and an agent's only where an admin allows it. |
| 1647 | scope = await this.callContext(run, exp, parent, !this.untrusted(run) && (!run.agent_origin || this.repo.policy.settings().agentRunSecrets)); |
| 1648 | } catch (e) { |
| 1649 | await this.completeJob(jobId, { conclusion: "failure", outputs: {}, failure: { message: (e as Error).message } }); |
| 1650 | throw new HttpError(409, (e as Error).message); |
| 1651 | } |
| 1652 | const m = this.repo.meta; |
| 1653 | // Deployment environment (`environment:`): its branch policy, secrets and variables, and a deployment record. |
| 1654 | let environment: { secrets: Record<string, string>; vars: Record<string, string> } = { secrets: {}, vars: {} }; |
| 1655 | if (def.environment) { |
| 1656 | try { |
| 1657 | const ctx = { github: this.githubContext(run, short, ""), inputs: scope.inputs, vars: this.variables(), matrix: JSON.parse(job.matrix), strategy: JSON.parse(job.strategy), needs: await this.needsResults(run, exp, exp.needs.get(job.job_key) ?? [], parent) }; |
| 1658 | // The name evaluated at launch, whose protection rules the job passed (older jobs evaluate it here). |
| 1659 | const name = job.environment ?? interpolate(def.environment.name, ctx); |
| 1660 | if (this.repo.deploys.protection(name) && job.gate_state !== "released") throw new Error(`The environment ${name} requires approval this job didn't get. Re-run the job to ask for it.`); |
| 1661 | const url = def.environment.url && !def.environment.url.includes("steps.") ? interpolate(def.environment.url, ctx) : undefined; |
| 1662 | const logUrl = `${this.env.PUBLIC_URL}/${m.owner}/${m.name}/actions/runs/${run.id}/job/${jobId}`; |
| 1663 | // Code an agent wrote deploys as that agent (environment rules hold it to the agent's limits). |
| 1664 | const actor = JSON.parse(run.actor) as Actor; |
| 1665 | environment = await this.repo.deploys.jobStarted({ id: jobId, ref: run.ref, sha: run.sha, logUrl, actor: run.agent_origin ? { ...actor, kind: "agent" } : actor }, { name, url }); |
| 1666 | } catch (e) { |
| 1667 | await this.completeJob(jobId, { conclusion: "failure", outputs: {}, failure: { message: (e as Error).message } }); |
| 1668 | throw new HttpError(409, (e as Error).message); |
| 1669 | } |
| 1670 | } |
| 1671 | const t = now(); |
| 1672 | this.sql.exec("UPDATE jobs SET status = 'in_progress', started_at = ? WHERE id = ?", t, jobId); |
| 1673 | this.repo.upsertCheck({ sha: run.head_sha ?? run.sha, name: this.checkName(run, job), status: "in_progress", runId: run.id, jobId, detailsUrl: `${this.env.PUBLIC_URL}/${m.owner}/${m.name}/actions/runs/${run.id}/job/${jobId}` }); |
| 1674 | this.repo.broadcast(`run:${run.id}`, { action: "job", jobId }); |
| 1675 | // Only the secrets the job's expressions name reach the runner, as on GitHub (plus the runner's own debug switches). |
| 1676 | const used = referencedSecrets(def, wf.env); |
| 1677 | if (used !== "all") for (const name of RUNNER_SECRETS) used.add(name); |
| 1678 | const secrets: Record<string, string> = { ...secretsFor({ ...scope.secrets, ...(this.untrusted(run) ? {} : environment.secrets) }, used), GITHUB_TOKEN: token, DIFFINERY_TOKEN: token }; |
| 1679 | // The job token can do what the effective `permissions:` allow (job level wins, like GitHub). |
| 1680 | const permissions = permissionsForJob(exp, job.job_key, { untrusted: this.untrusted(run), repoDefault: this.repo.policy.settings().defaultPermissions }); |
| 1681 | this.repo.runtime.grant(jobId, permissions); |
| 1682 | const idToken = permissions["id-token"] === "write"; |
| 1683 | return { |
| 1684 | runId: run.id, |
| 1685 | jobId, |
| 1686 | jobKey: short, |
| 1687 | name: job.name, |
| 1688 | repo: { owner: m.owner, name: m.name, cloneUrl: `${this.env.PUBLIC_URL}/${m.owner}/${m.name}.git`, defaultBranch: m.defaultBranch }, |
| 1689 | sha: run.sha, |
| 1690 | ref: run.ref, |
| 1691 | workflowPath: run.workflow_path, |
| 1692 | github: this.githubContext(run, short, token), |
| 1693 | env: { ...wf.env, ...def.env }, |
| 1694 | defaults: { run: { ...wf.defaults.run, ...def.defaults.run } }, |
| 1695 | matrix: JSON.parse(job.matrix), |
| 1696 | strategy: JSON.parse(job.strategy), |
| 1697 | needs: await this.needsResults(run, exp, exp.needs.get(job.job_key) ?? [], parent), |
| 1698 | inputs: scope.inputs, |
| 1699 | vars: { ...this.variables(), ...environment.vars }, |
| 1700 | secrets, |
| 1701 | outputs: def.outputs, |
| 1702 | steps: def.steps.map((s) => ({ ...s, "timeout-minutes": s["timeout-minutes"] === undefined ? undefined : Number(s["timeout-minutes"]) })), |
| 1703 | container: def.container, |
| 1704 | services: def.services, |
| 1705 | timeoutMinutes: job.timeout_minutes, |
| 1706 | continueOnError: !!job.continue_on_error, |
| 1707 | idToken: !!idToken, |
| 1708 | apiUrl: this.env.PUBLIC_URL, |
| 1709 | token, |
| 1710 | runtime: await this.runtimeServices(run, job), |
| 1711 | }; |
| 1712 | } |
| 1713 | |
| 1714 | /** ACTIONS_RUNTIME_TOKEN and the service URLs; the token expires when the job would time out. */ |
| 1715 | private async runtimeServices(run: RunRow, job: JobRow): Promise<JobSpec["runtime"]> { |
| 1716 | if (!this.env.MASTER_KEY) return undefined; |
| 1717 | const origin = new URL(this.env.PUBLIC_URL).origin; |
| 1718 | const exp = Math.floor(Date.now() / 1000) + (job.timeout_minutes + 30) * 60; |
| 1719 | const { scopes } = this.repo.runtime.job(job.id); |
| 1720 | const token = await signRuntimeToken(this.env.MASTER_KEY, origin, { repo: this.repo.meta.repoId, run: run.id, job: job.id, exp }, scopes); |
| 1721 | return { token, resultsUrl: `${origin}/`, cacheUrl: `${origin}/_runtime/`, runtimeUrl: `${origin}/_runtime/` }; |
| 1722 | } |
| 1723 | |
| 1724 | async runnerLogs(jobId: string, token: string, lines: LogLine[]): Promise<void> { |
| 1725 | const job = await this.authJob(jobId, token); |
| 1726 | let chars = job.log_chars ?? 0; |
| 1727 | if (chars > MAX_JOB_LOG) return; |
| 1728 | let seq = this.sql.exec<{ n: number | null }>("SELECT max(seq) AS n FROM logs WHERE job_id = ?", jobId).one().n ?? -1; |
| 1729 | const out: Array<LogLine & { seq: number }> = []; |
| 1730 | for (const l of lines.slice(0, 5000)) { |
| 1731 | seq++; |
| 1732 | const t = String(l.t ?? "").slice(0, 40); |
| 1733 | const step = Number.isInteger(l.step) ? l.step : 0; |
| 1734 | let text = String(l.text ?? ""); |
| 1735 | if (text.length > 16384) text = `${text.slice(0, 16384)}…`; |
| 1736 | // Past the cap, one last line says so and the rest is dropped: the whole log is read into |
| 1737 | // memory when the job completes, to archive it. |
| 1738 | if (chars + text.length > MAX_JOB_LOG) { |
| 1739 | text = `The log is cut off here: a job's log is limited to ${MAX_JOB_LOG / 1048576} MiB of text.`; |
| 1740 | chars = MAX_JOB_LOG + 1; |
| 1741 | } else chars += text.length; |
| 1742 | this.sql.exec("INSERT INTO logs (job_id, seq, t, step, text) VALUES (?, ?, ?, ?, ?)", jobId, seq, t, step, text); |
| 1743 | out.push({ t, step, text, seq }); |
| 1744 | if (chars > MAX_JOB_LOG) break; |
| 1745 | } |
| 1746 | this.sql.exec("UPDATE jobs SET log_chars = ? WHERE id = ?", chars, jobId); |
| 1747 | this.repo.broadcast(`job:${jobId}`, { action: "logs", lines: out }); |
| 1748 | } |
| 1749 | |
| 1750 | async runnerStep(jobId: string, token: string, index: number, report: StepReport): Promise<void> { |
| 1751 | const job = await this.authJob(jobId, token); |
| 1752 | const steps = JSON.parse(job.steps) as Array<StepReport & { index: number }>; |
| 1753 | const existing = steps.findIndex((s) => s.index === index); |
| 1754 | const next = { ...(existing >= 0 ? steps[existing] : {}), ...report, index }; |
| 1755 | if (existing >= 0) steps[existing] = next; |
| 1756 | else steps.push(next); |
| 1757 | steps.sort((a, b) => a.index - b.index); |
| 1758 | this.sql.exec("UPDATE jobs SET steps = ? WHERE id = ?", JSON.stringify(steps), jobId); |
| 1759 | this.repo.broadcast(`job:${jobId}`, { action: "step", step: next }); |
| 1760 | this.repo.broadcast(`run:${job.run_id}`, { action: "job", jobId }); |
| 1761 | } |
| 1762 | |
| 1763 | async runnerComplete(jobId: string, token: string, completion: JobCompletion): Promise<void> { |
| 1764 | await this.authJob(jobId, token); |
| 1765 | await this.completeJob(jobId, completion); |
| 1766 | } |
| 1767 | |
| 1768 | private async completeJob(jobId: string, completion: JobCompletion, override?: "timed_out"): Promise<void> { |
| 1769 | const job = this.jobRow(jobId); |
| 1770 | if (job.status === "completed") return; |
| 1771 | const gated = job.status === "approval"; |
| 1772 | const run = this.runRow(job.run_id); |
| 1773 | const t = now(); |
| 1774 | const conclusion = override ?? completion.conclusion; |
| 1775 | this.sql.exec( |
| 1776 | "UPDATE jobs SET status = 'completed', conclusion = ?, completed_at = ?, outputs = ?, failure = ?, summary = ?, token_hash = NULL, started_at = COALESCE(started_at, ?) WHERE id = ?", |
| 1777 | conclusion, |
| 1778 | t, |
| 1779 | JSON.stringify(completion.outputs ?? {}), |
| 1780 | completion.failure ? JSON.stringify(completion.failure) : null, |
| 1781 | completion.summary ?? null, |
| 1782 | t, |
| 1783 | jobId, |
| 1784 | ); |
| 1785 | this.repo.deploys.jobCompleted(jobId, conclusion); |
| 1786 | const m = this.repo.meta; |
| 1787 | this.repo.upsertCheck({ |
| 1788 | sha: run.head_sha ?? run.sha, |
| 1789 | name: this.checkName(run, job), |
| 1790 | status: "completed", |
| 1791 | conclusion: conclusion === "timed_out" ? "timed_out" : conclusion, |
| 1792 | runId: run.id, |
| 1793 | jobId, |
| 1794 | detailsUrl: `${this.env.PUBLIC_URL}/${m.owner}/${m.name}/actions/runs/${run.id}/job/${jobId}`, |
| 1795 | output: completion.failure ? { title: "Failed", summary: completion.failure.message } : { title: conclusion }, |
| 1796 | }); |
| 1797 | // Archive logs to R2 and free the Durable Object's storage. |
| 1798 | const lines = this.sql.exec<LogLine & { seq: number }>("SELECT seq, t, step, text FROM logs WHERE job_id = ? ORDER BY seq", jobId).toArray(); |
| 1799 | if (lines.length) { |
| 1800 | await this.env.BUCKET.put(this.logKey(jobId), JSON.stringify(lines), { httpMetadata: { contentType: "application/json" } }); |
| 1801 | this.sql.exec("DELETE FROM logs WHERE job_id = ?", jobId); |
| 1802 | } |
| 1803 | // Uploads the job left unfinished stop counting against the budgets, and their bytes go. |
| 1804 | await this.repo.runtime.sweepJob(jobId).catch((e) => console.error("runtime sweep failed", e)); |
| 1805 | this.repo.broadcast(`job:${jobId}`, { action: "completed", conclusion }); |
| 1806 | this.repo.broadcast(`run:${run.id}`, { action: "job", jobId }); |
| 1807 | |
| 1808 | // fail-fast: cancel unfinished siblings of a failed matrix job (unless the failed leg may continue on error). |
| 1809 | const siblings = this.sql.exec<JobRow>("SELECT * FROM jobs WHERE run_id = ? AND job_key = ? AND id != ? AND status != 'completed'", run.id, job.job_key, jobId).toArray(); |
| 1810 | const strategy = JSON.parse(job.strategy || "{}") as { "fail-fast"?: boolean }; |
| 1811 | if ((conclusion === "failure" || conclusion === "timed_out") && !job.continue_on_error && strategy["fail-fast"] !== false && siblings.length) { |
| 1812 | for (const s of siblings) { |
| 1813 | await this.stopContainer(s); |
| 1814 | await this.completeJob(s.id, { conclusion: "cancelled", outputs: {}, failure: { message: `Cancelled because ${job.name} failed (fail-fast).` } }); |
| 1815 | } |
| 1816 | } else { |
| 1817 | const queued = siblings.find((s) => s.status === "queued"); |
| 1818 | if (queued) await this.launch(queued.id); |
| 1819 | } |
| 1820 | // An ephemeral runner is done once its one job is. |
| 1821 | if (job.runner_id) this.sql.exec("DELETE FROM runners WHERE id = ? AND ephemeral = 1 AND token_hash IS NULL", job.runner_id); |
| 1822 | // A hosted job hands its slot to the oldest job waiting for one (the repository's or the account's). |
| 1823 | if (!job.labels && job.container) { |
| 1824 | await endCompute(this.env, jobId).catch((e: unknown) => console.error("compute usage failed", e)); |
| 1825 | const held = this.sql.exec<{ id: string }>("SELECT id FROM jobs WHERE fleet_held > 0 AND status = 'queued' ORDER BY rowid LIMIT 1").toArray()[0]; |
| 1826 | if (held) await this.launch(held.id); |
| 1827 | } |
| 1828 | // A job that held its concurrency group hands it to the oldest pending job (of any run). |
| 1829 | if (job.concurrency_group && (job.status === "waiting" || job.status === "in_progress")) { |
| 1830 | const next = this.sql.exec<{ id: string }>("SELECT id FROM jobs WHERE concurrency_group = ? AND status = 'pending' ORDER BY rowid LIMIT 1", job.concurrency_group).toArray()[0]; |
| 1831 | if (next) await this.launch(next.id); |
| 1832 | } |
| 1833 | try { |
| 1834 | const instance = await this.env.RUN_WORKFLOW.get(run.id); |
| 1835 | await instance.sendEvent({ type: `job-${jobId}`, payload: { conclusion } }); |
| 1836 | } catch (e) { |
| 1837 | console.error("failed to notify workflow", e); |
| 1838 | } |
| 1839 | // The Workflow waits for a job held for approval on its own event. |
| 1840 | if (gated) { |
| 1841 | this.syncWaiting(run.id); |
| 1842 | await this.wakeGate(run.id, jobId); |
| 1843 | } |
| 1844 | } |
| 1845 | |
| 1846 | // ---------------------------------------------------------------- secrets and variables |
| 1847 | |
| 1848 | private async repoSecrets(): Promise<Record<string, string>> { |
| 1849 | const out: Record<string, string> = {}; |
| 1850 | for (const r of this.sql.exec<{ name: string; ciphertext: string }>("SELECT name, ciphertext FROM secrets").toArray()) { |
| 1851 | out[r.name] = await decryptSecret(this.env.MASTER_KEY, `repo:${this.repo.meta.repoId}`, r.ciphertext); |
| 1852 | } |
| 1853 | return out; |
| 1854 | } |
| 1855 | |
| 1856 | private async orgSecrets(): Promise<Record<string, string>> { |
| 1857 | const m = this.repo.meta; |
| 1858 | if (m.ownerKind !== "org") return {}; |
| 1859 | // Only the secrets whose repository selection includes this repository (visibility all, private or selected). |
| 1860 | const rows = await this.env.DB.prepare( |
| 1861 | `SELECT s.name, s.ciphertext FROM org_secrets s WHERE s.org_id = ?1 AND (s.visibility = 'all' |
| 1862 | OR (s.visibility = 'private' AND EXISTS (SELECT 1 FROM repos r WHERE r.id = ?2 AND r.visibility != 'public')) |
| 1863 | OR (s.visibility = 'selected' AND EXISTS (SELECT 1 FROM org_secret_repos x WHERE x.org_id = s.org_id AND x.name = s.name AND x.repo_id = ?2)))`, |
| 1864 | ) |
| 1865 | .bind(m.ownerId, m.repoId) |
| 1866 | .all<{ name: string; ciphertext: string }>(); |
| 1867 | const out: Record<string, string> = {}; |
| 1868 | for (const r of rows.results) out[r.name] = await decryptSecret(this.env.MASTER_KEY, `org:${m.ownerId}`, r.ciphertext); |
| 1869 | return out; |
| 1870 | } |
| 1871 | |
| 1872 | async setSecret(name: string, value: string): Promise<void> { |
| 1873 | if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(name) || /^GITHUB_/i.test(name)) throw new HttpError(422, "invalid secret name"); |
| 1874 | this.sql.exec("INSERT OR REPLACE INTO secrets (name, ciphertext, updated_at) VALUES (?, ?, ?)", name.toUpperCase(), await encryptSecret(this.env.MASTER_KEY, `repo:${this.repo.meta.repoId}`, value), now()); |
| 1875 | } |
| 1876 | |
| 1877 | deleteSecret(name: string): void { |
| 1878 | this.sql.exec("DELETE FROM secrets WHERE name = ?", name.toUpperCase()); |
| 1879 | } |
| 1880 | |
| 1881 | secretNames(): Array<{ name: string; updatedAt: string }> { |
| 1882 | return this.sql.exec<{ name: string; updated_at: string }>("SELECT name, updated_at FROM secrets ORDER BY name").toArray().map((r) => ({ name: r.name, updatedAt: r.updated_at })); |
| 1883 | } |
| 1884 | |
| 1885 | setVariable(name: string, value: string | null): void { |
| 1886 | if (value === null) this.sql.exec("DELETE FROM variables WHERE name = ?", name.toUpperCase()); |
| 1887 | else this.sql.exec("INSERT OR REPLACE INTO variables (name, value, updated_at) VALUES (?, ?, ?)", name.toUpperCase(), value, now()); |
| 1888 | } |
| 1889 | |
| 1890 | /** Variables for `vars.*`: the organization's, overridden by the repository's. `withOrg = false` lists only the repository's own. */ |
| 1891 | variables(withOrg = true): Record<string, string> { |
| 1892 | const own = Object.fromEntries(this.sql.exec<{ name: string; value: string }>("SELECT name, value FROM variables").toArray().map((r) => [r.name, r.value])); |
| 1893 | if (!withOrg) return own; |
| 1894 | const org = Object.fromEntries(this.sql.exec<{ name: string; value: string }>("SELECT name, value FROM org_variables").toArray().map((r) => [r.name, r.value])); |
| 1895 | return { ...org, ...own }; |
| 1896 | } |
| 1897 | |
| 1898 | // ---------------------------------------------------------------- cache and artifacts |
| 1899 | |
| 1900 | /** |
| 1901 | * An older runner's cache (the runner API before the toolkit's cache service). |
| 1902 | * Entries are stored as `<ref>\n<key>` and scoped like the cache service's |
| 1903 | * (repo/runtime.ts): a job saves to its own ref and restores from its own ref, |
| 1904 | * then the pull request's base, then the default branch, so one branch or pull |
| 1905 | * request never sees another's caches. Entries saved before scoping (no ref) |
| 1906 | * are never restored, so they go a week after their last use. |
| 1907 | */ |
| 1908 | async cacheLookup(jobId: string, token: string, key: string, restoreKeys: string[]): Promise<{ key: string; stored: string } | null> { |
| 1909 | await this.authJob(jobId, token); |
| 1910 | const { scopes } = this.repo.runtime.job(jobId); |
| 1911 | // An entry whose object is gone is forgotten, and the lookup moves on. |
| 1912 | for (let attempt = 0; attempt < 5; attempt++) { |
| 1913 | const hit = this.legacyCacheMatch(scopes, key, restoreKeys); |
| 1914 | if (!hit) return null; |
| 1915 | if (!(await headFirst(this.env.BUCKET, this.cacheR2Keys(hit)))) { |
| 1916 | this.sql.exec("DELETE FROM caches WHERE key = ?", hit); |
| 1917 | continue; |
| 1918 | } |
| 1919 | this.sql.exec("UPDATE caches SET last_used = ? WHERE key = ?", now(), hit); |
| 1920 | return { key: hit.slice(hit.indexOf("\n") + 1), stored: hit }; |
| 1921 | } |
| 1922 | return null; |
| 1923 | } |
| 1924 | |
| 1925 | /** In each scope in turn: the exact key, then the newest entry for each restore key prefix. */ |
| 1926 | private legacyCacheMatch(scopes: string[], key: string, restoreKeys: string[]): string | null { |
| 1927 | for (const scope of scopes) { |
| 1928 | const exact = this.sql.exec<{ key: string }>("SELECT key FROM caches WHERE key = ?", `${scope}\n${key}`).toArray()[0]; |
| 1929 | if (exact) return exact.key; |
| 1930 | for (const prefix of restoreKeys) { |
| 1931 | const p = `${scope}\n${prefix}`; |
| 1932 | const r = this.sql.exec<{ key: string }>("SELECT key FROM caches WHERE substr(key, 1, ?) = ? ORDER BY created_at DESC LIMIT 1", p.length, p).toArray()[0]; |
| 1933 | if (r) return r.key; |
| 1934 | } |
| 1935 | } |
| 1936 | return null; |
| 1937 | } |
| 1938 | |
| 1939 | /** Where a job saves `key` (its own ref's scope): the stored key and its R2 keys. */ |
| 1940 | async cacheSaveKeys(jobId: string, token: string, key: string): Promise<{ stored: string; r2Keys: string[] }> { |
| 1941 | await this.authJob(jobId, token); |
| 1942 | const stored = `${this.repo.runtime.job(jobId).scopes[0]}\n${key}`; |
| 1943 | return { stored, r2Keys: this.cacheR2Keys(stored) }; |
| 1944 | } |
| 1945 | |
| 1946 | /** The R2 keys of entry `stored`, when the job may read its scope. */ |
| 1947 | async cacheReadKeys(jobId: string, token: string, stored: string): Promise<string[] | null> { |
| 1948 | await this.authJob(jobId, token); |
| 1949 | const nl = stored.indexOf("\n"); |
| 1950 | if (nl < 0 || !this.repo.runtime.job(jobId).scopes.includes(stored.slice(0, nl))) return null; |
| 1951 | return this.cacheR2Keys(stored); |
| 1952 | } |
| 1953 | |
| 1954 | /** Where an older runner's cache entry is stored: the current key, then the old layout's. */ |
| 1955 | cacheR2Keys(key: string): string[] { |
| 1956 | return legacyCacheKeys(this.repo.meta.repoId, key); |
| 1957 | } |
| 1958 | |
| 1959 | /** Records a saved entry; returns the R2 keys of the entries it evicted, for the caller to delete. */ |
| 1960 | async cacheSaved(jobId: string, token: string, key: string, size: number): Promise<string[]> { |
| 1961 | const { stored } = await this.cacheSaveKeys(jobId, token, key); |
| 1962 | this.sql.exec("INSERT OR REPLACE INTO caches (key, size, created_at, last_used) VALUES (?, ?, ?, ?)", stored, size, now(), now()); |
| 1963 | // Like the cache service (repo/runtime.ts): entries unused for a week go, then the |
| 1964 | // least recently used ones above 10 GB per repository. |
| 1965 | const stale = this.sql.exec<{ key: string }>("SELECT key FROM caches WHERE last_used < ?", new Date(Date.now() - CACHE_IDLE_MS).toISOString()).toArray().map((r) => r.key); |
| 1966 | for (const k of stale) this.sql.exec("DELETE FROM caches WHERE key = ?", k); |
| 1967 | const evicted = stale.flatMap((k) => this.cacheR2Keys(k)); |
| 1968 | let total = this.sql.exec<{ n: number | null }>("SELECT sum(size) AS n FROM caches").one().n ?? 0; |
| 1969 | while (total > CACHE_LIMIT_BYTES) { |
| 1970 | const old = this.sql.exec<{ key: string; size: number }>("SELECT key, size FROM caches ORDER BY last_used LIMIT 1").one(); |
| 1971 | this.sql.exec("DELETE FROM caches WHERE key = ?", old.key); |
| 1972 | evicted.push(...this.cacheR2Keys(old.key)); |
| 1973 | total -= old.size; |
| 1974 | } |
| 1975 | return evicted; |
| 1976 | } |
| 1977 | |
| 1978 | /** |
| 1979 | * Where job `jobId` (authenticated by its token) stores artifact `name` of its run, and how many bytes |
| 1980 | * it may store there (what is left of the artifact quota). Refused when the run |
| 1981 | * already has an artifact of that name from the artifact service. |
| 1982 | */ |
| 1983 | async artifactUploadKeys(jobId: string, token: string, name: string): Promise<{ r2Keys: string[]; room: number }> { |
| 1984 | const job = await this.authJob(jobId, token); |
| 1985 | this.repo.runtime.legacyArtifactCheck(job.run_id, name); |
| 1986 | return { r2Keys: this.artifactR2Keys(job.run_id, name), room: await this.repo.runtime.artifactRoom() }; |
| 1987 | } |
| 1988 | |
| 1989 | /** Records an older runner's artifact, stored at its legacy key, with the artifact service's (repo/runtime.ts). */ |
| 1990 | async artifactSaved(jobId: string, token: string, name: string, size: number): Promise<{ id: number }> { |
| 1991 | const job = await this.authJob(jobId, token); |
| 1992 | return { id: this.repo.runtime.legacyArtifactSaved(job.run_id, jobId, name, size).id }; |
| 1993 | } |
| 1994 | |
| 1995 | /** Where job `jobId` (authenticated by its token) reads artifact `name` of its run. */ |
| 1996 | async artifactKeysFor(jobId: string, token: string, name: string): Promise<string[]> { |
| 1997 | const job = await this.authJob(jobId, token); |
| 1998 | return this.artifactR2Keys(job.run_id, name); |
| 1999 | } |
| 2000 | |
| 2001 | /** Where an older runner's artifact is stored: the current key, then the old layout's. */ |
| 2002 | artifactR2Keys(runId: string, name: string): string[] { |
| 2003 | return legacyArtifactKeys(this.repo.meta.repoId, runId, name); |
| 2004 | } |
| 2005 | |
| 2006 | /** |
| 2007 | * A run's artifacts: those uploaded through the artifact service (zip files, or |
| 2008 | * single unarchived files) and older runners' tarballs. |
| 2009 | */ |
| 2010 | artifacts(runId: string): Array<{ name: string; size: number; createdAt: string; r2Keys: string[]; id: number; contentType: string; fileName: string }> { |
| 2011 | runId = this.runRow(runId).id; |
| 2012 | return this.repo.runtime |
| 2013 | .runArtifacts(runId) |
| 2014 | .map((a) => ({ |
| 2015 | name: a.name, |
| 2016 | size: a.size, |
| 2017 | createdAt: a.createdAt, |
| 2018 | r2Keys: artifactObjectKeys(this.repo.meta.repoId, a), |
| 2019 | id: a.id, |
| 2020 | contentType: a.mimeType, |
| 2021 | fileName: artifactFileName(a), |
| 2022 | })) |
| 2023 | .sort((a, b) => a.name.localeCompare(b.name)); |
| 2024 | } |
| 2025 | |
| 2026 | // ---------------------------------------------------------------- sandboxes for agents |
| 2027 | |
| 2028 | /** |
| 2029 | * `task` is set only by agentic/ (never from a request): the sandbox of an agent task, started for |
| 2030 | * the agent `caller` by the person `task.personId`. No public sandbox call reaches it. An agent |
| 2031 | * review's sandbox is `readOnly`: stored with a `read` caller, its git credential can't push. |
| 2032 | * A web terminal's sandbox (`kind: "terminal"`) is a task sandbox one person works in directly. |
| 2033 | */ |
| 2034 | async createSandbox(caller: Caller, ref?: string, task?: { personId: string; readOnly?: boolean; kind?: "task" | "terminal" }): Promise<{ id: string; ref: string }> { |
| 2035 | if (!atLeast(caller.role, "write")) throw new HttpError(403, "write access is required to create a sandbox"); |
| 2036 | // A review's sandbox acts with read access only: its git credential can clone but never push. |
| 2037 | if (task?.readOnly) caller = { ...caller, role: "read" }; |
| 2038 | if (!task) { |
| 2039 | const since = new Date(Date.now() - SANDBOX_TOKEN_TTL_MS).toISOString(); |
| 2040 | const open = this.sql |
| 2041 | .exec<{ n: number }>("SELECT count(*) AS n FROM sandboxes WHERE kind = 'api' AND status = 'running' AND created_at > ? AND json_extract(created_by, '$.id') = ?", since, caller.id) |
| 2042 | .one().n; |
| 2043 | if (open >= MAX_OPEN_SANDBOXES) throw new HttpError(429, `you have ${open} open sandboxes in this repository, the most allowed; close one first`); |
| 2044 | } |
| 2045 | await this.refuseIfBlocked("sandbox", false); |
| 2046 | // The account that owns the repository pays for sandboxes too (lib/quotas.ts). |
| 2047 | const quota = await mayStart(this.env, this.repo.meta.ownerId, "sandbox"); |
| 2048 | if (!quota.ok) throw new HttpError(429, quota.message); |
| 2049 | const sha = await this.repo.resolve(ref ?? "HEAD"); |
| 2050 | if (!sha) throw new HttpError(422, "unknown ref"); |
| 2051 | const id = newId(); |
| 2052 | const m = this.repo.meta; |
| 2053 | const token = `dst_${m.repoId}_${id}_${randomToken(24)}`; |
| 2054 | const secret = randomToken(32); |
| 2055 | this.sql.exec( |
| 2056 | "INSERT INTO sandboxes (id, secret, created_by, ref, created_at, status, kind, task_person) VALUES (?, ?, ?, ?, ?, 'running', ?, ?)", |
| 2057 | id, |
| 2058 | await sha256(token), |
| 2059 | JSON.stringify(caller), |
| 2060 | ref ?? m.defaultBranch, |
| 2061 | now(), |
| 2062 | task ? (task.kind ?? "task") : "api", |
| 2063 | task?.personId ?? null, |
| 2064 | ); |
| 2065 | await startCompute(this.env, { id, accountId: m.ownerId, repoId: m.repoId, kind: "sandbox" }); |
| 2066 | const stub = this.env.SANDBOX.get(this.env.SANDBOX.idFromName(id)); |
| 2067 | try { |
| 2068 | await stub.boot({ |
| 2069 | DIFFINERY_CLONE_URL: `${this.env.PUBLIC_URL}/${m.owner}/${m.name}.git`, |
| 2070 | DIFFINERY_REF: ref ?? m.defaultBranch, |
| 2071 | DIFFINERY_JOB_TOKEN: token, |
| 2072 | DIFFINERY_SANDBOX_SECRET: secret, |
| 2073 | DIFFINERY_API_URL: this.env.PUBLIC_URL, |
| 2074 | GIT_AUTHOR_NAME: caller.login, |
| 2075 | GIT_AUTHOR_EMAIL: `${caller.login}@users.noreply.diffinery.com`, |
| 2076 | GIT_COMMITTER_NAME: caller.login, |
| 2077 | GIT_COMMITTER_EMAIL: `${caller.login}@users.noreply.diffinery.com`, |
| 2078 | }, secret, id, { id, accountId: m.ownerId, repoId: m.repoId }); |
| 2079 | } catch (e) { |
| 2080 | console.error("sandbox boot failed", e); |
| 2081 | this.sql.exec("UPDATE sandboxes SET status = 'failed' WHERE id = ?", id); |
| 2082 | await endCompute(this.env, id).catch(() => {}); |
| 2083 | await stub.shutdown().catch(() => {}); |
| 2084 | throw new HttpError(503, "the sandbox container did not start; try again"); |
| 2085 | } |
| 2086 | return { id, ref: ref ?? m.defaultBranch }; |
| 2087 | } |
| 2088 | |
| 2089 | getSandbox(id: string, caller: Caller): { id: string; ref: string; status: string; createdAt: string } { |
| 2090 | const s = this.sql.exec<{ id: string; ref: string; status: string; created_at: string; created_by: string; kind: string }>("SELECT * FROM sandboxes WHERE id = ?", id).toArray()[0]; |
| 2091 | // A task's sandbox holds a credential for the person's model access, and a terminal's may hold their |
| 2092 | // provider sign-in: nobody reaches either through the sandbox API, admins included. |
| 2093 | if (!s || s.kind !== "api") throw new HttpError(404, "sandbox not found"); |
| 2094 | const owner = JSON.parse(s.created_by) as Caller; |
| 2095 | if (owner.id !== caller.id && !atLeast(caller.role, "admin")) throw new HttpError(403, "not your sandbox"); |
| 2096 | return { id: s.id, ref: s.ref, status: s.status, createdAt: s.created_at }; |
| 2097 | } |
| 2098 | |
| 2099 | async closeSandbox(id: string, caller: Caller): Promise<void> { |
| 2100 | this.getSandbox(id, caller); |
| 2101 | await this.endSandbox(id); |
| 2102 | } |
| 2103 | |
| 2104 | /** Close a sandbox: its token stops working, its time stops counting and its container stops. */ |
| 2105 | private async endSandbox(id: string): Promise<void> { |
| 2106 | this.sql.exec("UPDATE sandboxes SET status = 'closed' WHERE id = ?", id); |
| 2107 | await endCompute(this.env, id); |
| 2108 | await this.env.SANDBOX.get(this.env.SANDBOX.idFromName(id)).shutdown(); |
| 2109 | } |
| 2110 | |
| 2111 | /** Close an agent task's or a web terminal's sandbox (agentic/), which revokes its git credential. */ |
| 2112 | async closeTaskSandbox(id: string): Promise<void> { |
| 2113 | const s = this.sql.exec<{ kind: string }>("SELECT kind FROM sandboxes WHERE id = ?", id).toArray()[0]; |
| 2114 | if (s?.kind !== "task" && s?.kind !== "terminal") return; |
| 2115 | await this.endSandbox(id); |
| 2116 | } |
| 2117 | |
| 2118 | // ---------------------------------------------------------------- kill switches (lib/killswitch.ts) |
| 2119 | |
| 2120 | /** |
| 2121 | * Why `work` may not start here now, or null. One D1 read. A failed read lets work start |
| 2122 | * (`failOpen`), as quotas do for jobs: a D1 outage shouldn't stop every repository's CI. |
| 2123 | */ |
| 2124 | private async blocked(work: Work, failOpen = true): Promise<string | null> { |
| 2125 | try { |
| 2126 | return blockedBy(await killSwitches(this.env, this.repo.meta.repoId), work); |
| 2127 | } catch (e) { |
| 2128 | if (!failOpen) throw e; |
| 2129 | console.error("kill switch check failed", e); |
| 2130 | return null; |
| 2131 | } |
| 2132 | } |
| 2133 | |
| 2134 | /** Refuse (403, with the reason) what someone asked to start while a kill switch stops it. */ |
| 2135 | async refuseIfBlocked(work: Work, failOpen = true): Promise<void> { |
| 2136 | const block = await this.blocked(work, failOpen); |
| 2137 | if (block) throw new HttpError(403, block); |
| 2138 | } |
| 2139 | |
| 2140 | /** |
| 2141 | * A site administrator just suspended Actions here or for the owner (`hostedOnly` false): every |
| 2142 | * unfinished run is cancelled and every open sandbox closed. Or paused hosted compute on the |
| 2143 | * instance (`hostedOnly`): running hosted jobs fail and open sandboxes close; self-hosted jobs go on. |
| 2144 | * `message` says why. Jobs still queued fail when they would start (launch). |
| 2145 | */ |
| 2146 | async stopActions(o: { hostedOnly: boolean; message: string }): Promise<{ runs: number; jobs: number; sandboxes: number }> { |
| 2147 | let runs = 0; |
| 2148 | let jobs = 0; |
| 2149 | let sandboxes = 0; |
| 2150 | if (o.hostedOnly) { |
| 2151 | for (const j of this.sql.exec<JobRow>("SELECT * FROM jobs WHERE labels IS NULL AND container IS NOT NULL AND status IN ('waiting', 'in_progress') ORDER BY rowid").toArray()) { |
| 2152 | await this.stopContainer(j); |
| 2153 | await this.completeJob(j.id, { conclusion: "failure", outputs: {}, failure: { message: `The job was stopped: ${o.message}.` } }); |
| 2154 | jobs++; |
| 2155 | } |
| 2156 | } else { |
| 2157 | for (const r of this.sql.exec<{ id: string }>("SELECT id FROM runs WHERE status != 'completed' ORDER BY rowid").toArray()) { |
| 2158 | jobs += this.sql.exec<{ n: number }>("SELECT count(*) AS n FROM jobs WHERE run_id = ? AND status != 'completed'", r.id).one().n; |
| 2159 | await this.cancelInternal(r.id, `Cancelled: ${o.message}.`); |
| 2160 | runs++; |
| 2161 | } |
| 2162 | } |
| 2163 | for (const s of this.sql.exec<{ id: string }>("SELECT id FROM sandboxes WHERE status = 'running'").toArray()) { |
| 2164 | await this.endSandbox(s.id).catch((e: unknown) => console.error("closing a sandbox failed", e)); |
| 2165 | sandboxes++; |
| 2166 | } |
| 2167 | return { runs, jobs, sandboxes }; |
| 2168 | } |
| 2169 | } |
| 2170 | |
| 2171 | /** |
| 2172 | * One minute of a wait timer: a real minute, except in the end-to-end tests (WAIT_TIMER_MINUTE_MS). The setting |
| 2173 | * shortens a protection rule, so it counts only on a Worker that runs on this machine (auth/dev.ts): a deployed |
| 2174 | * one ignores it. |
| 2175 | */ |
| 2176 | export function timerMinute(env: Pick<Env, "WAIT_TIMER_MINUTE_MS" | "PUBLIC_URL" | "ACCESS_AUD">): number { |
| 2177 | if (!env.WAIT_TIMER_MINUTE_MS || !localOnly(env)) return 60_000; |
| 2178 | const ms = Number(env.WAIT_TIMER_MINUTE_MS); |
| 2179 | return Number.isInteger(ms) && ms > 0 && ms <= 60_000 ? ms : 60_000; |
| 2180 | } |