| | @@ -12,7 +12,7 @@ |
| 12 | 12 | // and complaint intake, no email to a suppressed address, lifting it again; the |
| 13 | 13 | // morning digest on its schedule (phase 5c) and Today at phone width. |
| 14 | 14 | import "./fresh-connections.mjs"; |
| 15 | | -import { spawn, execFileSync } from "node:child_process"; |
| 15 | +import { spawn, spawnSync, execFileSync } from "node:child_process"; |
| 16 | 16 | import { mkdtempSync, writeFileSync, mkdirSync, existsSync, rmSync } from "node:fs"; |
| 17 | 17 | import { tmpdir } from "node:os"; |
| 18 | 18 | import { createHmac, randomBytes } from "node:crypto"; |
| | @@ -77,7 +77,8 @@ |
| 77 | 77 | } |
| 78 | 78 | |
| 79 | 79 | const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); |
| 80 | | -async function until(what, fn, timeoutMs = 60_000) { |
| 80 | +/** `explain`, if given, says on a timeout where things stood (the polling's own log lines are left out of the tail). */ |
| 81 | +async function until(what, fn, timeoutMs = 60_000, explain = async () => "") { |
| 81 | 82 | const end = Date.now() + timeoutMs; |
| 82 | 83 | let last; |
| 83 | 84 | while (Date.now() < end) { |
| | @@ -85,9 +86,38 @@ |
| 85 | 86 | if (last) return last; |
| 86 | 87 | await sleep(500); |
| 87 | 88 | } |
| 88 | | - assert.fail(`timed out waiting for ${what}\n--- wrangler log tail ---\n${devLog.slice(-4000)}`); |
| 89 | + const tail = devLog |
| 90 | + .split("\n") |
| 91 | + .filter((l) => !/runners\/claim 204/.test(l)) |
| 92 | + .join("\n") |
| 93 | + .slice(-4000); |
| 94 | + assert.fail(`timed out waiting for ${what}\n${await explain().catch((e) => `(couldn't say where things stood: ${e.message})`)}\n--- wrangler log tail ---\n${tail}`); |
| 89 | 95 | } |
| 90 | 96 | |
| 97 | +/** |
| 98 | + * Waits for a self-hosted job of the repository to claim; on a timeout, says what the push printed and |
| 99 | + * where the repository's runs and their jobs stood, so a job that never became claimable can be told apart. |
| 100 | + */ |
| 101 | +async function claimJob(repoPath, pushed = "") { |
| 102 | + return until( |
| 103 | + "a job to claim", |
| 104 | + async () => { |
| 105 | + const r = await runnerClaim(repoPath, ["linux"]); |
| 106 | + return r.status === 200 ? r.json : null; |
| 107 | + }, |
| 108 | + 60_000, |
| 109 | + async () => { |
| 110 | + const runs = (await ok("GET", `${repoPath}/actions/runs?per_page=10`)).workflow_runs; |
| 111 | + const lines = [`push output: ${pushed.trim() || "(none)"}`, `runs: ${runs.length}`]; |
| 112 | + for (const r of runs) { |
| 113 | + const jobs = (await ok("GET", `${repoPath}/actions/runs/${r.id}/jobs?filter=all`)).jobs; |
| 114 | + lines.push(`- run ${r.run_number} ${r.head_sha.slice(0, 7)} ${r.status}/${r.conclusion}: ${jobs.map((j) => `${j.name} ${j.status}/${j.conclusion}`).join(", ") || "no jobs"}`); |
| 115 | + } |
| 116 | + return lines.join("\n"); |
| 117 | + }, |
| 118 | + ); |
| 119 | +} |
| 120 | + |
| 91 | 121 | /** My emails, newest first, with the full message (kept in local development only). */ |
| 92 | 122 | const emails = async (q = "") => ok("GET", `/diffinery/emails?per_page=200${q}`); |
| 93 | 123 | /** Wait for an email matching `pred` and return it. */ |
| | @@ -103,6 +133,18 @@ |
| 103 | 133 | }); |
| 104 | 134 | } |
| 105 | 135 | |
| 136 | +/** Pushes main and returns what the server said (a workflow that failed to start says so here). */ |
| 137 | +function gitPush(cwd) { |
| 138 | + const auth = Buffer.from(`x-token:${token}`).toString("base64"); |
| 139 | + const r = spawnSync("git", ["-c", `http.extraHeader=Authorization: Basic ${auth}`, "-c", "protocol.version=2", "push", "origin", "main"], { |
| 140 | + cwd, |
| 141 | + encoding: "utf8", |
| 142 | + env: { ...process.env, GIT_TERMINAL_PROMPT: "0" }, |
| 143 | + }); |
| 144 | + assert.equal(r.status, 0, `git push failed: ${r.stderr}`); |
| 145 | + return r.stderr.split("\n").filter((l) => l.startsWith("remote:")).join("\n"); |
| 146 | +} |
| 147 | + |
| 106 | 148 | function spawnAsync(cmd, args, env, timeoutMs) { |
| 107 | 149 | return new Promise((done) => { |
| 108 | 150 | const child = spawn(cmd, args, { env: { ...process.env, ...env } }); |
| | @@ -277,11 +319,7 @@ |
| 277 | 319 | git(src, ["add", "-A"]); |
| 278 | 320 | git(src, ["commit", "-q", "-m", "Add CI that fails"]); |
| 279 | 321 | git(src, ["remote", "add", "origin", `${base}/${owner}/mailbox.git`]); |
| 280 | | - git(src, ["push", "-q", "origin", "main"]); |
| 281 | | - const claim = await until("a job to claim", async () => { |
| 282 | | - const r = await runnerClaim(`/repos/${owner}/mailbox`, ["linux"]); |
| 283 | | - return r.status === 200 ? r.json : null; |
| 284 | | - }); |
| 322 | + const claim = await claimJob(`/repos/${owner}/mailbox`, gitPush(src)); |
| 285 | 323 | await spawnAsync(process.execPath, [AGENT], { DIFFINERY_API_URL: claim.api_url, DIFFINERY_JOB_ID: claim.job_id, DIFFINERY_JOB_TOKEN: claim.token, DIFFINERY_WORK_ROOT: join(work, `job-${claim.job_id}`) }, 180_000); |
| 286 | 324 | const failed = await email("run failed", (e) => e.kind === "run" && e.subject.includes(`${owner}/mailbox`)); |
| 287 | 325 | assert.equal(failed.status, "sent"); |
| | @@ -294,11 +332,7 @@ |
| 294 | 332 | writeFileSync(join(src, file), `${file}\n`); |
| 295 | 333 | git(src, ["add", "-A"]); |
| 296 | 334 | git(src, ["commit", "-q", "-m", `Add ${file}`]); |
| 297 | | - git(src, ["push", "-q", "origin", "main"]); |
| 298 | | - const c = await until("a job to claim", async () => { |
| 299 | | - const r = await runnerClaim(`/repos/${owner}/mailbox`, ["linux"]); |
| 300 | | - return r.status === 200 ? r.json : null; |
| 301 | | - }); |
| 335 | + const c = await claimJob(`/repos/${owner}/mailbox`, gitPush(src)); |
| 302 | 336 | await spawnAsync(process.execPath, [AGENT], { DIFFINERY_API_URL: c.api_url, DIFFINERY_JOB_ID: c.job_id, DIFFINERY_JOB_TOKEN: c.token, DIFFINERY_WORK_ROOT: join(work, `job-${c.job_id}`) }, 180_000); |
| 303 | 337 | }; |
| 304 | 338 | await ok("PUT", `/repos/${owner}/mailbox/subscription`, { ignored: true }); |