| 1 | // End-to-end status page: /status and GET /api/v3/diffinery/status for anyone, and incidents that a |
| 2 | // site administrator opens, updates and resolves on Settings → Status and through the admin API, on |
| 3 | // a real `wrangler dev` Worker with no fixed dev identity (so callers can be anonymous). Also covers |
| 4 | // 404 for everyone else, the audit log, and the hosted compute pause showing as maintenance. |
| 5 | // |
| 6 | // pnpm --filter @diffinery/app exec node --test test/e2e/status.test.mjs |
| 7 | import "./fresh-connections.mjs"; |
| 8 | import { spawn, execFileSync } from "node:child_process"; |
| 9 | import { createHash, randomBytes } from "node:crypto"; |
| 10 | import { mkdtempSync, rmSync } from "node:fs"; |
| 11 | import { tmpdir } from "node:os"; |
| 12 | import { join, dirname, resolve } from "node:path"; |
| 13 | import { fileURLToPath } from "node:url"; |
| 14 | import { test, after } from "node:test"; |
| 15 | import assert from "node:assert/strict"; |
| 16 | import { migratedState } from "./d1-state.mjs"; |
| 17 | import { freePort } from "./ports.mjs"; |
| 18 | |
| 19 | const APP = resolve(dirname(fileURLToPath(import.meta.url)), "../.."); |
| 20 | const work = mkdtempSync(join(tmpdir(), "diffinery-status-e2e-")); |
| 21 | const running = []; |
| 22 | |
| 23 | /** Start a Worker with extra `--var`s and seed SQL; resolves to its base URL once /healthz answers. */ |
| 24 | async function worker(name, vars, seed = []) { |
| 25 | const port = await freePort(); |
| 26 | const base = `http://127.0.0.1:${port}`; |
| 27 | const persist = join(work, name); |
| 28 | const wrangler = (...args) => execFileSync("pnpm", ["exec", "wrangler", ...args], { cwd: APP, stdio: "pipe", env: { ...process.env, CI: "1" } }); |
| 29 | migratedState(APP, persist); |
| 30 | if (seed.length) wrangler("d1", "execute", "diffinery-dev", "--local", "-e", "dev", "--persist-to", persist, "--command", seed.join("; ")); |
| 31 | const args = ["exec", "wrangler", "dev", "-e", "dev", "--ip", "127.0.0.1", "--port", String(port), "--inspector-port", String(await freePort()), "--persist-to", persist, "--var", `PUBLIC_URL:${base}`, "--show-interactive-dev-session=false"]; |
| 32 | for (const [k, v] of Object.entries(vars)) args.push("--var", `${k}:${v}`); |
| 33 | const dev = spawn("pnpm", args, { cwd: APP, env: { ...process.env, CI: "1", WRANGLER_SEND_METRICS: "false" }, stdio: ["ignore", "pipe", "pipe"], detached: true }); |
| 34 | let log = ""; |
| 35 | dev.stdout.on("data", (d) => (log += d)); |
| 36 | dev.stderr.on("data", (d) => (log += d)); |
| 37 | running.push(dev); |
| 38 | const end = Date.now() + 120_000; |
| 39 | while (Date.now() < end) { |
| 40 | try { |
| 41 | if ((await fetch(`${base}/healthz`)).ok) return { base, stop: () => stopWorker(dev), log: () => log }; |
| 42 | } catch {} |
| 43 | await new Promise((r) => setTimeout(r, 500)); |
| 44 | } |
| 45 | assert.fail(`wrangler dev did not start\n${log.slice(-4000)}`); |
| 46 | } |
| 47 | |
| 48 | function stopWorker(dev) { |
| 49 | try { |
| 50 | process.kill(-dev.pid, "SIGTERM"); |
| 51 | } catch {} |
| 52 | } |
| 53 | |
| 54 | after(() => { |
| 55 | for (const dev of running) stopWorker(dev); |
| 56 | if (process.env.E2E_KEEP !== "1") rmSync(work, { recursive: true, force: true }); |
| 57 | }); |
| 58 | |
| 59 | const hash = (s) => createHash("sha256").update(s).digest("hex"); |
| 60 | |
| 61 | /** The Worker, with a site administrator (root) and a person (ann); started once. */ |
| 62 | let statusWorker; |
| 63 | const T = { root: `dfy_root_${randomBytes(12).toString("hex")}`, ann: `dfy_ann_${randomBytes(12).toString("hex")}` }; |
| 64 | const S = { root: `root-session-${randomBytes(12).toString("hex")}`, ann: `ann-session-${randomBytes(12).toString("hex")}` }; |
| 65 | const ids = {}; |
| 66 | |
| 67 | async function status() { |
| 68 | if (statusWorker) return statusWorker; |
| 69 | const at = new Date().toISOString(); |
| 70 | const later = new Date(Date.now() + 30 * 86400_000).toISOString(); |
| 71 | const person = (id, login, email, admin) => `INSERT INTO accounts (id, login, kind, name, email, status, site_admin, created_at) VALUES ('${id}', '${login}', 'user', '${login}', '${email}', 'active', ${admin}, '${at}')`; |
| 72 | const token = (id, account, t) => `INSERT INTO tokens (id, account_id, name, token_hash, prefix, scopes, expires_at, created_at) VALUES ('${id}', '${account}', 'e2e', '${hash(t)}', '${t.slice(0, 10)}', '["admin"]', '${later}', '${at}')`; |
| 73 | const session = (account, cookie) => `INSERT INTO sessions (id, account_id, method, created_at, last_seen_at, expires_at) VALUES ('${hash(cookie)}', '${account}', 'passkey', '${at}', '${at}', '${later}')`; |
| 74 | const w = await worker("status", { DEV_AUTH: "" }, [ |
| 75 | person("u-root", "root", "root@example.com", 1), |
| 76 | person("u-ann", "ann", "ann@example.com", 0), |
| 77 | token("t-root", "u-root", T.root), |
| 78 | token("t-ann", "u-ann", T.ann), |
| 79 | session("u-root", S.root), |
| 80 | session("u-ann", S.ann), |
| 81 | ]); |
| 82 | const cookieOf = (who) => (who ? `__Host-dfy_session=${S[who]}` : ""); |
| 83 | const api = async (method, path, body, tok) => { |
| 84 | const r = await fetch(`${w.base}/api/v3${path}`, { method, headers: { "content-type": "application/json", ...(tok ? { authorization: `Bearer ${tok}` } : {}) }, body: body === undefined ? undefined : JSON.stringify(body) }); |
| 85 | const text = await r.text(); |
| 86 | let json = null; |
| 87 | try { |
| 88 | json = text ? JSON.parse(text) : null; |
| 89 | } catch { |
| 90 | json = text; |
| 91 | } |
| 92 | return { status: r.status, json, headers: r.headers }; |
| 93 | }; |
| 94 | const page = (path, who) => fetch(`${w.base}${path}`, { headers: { cookie: cookieOf(who) }, redirect: "manual" }); |
| 95 | /** Submit a form on `pagePath` to `action` the way a browser does: the page's CSRF pair, same-origin. `fields` is a list of [name, value]. */ |
| 96 | const submit = async (pagePath, action, fields, who) => { |
| 97 | const res = await page(pagePath, who); |
| 98 | assert.equal(res.status, 200, `GET ${pagePath} -> ${res.status}`); |
| 99 | const csrf = /name="_csrf" value="([^"]+)"/.exec(await res.text())?.[1]; |
| 100 | const pair = /dfy_csrf=([^;]+)/.exec(res.headers.get("set-cookie") ?? "")?.[1]; |
| 101 | assert.ok(csrf && pair, `no CSRF pair on ${pagePath}`); |
| 102 | const r = await fetch(`${w.base}${action}`, { method: "POST", body: new URLSearchParams([["_csrf", csrf], ...fields]), redirect: "manual", headers: { cookie: [cookieOf(who), `dfy_csrf=${pair}`].filter(Boolean).join("; "), "sec-fetch-site": "same-origin" } }); |
| 103 | return { status: r.status, location: r.headers.get("location"), setCookie: r.headers.get("set-cookie") ?? "", html: await r.text() }; |
| 104 | }; |
| 105 | const summary = async () => { |
| 106 | const r = await api("GET", "/diffinery/status"); |
| 107 | assert.equal(r.status, 200, JSON.stringify(r.json)); |
| 108 | return r.json; |
| 109 | }; |
| 110 | statusWorker = { ...w, api, page, submit, summary }; |
| 111 | return statusWorker; |
| 112 | } |
| 113 | |
| 114 | /** The flash message a redirect sets, decoded. */ |
| 115 | function flashOf(setCookie) { |
| 116 | let v = /dfy_flash=([^;]+)/.exec(setCookie)?.[1] ?? ""; |
| 117 | for (let i = 0; i < 3 && /%[0-9A-F]{2}/i.test(v); i++) v = decodeURIComponent(v); |
| 118 | return v; |
| 119 | } |
| 120 | const componentOf = (s, id) => s.components.find((c) => c.id === id); |
| 121 | /** The part of the public page from its "Past incidents" heading on. */ |
| 122 | const pastPart = (html) => html.slice(html.indexOf("<h2>Past incidents</h2>")); |
| 123 | const activePart = (html) => html.slice(0, html.indexOf("<h2>Components</h2>")); |
| 124 | |
| 125 | const apiTitle = `Pushes are failing <b>${randomBytes(4).toString("hex")}</b>`; |
| 126 | const webTitle = `Emails are delayed ${randomBytes(4).toString("hex")}`; |
| 127 | |
| 128 | test("status page: anyone can read it, and everything is operational at first", async () => { |
| 129 | const { page, summary, api } = await status(); |
| 130 | const res = await page("/status"); |
| 131 | assert.equal(res.status, 200); |
| 132 | const html = await res.text(); |
| 133 | assert.match(html, /<h1>Status<\/h1>/); |
| 134 | assert.match(html, /All systems operational/); |
| 135 | for (const name of ["Website and API", "Git operations", "Actions", "Agents and sandboxes", "Packages", "Email"]) assert.ok(html.includes(name), `${name} is listed`); |
| 136 | assert.match(html, /No incidents in the last 14 days/); |
| 137 | // Every page links it from the footer. |
| 138 | assert.match(await (await page("/")).text(), /<footer class="site-foot">[\s\S]*href="\/status"/); |
| 139 | |
| 140 | const s = await summary(); |
| 141 | assert.deepEqual(s.status, { indicator: "none", description: "All systems operational" }); |
| 142 | assert.deepEqual( |
| 143 | s.components.map((c) => [c.id, c.status]), |
| 144 | ["web", "git", "actions", "agents", "packages", "email"].map((id) => [id, "operational"]), |
| 145 | ); |
| 146 | assert.deepEqual(s.incidents, []); |
| 147 | assert.ok(!Number.isNaN(Date.parse(s.updated_at))); |
| 148 | const r = await api("GET", "/diffinery/status"); |
| 149 | assert.equal(r.headers.get("cache-control"), "public, max-age=30"); |
| 150 | }); |
| 151 | |
| 152 | test("status page: only site administrators can manage incidents (404 for anyone else)", async () => { |
| 153 | const { api, page } = await status(); |
| 154 | const open = { title: "Nope", impact: "major", components: ["git"], state: "investigating", body: "x" }; |
| 155 | assert.equal((await api("GET", "/diffinery/admin/status/incidents", undefined, T.ann)).status, 404); |
| 156 | assert.equal((await api("POST", "/diffinery/admin/status/incidents", open, T.ann)).status, 404); |
| 157 | assert.equal((await api("POST", `/diffinery/admin/status/incidents/${"0".repeat(32)}/updates`, { state: "resolved", body: "x" }, T.ann)).status, 404); |
| 158 | assert.equal((await api("GET", "/diffinery/admin/status/incidents")).status, 401); |
| 159 | assert.equal((await api("POST", "/diffinery/admin/status/incidents", open)).status, 401); |
| 160 | assert.equal((await page("/settings/admin/status", "ann")).status, 404); |
| 161 | const anon = await page("/settings/admin/status"); |
| 162 | assert.equal(anon.status, 302, "a signed-out visitor is sent to sign in"); |
| 163 | assert.match(anon.headers.get("location") ?? "", /^\/login\?next=/); |
| 164 | assert.doesNotMatch(await (await page("/settings", "ann")).text(), /href="\/settings\/admin\/status"/); |
| 165 | // Nothing was opened. |
| 166 | assert.deepEqual((await api("GET", "/diffinery/admin/status/incidents", undefined, T.root)).json, []); |
| 167 | }); |
| 168 | |
| 169 | test("status page: a site administrator opens an incident through the API, and the page shows it", async () => { |
| 170 | const { api, page, summary } = await status(); |
| 171 | // Refused with sentences that say what to fix. |
| 172 | const bad = await api("POST", "/diffinery/admin/status/incidents", { title: "x", impact: "major", components: [], state: "investigating", body: "x" }, T.root); |
| 173 | assert.equal(bad.status, 422); |
| 174 | assert.equal(bad.json.message, "Choose at least one affected component."); |
| 175 | const unknown = await api("POST", "/diffinery/admin/status/incidents", { title: "x", impact: "major", components: ["database"], state: "investigating", body: "x" }, T.root); |
| 176 | assert.equal(unknown.status, 422); |
| 177 | assert.match(unknown.json.message, /There is no component called "database"/); |
| 178 | assert.equal((await api("POST", "/diffinery/admin/status/incidents", { title: "x", impact: "huge", components: ["git"], state: "investigating", body: "x" }, T.root)).status, 422); |
| 179 | |
| 180 | const r = await api("POST", "/diffinery/admin/status/incidents", { title: apiTitle, impact: "major", components: ["git"], state: "investigating", body: "We are investigating failed pushes.\n<script>alert(1)</script>" }, T.root); |
| 181 | assert.equal(r.status, 201, JSON.stringify(r.json)); |
| 182 | ids.api = r.json.id; |
| 183 | assert.match(ids.api, /^[0-9a-f]{32}$/); |
| 184 | assert.equal(r.json.created_by, "root"); |
| 185 | assert.equal(r.json.updates.length, 1); |
| 186 | assert.equal(r.json.updates[0].author, "root"); |
| 187 | |
| 188 | const s = await summary(); |
| 189 | assert.deepEqual(s.status, { indicator: "major", description: "Partial outage" }); |
| 190 | assert.equal(componentOf(s, "git").status, "partial_outage"); |
| 191 | assert.equal(componentOf(s, "web").status, "operational"); |
| 192 | assert.deepEqual( |
| 193 | s.incidents.map((i) => [i.id, i.title, i.state, i.components]), |
| 194 | [[ids.api, apiTitle, "investigating", ["git"]]], |
| 195 | ); |
| 196 | assert.equal(s.incidents[0].created_by, undefined, "the public JSON doesn't name administrators"); |
| 197 | assert.equal(s.incidents[0].updates[0].author, undefined); |
| 198 | |
| 199 | const html = await (await page("/status")).text(); |
| 200 | assert.match(html, /Partial outage/); |
| 201 | assert.match(html, /<li data-component="git" data-status="partial_outage">/); |
| 202 | assert.ok(activePart(html).includes(apiTitle.replace("<b>", "<b>").replace("</b>", "</b>")), "the title is escaped, under active incidents"); |
| 203 | assert.ok(!html.includes("<script>alert(1)</script>"), "the update is plain text"); |
| 204 | assert.match(html, /We are investigating failed pushes\./); |
| 205 | }); |
| 206 | |
| 207 | test("status page: the web form opens an incident with several components and posts updates", async () => { |
| 208 | const { submit, summary, api, page } = await status(); |
| 209 | const base = "/settings/admin/status"; |
| 210 | const missing = await submit(base, base, [["title", webTitle], ["impact", "minor"], ["state", "investigating"], ["body", "Looking."]], "root"); |
| 211 | assert.equal(missing.status, 303); |
| 212 | assert.match(flashOf(missing.setCookie), /Choose at least one affected component\./); |
| 213 | const opened = await submit( |
| 214 | base, |
| 215 | base, |
| 216 | [ |
| 217 | ["title", webTitle], |
| 218 | ["impact", "minor"], |
| 219 | ["components", "email"], |
| 220 | ["components", "web"], |
| 221 | ["state", "investigating"], |
| 222 | ["body", "Some emails are delayed by a few minutes."], |
| 223 | ], |
| 224 | "root", |
| 225 | ); |
| 226 | assert.equal(opened.status, 303); |
| 227 | assert.equal(opened.location, base); |
| 228 | let s = await summary(); |
| 229 | const web = s.incidents.find((i) => i.title === webTitle); |
| 230 | assert.ok(web, "the incident from the form is listed"); |
| 231 | ids.web = web.id; |
| 232 | assert.deepEqual(web.components, ["web", "email"]); |
| 233 | assert.equal(componentOf(s, "email").status, "degraded_performance"); |
| 234 | // Worst wins: git's partial outage outranks the email incident. |
| 235 | assert.deepEqual(s.status, { indicator: "major", description: "Partial outage" }); |
| 236 | |
| 237 | const adminHtml = await (await page(base, "root")).text(); |
| 238 | assert.ok(adminHtml.includes(webTitle)); |
| 239 | assert.ok(adminHtml.includes(`action="${base}/${ids.web}"`), "each open incident has an update form"); |
| 240 | assert.doesNotMatch(adminHtml, /<select|type="checkbox"/, "Kumo controls only"); |
| 241 | |
| 242 | const identified = await submit(base, `${base}/${ids.web}`, [["state", "identified"], ["body", "A mail queue was stuck; draining it."]], "root"); |
| 243 | assert.equal(identified.status, 303); |
| 244 | const resolved = await submit(base, `${base}/${ids.web}`, [["state", "resolved"], ["body", "Emails are on time again."]], "root"); |
| 245 | assert.equal(resolved.status, 303); |
| 246 | assert.match(flashOf(resolved.setCookie), /Incident resolved/); |
| 247 | const listed = (await api("GET", "/diffinery/admin/status/incidents", undefined, T.root)).json.find((i) => i.id === ids.web); |
| 248 | assert.equal(listed.state, "resolved"); |
| 249 | assert.ok(listed.resolved_at); |
| 250 | assert.deepEqual( |
| 251 | listed.updates.map((u) => u.state), |
| 252 | ["resolved", "identified", "investigating"], |
| 253 | ); |
| 254 | s = await summary(); |
| 255 | assert.equal(componentOf(s, "email").status, "operational"); |
| 256 | assert.ok(!s.incidents.some((i) => i.id === ids.web)); |
| 257 | }); |
| 258 | |
| 259 | test("status page: resolving through the API returns the component to operational and moves the incident to past incidents", async () => { |
| 260 | const { api, page, summary } = await status(); |
| 261 | assert.ok(ids.api, "the incident from the test above"); |
| 262 | const bad = await api("POST", `/diffinery/admin/status/incidents/${ids.api}/updates`, { state: "fixed", body: "x" }, T.root); |
| 263 | assert.equal(bad.status, 422); |
| 264 | assert.equal((await api("POST", `/diffinery/admin/status/incidents/${"0".repeat(32)}/updates`, { state: "resolved", body: "x" }, T.root)).status, 404); |
| 265 | const r = await api("POST", `/diffinery/admin/status/incidents/${ids.api}/updates`, { state: "resolved", body: "Pushes work again." }, T.root); |
| 266 | assert.equal(r.status, 201, JSON.stringify(r.json)); |
| 267 | assert.equal(r.json.state, "resolved"); |
| 268 | |
| 269 | const s = await summary(); |
| 270 | assert.deepEqual(s.status, { indicator: "none", description: "All systems operational" }); |
| 271 | assert.equal(componentOf(s, "git").status, "operational"); |
| 272 | assert.deepEqual(s.incidents, []); |
| 273 | |
| 274 | const html = await (await page("/status")).text(); |
| 275 | assert.match(html, /All systems operational/); |
| 276 | assert.doesNotMatch(html, /Active incidents/); |
| 277 | const past = pastPart(html); |
| 278 | assert.ok(past.includes("Pushes are failing"), "the API incident is under past incidents"); |
| 279 | assert.ok(past.includes(webTitle), "so is the one from the form"); |
| 280 | assert.match(past, /Pushes work again\./); |
| 281 | }); |
| 282 | |
| 283 | test("status page: changes are audited without the updates' text", async () => { |
| 284 | const { page } = await status(); |
| 285 | assert.ok(ids.api && ids.web, "the incidents from the tests above"); |
| 286 | const audit = await (await page("/settings/audit", "root")).text(); |
| 287 | for (const action of ["status_incident.create", "status_incident.update", "status_incident.resolve"]) assert.ok(audit.includes(action), `${action} is audited`); |
| 288 | assert.ok(audit.includes(ids.api) && audit.includes(ids.web), "the audit log names the incidents"); |
| 289 | assert.ok(!audit.includes("Pushes work again"), "the audit log holds no update text"); |
| 290 | }); |
| 291 | |
| 292 | test("status page: pausing hosted compute shows Actions and agent sandboxes under maintenance", async () => { |
| 293 | const { api, summary, page } = await status(); |
| 294 | const paused = await api("PUT", "/admin/hosted-compute-pause", { reason: "internal runner image problem" }, T.root); |
| 295 | assert.equal(paused.status, 200, JSON.stringify(paused.json)); |
| 296 | try { |
| 297 | const s = await summary(); |
| 298 | assert.deepEqual(s.status, { indicator: "maintenance", description: "Service under maintenance" }); |
| 299 | assert.deepEqual(componentOf(s, "actions"), { id: "actions", name: "Actions", status: "under_maintenance", note: "Hosted jobs are paused by the site administrators. Jobs on self-hosted runners still run." }); |
| 300 | assert.equal(componentOf(s, "agents").status, "under_maintenance"); |
| 301 | assert.equal(componentOf(s, "git").status, "operational"); |
| 302 | const html = await (await page("/status")).text(); |
| 303 | assert.match(html, /Hosted jobs are paused by the site administrators\./); |
| 304 | assert.ok(!html.includes("internal runner image problem"), "the pause's reason isn't public"); |
| 305 | } finally { |
| 306 | assert.equal((await api("DELETE", "/admin/hosted-compute-pause", undefined, T.root)).status, 200); |
| 307 | } |
| 308 | assert.equal((await summary()).status.indicator, "none"); |
| 309 | }); |