| 1 | import { describe, expect, it } from "vitest"; |
| 2 | import type { Env } from "../../src/env.js"; |
| 3 | import { PAUSE_FLAG } from "../../src/lib/killswitch.js"; |
| 4 | import { |
| 5 | COMPONENT_IDS, |
| 6 | DB_DOWN, |
| 7 | MAX_BODY, |
| 8 | MAX_TITLE, |
| 9 | StatusIncidents, |
| 10 | componentStates, |
| 11 | incidentJson, |
| 12 | killSwitchSignals, |
| 13 | openIncident, |
| 14 | overall, |
| 15 | postUpdate, |
| 16 | statusSummary, |
| 17 | summaryJson, |
| 18 | validateIncident, |
| 19 | validateUpdate, |
| 20 | type Incident, |
| 21 | } from "../../src/lib/status.js"; |
| 22 | import { HttpError } from "../../src/lib/util.js"; |
| 23 | import { testD1 } from "./stubs/d1.js"; |
| 24 | |
| 25 | const PUB = "https://diffinery.example"; |
| 26 | const root = { id: "u-root", login: "root" }; |
| 27 | |
| 28 | function setup() { |
| 29 | const DB = testD1(); |
| 30 | const env = { DB, PUBLIC_URL: PUB } as unknown as Env; |
| 31 | const audits: Array<{ action: string; target?: string; actorLogin?: string; detail?: unknown }> = []; |
| 32 | const audit = async (e: (typeof audits)[number]) => void audits.push(e); |
| 33 | return { env, DB, audits, audit }; |
| 34 | } |
| 35 | |
| 36 | /** An incident as the store would return it. */ |
| 37 | const incident = (o: Partial<Incident>): Incident => ({ |
| 38 | id: "0".repeat(32), |
| 39 | title: "Something", |
| 40 | impact: "minor", |
| 41 | components: ["web"], |
| 42 | state: "investigating", |
| 43 | created_by: "root", |
| 44 | created_at: "2026-10-01T00:00:00.000Z", |
| 45 | updated_at: "2026-10-01T00:00:00.000Z", |
| 46 | resolved_at: null, |
| 47 | updates: [], |
| 48 | ...o, |
| 49 | }); |
| 50 | |
| 51 | const statusOf = (incidents: Incident[], signals: Parameters<typeof componentStates>[1] = []) => Object.fromEntries(componentStates(incidents, signals).map((c) => [c.id, c.status])); |
| 52 | |
| 53 | const refused = (f: () => unknown, msg: RegExp) => { |
| 54 | try { |
| 55 | f(); |
| 56 | } catch (e) { |
| 57 | expect(e).toBeInstanceOf(HttpError); |
| 58 | expect((e as HttpError).status).toBe(422); |
| 59 | expect((e as HttpError).detail).toMatch(msg); |
| 60 | return; |
| 61 | } |
| 62 | throw new Error("expected it to be refused"); |
| 63 | }; |
| 64 | |
| 65 | describe("component statuses", () => { |
| 66 | it("lists every component, operational when nothing is wrong", () => { |
| 67 | const c = componentStates([], []); |
| 68 | expect(c.map((x) => [x.id, x.name])).toEqual([ |
| 69 | ["web", "Website and API"], |
| 70 | ["git", "Git operations"], |
| 71 | ["actions", "Actions"], |
| 72 | ["agents", "Agents and sandboxes"], |
| 73 | ["packages", "Packages"], |
| 74 | ["email", "Email"], |
| 75 | ]); |
| 76 | expect(c.every((x) => x.status === "operational" && !("note" in x))).toBe(true); |
| 77 | expect(overall(c)).toEqual({ indicator: "none", description: "All systems operational" }); |
| 78 | }); |
| 79 | |
| 80 | it("maps each impact to Statuspage's component status, only for the components an incident names", () => { |
| 81 | expect(statusOf([incident({ impact: "critical", components: ["git"] })])).toMatchObject({ git: "major_outage", web: "operational" }); |
| 82 | expect(statusOf([incident({ impact: "major", components: ["actions", "agents"] })])).toMatchObject({ actions: "partial_outage", agents: "partial_outage", git: "operational" }); |
| 83 | expect(statusOf([incident({ impact: "minor", components: ["email"] })])).toMatchObject({ email: "degraded_performance" }); |
| 84 | expect(statusOf([incident({ impact: "maintenance", components: ["packages"] })])).toMatchObject({ packages: "under_maintenance" }); |
| 85 | }); |
| 86 | |
| 87 | it("takes the worst unresolved incident, and ignores resolved ones", () => { |
| 88 | const s = statusOf([ |
| 89 | incident({ impact: "minor", components: ["web", "git"] }), |
| 90 | incident({ impact: "critical", components: ["web"], state: "monitoring" }), |
| 91 | incident({ impact: "critical", components: ["git"], state: "resolved", resolved_at: "2026-10-02T00:00:00.000Z" }), |
| 92 | incident({ impact: "maintenance", components: ["git"] }), |
| 93 | ]); |
| 94 | expect(s).toMatchObject({ web: "major_outage", git: "degraded_performance" }); |
| 95 | }); |
| 96 | |
| 97 | it("gives the overall indicator and description of the worst component", () => { |
| 98 | const of = (impact: Incident["impact"]) => overall(componentStates([incident({ impact }), incident({ impact: "maintenance", components: ["email"] })], [])); |
| 99 | expect(of("critical")).toEqual({ indicator: "critical", description: "Major outage" }); |
| 100 | expect(of("major")).toEqual({ indicator: "major", description: "Partial outage" }); |
| 101 | expect(of("minor")).toEqual({ indicator: "minor", description: "Degraded performance" }); |
| 102 | expect(of("maintenance")).toEqual({ indicator: "maintenance", description: "Service under maintenance" }); |
| 103 | }); |
| 104 | }); |
| 105 | |
| 106 | describe("kill switches and failures", () => { |
| 107 | it("puts what an instance-wide switch stops under maintenance, with a note", () => { |
| 108 | expect(killSwitchSignals({ hostedJobs: false, selfHostedJobs: false, sandboxes: false })).toEqual([]); |
| 109 | const paused = killSwitchSignals({ hostedJobs: true, selfHostedJobs: false, sandboxes: true }); |
| 110 | const c = componentStates([], paused); |
| 111 | expect(c.find((x) => x.id === "actions")).toEqual({ id: "actions", name: "Actions", status: "under_maintenance", note: "Hosted jobs are paused by the site administrators. Jobs on self-hosted runners still run." }); |
| 112 | expect(c.find((x) => x.id === "agents")).toEqual({ id: "agents", name: "Agents and sandboxes", status: "under_maintenance", note: "Agent sandboxes are paused by the site administrators." }); |
| 113 | expect(overall(c).indicator).toBe("maintenance"); |
| 114 | expect(killSwitchSignals({ hostedJobs: true, selfHostedJobs: true, sandboxes: false })).toEqual([{ component: "actions", status: "under_maintenance", note: "Actions jobs are paused by the site administrators." }]); |
| 115 | expect(killSwitchSignals({ hostedJobs: false, selfHostedJobs: true, sandboxes: false })[0]!.note).toBe("Jobs on self-hosted runners are paused by the site administrators."); |
| 116 | }); |
| 117 | |
| 118 | it("lets a worse incident win over a switch, keeping the switch's note", () => { |
| 119 | const signals = killSwitchSignals({ hostedJobs: true, selfHostedJobs: false, sandboxes: true }); |
| 120 | const c = componentStates([incident({ impact: "major", components: ["actions"] }), incident({ impact: "maintenance", components: ["agents"] })], signals); |
| 121 | expect(c.find((x) => x.id === "actions")).toMatchObject({ status: "partial_outage", note: expect.stringMatching(/^Hosted jobs are paused/) }); |
| 122 | expect(c.find((x) => x.id === "agents")!.status).toBe("under_maintenance"); |
| 123 | }); |
| 124 | |
| 125 | it("reads the instance's hosted compute pause into the summary", async () => { |
| 126 | const { env, DB } = setup(); |
| 127 | expect((await statusSummary(env)).status).toEqual({ indicator: "none", description: "All systems operational" }); |
| 128 | DB.sqlite.prepare("INSERT INTO site_flags (key, value, updated_at, updated_by) VALUES (?, 'a secret reason', ?, 'root')").run(PAUSE_FLAG, new Date().toISOString()); |
| 129 | const s = await statusSummary(env); |
| 130 | expect(s.status.indicator).toBe("maintenance"); |
| 131 | expect(s.components.find((c) => c.id === "actions")).toMatchObject({ status: "under_maintenance", note: expect.stringMatching(/^Hosted jobs are paused by the site administrators/) }); |
| 132 | expect(s.components.find((c) => c.id === "agents")!.status).toBe("under_maintenance"); |
| 133 | expect(s.components.find((c) => c.id === "git")!.status).toBe("operational"); |
| 134 | // The administrators' reason isn't public. |
| 135 | expect(JSON.stringify(summaryJson(env, s))).not.toContain("secret"); |
| 136 | }); |
| 137 | |
| 138 | it("still answers when the database can't be read: the website is down and there are no incidents", async () => { |
| 139 | const DB = { prepare: () => ({ bind: () => ({ all: async () => { throw new Error("D1_ERROR: unavailable"); }, first: async () => { throw new Error("D1_ERROR: unavailable"); } }), all: async () => { throw new Error("D1_ERROR: unavailable"); }, first: async () => { throw new Error("D1_ERROR: unavailable"); } }) }; |
| 140 | const env = { DB, PUBLIC_URL: PUB } as unknown as Env; |
| 141 | const error = console.error; |
| 142 | console.error = () => {}; |
| 143 | try { |
| 144 | const s = await statusSummary(env, new Date("2026-10-14T12:00:00.000Z")); |
| 145 | expect(s.components.find((c) => c.id === "web")).toEqual({ id: "web", name: "Website and API", status: "major_outage", note: DB_DOWN }); |
| 146 | expect(s.components.filter((c) => c.status !== "operational").map((c) => c.id)).toEqual(["web"]); |
| 147 | expect(s.status).toEqual({ indicator: "critical", description: "Major outage" }); |
| 148 | expect(s).toMatchObject({ incidents: [], resolved: [], updated_at: "2026-10-14T12:00:00.000Z" }); |
| 149 | } finally { |
| 150 | console.error = error; |
| 151 | } |
| 152 | }); |
| 153 | }); |
| 154 | |
| 155 | describe("validation", () => { |
| 156 | const good = { title: " Pushes\n are failing ", impact: "major", components: ["git", "web", "git"], state: "investigating", body: "\n\nWe are looking into it.\r\nMore soon.\n\n " }; |
| 157 | |
| 158 | it("keeps a one-line title, the update's lines and the components in the page's order", () => { |
| 159 | expect(validateIncident(good)).toEqual({ title: "Pushes are failing", impact: "major", components: ["web", "git"], state: "investigating", body: "We are looking into it.\nMore soon." }); |
| 160 | expect(validateUpdate({ state: "resolved", body: "Fixed." })).toEqual({ state: "resolved", body: "Fixed." }); |
| 161 | }); |
| 162 | |
| 163 | it("refuses what is missing, too long or unknown, in full sentences", () => { |
| 164 | refused(() => validateIncident({ ...good, title: " \n " }), /^Give the incident a title\.$/); |
| 165 | refused(() => validateIncident({ ...good, title: "x".repeat(MAX_TITLE + 1) }), /200 characters/); |
| 166 | expect(validateIncident({ ...good, title: "x".repeat(MAX_TITLE) }).title).toHaveLength(MAX_TITLE); |
| 167 | refused(() => validateIncident({ ...good, impact: "huge" }), /^Choose the incident's impact: minor, major, critical or maintenance\.$/); |
| 168 | refused(() => validateIncident({ ...good, components: [] }), /^Choose at least one affected component\.$/); |
| 169 | refused(() => validateIncident({ ...good, components: "web" }), /at least one affected component/); |
| 170 | refused(() => validateIncident({ ...good, components: ["web", "database"] }), /^There is no component called "database"\. Choose from web, git, actions, agents, packages, email\.$/); |
| 171 | refused(() => validateIncident({ ...good, state: "closed" }), /^Choose the incident's state: investigating, identified, monitoring or resolved\.$/); |
| 172 | refused(() => validateIncident({ ...good, body: " \n " }), /^Write an update that says what is happening\.$/); |
| 173 | refused(() => validateUpdate({ state: "monitoring", body: "x".repeat(MAX_BODY + 1) }), /^Keep the update to 5,000 characters\.$/); |
| 174 | expect(validateUpdate({ state: "monitoring", body: "x".repeat(MAX_BODY) }).body).toHaveLength(MAX_BODY); |
| 175 | refused(() => validateUpdate({ state: undefined, body: "x" }), /state/); |
| 176 | refused(() => validateIncident({ ...good, title: 42 }), /title/); |
| 177 | expect(COMPONENT_IDS).toEqual(["web", "git", "actions", "agents", "packages", "email"]); |
| 178 | }); |
| 179 | }); |
| 180 | |
| 181 | describe("opening, updating and resolving incidents", () => { |
| 182 | it("opens an incident with its first update, audited, and the summary shows it", async () => { |
| 183 | const { env, audits, audit } = setup(); |
| 184 | const i = await openIncident(env.DB, audit, root, validateIncident({ title: "Pushes are failing", impact: "major", components: ["git"], state: "investigating", body: "Looking into it." }), "api"); |
| 185 | expect(i).toMatchObject({ title: "Pushes are failing", impact: "major", components: ["git"], state: "investigating", created_by: "root", resolved_at: null }); |
| 186 | expect(i.updates).toEqual([expect.objectContaining({ state: "investigating", body: "Looking into it.", author_login: "root" })]); |
| 187 | expect(audits).toEqual([{ actorId: "u-root", actorLogin: "root", action: "status_incident.create", target: i.id, detail: { title: "Pushes are failing", impact: "major", components: ["git"], state: "investigating", via: "api" } }]); |
| 188 | const s = await statusSummary(env); |
| 189 | expect(s.status).toEqual({ indicator: "major", description: "Partial outage" }); |
| 190 | expect(s.components.find((c) => c.id === "git")!.status).toBe("partial_outage"); |
| 191 | expect(s.incidents.map((x) => x.id)).toEqual([i.id]); |
| 192 | const json = summaryJson(env, s); |
| 193 | expect(json.incidents[0]).toEqual({ |
| 194 | id: i.id, |
| 195 | title: "Pushes are failing", |
| 196 | impact: "major", |
| 197 | state: "investigating", |
| 198 | components: ["git"], |
| 199 | created_at: i.created_at, |
| 200 | updated_at: i.updated_at, |
| 201 | resolved_at: null, |
| 202 | updates: [{ id: i.updates[0]!.id, state: "investigating", body: "Looking into it.", created_at: i.created_at }], |
| 203 | html_url: `${PUB}/status#incident-${i.id}`, |
| 204 | }); |
| 205 | expect(incidentJson(env, i, { admin: true })).toMatchObject({ created_by: "root", updates: [{ author: "root" }] }); |
| 206 | }); |
| 207 | |
| 208 | it("posts updates newest first, and resolving returns the components to operational", async () => { |
| 209 | const { env, audits, audit } = setup(); |
| 210 | const i = await openIncident(env.DB, audit, root, validateIncident({ title: "Slow pages", impact: "minor", components: ["web", "git"], state: "investigating", body: "Looking." }), "web"); |
| 211 | audits.length = 0; |
| 212 | const found = await postUpdate(env.DB, audit, root, i.id, validateUpdate({ state: "identified", body: "Found it." }), "web"); |
| 213 | expect(found.state).toBe("identified"); |
| 214 | const done = await postUpdate(env.DB, audit, root, i.id, validateUpdate({ state: "resolved", body: "Fixed." }), "api"); |
| 215 | expect(done.state).toBe("resolved"); |
| 216 | expect(done.resolved_at).toBeTruthy(); |
| 217 | expect(done.updates.map((u) => u.state)).toEqual(["resolved", "identified", "investigating"]); |
| 218 | expect(audits).toEqual([ |
| 219 | { actorId: "u-root", actorLogin: "root", action: "status_incident.update", target: i.id, detail: { from: "investigating", to: "identified", via: "web" } }, |
| 220 | { actorId: "u-root", actorLogin: "root", action: "status_incident.resolve", target: i.id, detail: { from: "identified", to: "resolved", via: "api" } }, |
| 221 | ]); |
| 222 | expect(JSON.stringify(audits)).not.toContain("Found it"); |
| 223 | const s = await statusSummary(env); |
| 224 | expect(s.components.every((c) => c.status === "operational")).toBe(true); |
| 225 | expect(s.incidents).toEqual([]); |
| 226 | expect(s.resolved.map((x) => x.id)).toEqual([i.id]); |
| 227 | // Fourteen days later it is off the page. |
| 228 | expect((await statusSummary(env, new Date(Date.now() + 15 * 86400_000))).resolved).toEqual([]); |
| 229 | // Reopening clears resolved_at. |
| 230 | const again = await postUpdate(env.DB, audit, root, i.id, validateUpdate({ state: "monitoring", body: "It came back." }), "api"); |
| 231 | expect(again).toMatchObject({ state: "monitoring", resolved_at: null }); |
| 232 | }); |
| 233 | |
| 234 | it("refuses an update to an incident that doesn't exist", async () => { |
| 235 | const { env, audit } = setup(); |
| 236 | await expect(postUpdate(env.DB, audit, root, "f".repeat(32), { state: "resolved", body: "x" }, "api")).rejects.toMatchObject({ status: 404 }); |
| 237 | await expect(postUpdate(env.DB, audit, root, "not-an-id", { state: "resolved", body: "x" }, "api")).rejects.toMatchObject({ status: 404 }); |
| 238 | expect(await new StatusIncidents(env.DB).list()).toEqual([]); |
| 239 | }); |
| 240 | }); |