Diffinery

codician/diffinerypublic

Open-source, agent-first GitHub and GitHub Actions alternative that runs on your own Cloudflare account: git hosting, pull requests, code review and drop-in GitHub Actions YAML on Workers, Durable Objects, R2 and Containers.

432 lines · 21.4 KBRaw History
1import { describe, expect, it, vi } from "vitest";
2import { DatabaseSync } from "node:sqlite";
3import { parseWorkflow, type Workflow } from "@diffinery/workflow-yaml";
4import { Actions, canonicalJson } from "../../src/repo/actions.js";
5import { REPO_SCHEMA, migrateRepoSchema } from "../../src/repo/schema.js";
6import { logKeys } from "../../src/lib/storage-keys.js";
7import { Runtime } from "../../src/repo/runtime.js";
8import { testD1 } from "./stubs/d1.js";
9import type { RepoDO } from "../../src/repo/RepoDO.js";
10import type { Env } from "../../src/env.js";
11import type { Caller } from "../../src/repo/types.js";
12
13/** The Durable Object's SqlStorage over node:sqlite: exec(query, ...bindings) with toArray() and one(). */
14function sqlStorage() {
15 const db = new DatabaseSync(":memory:");
16 return {
17 exec(query: string, ...args: unknown[]) {
18 const params = args.map((a) => (a === undefined ? null : a)) as Array<string | number | null>;
19 let rows: unknown[] = [];
20 if (!params.length && query.trim().replace(/;\s*$/, "").includes(";")) db.exec(query);
21 else rows = db.prepare(query).all(...params);
22 return {
23 toArray: () => rows as never[],
24 one: () => {
25 if (rows.length !== 1) throw new Error(`expected one row, got ${rows.length}`);
26 return rows[0] as never;
27 },
28 };
29 },
30 };
31}
32
33const parse = (yaml: string, path: string): Workflow => {
34 const { workflow, errors } = parseWorkflow(yaml, path);
35 if (!workflow) throw new Error(errors.map((e) => e.message).join("; "));
36 return workflow;
37};
38
39const WORKFLOW = `name: w
40on: push
41jobs:
42 lint:
43 runs-on: ubuntu-latest
44 steps: [{ run: echo lint }]
45 test:
46 runs-on: ubuntu-latest
47 strategy:
48 matrix:
49 os: [a, b, c]
50 steps: [{ run: echo test }]
51 deploy:
52 needs: test
53 runs-on: ubuntu-latest
54 steps: [{ run: echo deploy }]
55 report:
56 needs: lint
57 runs-on: ubuntu-latest
58 steps: [{ run: echo report }]
59`;
60
61type Row = { id: string; run_id: string; job_key: string; name: string; status: string; conclusion: string | null; matrix: string; carried_from: string | null; container: string | null };
62type Outcome = (job: Row) => "success" | "failure";
63
64const ADA: Caller = { id: "u1", login: "ada", kind: "user", role: "write" };
65
66/**
67 * Actions over an in-memory repository and D1, with the workflow files given, and a stand-in for
68 * RunWorkflow (`drive`) that walks the job graph and finishes each started job as `outcome` says.
69 */
70function fixture(files: Record<string, string>) {
71 const sql = sqlStorage();
72 sql.exec(REPO_SCHEMA);
73 migrateRepoSchema(sql as unknown as SqlStorage);
74 const DB = testD1();
75 DB.sqlite.prepare("INSERT INTO accounts (id, login, kind, status, created_at) VALUES ('acme', 'acme', 'org', 'active', ?)").run(new Date().toISOString());
76 const objects = new Map<string, string>();
77 const started: string[] = [];
78 const checks: Array<{ name: string; status: string; conclusion?: string | null; runId?: string; jobId?: string }> = [];
79 const repo = {
80 sql,
81 meta: { repoId: "r", owner: "acme", ownerId: "acme", ownerKind: "org", name: "app", defaultBranch: "main" },
82 headRef: "refs/heads/main",
83 getRef: () => null,
84 // Like Collab.upsertCheck: one check run per commit and name.
85 upsertCheck: (c: (typeof checks)[number] & { sha: string }) => {
86 checks.push(c);
87 sql.exec("DELETE FROM checks WHERE sha = ? AND name = ?", c.sha, c.name);
88 sql.exec("INSERT INTO checks (id, sha, name, status, conclusion, run_id, job_id) VALUES (?, ?, ?, ?, ?, ?, ?)", `${c.sha}/${c.name}`, c.sha, c.name, c.status, c.conclusion ?? null, c.runId ?? null, c.jobId ?? null);
89 },
90 broadcast: () => {},
91 emit: () => {},
92 deploys: { jobCompleted: () => {}, protection: () => null },
93 protectionFor: (branch: string) => (branch === "release" ? {} : null),
94 runtime: { sweepJob: async () => {} },
95 policy: { agentOrigin: () => false },
96 alarmBy: () => {},
97 } as unknown as RepoDO;
98 const env = {
99 DB,
100 PUBLIC_URL: "https://d.test",
101 BUCKET: {
102 put: async (k: string, v: string) => void objects.set(k, v),
103 get: async (k: string) => (objects.has(k) ? { json: async () => JSON.parse(objects.get(k)!) } : null),
104 delete: async (keys: string | string[]) => void [keys].flat().forEach((k) => objects.delete(k)),
105 },
106 RUNNER: { idFromName: (n: string) => n, get: (n: string) => ({ startJob: async () => void started.push(n), stopJob: async () => {} }) },
107 RUN_WORKFLOW: { create: async () => {}, get: async () => ({ sendEvent: async () => {} }) },
108 } as unknown as Env;
109 const actions = new Actions(repo, env);
110 const workflows = Object.entries(files).map(([path, text]) => ({ path, workflow: parse(text, path), errors: [] as string[] }));
111 Object.defineProperty(actions, "loadWorkflows", { value: async () => workflows });
112 Object.defineProperty(actions, "loadCalled", { value: async (uses: string) => ({ workflow: workflows.find((w) => w.path === uses.replace(/^\.\//, ""))!.workflow, source: { repoId: null, sha: "abc" } }) });
113 const internal = actions as unknown as { completeJob(id: string, c: unknown): Promise<void> };
114 const jobs = (runId: string) => sql.exec("SELECT * FROM jobs WHERE run_id = ? ORDER BY rowid", runId).toArray() as Row[];
115 const leg = (j: Row) => `${j.job_key}${j.matrix === "{}" ? "" : ` ${Object.values(JSON.parse(j.matrix)).join(",")}`}`;
116 /** Each job (by `leg`) with its conclusion, and whether it was kept from an earlier attempt. */
117 const summary = (runId: string) => Object.fromEntries(jobs(runId).map((j) => [leg(j), `${j.conclusion}${j.carried_from ? " (kept)" : ""}`]));
118
119 async function drive(runId: string, outcome: Outcome) {
120 const plan = await actions.plan(runId);
121 expect(plan.error).toBeUndefined();
122 const done = new Set<string>();
123 while (done.size < plan.jobs.length) {
124 const next = plan.jobs.find((n) => !done.has(n.key) && n.needs.every((k) => done.has(k)))!;
125 await actions.startJobKey(runId, next.key);
126 // Finish the key's started legs one at a time, in order, as their runners would.
127 for (;;) {
128 const running = jobs(runId).find((j) => j.job_key === next.key && j.status === "waiting");
129 if (!running) break;
130 await internal.completeJob(running.id, { conclusion: outcome(running), outputs: {} });
131 }
132 expect(jobs(runId).filter((j) => j.job_key === next.key && j.status !== "completed")).toEqual([]);
133 done.add(next.key);
134 }
135 return actions.finishRun(runId);
136 }
137
138 async function firstRun(outcome: Outcome, ref = "refs/heads/main") {
139 const run = (await actions.createRun(workflows[0]!.workflow, { event: "push", ref, sha: "abc", payload: {}, caller: ADA, title: "first" }))!;
140 await drive(run.id, outcome);
141 return run.id;
142 }
143 return { sql, actions, started, checks, objects, jobs, leg, summary, drive, firstRun };
144}
145
146const failB: Outcome = (j) => (j.job_key === "test" && JSON.parse(j.matrix).os === "b" ? "failure" : "success");
147const allPass: Outcome = () => "success";
148
149describe("re-running failed jobs", () => {
150 it("keeps the legs that passed and re-runs the failed and cancelled ones, with what depends on them", async () => {
151 const f = fixture({ ".github/workflows/w.yml": WORKFLOW });
152 const first = await f.firstRun(failB);
153 // fail-fast: b's failure cancelled c, and deploy (needs test) was skipped.
154 expect(f.summary(first)).toEqual({ lint: "success", "test a": "success", "test b": "failure", "test c": "cancelled", deploy: "skipped", report: "success" });
155 expect(f.actions.getRun(first).conclusion).toBe("failure");
156
157 const before = f.started.length;
158 const second = await f.actions.rerun(first, ADA, "failed");
159 expect(second).toMatchObject({ attempt: 2, number: f.actions.getRun(first).number });
160 const done = await f.drive(second.id, allPass);
161 expect(done.conclusion).toBe("success");
162 expect(f.summary(second.id)).toEqual({ lint: "success (kept)", report: "success (kept)", "test a": "success (kept)", "test b": "success", "test c": "success", deploy: "success" });
163 // Only the re-run legs started a container.
164 const ran = f.jobs(second.id).filter((j) => !j.carried_from);
165 expect(f.started.slice(before).sort()).toEqual(ran.map((j) => `job-${j.id}`).sort());
166 // A kept job is a copy of the one that ran, pointing at it.
167 const old = Object.fromEntries(f.jobs(first).map((j) => [f.leg(j), j]));
168 for (const j of f.jobs(second.id).filter((x) => x.carried_from)) {
169 expect(j.carried_from).toBe(old[f.leg(j)]!.id);
170 expect(j.name).toBe(old[f.leg(j)]!.name);
171 }
172 // Kept jobs use no minutes.
173 const usage = (await (f.actions as unknown as { env: Env }).env.DB.prepare("SELECT id FROM compute_usage").all<{ id: string }>()).results.map((r) => r.id);
174 for (const j of f.jobs(second.id)) expect(usage.includes(j.id)).toBe(!j.carried_from);
175 // The kept jobs' check runs now point at the new attempt, completed with their conclusion.
176 const lintCheck = f.checks.filter((c) => c.name === "w / lint").at(-1)!;
177 expect(lintCheck).toMatchObject({ status: "completed", conclusion: "success", runId: second.id });
178 });
179
180 it("refuses a run without failed jobs, and one that hasn't finished", async () => {
181 const f = fixture({ ".github/workflows/w.yml": WORKFLOW });
182 const first = await f.firstRun(allPass);
183 await expect(f.actions.rerun(first, ADA, "failed")).rejects.toMatchObject({ status: 409, detail: "this run has no failed jobs to re-run" });
184 const second = await f.actions.rerun(first, ADA);
185 await expect(f.actions.rerun(second.id, ADA, "failed")).rejects.toMatchObject({ status: 409 });
186 await expect(f.actions.rerunJob(f.jobs(first)[0]!.id, ADA)).resolves.toMatchObject({ attempt: 2 });
187 });
188
189 it("checks permissions like a full re-run", async () => {
190 const f = fixture({ ".github/workflows/w.yml": WORKFLOW });
191 const first = await f.firstRun(failB);
192 await expect(f.actions.rerun(first, { ...ADA, role: "read" }, "failed")).rejects.toMatchObject({ status: 403 });
193 await expect(f.actions.rerun(first, { ...ADA, id: "job:x" }, "failed")).rejects.toMatchObject({ status: 403, detail: "a workflow's GITHUB_TOKEN cannot re-run workflows" });
194 await expect(f.actions.rerunJob(f.jobs(first)[0]!.id, { ...ADA, role: "read" })).rejects.toMatchObject({ status: 403 });
195 });
196
197 it("re-runs a full run as before", async () => {
198 const f = fixture({ ".github/workflows/w.yml": WORKFLOW });
199 const first = await f.firstRun(failB);
200 const second = await f.actions.rerun(first, ADA);
201 await f.drive(second.id, allPass);
202 expect(Object.values(f.summary(second.id))).toEqual(Array(6).fill("success"));
203 expect((f.sql.exec("SELECT rerun_carry FROM runs WHERE id = ?", second.id).one() as { rerun_carry: string | null }).rerun_carry).toBeNull();
204 });
205
206 it("starts up to max-parallel re-run legs; kept legs don't count", async () => {
207 const f = fixture({
208 ".github/workflows/w.yml": `name: w
209on: push
210jobs:
211 test:
212 runs-on: ubuntu-latest
213 strategy:
214 fail-fast: false
215 max-parallel: 1
216 matrix:
217 n: [1, 2, 3, 4]
218 steps: [{ run: echo }]
219`,
220 });
221 const first = await f.firstRun((j) => (JSON.parse(j.matrix).n <= 2 ? "failure" : "success"));
222 const second = await f.actions.rerun(first, ADA, "failed");
223 await f.actions.plan(second.id);
224 await f.actions.startJobKey(second.id, "test");
225 expect(f.jobs(second.id).map((j) => [f.leg(j), j.status, !!j.carried_from])).toEqual([
226 ["test 1", "waiting", false],
227 ["test 2", "queued", false],
228 ["test 3", "completed", true],
229 ["test 4", "completed", true],
230 ]);
231 });
232});
233
234describe("re-running one job", () => {
235 it("re-runs that job and its dependents and keeps the rest, failures included", async () => {
236 const f = fixture({ ".github/workflows/w.yml": WORKFLOW });
237 const first = await f.firstRun(failB);
238 const lint = f.jobs(first).find((j) => j.job_key === "lint")!;
239 const second = await f.actions.rerunJob(lint.id, ADA);
240 await f.drive(second.id, allPass);
241 expect(f.summary(second.id)).toEqual({ lint: "success", report: "success", "test a": "success (kept)", "test b": "failure (kept)", "test c": "cancelled (kept)", deploy: "skipped (kept)" });
242 expect(f.actions.getRun(second.id).conclusion).toBe("failure");
243
244 // One matrix leg: the other legs are kept. deploy needs test, so it runs again, and is skipped: test c is still cancelled.
245 const b = f.jobs(second.id).find((j) => f.leg(j) === "test b")!;
246 const third = await f.actions.rerunJob(b.id, ADA);
247 expect(third.attempt).toBe(3);
248 await f.drive(third.id, allPass);
249 expect(f.summary(third.id)).toEqual({ lint: "success (kept)", report: "success (kept)", "test a": "success (kept)", "test b": "success", "test c": "cancelled (kept)", deploy: "skipped" });
250 // A job kept twice points at the job that actually ran, in the first attempt.
251 const a1 = f.jobs(first).find((j) => f.leg(j) === "test a")!;
252 expect(f.jobs(third.id).find((j) => f.leg(j) === "test a")!.carried_from).toBe(a1.id);
253 });
254
255 it("serves a kept job's log from the job that ran", async () => {
256 const f = fixture({ ".github/workflows/w.yml": WORKFLOW });
257 const first = await f.firstRun((j) => {
258 if (j.job_key === "lint") f.sql.exec("INSERT INTO logs (job_id, seq, t, step, text) VALUES (?, 0, 't', 0, 'linted')", j.id);
259 return failB(j);
260 });
261 const lint = f.jobs(first).find((j) => j.job_key === "lint")!;
262 expect(f.objects.has(logKeys("r", lint.id)[0]!)).toBe(true);
263 const second = await f.actions.rerun(first, ADA, "failed");
264 await f.drive(second.id, allPass);
265 const kept = f.jobs(second.id).find((j) => j.job_key === "lint")!;
266 expect(kept.carried_from).toBe(lint.id);
267 const logs = await f.actions.logs(kept.id);
268 expect(logs).toMatchObject({ complete: true, lines: [{ text: "linted" }] });
269 expect(f.actions.getJob(kept.id).carriedFrom).toBe(lint.id);
270 });
271
272 it("lists the kept jobs' artifacts with the new attempt's", async () => {
273 const f = fixture({ ".github/workflows/w.yml": WORKFLOW });
274 const first = await f.firstRun(failB);
275 const byLeg = Object.fromEntries(f.jobs(first).map((j) => [f.leg(j), j.id]));
276 const t = new Date().toISOString();
277 const later = new Date(Date.now() + 86400_000).toISOString();
278 const add = (runId: string, jobId: string, name: string) =>
279 f.sql.exec("INSERT INTO rt_artifacts (run_id, job_id, name, size, status, created_at, updated_at, expires_at) VALUES (?, ?, ?, 1, 'ready', ?, ?, ?)", runId, jobId, name, t, t, later);
280 add(first, byLeg.lint!, "lint-report");
281 add(first, byLeg["test b"]!, "failed-b");
282 const second = await f.actions.rerun(first, ADA, "failed");
283 await f.drive(second.id, allPass);
284 add(second.id, f.jobs(second.id).find((j) => f.leg(j) === "test b")!.id, "passed-b");
285 const rt = new Runtime({ sql: f.sql } as unknown as RepoDO, {} as Env);
286 // The failed leg ran again, so its first attempt's artifact isn't the new attempt's.
287 expect(rt.runArtifacts(second.id).map((a) => a.name).sort()).toEqual(["lint-report", "passed-b"]);
288 expect(rt.runArtifacts(first).map((a) => a.name).sort()).toEqual(["failed-b", "lint-report"]);
289 });
290});
291
292describe("re-running failed jobs of a reusable workflow call", () => {
293 it("treats the call's jobs like any other and re-runs what needs the call", async () => {
294 const f = fixture({
295 ".github/workflows/w.yml": `name: w
296on: push
297jobs:
298 build:
299 uses: ./.github/workflows/lib.yml
300 ship:
301 needs: build
302 runs-on: ubuntu-latest
303 steps: [{ run: echo ship }]
304 docs:
305 runs-on: ubuntu-latest
306 steps: [{ run: echo docs }]
307`,
308 ".github/workflows/lib.yml": `on: workflow_call
309jobs:
310 compile:
311 runs-on: ubuntu-latest
312 steps: [{ run: echo compile }]
313 check:
314 needs: compile
315 runs-on: ubuntu-latest
316 steps: [{ run: echo check }]
317`,
318 });
319 const first = await f.firstRun((j) => (j.job_key === "build/check" ? "failure" : "success"));
320 expect(f.summary(first)).toEqual({ "build/compile": "success", "build/check": "failure", ship: "skipped", docs: "success" });
321 const second = await f.actions.rerun(first, ADA, "failed");
322 await f.drive(second.id, allPass);
323 expect(f.summary(second.id)).toEqual({ "build/compile": "success (kept)", "build/check": "success", ship: "success", docs: "success (kept)" });
324 });
325});
326
327describe("canonicalJson", () => {
328 it("doesn't depend on key order", () => {
329 expect(canonicalJson({ b: 1, a: { d: [1, { y: 2, x: 1 }], c: null } })).toBe(canonicalJson({ a: { c: null, d: [1, { x: 1, y: 2 }] }, b: 1 }));
330 expect(canonicalJson({ a: 1 })).not.toBe(canonicalJson({ a: "1" }));
331 });
332});
333
334describe("starting a run's workflow", () => {
335 const start = async (create: () => Promise<void>, exists: () => boolean) => {
336 const f = fixture({ ".github/workflows/w.yml": WORKFLOW });
337 let calls = 0;
338 (f.actions as unknown as { env: Env }).env.RUN_WORKFLOW = {
339 create: async () => {
340 calls++;
341 await create();
342 },
343 get: async () => ({
344 status: async () => {
345 if (!exists()) throw new Error("instance not found");
346 return { status: "running" };
347 },
348 sendEvent: async () => {},
349 }),
350 } as unknown as Env["RUN_WORKFLOW"];
351 vi.useFakeTimers();
352 try {
353 const pending = f.actions.createRun(parse(WORKFLOW, ".github/workflows/w.yml"), { event: "push", ref: "refs/heads/main", sha: "abc", payload: {}, caller: ADA, title: "t" });
354 await vi.runAllTimersAsync();
355 return { f, run: (await pending)!, calls: () => calls };
356 } finally {
357 vi.useRealTimers();
358 }
359 };
360
361 it("tries again when the platform fails the call, and counts a call that failed after creating it", async () => {
362 let fails = 1;
363 const a = await start(async () => {
364 if (fails-- > 0) throw new Error("internal error; reference = abc");
365 }, () => false);
366 expect(a.calls()).toBe(2);
367 expect(a.run.status).toBe("queued");
368 let created = false;
369 const b = await start(async () => {
370 created = true;
371 throw new Error("internal error; reference = def");
372 }, () => created);
373 expect(b.calls()).toBe(1);
374 expect(b.run.status).toBe("queued");
375 });
376
377 it("fails the run, saying why, rather than leave it queued for good", async () => {
378 const { f, run, calls } = await start(async () => {
379 throw new Error("internal error; reference = ghi");
380 }, () => false);
381 expect(calls()).toBe(3);
382 expect(run).toMatchObject({ status: "completed", conclusion: "failure" });
383 expect(f.jobs(run.id).map((j) => [j.name, j.conclusion])).toEqual([["Set up workflow", "failure"]]);
384 const failure = f.sql.exec<{ failure: string }>("SELECT failure FROM jobs WHERE run_id = ?", run.id).one().failure;
385 expect(JSON.parse(failure).message).toBe("Diffinery couldn't start this run (internal error; reference = ghi). Re-run the workflow.");
386 });
387});
388
389describe("deleting a run's logs", () => {
390 const logged = (f: ReturnType<typeof fixture>) => (j: Row) => {
391 f.sql.exec("INSERT INTO logs (job_id, seq, t, step, text) VALUES (?, 0, 't', 0, ?)", j.id, `log of ${j.job_key}`);
392 f.sql.exec("UPDATE jobs SET summary = 'a summary' WHERE id = ?", j.id);
393 return "success" as const;
394 };
395
396 it("deletes every job's lines, archived copy and summary, and keeps the run and its jobs", async () => {
397 const f = fixture({ ".github/workflows/w.yml": WORKFLOW });
398 const first = await f.firstRun(logged(f));
399 const jobs = f.jobs(first);
400 expect(jobs.every((j) => f.objects.has(logKeys("r", j.id)[0]!))).toBe(true);
401 expect(await f.actions.deleteLogs(first, ADA)).toEqual(jobs.map((j) => j.id));
402 for (const j of jobs) {
403 expect(f.objects.has(logKeys("r", j.id)[0]!)).toBe(false);
404 expect(await f.actions.logs(j.id)).toEqual({ lines: [], complete: true });
405 }
406 expect(f.sql.exec("SELECT count(*) AS n FROM logs").one()).toEqual({ n: 0 });
407 expect(f.sql.exec("SELECT count(*) AS n FROM jobs WHERE run_id = ? AND summary IS NOT NULL", first).one()).toEqual({ n: 0 });
408 expect(f.actions.getRun(first)).toMatchObject({ conclusion: "success", jobs: expect.arrayContaining([expect.objectContaining({ conclusion: "success" })]) });
409 });
410
411 it("needs write access, refuses job tokens and runs that haven't finished", async () => {
412 const f = fixture({ ".github/workflows/w.yml": WORKFLOW });
413 const first = await f.firstRun(allPass);
414 await expect(f.actions.deleteLogs(first, { ...ADA, role: "read" })).rejects.toMatchObject({ status: 403 });
415 await expect(f.actions.deleteLogs(first, { ...ADA, id: "job:x" })).rejects.toMatchObject({ status: 403 });
416 const second = await f.actions.rerun(first, ADA);
417 await expect(f.actions.deleteLogs(second.id, ADA)).rejects.toMatchObject({ status: 409 });
418 });
419
420 it("lets agents delete logs only of runs off the default and protected branches", async () => {
421 const BOT: Caller = { id: "a1", login: "bot", kind: "agent", role: "write" };
422 for (const [ref, ok] of [["refs/heads/main", false], ["refs/heads/release", false], ["refs/heads/feature", true], ["refs/pull/3/merge", true]] as const) {
423 const f = fixture({ ".github/workflows/w.yml": WORKFLOW });
424 const run = await f.firstRun(logged(f), ref);
425 if (ok) await expect(f.actions.deleteLogs(run, BOT)).resolves.toHaveLength(6);
426 else await expect(f.actions.deleteLogs(run, BOT)).rejects.toMatchObject({ status: 403 });
427 }
428 // People may delete logs of any branch's runs.
429 const f = fixture({ ".github/workflows/w.yml": WORKFLOW });
430 await expect(f.actions.deleteLogs(await f.firstRun(logged(f)), ADA)).resolves.toHaveLength(6);
431 });
432});