Diffinery

codician/diffinerypublic

Open-source, agent-first GitHub and GitHub Actions alternative that runs on your own Cloudflare account: git hosting, pull requests, code review and drop-in GitHub Actions YAML on Workers, Durable Objects, R2 and Containers.

339 lines · 20.6 KBRaw History
1// Takes the screenshots of Diffinery's agent features (Today, an agent task, an agent review, a pull request that
2// merges on green) from a real local Worker with sample data, for the README and the docs:
3//
4// pnpm --filter @diffinery/app build && pnpm --filter @diffinery/runner build
5// node scripts/agent-screenshots.mjs
6//
7// It runs the real thing end to end: a sandbox pool (`agent.mjs --sandbox-pool`), an agent task with the Claude Code
8// runtime, the automatic agent review of the pull request it opens, and merge on green. The one stand-in is Claude Code
9// itself: a script with its name prints what Claude Code prints (`stream-json`) and changes the files, so no model or
10// subscription is needed. Then the Worker restarts as https://diffinery.com, so the pages show the addresses people see.
11//
12// Writes docs/images/{today,agent-task,agent-pull-request,merge-on-green}.webp here and the same pictures as PNG in the
13// repository's docs/images/ (the README's).
14import { spawn, execFileSync } from "node:child_process";
15import { createHash, randomBytes } from "node:crypto";
16import { appendFileSync, chmodSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
17import { tmpdir } from "node:os";
18import { join, dirname, resolve } from "node:path";
19import { fileURLToPath } from "node:url";
20import { chromium } from "playwright";
21import { migratedState } from "../test/e2e/d1-state.mjs";
22import { freePort } from "../test/e2e/ports.mjs";
23
24const APP = resolve(dirname(fileURLToPath(import.meta.url)), "..");
25const AGENT = resolve(APP, "../../containers/runner/dist/agent.mjs");
26const OUT = join(APP, "docs/images");
27const README_OUT = resolve(APP, "../../docs/images");
28const top = mkdtempSync(join(tmpdir(), "diffinery-agent-shots-"));
29const work = join(top, "work");
30const poolRoot = join(top, "pool");
31const poolHome = join(top, "home");
32const persist = join(work, "state");
33const bin = join(work, "bin");
34for (const d of [work, poolHome, bin]) mkdirSync(d, { recursive: true });
35const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
36const hash = (s) => createHash("sha256").update(s).digest("hex");
37const SESSION = `agent-shots-${randomBytes(12).toString("hex")}`;
38const procs = [];
39let log = "";
40
41function start(cmd, args, env) {
42 const p = spawn(cmd, args, { cwd: APP, env: { ...process.env, CI: "1", WRANGLER_SEND_METRICS: "false", ...env }, stdio: ["ignore", "pipe", "pipe"], detached: true });
43 // LOG=<file> keeps everything the Worker and the pool print, for when a step times out.
44 const keep = (d) => {
45 log += d;
46 if (process.env.LOG) appendFileSync(process.env.LOG, d);
47 };
48 p.stdout.on("data", keep);
49 p.stderr.on("data", keep);
50 procs.push(p);
51 return p;
52}
53const stop = (p) => {
54 try {
55 process.kill(-p.pid, "SIGTERM");
56 } catch {}
57};
58async function until(what, fn, ms = 120_000) {
59 let v;
60 for (const end = Date.now() + ms; Date.now() < end; await sleep(500)) {
61 v = await fn();
62 if (v) return v;
63 }
64 throw new Error(`timed out waiting for ${what}\n${log.slice(-4000)}`);
65}
66const sql = (command) => execFileSync("pnpm", ["exec", "wrangler", "d1", "execute", "diffinery-dev", "--local", "-e", "dev", "--persist-to", persist, "--command", command], { cwd: APP, stdio: "pipe", env: { ...process.env, CI: "1" } });
67
68// ---------------------------------------------------------------- the stand-in for Claude Code
69
70// `npm install --global @anthropic-ai/claude-code@…` (the runtime's install step) and `claude`, as the jail runs them.
71writeFileSync(join(bin, "npm"), "#!/bin/sh\nexit 0\n");
72writeFileSync(
73 join(bin, "claude"),
74 `#!/usr/bin/env node
75const fs = require("fs");
76if (process.argv.includes("--version")) return console.log("2.1.296 (Claude Code)");
77const prompt = process.env.DIFFINERY_TASK_PROMPT || "";
78const say = (o) => console.log(JSON.stringify(o));
79const text = (t) => say({ type: "assistant", message: { content: [{ type: "text", text: t }] } });
80const tool = (name, input, out) => {
81 say({ type: "assistant", message: { content: [{ type: "tool_use", name, input }] } });
82 if (out) say({ type: "user", message: { content: [{ type: "tool_result", content: out }] } });
83};
84say({ type: "system", subtype: "init", model: "claude-sonnet" });
85if (prompt.startsWith("You are reviewing")) {
86 text("I'll read the changes and the tests around them.");
87 tool("Read", { file_path: "src/discounts.js" });
88 tool("Read", { file_path: "src/checkout.js" });
89 tool("Bash", { command: "npm test" }, "✔ 9 tests passed");
90 const findings = {
91 summary: "Adds discount codes at checkout. One case worth covering: codes are matched case-sensitively.",
92 findings: [{ path: "src/discounts.js", line: 8, side: "RIGHT", severity: "not_blocking", title: "Codes typed in lower case are rejected", body: "Customers often type codes as they read them in an email, such as \`welcome10\`. \`CODES[code]\` only finds \`WELCOME10\`, so the code is refused with no hint why.", suggestion: " const rule = CODES[code.trim().toUpperCase()];" }],
93 };
94 say({ type: "result", subtype: "success", num_turns: 4, result: "Reviewed the changes.\\n\`\`\`json\\n" + JSON.stringify(findings) + "\\n\`\`\`" });
95 return;
96}
97text("I'll add discount codes to checkout: a table of codes, applying one to the cart total, and tests.");
98tool("Read", { file_path: "src/checkout.js" });
99tool("Bash", { command: "npm test" }, "✔ 6 tests passed");
100fs.writeFileSync("src/discounts.js", \`// Discount codes customers can enter at checkout.
101const CODES = {
102 WELCOME10: { percent: 10 },
103 FREESHIP: { freeShipping: true },
104};
105
106export function applyDiscount(cart, code) {
107 const rule = CODES[code];
108 if (!rule) throw new Error(\\\`Unknown discount code: \\\${code}\\\`);
109 const subtotal = rule.percent ? Math.round(cart.subtotal * (100 - rule.percent)) / 100 : cart.subtotal;
110 return { ...cart, subtotal, shipping: rule.freeShipping ? 0 : cart.shipping, code };
111}
112\`);
113tool("Write", { file_path: "src/discounts.js" });
114const checkout = fs.readFileSync("src/checkout.js", "utf8");
115if (!checkout.includes("totalWithCode")) fs.writeFileSync("src/checkout.js", checkout.replace("export function total(cart) {", 'import { applyDiscount } from "./discounts.js";\\n\\nexport function totalWithCode(cart, code) {\\n return total(code ? applyDiscount(cart, code) : cart);\\n}\\n\\nexport function total(cart) {'));
116tool("Edit", { file_path: "src/checkout.js" });
117fs.writeFileSync("test/discounts.test.js", \`import { test } from "node:test";
118import assert from "node:assert/strict";
119import { applyDiscount } from "../src/discounts.js";
120
121test("WELCOME10 takes 10% off the subtotal", () => {
122 assert.equal(applyDiscount({ subtotal: 50, shipping: 5 }, "WELCOME10").subtotal, 45);
123});
124
125test("FREESHIP removes shipping", () => {
126 assert.equal(applyDiscount({ subtotal: 50, shipping: 5 }, "FREESHIP").shipping, 0);
127});
128
129test("an unknown code is refused", () => {
130 assert.throws(() => applyDiscount({ subtotal: 50, shipping: 5 }, "NOPE"), /Unknown discount code/);
131});
132\`);
133tool("Write", { file_path: "test/discounts.test.js" });
134tool("Bash", { command: "npm test" }, "✔ 9 tests passed");
135say({ type: "result", subtype: "success", num_turns: 9, result: "Added discount codes at checkout: src/discounts.js with WELCOME10 and FREESHIP, totalWithCode() in src/checkout.js, and three tests. All 9 tests pass." });
136`,
137);
138chmodSync(join(bin, "npm"), 0o755);
139chmodSync(join(bin, "claude"), 0o755);
140
141// ---------------------------------------------------------------- a Worker with a sandbox pool
142
143const poolPort = await freePort();
144const poolSecret = randomBytes(16).toString("hex");
145const pool = start("node", [AGENT, "--sandbox-pool", "--work-root", poolRoot], { HOME: poolHome, PATH: `${bin}:${process.env.PATH}`, DIFFINERY_SANDBOX_POOL_SECRET: poolSecret, DIFFINERY_SANDBOX_PORT: String(poolPort), DIFFINERY_JAIL_SHARED_FOR_TESTS: work });
146await until("the sandbox pool", async () => log.includes("listening"), 30_000);
147
148migratedState(APP, persist);
149const at = (hoursAgo) => new Date(Date.now() - hoursAgo * 3600_000).toISOString();
150sql(`INSERT INTO accounts (id, login, kind, name, email, status, site_admin, created_at) VALUES ('ada1', 'ada', 'user', 'Ada Lovelace', 'ada@example.com', 'active', 1, '${at(24 * 30)}')`);
151
152const MASTER_KEY = randomBytes(32).toString("base64");
153async function worker(varsFor) {
154 const port = await freePort();
155 const base = `http://127.0.0.1:${port}`;
156 const vars = varsFor(base);
157 const p = start("pnpm", ["exec", "wrangler", "dev", "-e", "dev", "--ip", "127.0.0.1", "--port", String(port), "--inspector-port", String(await freePort()), "--persist-to", persist, ...Object.entries({ MASTER_KEY, ...vars }).flatMap(([k, v]) => ["--var", `${k}:${v}`]), "--show-interactive-dev-session=false"]);
158 await until("wrangler dev", async () => {
159 try {
160 return (await fetch(`${base}/healthz`)).ok;
161 } catch {
162 return false;
163 }
164 });
165 return { p, base };
166}
167
168// Phase 1: the Worker at its own address, which the sandboxes clone from, signed in as Ada (DEV_AUTH).
169let { p: dev, base } = await worker((b) => ({ PUBLIC_URL: b, DEV_AUTH: "ada@example.com", SANDBOX_POOL_URL: `http://127.0.0.1:${poolPort}`, SANDBOX_POOL_SECRET: poolSecret, AGENT_POLL_MS: "500" }));
170let token;
171async function api(method, path, body, tok = token) {
172 const res = await fetch(`${base}/api/v3${path}`, { method, headers: { ...(tok ? { authorization: `Bearer ${tok}` } : {}), "content-type": "application/json" }, body: body === undefined ? undefined : JSON.stringify(body) });
173 const text = await res.text();
174 if (!res.ok) throw new Error(`${method} ${path} -> ${res.status} ${text}`);
175 return text ? JSON.parse(text) : null;
176}
177const asAda = (method, path, body) => api(method, path, body, null);
178function git(cwd, args, tok = token) {
179 const auth = Buffer.from(`x-token:${tok}`).toString("base64");
180 return execFileSync("git", ["-c", `http.extraHeader=Authorization: Basic ${auth}`, ...args], { cwd, encoding: "utf8", stdio: ["ignore", "pipe", "pipe"], env: { ...process.env, GIT_TERMINAL_PROMPT: "0", GIT_AUTHOR_NAME: "Ada Lovelace", GIT_AUTHOR_EMAIL: "ada@example.com", GIT_COMMITTER_NAME: "Ada Lovelace", GIT_COMMITTER_EMAIL: "ada@example.com" } });
181}
182
183const browser = await chromium.launch(process.env.CHROMIUM_PATH ? { executablePath: process.env.CHROMIUM_PATH } : {});
184try {
185 token = (await asAda("POST", "/diffinery/tokens", { name: "screenshots", scopes: ["admin"] })).token;
186 await asAda("POST", "/user/orgs", { login: "acme", name: "Acme" });
187 await api("POST", "/orgs/acme/repos", { name: "storefront", description: "The Acme web shop", private: true });
188 const R = "/repos/acme/storefront";
189
190 // A small shop, with CI.
191 const src = join(work, "storefront");
192 const files = {
193 "README.md": "# storefront\n\nThe Acme web shop: cart, checkout and payments.\n",
194 "package.json": JSON.stringify({ name: "storefront", private: true, type: "module", scripts: { test: "node --test" } }, null, 2) + "\n",
195 "src/checkout.js": "export function total(cart) {\n return Math.round((cart.subtotal + cart.shipping) * 100) / 100;\n}\n",
196 "src/cart.js": "export function addItem(cart, item) {\n const subtotal = cart.subtotal + item.price * item.quantity;\n return { ...cart, items: [...cart.items, item], subtotal };\n}\n",
197 "test/checkout.test.js": 'import { test } from "node:test";\nimport assert from "node:assert/strict";\nimport { total } from "../src/checkout.js";\n\ntest("adds shipping", () => assert.equal(total({ subtotal: 10, shipping: 5 }), 15));\n',
198 "REVIEW.md": "Prices are in dollars with two decimals. Customer-facing errors must say what to do next.\n",
199 // A self-hosted runner here (this script runs its jobs with the real runner agent); on diffinery.com it would be ubuntu-latest.
200 ".github/workflows/ci.yml": "name: CI\non:\n push:\n branches: [main]\n pull_request:\njobs:\n test:\n runs-on: self-hosted\n steps:\n - uses: actions/checkout@v4\n - run: node --test\n",
201 };
202 for (const [f, c] of Object.entries(files)) {
203 mkdirSync(dirname(join(src, f)), { recursive: true });
204 writeFileSync(join(src, f), c);
205 }
206 git(src, ["init", "-q", "-b", "main"]);
207 git(src, ["add", "-A"]);
208 git(src, ["commit", "-q", "-m", "Cart and checkout"]);
209 git(src, ["push", "-q", `${base}/acme/storefront.git`, "main"]);
210
211 // Ada's model key (a stand-in: the stand-in Claude Code never calls a model), and two agents of Acme's.
212 await asAda("PUT", "/diffinery/model-keys/anthropic", { key: `sk-ant-api03-${randomBytes(30).toString("hex")}`, accept_terms: true });
213 const coder = (await asAda("POST", "/diffinery/agents", { login: "acme-coder", name: "Acme coder", org: "acme", auto_merge: true })).token.token;
214 await asAda("POST", "/diffinery/agents", { login: "acme-reviewer", name: "Acme reviewer", org: "acme" });
215
216 // CI: a self-hosted runner that runs every job the repository queues with the real runner agent, as `diffinery runner` does.
217 const reg = await api("POST", `${R}/actions/runners/registration-token`);
218 const runner = (await api("POST", `${R}/actions/runners/register`, { registration_token: reg.token, name: "ci-1", labels: ["linux"] })).token;
219 let running = true;
220 const jobs = (async () => {
221 while (running) {
222 const r = await fetch(`${base}/api/v3${R}/actions/runners/claim`, { method: "POST", headers: { authorization: `Bearer ${runner}`, "content-type": "application/json" }, body: "{}" }).catch(() => null);
223 if (r?.status !== 200) {
224 await sleep(1000);
225 continue;
226 }
227 const claim = await r.json();
228 await new Promise((done) => {
229 const job = spawn(process.execPath, [AGENT], { env: { ...process.env, DIFFINERY_API_URL: claim.api_url, DIFFINERY_JOB_ID: claim.job_id, DIFFINERY_JOB_TOKEN: claim.token, DIFFINERY_WORK_ROOT: join(work, `job-${claim.job_id}`) }, stdio: "ignore" });
230 job.on("close", done);
231 });
232 }
233 })();
234 const green = (sha) =>
235 until(`CI on ${sha.slice(0, 7)}`, async () => {
236 const runs = ((await api("GET", `${R}/commits/${sha}/check-runs`)).check_runs ?? []).filter((c) => c.name.startsWith("CI"));
237 if (runs.some((c) => c.conclusion && c.conclusion !== "success")) throw new Error(`CI failed on ${sha}: ${JSON.stringify(runs)}`);
238 return runs.length && runs.every((c) => c.conclusion === "success");
239 }, 300_000);
240
241 // An agent pull request from earlier today, approved by Ada and merged.
242 git(src, ["checkout", "-q", "-b", "fix-rounding"]);
243 writeFileSync(join(src, "src/cart.js"), files["src/cart.js"].replace("cart.subtotal + item.price * item.quantity", "Math.round((cart.subtotal + item.price * item.quantity) * 100) / 100"));
244 git(src, ["commit", "-qam", "Round the cart subtotal to cents"]);
245 git(src, ["push", "-q", `${base}/acme/storefront.git`, "fix-rounding"], coder);
246 const early = await api("POST", `${R}/pulls`, { title: "Round the cart subtotal to cents", head: "fix-rounding", base: "main", body: "Adding 0.1 + 0.2 showed $0.30000000000000004 in the cart.", intent: "Cart totals always show whole cents." }, coder);
247 await green(early.head.sha);
248 await api("POST", `${R}/pulls/${early.number}/reviews`, { event: "APPROVE", body: "Thanks!" });
249 await until("the first pull request to merge", async () => (await api("GET", `${R}/pulls/${early.number}`)).merged || (await api("PUT", `${R}/pulls/${early.number}/merge`, { merge_method: "squash" }).catch(() => null)));
250 git(src, ["checkout", "-q", "main"]);
251
252 // Merge on green: the CI checks must pass, and acme-coder's pull requests land once a person approves.
253 const page = await browser.newPage();
254 await asAda("PUT", `${R}/diffinery/agent-review`, { agent: "acme-reviewer", runtime: "claude-code", trigger: "push", autofix: false, accept_terms: true });
255 await page.goto(`${base}/acme/storefront/settings/merge-on-green`);
256 await page.waitForLoadState("networkidle");
257 await page.locator('textarea[name="otherChecks"]').fill("CI / test");
258 const others = page.locator('input[name="otherAgents"]');
259 if (await others.count()) await others.fill("acme-coder");
260 // Agents' pull requests still wait for a person's approval (the page's default lands them as soon as the checks pass).
261 await page.getByText("Once the checks pass and a person approved").click();
262 await page.waitForFunction(() => document.querySelector('input[name="approval"]')?.value === "approval");
263 await page.getByRole("button", { name: /Turn on merge on green|Save/ }).click();
264 await page.waitForLoadState("networkidle");
265
266 // The task: Ada asks acme-coder for a feature, Claude Code writes it in a sandbox and opens a pull request.
267 const task = await asAda("POST", `${R}/diffinery/tasks`, { goal: "Add discount codes at checkout\n\nLet customers enter a code in the cart. Support WELCOME10 (10% off) and FREESHIP, with tests.", runtime: "claude-code", agent: "acme-coder" });
268 const done = await until("the task", async () => {
269 const t = await api("GET", `${R}/diffinery/tasks/${task.number}`);
270 if (t.state === "failed") throw new Error(`the task failed: ${JSON.stringify(await api("GET", `${R}/diffinery/tasks/${task.number}/transcript`))}`);
271 return t.state !== "running" && t.pull_request ? t : null;
272 }, 300_000);
273 const pr = await api("GET", `${R}/pulls/${done.pull_request.number}`);
274 await green(pr.head.sha);
275 await until("the agent review", async () => {
276 const r = await api("GET", `${R}/pulls/${pr.number}/diffinery/agent-review`);
277 if (process.env.DEBUG && r.task) console.error("review", JSON.stringify((await api("GET", `${R}/diffinery/tasks/${r.task.number}`)).runs));
278 return r.findings?.length && r.reviewed_sha === pr.head.sha ? r : null;
279 }, 300_000);
280
281 // Another agent pull request, still being worked on.
282 git(src, ["checkout", "-q", "-b", "agent/order-emails"]);
283 writeFileSync(join(src, "src/emails.js"), "export const orderConfirmation = (order) => `Thanks for your order #${order.id}!`;\n");
284 git(src, ["add", "-A"]);
285 git(src, ["commit", "-qm", "Order confirmation email"]);
286 git(src, ["push", "-q", `${base}/acme/storefront.git`, "agent/order-emails"], coder);
287 const draft = await api("POST", `${R}/pulls`, { title: "Send an order confirmation email", head: "agent/order-emails", base: "main", draft: true, intent: "Customers get an email with their order number right after paying." }, coder);
288 await green(draft.head.sha);
289 running = false;
290 await jobs;
291
292 // Phase 2: the same data, served as https://diffinery.com to Ada's signed-in browser.
293 stop(dev);
294 await sleep(2000);
295 sql(`INSERT INTO sessions (id, account_id, method, created_at, last_seen_at, expires_at) VALUES ('${hash(SESSION)}', 'ada1', 'passkey', '${at(0)}', '${at(0)}', '${new Date(Date.now() + 86400_000).toISOString()}')`);
296 ({ p: dev, base } = await worker(() => ({ PUBLIC_URL: "https://diffinery.com", DEV_AUTH: "" })));
297
298 const shots = [];
299 for (const scheme of ["light"]) {
300 const context = await browser.newContext({ viewport: { width: 1280, height: 900 }, deviceScaleFactor: 2, colorScheme: scheme });
301 // localhost is a secure context, so the browser keeps the __Host- cookie over plain HTTP.
302 const site = base.replace("127.0.0.1", "localhost");
303 await context.addCookies([{ name: "__Host-dfy_session", value: SESSION, domain: "localhost", path: "/", secure: true, sameSite: "Lax" }]);
304 const p = await context.newPage();
305 const shot = async (name, path, { clip, locator, height = 900 } = {}) => {
306 await p.setViewportSize({ width: 1280, height });
307 await p.goto(`${site}${path}`);
308 await p.waitForLoadState("networkidle");
309 await sleep(500);
310 const png = locator ? await p.locator(locator).first().screenshot() : await p.screenshot({ clip });
311 shots.push({ name, png, page: p });
312 };
313 await shot("today", "/", { height: 1000 });
314 await shot("agent-task", `/acme/storefront/tasks/${task.number}`, { height: 1100 });
315 await shot("agent-pull-request", `/acme/storefront/pull/${pr.number}`, { height: 1200 });
316 await shot("merge-on-green", `/acme/storefront/pull/${pr.number}`, { locator: ".mergebox" });
317 }
318
319 for (const { name, png, page: p } of shots) {
320 writeFileSync(join(README_OUT, `${name}.png`), png);
321 const webp = await p.evaluate(async (b64) => {
322 const img = new Image();
323 img.src = `data:image/png;base64,${b64}`;
324 await img.decode();
325 const c = document.createElement("canvas");
326 c.width = img.width;
327 c.height = img.height;
328 c.getContext("2d").drawImage(img, 0, 0);
329 return c.toDataURL("image/webp", 0.86).split(",")[1];
330 }, png.toString("base64"));
331 writeFileSync(join(OUT, `${name}.webp`), Buffer.from(webp, "base64"));
332 console.log(`${name}: png ${Math.round(png.length / 1024)} KiB, webp ${Math.round((webp.length * 3) / 4 / 1024)} KiB`);
333 }
334} finally {
335 await browser.close();
336 for (const p of procs) stop(p);
337 if (process.env.KEEP !== "1") rmSync(top, { recursive: true, force: true, maxRetries: 5, retryDelay: 200 });
338 else console.log(`kept ${top}`);
339}