Diffinery

codician/diffinerypublic

Open-source, agent-first GitHub and GitHub Actions alternative that runs on your own Cloudflare account: git hosting, pull requests, code review and drop-in GitHub Actions YAML on Workers, Durable Objects, R2 and Containers.

327 lines · 21.3 KBRaw History
1// End-to-end web terminals, kill switches on them, and tasks run in them (phase 4b), on the same setup as tasks.test.mjs
2// (agents-harness.mjs). In a file of their own so each file stays well under the test runner's per-file limit.
3//
4// pnpm --filter @diffinery/app e2e
5import "./fresh-connections.mjs";
6import { spawn, execFileSync } from "node:child_process";
7import { mkdtempSync, writeFileSync, mkdirSync, rmSync, readdirSync, statSync, readFileSync, existsSync } from "node:fs";
8import { tmpdir } from "node:os";
9import { createHash, randomBytes } from "node:crypto";
10import { createRequire } from "node:module";
11import { join, dirname, resolve } from "node:path";
12import { fileURLToPath } from "node:url";
13import { createServer as netServer } from "node:net";
14import { createServer as httpServer } from "node:http";
15import { test } from "node:test";
16import assert from "node:assert/strict";
17import {
18 APP,
19 AGENT,
20 work,
21 poolRoot,
22 poolHome,
23 WS,
24 RIVAL_SESSION,
25 ANTHROPIC_KEY,
26 OPENROUTER_KEY,
27 base,
28 dev,
29 pool,
30 upstream,
31 devLog,
32 poolLog,
33 token,
34 modelRequests,
35 api,
36 ok,
37 asPerson,
38 sleep,
39 until,
40 git,
41 mcp,
42 poolSandboxes,
43} from "./agents-harness.mjs";
44import { freePort } from "./ports.mjs";
45
46/** Opens a terminal's WebSocket; resolves with the HTTP status when the upgrade is refused. */
47function termSocket(url, headers) {
48 return new Promise((resolve) => {
49 const ws = new WS(url, { headers });
50 let out = "";
51 ws.on("message", (d, binary) => {
52 if (binary) out += String(d);
53 });
54 let closedWith = null;
55 const closed = new Promise((r) => ws.on("close", (code, reason) => r((closedWith = { code, reason: String(reason) }))));
56 ws.once("unexpected-response", (_req, res) => resolve({ status: res.statusCode }));
57 ws.once("error", (e) => resolve({ status: 0, error: e.message }));
58 ws.once("open", () =>
59 resolve({
60 status: 101,
61 ws,
62 closed,
63 out: () => out,
64 type: (text) => ws.send(Buffer.from(text)),
65 until: (re, ms = 30_000) => until(`${re} in the terminal (got ${JSON.stringify(out.slice(-300))}, closed ${JSON.stringify(closedWith)})`, async () => re.test(out), ms),
66 }),
67 );
68 });
69}
70
71test("a web terminal: only its opener attaches, the sandbox API can't reach it, and End takes it away", async () => {
72 const me = await ok("GET", "/user");
73 const owner = me.login;
74 await ok("POST", "/user/repos", { name: "term-demo" });
75 await ok("PUT", `/repos/${owner}/term-demo/contents/README.md`, { message: "init", content: Buffer.from("# term\n").toString("base64") });
76 assert.equal((await asPerson("POST", "/diffinery/agents", { login: "term-bot", name: "Term Bot" })).status, 201);
77 const page = (path) => fetch(`${base}/${owner}/term-demo/${path}`).then(async (r) => ({ status: r.status, text: await r.text(), headers: r.headers }));
78 async function form(path, fields, headers = {}) {
79 const res = await fetch(`${base}/${owner}/term-demo/terminals`, { headers });
80 const csrf = /name="_csrf" value="([^"]+)"/.exec(await res.text())?.[1];
81 const cookie = /dfy_csrf=([^;]+)/.exec(res.headers.get("set-cookie") ?? "")?.[1];
82 return fetch(`${base}/${owner}/term-demo/${path}`, { method: "POST", redirect: "manual", headers: { ...headers, cookie: [headers.cookie, `dfy_csrf=${cookie}`].filter(Boolean).join("; "), "sec-fetch-site": "same-origin" }, body: new URLSearchParams({ _csrf: csrf ?? "", ...fields }) });
83 }
84
85 // The notice comes first, and it is a Kumo checkbox, not a native one.
86 const list = await page("terminals");
87 assert.equal(list.status, 200);
88 assert.match(list.text, /Open a terminal/);
89 assert.match(list.text, /my own Anthropic or OpenAI account/);
90 assert.ok(!/<input[^>]+type="checkbox"/.test(list.text.replace(/<input[^>]+data-k[^>]*>/g, "")), "no native checkbox");
91 const refused = await form("terminals", { agent: "term-bot", ref: "main" });
92 assert.equal(refused.status, 422);
93 assert.match(await refused.text(), /accept the notice first/);
94 const opened = await form("terminals", { agent: "term-bot", ref: "main", accept: "1" });
95 assert.equal(opened.status, 303, await opened.text());
96 const path = new URL(opened.headers.get("location"), base).pathname;
97 const id = path.split("/").pop();
98 const running = await until("the terminal to start", async () => {
99 const p = await page(`terminals/${id}`);
100 return p.text.includes("data-socket") ? p : null;
101 });
102 assert.match(running.text, /Start Claude Code/);
103 assert.match(running.text, /Start Codex/, "a private repository offers Codex");
104 assert.match(running.text, /\/-\/terminal\/[a-z0-9]+\.js/);
105 const sandboxId = poolSandboxes()[0];
106
107 // The socket: this site's Origin, a browser session, and only the opener.
108 const url = `${base.replace("http", "ws")}/${owner}/term-demo/terminals/${id}/socket?cols=100&rows=30`;
109 const origin = new URL(base).origin;
110 assert.equal((await termSocket(url, {})).status, 403, "no Origin");
111 assert.equal((await termSocket(url, { origin: "https://evil.example" })).status, 403, "another Origin");
112 assert.equal((await termSocket(url, { origin, authorization: `Bearer ${token}` })).status, 403, "a token, even the opener's own admin token");
113 assert.equal((await termSocket(url, { origin, cookie: `__Host-dfy_session=${RIVAL_SESSION}` })).status, 404, "another person");
114 const t = await termSocket(url, { origin });
115 assert.equal(t.status, 101, JSON.stringify(t));
116 t.type("stty size; echo \"s=[$DIFFINERY_SANDBOX_SECRET] t=[$DIFFINERY_JOB_TOKEN]\"; git rev-parse --abbrev-ref HEAD; echo hi-$((6*7))\n");
117 await t.until(/hi-42/);
118 assert.match(t.out(), /30 100/);
119 assert.match(t.out(), /s=\[\] t=\[\]/);
120 assert.match(t.out(), /\bmain\b/);
121
122 // Another person, even one the owner made an admin of the repository, sees nothing of it.
123 const invite = await ok("PUT", `/repos/${owner}/term-demo/collaborators/rival`, { permission: "admin" });
124 if (invite?.id) await fetch(`${base}/api/v3/user/repository_invitations/${invite.id}`, { method: "PATCH", headers: { cookie: `__Host-dfy_session=${RIVAL_SESSION}`, "sec-fetch-site": "same-origin" } });
125 const rival = { cookie: `__Host-dfy_session=${RIVAL_SESSION}` };
126 assert.equal((await fetch(`${base}/${owner}/term-demo/terminals/${id}`, { headers: rival })).status, 404);
127 assert.ok(!(await (await fetch(`${base}/${owner}/term-demo/terminals`, { headers: rival })).text()).includes(id));
128 assert.equal((await form(`terminals/${id}/end`, {}, rival)).status, 404);
129 assert.equal((await termSocket(url, { origin, ...rival })).status, 404);
130
131 // The sandbox API and MCP never reach a terminal's sandbox, not even for a site admin's admin token.
132 assert.equal((await api("POST", `/repos/${owner}/term-demo/diffinery/sandboxes/${sandboxId}/exec`, { cmd: "cat ~/.claude/.credentials.json" })).status, 404);
133 assert.equal((await api("POST", `/repos/${owner}/term-demo/diffinery/sandboxes/${sandboxId}/read`, { path: "/root/.codex/auth.json" })).status, 404);
134 assert.equal((await api("POST", `/repos/${owner}/term-demo/diffinery/sandboxes/${sandboxId}/close`)).status, 404);
135 const viaMcp = await mcp("sandbox_exec", { owner, repo: "term-demo", sandbox_id: sandboxId, command: "id" });
136 assert.ok(viaMcp?.isError, JSON.stringify(viaMcp));
137
138 // Limits: two open per repository.
139 const second = await form("terminals", { agent: "term-bot", ref: "main" });
140 assert.equal(second.status, 303);
141 const third = await form("terminals", { agent: "term-bot", ref: "main" });
142 assert.equal(third.status, 422);
143 assert.match(await third.text(), /2 terminals open in this repository/);
144 const secondId = new URL(second.headers.get("location"), base).pathname.split("/").pop();
145
146 // End: the socket closes, the sandbox and its git credential go, and it can't be attached again.
147 assert.equal((await form(`terminals/${id}/end`, {})).status, 303);
148 const closed = await t.closed;
149 assert.ok([1000, 1006, 1011].includes(closed.code), JSON.stringify(closed));
150 assert.equal((await termSocket(url, { origin })).status, 409);
151 assert.match((await page(`terminals/${id}`)).text, /You ended it/);
152 await until("the pool to stop the sandbox", async () => !poolSandboxes().includes(sandboxId) || (await fetch(`${base}/${owner}/term-demo/terminals/${id}`)).ok);
153
154 // Losing access ends a terminal too: the agent's write access goes, and the next check ends it.
155 await until("the second terminal to start", async () => (await page(`terminals/${secondId}`)).text.includes("data-socket"));
156 assert.equal((await api("DELETE", "/diffinery/agents/term-bot")).status, 204);
157 await until("the second terminal to end", async () => /no longer have write access|ended/i.test((await page(`terminals/${secondId}`)).text) && !(await page(`terminals/${secondId}`)).text.includes("data-socket"), 150_000);
158});
159
160test("a kill switch ends web terminals and refuses new ones until it's lifted", async () => {
161 const owner = (await ok("GET", "/user")).login;
162 assert.equal((await asPerson("POST", "/diffinery/agents", { login: "term-bot-2", name: "Term Bot 2" })).status, 201);
163 const page = (path) => fetch(`${base}/${owner}/term-demo/${path}`).then(async (r) => ({ status: r.status, text: await r.text() }));
164 async function form(path, fields) {
165 const res = await fetch(`${base}/${owner}/term-demo/terminals`);
166 const csrf = /name="_csrf" value="([^"]+)"/.exec(await res.text())?.[1];
167 const cookie = /dfy_csrf=([^;]+)/.exec(res.headers.get("set-cookie") ?? "")?.[1];
168 return fetch(`${base}/${owner}/term-demo/${path}`, { method: "POST", redirect: "manual", headers: { cookie: `dfy_csrf=${cookie}`, "sec-fetch-site": "same-origin" }, body: new URLSearchParams({ _csrf: csrf ?? "", ...fields }) });
169 }
170 const opened = await form("terminals", { agent: "term-bot-2", ref: "main" });
171 assert.equal(opened.status, 303, await opened.text());
172 const id = new URL(opened.headers.get("location"), base).pathname.split("/").pop();
173 await until("the terminal to start", async () => (await page(`terminals/${id}`)).text.includes("data-socket"));
174 const url = `${base.replace("http", "ws")}/${owner}/term-demo/terminals/${id}/socket`;
175 const origin = new URL(base).origin;
176
177 // Suspending the repository's Actions ends the terminal, refuses attaching and opening another.
178 const S = `/admin/repos/${owner}/term-demo/actions-suspension`;
179 assert.equal((await api("PUT", S, { reason: "abuse report" })).status, 200);
180 // Nor does it start a task in the terminal (phase 4b).
181 const before = (await ok("GET", `/repos/${owner}/term-demo/diffinery/tasks`)).length;
182 assert.equal((await form(`terminals/${id}/task`, { goal: "Do it while suspended", runtime: "claude-code" })).status, 303);
183 assert.equal((await ok("GET", `/repos/${owner}/term-demo/diffinery/tasks`)).length, before, "a suspended repository's terminal starts no task");
184 const refused = await termSocket(url, { origin });
185 assert.ok([403, 409].includes(refused.status), JSON.stringify(refused));
186 const again = await form("terminals", { agent: "term-bot-2", ref: "main" });
187 assert.equal(again.status, 422);
188 assert.match(await again.text(), /Actions are suspended for this repository by a site administrator: abuse report/);
189 await until("the terminal to end", async () => /suspended Actions and sandboxes here|Its sandbox stopped/.test((await page(`terminals/${id}`)).text), 150_000);
190
191 // Lifted, a terminal opens again.
192 assert.equal((await api("DELETE", S)).status, 200);
193 const after = await form("terminals", { agent: "term-bot-2", ref: "main" });
194 assert.equal(after.status, 303, await after.text());
195 assert.equal((await form(`${new URL(after.headers.get("location"), base).pathname.split("/").slice(3).join("/")}/end`, {})).status, 303);
196});
197
198test("a task in my terminal: only on my click, with the CLI signed in there, in a checkout of its own", async () => {
199 const owner = (await ok("GET", "/user")).login;
200 await ok("POST", "/user/repos", { name: "term-tasks" });
201 await ok("PUT", `/repos/${owner}/term-tasks/contents/README.md`, { message: "init", content: Buffer.from("# term tasks\n").toString("base64") });
202 assert.equal((await asPerson("POST", "/diffinery/agents", { login: "term-hand", name: "Term Hand" })).status, 201);
203 const R = `/repos/${owner}/term-tasks/diffinery/tasks`;
204 const page = (path, headers = {}) => fetch(`${base}/${owner}/term-tasks/${path}`, { headers }).then(async (r) => ({ status: r.status, text: await r.text() }));
205 async function form(path, fields, headers = {}) {
206 const res = await fetch(`${base}/${owner}/term-tasks/terminals`, { headers });
207 const csrf = /name="_csrf" value="([^"]+)"/.exec(await res.text())?.[1];
208 const cookie = /dfy_csrf=([^;]+)/.exec(res.headers.get("set-cookie") ?? "")?.[1];
209 return fetch(`${base}/${owner}/term-tasks/${path}`, { method: "POST", redirect: "manual", headers: { ...headers, cookie: [headers.cookie, `dfy_csrf=${cookie}`].filter(Boolean).join("; "), "sec-fetch-site": "same-origin" }, body: new URLSearchParams({ _csrf: csrf ?? "", ...fields }) });
210 }
211 // The terminal opens on a branch whose name would run a command wherever it reached a shell unquoted.
212 const EVIL = "t$(touch${IFS}injected)";
213 const mainSha = (await ok("GET", `/repos/${owner}/term-tasks/git/ref/heads/main`)).object.sha;
214 await ok("POST", `/repos/${owner}/term-tasks/git/refs`, { ref: `refs/heads/${EVIL}`, sha: mainSha });
215 const opened = await form("terminals", { agent: "term-hand", ref: EVIL, accept: "1", tasks: "1" });
216 assert.equal(opened.status, 303, await opened.text());
217 const id = new URL(opened.headers.get("location"), base).pathname.split("/").pop();
218 const running = await until("the terminal to start", async () => {
219 const p = await page(`terminals/${id}`);
220 return p.text.includes("data-socket") ? p : null;
221 });
222 assert.match(running.text, /Give it a task/);
223 assert.match(running.text, /Run with my subscription/);
224 assert.match(running.text, /on your own subscription/);
225
226 // Someone else, even an admin of the repository, can't send a task to it; nor can a token (there is no API for it).
227 const invite = await ok("PUT", `/repos/${owner}/term-tasks/collaborators/rival`, { permission: "admin" });
228 if (invite?.id) await fetch(`${base}/api/v3/user/repository_invitations/${invite.id}`, { method: "PATCH", headers: { cookie: `__Host-dfy_session=${RIVAL_SESSION}`, "sec-fetch-site": "same-origin" } });
229 const rival = { cookie: `__Host-dfy_session=${RIVAL_SESSION}` };
230 assert.equal((await form(`terminals/${id}/task`, { goal: "Do it", runtime: "claude-code" }, rival)).status, 303);
231 assert.deepEqual(await ok("GET", R), [], "someone else's click starts nothing");
232 const viaToken = await api("POST", R, { goal: "Do it", runtime: "claude-code", agent: "term-hand", terminal: id });
233 assert.equal(viaToken.status, 403, "a token starts no task, in a terminal or not");
234 // A terminal opened without "Will run agent tasks" never runs one.
235 const plainOpened = await form("terminals", { agent: "term-hand", ref: "main" });
236 assert.equal(plainOpened.status, 303, await plainOpened.text());
237 const plainId = new URL(plainOpened.headers.get("location"), base).pathname.split("/").pop();
238 const plain = await until("the plain terminal to start", async () => {
239 const p = await page(`terminals/${plainId}`);
240 return p.text.includes("data-socket") ? p : null;
241 });
242 assert.match(plain.text, /opened without agent tasks/);
243 assert.doesNotMatch(plain.text, /Run with my subscription/);
244 assert.equal((await form(`terminals/${plainId}/task`, { goal: "Do it", runtime: "claude-code" })).status, 303);
245 assert.deepEqual(await ok("GET", R), [], "a terminal opened without tasks runs none");
246 assert.equal((await form(`terminals/${plainId}/end`, {})).status, 303);
247 // Only Claude Code or Codex, as signed in there.
248 const custom = await form(`terminals/${id}/task`, { goal: "Do it", runtime: "custom" });
249 assert.equal(custom.status, 303);
250 assert.deepEqual(await ok("GET", R), [], "a custom command doesn't run in a terminal");
251
252 // My click: the task runs in my terminal with the CLI there, with no model key or proxy, in its own checkout.
253 const started = await form(`terminals/${id}/task`, { goal: "Add a terminal file", runtime: "claude-code" });
254 assert.equal(started.status, 303);
255 const n = Number(new URL(started.headers.get("location"), base).pathname.split("/").pop());
256 assert.ok(n > 0, started.headers.get("location"));
257 const done = await until("the terminal task to finish", async () => {
258 const t = await ok("GET", `${R}/${n}`);
259 return t.state !== "running" && t.state !== "queued" ? t : null;
260 });
261 const text = (await ok("GET", `${R}/${n}/transcript`)).map((l) => `[${l.stream}] ${l.text}`).join("\n");
262 assert.equal(done.state, "ready", text);
263 assert.ok(done.pull_request, text);
264 assert.match(text, /Running in .*'s terminal/);
265 assert.match(text, /signed-in claude: key=none/, "no model key or token reaches it");
266 assert.ok(!/base=[^ ]*\/-\/model\//.test(text), "nor the model proxy");
267 assert.match(text, new RegExp(`dir=task-${n}-1`), "in a checkout of its own");
268 assert.match(text, /\[out\] Add a terminal file/, "the goal reaches it");
269 assert.match(text, /--permission-mode bypassPermissions --disallowedTools WebSearch,WebFetch --settings \{/, "no web tools");
270 assert.match(text, /"Read\(~\/\.claude\/\*\*\)"/, "and deny rules on the sign-ins");
271 // Its jail: the CLI's own sign-in, the proxy as its only way out, and nothing else of the terminal's.
272 assert.match(text, /jail: proxy=http:\/\/127\.0\.0\.1:\d+/);
273 assert.match(text, /jail: signin=\{"fake":"signin"\}/, "the CLI signs itself in");
274 assert.match(text, /jail: history=hidden/, "the rest of the home directory isn't there");
275 assert.match(text, /jail: credential=hidden/, "nor the checkout's git config, with the sandbox's credential");
276 assert.match(text, /jail: direct=(ENETUNREACH|EHOSTUNREACH|ECONNREFUSED)/, "and there is no route out but the proxy");
277 const file = await ok("GET", `/repos/${owner}/term-tasks/contents/terminal.txt?ref=${encodeURIComponent(done.branch)}`);
278 assert.equal(Buffer.from(file.content, "base64").toString(), "made in the terminal\n");
279 assert.equal(done.runs[0].in_terminal, true);
280 assert.equal(done.base, EVIL, "checked out from the terminal's branch, its name passed as data");
281
282 // The terminal's own checkout is untouched, and the terminal is still mine to use.
283 const t = await termSocket(`${base.replace("http", "ws")}/${owner}/term-tasks/terminals/${id}/socket?cols=100&rows=30`, { origin: new URL(base).origin });
284 assert.equal(t.status, 101);
285 t.type("cat ~/.claude/.credentials.json; echo; ls terminal.txt 2>&1 | head -1; ls injected ../injected ../pwned-fsmonitor 2>&1 | grep -c 'No such file'; git status --short | wc -l; ls .. | grep -c task- ; echo check-$((6*7))\n");
286 await t.until(/check-42/);
287 assert.match(t.out(), /No such file/);
288 assert.match(t.out(), /\{"fake":"signin"\}/, "the terminal's own sign-in is untouched by what the task wrote");
289 assert.doesNotMatch(t.out(), /attacker/);
290 assert.equal(readFileSync(join(poolHome, ".claude", ".credentials.json"), "utf8"), '{"fake":"signin"}', "byte for byte");
291 assert.match(t.out(), /\n3\r?\n0\r?\n0\r?\n/, "the branch name ran nothing, nor the clone's config outside its jail");
292 assert.match(t.out(), /\n0\r?\n0\r?\n/, "a clean checkout, and the task's checkout removed");
293
294 // Its page offers to run it again there; doing so is another run in the terminal.
295 const taskPage = await page(`tasks/${n}`);
296 assert.match(taskPage.text, /Run in my terminal/);
297 assert.match(taskPage.text, /in [a-z0-9-]+(&#39;|')s terminal/);
298 assert.doesNotMatch((await page(`tasks/${n}`, rival)).text, /Run in my terminal/, "not for anyone else");
299 assert.equal((await form(`tasks/${n}/terminal`, { terminal: id }, rival)).status, 303);
300 assert.equal((await ok("GET", `${R}/${n}`)).runs.length, 1, "someone else's click starts nothing");
301 const again = await form(`tasks/${n}/terminal`, { terminal: id, body: "Say it again" });
302 assert.equal(again.status, 303);
303 const twice = await until("the second terminal run", async () => {
304 const x = await ok("GET", `${R}/${n}`);
305 return x.runs.length === 2 && x.state === "ready" ? x : null;
306 });
307 assert.equal(twice.runs[1].state, "done");
308 assert.equal(twice.runs[1].in_terminal, true);
309 assert.equal(twice.runs[1].kind, "message");
310 // Usage (phase 5a): no request passed Diffinery, so each run shows what the signed-in CLI reported, with its cost.
311 assert.deepEqual(twice.runs[0].usage, { source: "runtime", requests: 0, input_tokens: 40, output_tokens: 7, cache_read_tokens: 1200, cache_write_tokens: 90, cost_usd: 0.0312 });
312 assert.deepEqual(twice.usage, { requests: 0, input_tokens: 80, output_tokens: 14, cache_read_tokens: 2400, cache_write_tokens: 180, cost_usd: 0.0624 });
313 assert.match((await page(`tasks/${n}`)).text, /Claude Code in the terminal/);
314 const usage = await until("the terminal runs in the usage report", async () => {
315 const r = await ok("GET", "/diffinery/model-usage");
316 return r.repositories.find((x) => x.repository === `${owner}/term-tasks` && x.runs === 2) ?? null;
317 });
318 assert.equal(usage.reported_by_cli_runs, 2);
319 assert.equal(usage.cost_usd, 0.0624);
320
321 // Ending the terminal ends what it can run: the page no longer offers it, and a click is refused.
322 assert.equal((await form(`terminals/${id}/end`, {})).status, 303);
323 assert.doesNotMatch((await page(`tasks/${n}`)).text, /Run in my terminal/);
324 assert.equal((await form(`tasks/${n}/terminal`, { terminal: id })).status, 303);
325 assert.equal((await ok("GET", `${R}/${n}`)).runs.length, 2);
326 t.close?.();
327});