| 1 | // End-to-end web terminals, kill switches on them, and tasks run in them (phase 4b), on the same setup as tasks.test.mjs |
| 2 | // (agents-harness.mjs). In a file of their own so each file stays well under the test runner's per-file limit. |
| 3 | // |
| 4 | // pnpm --filter @diffinery/app e2e |
| 5 | import { spawn, execFileSync } from "node:child_process"; |
| 6 | import { mkdtempSync, writeFileSync, mkdirSync, rmSync, readdirSync, statSync, readFileSync, existsSync } from "node:fs"; |
| 7 | import { tmpdir } from "node:os"; |
| 8 | import { createHash, randomBytes } from "node:crypto"; |
| 9 | import { createRequire } from "node:module"; |
| 10 | import { join, dirname, resolve } from "node:path"; |
| 11 | import { fileURLToPath } from "node:url"; |
| 12 | import { createServer as netServer } from "node:net"; |
| 13 | import { createServer as httpServer } from "node:http"; |
| 14 | import { test } from "node:test"; |
| 15 | import assert from "node:assert/strict"; |
| 16 | import { |
| 17 | APP, |
| 18 | AGENT, |
| 19 | work, |
| 20 | poolRoot, |
| 21 | WS, |
| 22 | RIVAL_SESSION, |
| 23 | ANTHROPIC_KEY, |
| 24 | OPENROUTER_KEY, |
| 25 | base, |
| 26 | dev, |
| 27 | pool, |
| 28 | upstream, |
| 29 | devLog, |
| 30 | poolLog, |
| 31 | token, |
| 32 | modelRequests, |
| 33 | freePort, |
| 34 | api, |
| 35 | ok, |
| 36 | asPerson, |
| 37 | sleep, |
| 38 | until, |
| 39 | git, |
| 40 | mcp, |
| 41 | poolSandboxes, |
| 42 | } from "./agents-harness.mjs"; |
| 43 | |
| 44 | /** Opens a terminal's WebSocket; resolves with the HTTP status when the upgrade is refused. */ |
| 45 | function termSocket(url, headers) { |
| 46 | return new Promise((resolve) => { |
| 47 | const ws = new WS(url, { headers }); |
| 48 | let out = ""; |
| 49 | ws.on("message", (d, binary) => { |
| 50 | if (binary) out += String(d); |
| 51 | }); |
| 52 | let closedWith = null; |
| 53 | const closed = new Promise((r) => ws.on("close", (code, reason) => r((closedWith = { code, reason: String(reason) })))); |
| 54 | ws.once("unexpected-response", (_req, res) => resolve({ status: res.statusCode })); |
| 55 | ws.once("error", (e) => resolve({ status: 0, error: e.message })); |
| 56 | ws.once("open", () => |
| 57 | resolve({ |
| 58 | status: 101, |
| 59 | ws, |
| 60 | closed, |
| 61 | out: () => out, |
| 62 | type: (text) => ws.send(Buffer.from(text)), |
| 63 | until: (re, ms = 30_000) => until(`${re} in the terminal (got ${JSON.stringify(out.slice(-300))}, closed ${JSON.stringify(closedWith)})`, async () => re.test(out), ms), |
| 64 | }), |
| 65 | ); |
| 66 | }); |
| 67 | } |
| 68 | |
| 69 | test("a web terminal: only its opener attaches, the sandbox API can't reach it, and End takes it away", async () => { |
| 70 | const me = await ok("GET", "/user"); |
| 71 | const owner = me.login; |
| 72 | await ok("POST", "/user/repos", { name: "term-demo" }); |
| 73 | await ok("PUT", `/repos/${owner}/term-demo/contents/README.md`, { message: "init", content: Buffer.from("# term\n").toString("base64") }); |
| 74 | assert.equal((await asPerson("POST", "/diffinery/agents", { login: "term-bot", name: "Term Bot" })).status, 201); |
| 75 | const page = (path) => fetch(`${base}/${owner}/term-demo/${path}`).then(async (r) => ({ status: r.status, text: await r.text(), headers: r.headers })); |
| 76 | async function form(path, fields, headers = {}) { |
| 77 | const res = await fetch(`${base}/${owner}/term-demo/terminals`, { headers }); |
| 78 | const csrf = /name="_csrf" value="([^"]+)"/.exec(await res.text())?.[1]; |
| 79 | const cookie = /dfy_csrf=([^;]+)/.exec(res.headers.get("set-cookie") ?? "")?.[1]; |
| 80 | return fetch(`${base}/${owner}/term-demo/${path}`, { method: "POST", redirect: "manual", headers: { ...headers, cookie: [headers.cookie, `dfy_csrf=${cookie}`].filter(Boolean).join("; "), "sec-fetch-site": "same-origin" }, body: new URLSearchParams({ _csrf: csrf ?? "", ...fields }) }); |
| 81 | } |
| 82 | |
| 83 | // The notice comes first, and it is a Kumo checkbox, not a native one. |
| 84 | const list = await page("terminals"); |
| 85 | assert.equal(list.status, 200); |
| 86 | assert.match(list.text, /Open a terminal/); |
| 87 | assert.match(list.text, /my own Anthropic or OpenAI account/); |
| 88 | assert.ok(!/<input[^>]+type="checkbox"/.test(list.text.replace(/<input[^>]+data-k[^>]*>/g, "")), "no native checkbox"); |
| 89 | const refused = await form("terminals", { agent: "term-bot", ref: "main" }); |
| 90 | assert.equal(refused.status, 422); |
| 91 | assert.match(await refused.text(), /accept the notice first/); |
| 92 | const opened = await form("terminals", { agent: "term-bot", ref: "main", accept: "1" }); |
| 93 | assert.equal(opened.status, 303, await opened.text()); |
| 94 | const path = new URL(opened.headers.get("location"), base).pathname; |
| 95 | const id = path.split("/").pop(); |
| 96 | const running = await until("the terminal to start", async () => { |
| 97 | const p = await page(`terminals/${id}`); |
| 98 | return p.text.includes("data-socket") ? p : null; |
| 99 | }); |
| 100 | assert.match(running.text, /Start Claude Code/); |
| 101 | assert.match(running.text, /Start Codex/, "a private repository offers Codex"); |
| 102 | assert.match(running.text, /\/-\/terminal\/[a-z0-9]+\.js/); |
| 103 | const sandboxId = poolSandboxes()[0]; |
| 104 | |
| 105 | // The socket: this site's Origin, a browser session, and only the opener. |
| 106 | const url = `${base.replace("http", "ws")}/${owner}/term-demo/terminals/${id}/socket?cols=100&rows=30`; |
| 107 | const origin = new URL(base).origin; |
| 108 | assert.equal((await termSocket(url, {})).status, 403, "no Origin"); |
| 109 | assert.equal((await termSocket(url, { origin: "https://evil.example" })).status, 403, "another Origin"); |
| 110 | assert.equal((await termSocket(url, { origin, authorization: `Bearer ${token}` })).status, 403, "a token, even the opener's own admin token"); |
| 111 | assert.equal((await termSocket(url, { origin, cookie: `__Host-dfy_session=${RIVAL_SESSION}` })).status, 404, "another person"); |
| 112 | const t = await termSocket(url, { origin }); |
| 113 | assert.equal(t.status, 101, JSON.stringify(t)); |
| 114 | t.type("stty size; echo \"s=[$DIFFINERY_SANDBOX_SECRET] t=[$DIFFINERY_JOB_TOKEN]\"; git rev-parse --abbrev-ref HEAD; echo hi-$((6*7))\n"); |
| 115 | await t.until(/hi-42/); |
| 116 | assert.match(t.out(), /30 100/); |
| 117 | assert.match(t.out(), /s=\[\] t=\[\]/); |
| 118 | assert.match(t.out(), /\bmain\b/); |
| 119 | |
| 120 | // Another person, even one the owner made an admin of the repository, sees nothing of it. |
| 121 | const invite = await ok("PUT", `/repos/${owner}/term-demo/collaborators/rival`, { permission: "admin" }); |
| 122 | if (invite?.id) await fetch(`${base}/api/v3/user/repository_invitations/${invite.id}`, { method: "PATCH", headers: { cookie: `__Host-dfy_session=${RIVAL_SESSION}`, "sec-fetch-site": "same-origin" } }); |
| 123 | const rival = { cookie: `__Host-dfy_session=${RIVAL_SESSION}` }; |
| 124 | assert.equal((await fetch(`${base}/${owner}/term-demo/terminals/${id}`, { headers: rival })).status, 404); |
| 125 | assert.ok(!(await (await fetch(`${base}/${owner}/term-demo/terminals`, { headers: rival })).text()).includes(id)); |
| 126 | assert.equal((await form(`terminals/${id}/end`, {}, rival)).status, 404); |
| 127 | assert.equal((await termSocket(url, { origin, ...rival })).status, 404); |
| 128 | |
| 129 | // The sandbox API and MCP never reach a terminal's sandbox, not even for a site admin's admin token. |
| 130 | assert.equal((await api("POST", `/repos/${owner}/term-demo/diffinery/sandboxes/${sandboxId}/exec`, { cmd: "cat ~/.claude/.credentials.json" })).status, 404); |
| 131 | assert.equal((await api("POST", `/repos/${owner}/term-demo/diffinery/sandboxes/${sandboxId}/read`, { path: "/root/.codex/auth.json" })).status, 404); |
| 132 | assert.equal((await api("POST", `/repos/${owner}/term-demo/diffinery/sandboxes/${sandboxId}/close`)).status, 404); |
| 133 | const viaMcp = await mcp("sandbox_exec", { owner, repo: "term-demo", sandbox_id: sandboxId, command: "id" }); |
| 134 | assert.ok(viaMcp?.isError, JSON.stringify(viaMcp)); |
| 135 | |
| 136 | // Limits: two open per repository. |
| 137 | const second = await form("terminals", { agent: "term-bot", ref: "main" }); |
| 138 | assert.equal(second.status, 303); |
| 139 | const third = await form("terminals", { agent: "term-bot", ref: "main" }); |
| 140 | assert.equal(third.status, 422); |
| 141 | assert.match(await third.text(), /2 terminals open in this repository/); |
| 142 | const secondId = new URL(second.headers.get("location"), base).pathname.split("/").pop(); |
| 143 | |
| 144 | // End: the socket closes, the sandbox and its git credential go, and it can't be attached again. |
| 145 | assert.equal((await form(`terminals/${id}/end`, {})).status, 303); |
| 146 | const closed = await t.closed; |
| 147 | assert.ok([1000, 1006, 1011].includes(closed.code), JSON.stringify(closed)); |
| 148 | assert.equal((await termSocket(url, { origin })).status, 409); |
| 149 | assert.match((await page(`terminals/${id}`)).text, /You ended it/); |
| 150 | await until("the pool to stop the sandbox", async () => !poolSandboxes().includes(sandboxId) || (await fetch(`${base}/${owner}/term-demo/terminals/${id}`)).ok); |
| 151 | |
| 152 | // Losing access ends a terminal too: the agent's write access goes, and the next check ends it. |
| 153 | await until("the second terminal to start", async () => (await page(`terminals/${secondId}`)).text.includes("data-socket")); |
| 154 | assert.equal((await api("DELETE", "/diffinery/agents/term-bot")).status, 204); |
| 155 | await until("the second terminal to end", async () => /no longer have write access|ended/i.test((await page(`terminals/${secondId}`)).text) && !(await page(`terminals/${secondId}`)).text.includes("data-socket"), 150_000); |
| 156 | }); |
| 157 | |
| 158 | test("a kill switch ends web terminals and refuses new ones until it's lifted", async () => { |
| 159 | const owner = (await ok("GET", "/user")).login; |
| 160 | assert.equal((await asPerson("POST", "/diffinery/agents", { login: "term-bot-2", name: "Term Bot 2" })).status, 201); |
| 161 | const page = (path) => fetch(`${base}/${owner}/term-demo/${path}`).then(async (r) => ({ status: r.status, text: await r.text() })); |
| 162 | async function form(path, fields) { |
| 163 | const res = await fetch(`${base}/${owner}/term-demo/terminals`); |
| 164 | const csrf = /name="_csrf" value="([^"]+)"/.exec(await res.text())?.[1]; |
| 165 | const cookie = /dfy_csrf=([^;]+)/.exec(res.headers.get("set-cookie") ?? "")?.[1]; |
| 166 | return fetch(`${base}/${owner}/term-demo/${path}`, { method: "POST", redirect: "manual", headers: { cookie: `dfy_csrf=${cookie}`, "sec-fetch-site": "same-origin" }, body: new URLSearchParams({ _csrf: csrf ?? "", ...fields }) }); |
| 167 | } |
| 168 | const opened = await form("terminals", { agent: "term-bot-2", ref: "main" }); |
| 169 | assert.equal(opened.status, 303, await opened.text()); |
| 170 | const id = new URL(opened.headers.get("location"), base).pathname.split("/").pop(); |
| 171 | await until("the terminal to start", async () => (await page(`terminals/${id}`)).text.includes("data-socket")); |
| 172 | const url = `${base.replace("http", "ws")}/${owner}/term-demo/terminals/${id}/socket`; |
| 173 | const origin = new URL(base).origin; |
| 174 | |
| 175 | // Suspending the repository's Actions ends the terminal, refuses attaching and opening another. |
| 176 | const S = `/admin/repos/${owner}/term-demo/actions-suspension`; |
| 177 | assert.equal((await api("PUT", S, { reason: "abuse report" })).status, 200); |
| 178 | // Nor does it start a task in the terminal (phase 4b). |
| 179 | const before = (await ok("GET", `/repos/${owner}/term-demo/diffinery/tasks`)).length; |
| 180 | assert.equal((await form(`terminals/${id}/task`, { goal: "Do it while suspended", runtime: "claude-code" })).status, 303); |
| 181 | assert.equal((await ok("GET", `/repos/${owner}/term-demo/diffinery/tasks`)).length, before, "a suspended repository's terminal starts no task"); |
| 182 | const refused = await termSocket(url, { origin }); |
| 183 | assert.ok([403, 409].includes(refused.status), JSON.stringify(refused)); |
| 184 | const again = await form("terminals", { agent: "term-bot-2", ref: "main" }); |
| 185 | assert.equal(again.status, 422); |
| 186 | assert.match(await again.text(), /Actions are suspended for this repository by a site administrator: abuse report/); |
| 187 | await until("the terminal to end", async () => /suspended Actions and sandboxes here|Its sandbox stopped/.test((await page(`terminals/${id}`)).text), 150_000); |
| 188 | |
| 189 | // Lifted, a terminal opens again. |
| 190 | assert.equal((await api("DELETE", S)).status, 200); |
| 191 | const after = await form("terminals", { agent: "term-bot-2", ref: "main" }); |
| 192 | assert.equal(after.status, 303, await after.text()); |
| 193 | assert.equal((await form(`${new URL(after.headers.get("location"), base).pathname.split("/").slice(3).join("/")}/end`, {})).status, 303); |
| 194 | }); |
| 195 | |
| 196 | test("a task in my terminal: only on my click, with the CLI signed in there, in a checkout of its own", async () => { |
| 197 | const owner = (await ok("GET", "/user")).login; |
| 198 | await ok("POST", "/user/repos", { name: "term-tasks" }); |
| 199 | await ok("PUT", `/repos/${owner}/term-tasks/contents/README.md`, { message: "init", content: Buffer.from("# term tasks\n").toString("base64") }); |
| 200 | assert.equal((await asPerson("POST", "/diffinery/agents", { login: "term-hand", name: "Term Hand" })).status, 201); |
| 201 | const R = `/repos/${owner}/term-tasks/diffinery/tasks`; |
| 202 | const page = (path, headers = {}) => fetch(`${base}/${owner}/term-tasks/${path}`, { headers }).then(async (r) => ({ status: r.status, text: await r.text() })); |
| 203 | async function form(path, fields, headers = {}) { |
| 204 | const res = await fetch(`${base}/${owner}/term-tasks/terminals`, { headers }); |
| 205 | const csrf = /name="_csrf" value="([^"]+)"/.exec(await res.text())?.[1]; |
| 206 | const cookie = /dfy_csrf=([^;]+)/.exec(res.headers.get("set-cookie") ?? "")?.[1]; |
| 207 | return fetch(`${base}/${owner}/term-tasks/${path}`, { method: "POST", redirect: "manual", headers: { ...headers, cookie: [headers.cookie, `dfy_csrf=${cookie}`].filter(Boolean).join("; "), "sec-fetch-site": "same-origin" }, body: new URLSearchParams({ _csrf: csrf ?? "", ...fields }) }); |
| 208 | } |
| 209 | // The terminal opens on a branch whose name would run a command wherever it reached a shell unquoted. |
| 210 | const EVIL = "t$(touch${IFS}injected)"; |
| 211 | const mainSha = (await ok("GET", `/repos/${owner}/term-tasks/git/ref/heads/main`)).object.sha; |
| 212 | await ok("POST", `/repos/${owner}/term-tasks/git/refs`, { ref: `refs/heads/${EVIL}`, sha: mainSha }); |
| 213 | const opened = await form("terminals", { agent: "term-hand", ref: EVIL, accept: "1" }); |
| 214 | assert.equal(opened.status, 303, await opened.text()); |
| 215 | const id = new URL(opened.headers.get("location"), base).pathname.split("/").pop(); |
| 216 | const running = await until("the terminal to start", async () => { |
| 217 | const p = await page(`terminals/${id}`); |
| 218 | return p.text.includes("data-socket") ? p : null; |
| 219 | }); |
| 220 | assert.match(running.text, /Give it a task/); |
| 221 | assert.match(running.text, /Run with my subscription/); |
| 222 | assert.match(running.text, /on your own subscription/); |
| 223 | |
| 224 | // Someone else, even an admin of the repository, can't send a task to it; nor can a token (there is no API for it). |
| 225 | const invite = await ok("PUT", `/repos/${owner}/term-tasks/collaborators/rival`, { permission: "admin" }); |
| 226 | if (invite?.id) await fetch(`${base}/api/v3/user/repository_invitations/${invite.id}`, { method: "PATCH", headers: { cookie: `__Host-dfy_session=${RIVAL_SESSION}`, "sec-fetch-site": "same-origin" } }); |
| 227 | const rival = { cookie: `__Host-dfy_session=${RIVAL_SESSION}` }; |
| 228 | assert.equal((await form(`terminals/${id}/task`, { goal: "Do it", runtime: "claude-code" }, rival)).status, 303); |
| 229 | assert.deepEqual(await ok("GET", R), [], "someone else's click starts nothing"); |
| 230 | const viaToken = await api("POST", R, { goal: "Do it", runtime: "claude-code", agent: "term-hand", terminal: id }); |
| 231 | assert.equal(viaToken.status, 403, "a token starts no task, in a terminal or not"); |
| 232 | // Only Claude Code or Codex, as signed in there. |
| 233 | const custom = await form(`terminals/${id}/task`, { goal: "Do it", runtime: "custom" }); |
| 234 | assert.equal(custom.status, 303); |
| 235 | assert.deepEqual(await ok("GET", R), [], "a custom command doesn't run in a terminal"); |
| 236 | |
| 237 | // My click: the task runs in my terminal with the CLI there, with no model key or proxy, in its own checkout. |
| 238 | const started = await form(`terminals/${id}/task`, { goal: "Add a terminal file", runtime: "claude-code" }); |
| 239 | assert.equal(started.status, 303); |
| 240 | const n = Number(new URL(started.headers.get("location"), base).pathname.split("/").pop()); |
| 241 | assert.ok(n > 0, started.headers.get("location")); |
| 242 | const done = await until("the terminal task to finish", async () => { |
| 243 | const t = await ok("GET", `${R}/${n}`); |
| 244 | return t.state !== "running" && t.state !== "queued" ? t : null; |
| 245 | }); |
| 246 | const text = (await ok("GET", `${R}/${n}/transcript`)).map((l) => `[${l.stream}] ${l.text}`).join("\n"); |
| 247 | assert.equal(done.state, "ready", text); |
| 248 | assert.ok(done.pull_request, text); |
| 249 | assert.match(text, /Running in .*'s terminal/); |
| 250 | assert.match(text, /signed-in claude: key=none/, "no model key or token reaches it"); |
| 251 | assert.ok(!/base=[^ ]*\/-\/model\//.test(text), "nor the model proxy"); |
| 252 | assert.match(text, new RegExp(`dir=task-${n}-1`), "in a checkout of its own"); |
| 253 | assert.match(text, /\[out\] Add a terminal file/, "the goal reaches it"); |
| 254 | const file = await ok("GET", `/repos/${owner}/term-tasks/contents/terminal.txt?ref=${encodeURIComponent(done.branch)}`); |
| 255 | assert.equal(Buffer.from(file.content, "base64").toString(), "made in the terminal\n"); |
| 256 | assert.equal(done.runs[0].in_terminal, true); |
| 257 | assert.equal(done.base, EVIL, "checked out from the terminal's branch, its name passed as data"); |
| 258 | |
| 259 | // The terminal's own checkout is untouched, and the terminal is still mine to use. |
| 260 | const t = await termSocket(`${base.replace("http", "ws")}/${owner}/term-tasks/terminals/${id}/socket?cols=100&rows=30`, { origin: new URL(base).origin }); |
| 261 | assert.equal(t.status, 101); |
| 262 | t.type("ls terminal.txt 2>&1 | head -1; ls injected ../injected 2>&1 | grep -c 'No such file'; git status --short | wc -l; ls .. | grep -c task- ; echo check-$((6*7))\n"); |
| 263 | await t.until(/check-42/); |
| 264 | assert.match(t.out(), /No such file/); |
| 265 | assert.match(t.out(), /\n2\r?\n0\r?\n0\r?\n/, "the branch name ran nothing"); |
| 266 | assert.match(t.out(), /\n0\r?\n0\r?\n/, "a clean checkout, and the task's checkout removed"); |
| 267 | |
| 268 | // Its page offers to run it again there; doing so is another run in the terminal. |
| 269 | const taskPage = await page(`tasks/${n}`); |
| 270 | assert.match(taskPage.text, /Run in my terminal/); |
| 271 | assert.match(taskPage.text, /in [a-z0-9-]+('|')s terminal/); |
| 272 | assert.doesNotMatch((await page(`tasks/${n}`, rival)).text, /Run in my terminal/, "not for anyone else"); |
| 273 | assert.equal((await form(`tasks/${n}/terminal`, { terminal: id }, rival)).status, 303); |
| 274 | assert.equal((await ok("GET", `${R}/${n}`)).runs.length, 1, "someone else's click starts nothing"); |
| 275 | const again = await form(`tasks/${n}/terminal`, { terminal: id, body: "Say it again" }); |
| 276 | assert.equal(again.status, 303); |
| 277 | const twice = await until("the second terminal run", async () => { |
| 278 | const x = await ok("GET", `${R}/${n}`); |
| 279 | return x.runs.length === 2 && x.state === "ready" ? x : null; |
| 280 | }); |
| 281 | assert.equal(twice.runs[1].state, "done"); |
| 282 | assert.equal(twice.runs[1].in_terminal, true); |
| 283 | assert.equal(twice.runs[1].kind, "message"); |
| 284 | |
| 285 | // Ending the terminal ends what it can run: the page no longer offers it, and a click is refused. |
| 286 | assert.equal((await form(`terminals/${id}/end`, {})).status, 303); |
| 287 | assert.doesNotMatch((await page(`tasks/${n}`)).text, /Run in my terminal/); |
| 288 | assert.equal((await form(`tasks/${n}/terminal`, { terminal: id })).status, 303); |
| 289 | assert.equal((await ok("GET", `${R}/${n}`)).runs.length, 2); |
| 290 | t.close?.(); |
| 291 | }); |