| 1 | // Shared setup for the agent e2e files (tasks.test.mjs, terminals.test.mjs): a real `wrangler dev` |
| 2 | // Worker, sandboxes in an `agent.mjs --sandbox-pool` process, a fake model provider, and helpers. |
| 3 | // Importing it registers the before/after hooks that start and stop them. |
| 4 | import { spawn, execFileSync } from "node:child_process"; |
| 5 | import { mkdtempSync, writeFileSync, mkdirSync, rmSync, readdirSync, statSync, readFileSync, existsSync } from "node:fs"; |
| 6 | import { tmpdir } from "node:os"; |
| 7 | import { createHash, randomBytes } from "node:crypto"; |
| 8 | import { createRequire } from "node:module"; |
| 9 | import { join, dirname, resolve } from "node:path"; |
| 10 | import { fileURLToPath } from "node:url"; |
| 11 | import { createServer as httpServer } from "node:http"; |
| 12 | import { before, after } from "node:test"; |
| 13 | import assert from "node:assert/strict"; |
| 14 | import { migratedState } from "./d1-state.mjs"; |
| 15 | import { freePort } from "./ports.mjs"; |
| 16 | |
| 17 | export const APP = resolve(dirname(fileURLToPath(import.meta.url)), "../.."); |
| 18 | export const AGENT = resolve(APP, "../../containers/runner/dist/agent.mjs"); |
| 19 | // The test's files (scripts, fake CLIs, what commands write) are in `work`, which jails share; the pool's work root |
| 20 | // and home are beside it, since a jail refuses to share a directory that holds what it hides. |
| 21 | const top = mkdtempSync(join(tmpdir(), "diffinery-tasks-e2e-")); |
| 22 | export const work = join(top, "work"); |
| 23 | mkdirSync(work); |
| 24 | export const poolRoot = join(top, "pool"); |
| 25 | export const poolHome = join(top, "home"); |
| 26 | // The terminal test speaks WebSocket with headers (Origin, cookies), which Node's built-in client can't set. |
| 27 | export const { WebSocket: WS } = createRequire(resolve(APP, "../../containers/runner/package.json"))("ws"); |
| 28 | export const RIVAL_SESSION = `rival-${randomBytes(12).toString("hex")}`; |
| 29 | export const ANTHROPIC_KEY = `sk-ant-api03-${randomBytes(30).toString("hex")}`; |
| 30 | export const OPENROUTER_KEY = `sk-or-v1-${randomBytes(32).toString("hex")}`; |
| 31 | export let base; |
| 32 | export let dev; |
| 33 | export let pool; |
| 34 | export let upstream; |
| 35 | export let devLog = ""; |
| 36 | export let poolLog = ""; |
| 37 | export let token; |
| 38 | export const modelRequests = []; |
| 39 | |
| 40 | export async function api(method, path, body, tok = token) { |
| 41 | const res = await fetch(`${base}/api/v3${path}`, { |
| 42 | method, |
| 43 | headers: { ...(tok === null ? {} : { authorization: `Bearer ${tok}` }), "content-type": "application/json" }, |
| 44 | body: body === undefined ? undefined : JSON.stringify(body), |
| 45 | }); |
| 46 | const text = await res.text(); |
| 47 | let json = null; |
| 48 | try { |
| 49 | json = text ? JSON.parse(text) : null; |
| 50 | } catch { |
| 51 | json = text; |
| 52 | } |
| 53 | return { status: res.status, json }; |
| 54 | } |
| 55 | export async function ok(method, path, body, tok) { |
| 56 | const r = await api(method, path, body, tok); |
| 57 | assert.ok(r.status >= 200 && r.status < 300, `${method} ${path} -> ${r.status} ${JSON.stringify(r.json)}`); |
| 58 | return r.json; |
| 59 | } |
| 60 | /** As the signed-in person in a browser (the dev sign-in: no credentials). */ |
| 61 | export const asPerson = (method, path, body) => api(method, path, body, null); |
| 62 | |
| 63 | export const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); |
| 64 | export async function until(what, fn, timeoutMs = 90_000) { |
| 65 | const end = Date.now() + timeoutMs; |
| 66 | let last; |
| 67 | while (Date.now() < end) { |
| 68 | last = await fn(); |
| 69 | if (last) return last; |
| 70 | await sleep(500); |
| 71 | } |
| 72 | assert.fail(`timed out waiting for ${what}\n--- wrangler log tail ---\n${devLog.slice(-4000)}\n--- pool log tail ---\n${poolLog.slice(-3000)}`); |
| 73 | } |
| 74 | |
| 75 | export function git(cwd, args, tok = token) { |
| 76 | const auth = Buffer.from(`x-token:${tok}`).toString("base64"); |
| 77 | return execFileSync("git", ["-c", `http.extraHeader=Authorization: Basic ${auth}`, "-c", "protocol.version=2", ...args], { |
| 78 | cwd, |
| 79 | encoding: "utf8", |
| 80 | stdio: ["ignore", "pipe", "pipe"], |
| 81 | env: { ...process.env, GIT_TERMINAL_PROMPT: "0", GIT_AUTHOR_NAME: "E2E", GIT_AUTHOR_EMAIL: "e2e@example.com", GIT_COMMITTER_NAME: "E2E", GIT_COMMITTER_EMAIL: "e2e@example.com" }, |
| 82 | }); |
| 83 | } |
| 84 | |
| 85 | export async function mcp(name, args, tok = token) { |
| 86 | const r = await fetch(`${base}/mcp`, { method: "POST", headers: { authorization: `Bearer ${tok}`, "content-type": "application/json", accept: "application/json, text/event-stream" }, body: JSON.stringify({ jsonrpc: "2.0", id: 1, method: "tools/call", params: { name, arguments: args } }) }); |
| 87 | const { result } = await r.json(); |
| 88 | return result; |
| 89 | } |
| 90 | |
| 91 | /** |
| 92 | * A headless run's model token (phase 5d). The agent in its jail has only a stand-in, so the test reads the token |
| 93 | * where only the runner has it: the environment of the jail's own process outside, found by a path in its command. |
| 94 | * `file` is one the run's command writes $ANTHROPIC_API_KEY to, once it has started. |
| 95 | */ |
| 96 | export async function jailToken(file) { |
| 97 | await until(`the agent to write ${file}`, async () => existsSync(file)); |
| 98 | assert.equal(readFileSync(file, "utf8"), "diffinery-jail", "the agent holds a stand-in, never the run's token"); |
| 99 | return until(`the jail of the run writing ${file}`, async () => { |
| 100 | for (const pid of readdirSync("/proc").filter((p) => /^\d+$/.test(p))) { |
| 101 | try { |
| 102 | const argv = readFileSync(`/proc/${pid}/cmdline`, "utf8").split("\0"); |
| 103 | if (!argv.includes("--jail") || !argv.some((a) => a.includes(file))) continue; |
| 104 | const env = readFileSync(`/proc/${pid}/environ`, "utf8").split("\0"); |
| 105 | const tok = env.find((e) => e.startsWith("DIFFINERY_MODEL_TOKEN=")); |
| 106 | if (tok) return tok.slice("DIFFINERY_MODEL_TOKEN=".length); |
| 107 | } catch { |
| 108 | // gone, or not ours |
| 109 | } |
| 110 | } |
| 111 | return null; |
| 112 | }); |
| 113 | } |
| 114 | |
| 115 | /** The sandboxes the pool has started, newest first. */ |
| 116 | export const poolSandboxes = () => |
| 117 | existsSync(poolRoot) |
| 118 | ? readdirSync(poolRoot) |
| 119 | .map((id) => ({ id, at: statSync(join(poolRoot, id)).mtimeMs })) |
| 120 | .sort((a, b) => b.at - a.at) |
| 121 | .map((x) => x.id) |
| 122 | : []; |
| 123 | |
| 124 | before(async () => { |
| 125 | if (!existsSync(AGENT)) execFileSync("pnpm", ["--filter", "@diffinery/runner", "build"], { cwd: APP, stdio: "pipe" }); |
| 126 | // A stand-in for the provider: records what reaches it and answers like the Messages API. |
| 127 | upstream = httpServer((req, res) => { |
| 128 | let body = ""; |
| 129 | req.on("data", (d) => (body += d)); |
| 130 | req.on("end", () => { |
| 131 | modelRequests.push({ method: req.method, url: req.url, headers: req.headers, body }); |
| 132 | // A request that asks for it is answered after a while, so several can be on their way at once. |
| 133 | setTimeout(() => { |
| 134 | res.writeHead(200, { "content-type": "application/json", "set-cookie": "upstream=1" }); |
| 135 | res.end(JSON.stringify({ type: "message", content: [{ type: "text", text: "fake model says hi" }], usage: { input_tokens: 21, output_tokens: 4, cache_read_input_tokens: 300 } })); |
| 136 | }, body.includes('"slow"') ? 3000 : 0); |
| 137 | }); |
| 138 | }); |
| 139 | await new Promise((r) => upstream.listen(0, "127.0.0.1", r)); |
| 140 | const upstreamUrl = `http://127.0.0.1:${upstream.address().port}`; |
| 141 | |
| 142 | // A stand-in for a Claude Code someone signed in to in their terminal (phase 4b): it reports what it was given and |
| 143 | // where it ran, and changes a file, without any model. |
| 144 | const fakeBin = join(work, "bin"); |
| 145 | mkdirSync(fakeBin, { recursive: true }); |
| 146 | writeFileSync( |
| 147 | join(fakeBin, "claude"), |
| 148 | [ |
| 149 | "#!/bin/sh", |
| 150 | 'd=$(basename "$PWD")', |
| 151 | 'echo "{\\"type\\":\\"assistant\\",\\"message\\":{\\"content\\":[{\\"type\\":\\"text\\",\\"text\\":\\"signed-in claude: key=${ANTHROPIC_API_KEY:-none} base=${ANTHROPIC_BASE_URL:-none} dir=$d\\"}]}}"', |
| 152 | 'printf "%s\\n" "$DIFFINERY_TASK_PROMPT"', |
| 153 | 'printf "args: %s\\n" "$*"', |
| 154 | // What its jail lets it see and reach (containers/runner/src/jail.ts). |
| 155 | 'printf "jail: proxy=%s\\n" "$HTTPS_PROXY"', |
| 156 | 'printf "jail: signin=%s\\n" "$(cat "$HOME/.claude/.credentials.json" 2>/dev/null || echo none)"', |
| 157 | // Writing another account's credentials over the sign-in changes only the jail's copy. |
| 158 | 'echo \'{"attacker":"signin"}\' > "$HOME/.claude/.credentials.json"', |
| 159 | 'printf "jail: history=%s\\n" "$(cat "$HOME/.bash_history" 2>/dev/null || echo hidden)"', |
| 160 | 'printf "jail: credential=%s\\n" "$(cat ../repo/.git/config 2>/dev/null || echo hidden)"', |
| 161 | 'printf "jail: direct=%s\\n" "$(node -e "require(\\"net\\").connect(80, \\"1.1.1.1\\").on(\\"connect\\", () => (console.log(\\"open\\"), process.exit())).on(\\"error\\", (e) => (console.log(e.code), process.exit()))")"', |
| 162 | // A repository can set config that runs commands; in the task's clone it only ever runs in a jail. |
| 163 | "git config core.fsmonitor 'touch ../pwned-fsmonitor; false'", |
| 164 | 'echo "made in the terminal" > terminal.txt', |
| 165 | // What it reports about its own usage: a subscription's requests never pass Diffinery (phase 5a). |
| 166 | 'echo \'{"type":"result","subtype":"success","result":"Done in the terminal.","total_cost_usd":0.0312,"usage":{"input_tokens":40,"output_tokens":7,"cache_read_input_tokens":1200,"cache_creation_input_tokens":90}}\'', |
| 167 | ].join("\n"), |
| 168 | { mode: 0o755 }, |
| 169 | ); |
| 170 | // The pool's home is the terminal's: a provider sign-in, and shell history that a task's jail must not show. |
| 171 | mkdirSync(join(poolHome, ".claude"), { recursive: true }); |
| 172 | writeFileSync(join(poolHome, ".claude", ".credentials.json"), '{"fake":"signin"}'); |
| 173 | writeFileSync(join(poolHome, ".bash_history"), "HISTORY-SECRET\n"); |
| 174 | const poolPort = await freePort(); |
| 175 | const poolSecret = randomBytes(16).toString("hex"); |
| 176 | // Commands share the test's directory with their jails: their scripts are there, and what they write for the test to read. |
| 177 | pool = spawn("node", [AGENT, "--sandbox-pool", "--work-root", poolRoot], { env: { ...process.env, HOME: poolHome, PATH: `${fakeBin}:${process.env.PATH}`, DIFFINERY_SANDBOX_POOL_SECRET: poolSecret, DIFFINERY_SANDBOX_PORT: String(poolPort), DIFFINERY_JAIL_SHARED_FOR_TESTS: work }, stdio: ["ignore", "pipe", "pipe"], detached: true }); |
| 178 | pool.stdout.on("data", (d) => (poolLog += d)); |
| 179 | pool.stderr.on("data", (d) => (poolLog += d)); |
| 180 | await until("sandbox pool", async () => poolLog.includes("listening"), 30_000); |
| 181 | |
| 182 | const port = await freePort(); |
| 183 | base = `http://127.0.0.1:${port}`; |
| 184 | const persist = join(work, "state"); |
| 185 | migratedState(APP, persist); |
| 186 | // A second person, signed in with a session: not the terminal's opener, whatever their role. |
| 187 | { |
| 188 | const at = new Date().toISOString(); |
| 189 | const later = new Date(Date.now() + 30 * 86400_000).toISOString(); |
| 190 | const hash = createHash("sha256").update(RIVAL_SESSION).digest("hex"); |
| 191 | execFileSync("pnpm", ["exec", "wrangler", "d1", "execute", "diffinery-dev", "--local", "-e", "dev", "--persist-to", persist, "--command", `INSERT INTO accounts (id, login, kind, name, status, created_at) VALUES ('u-rival', 'rival', 'user', 'Rival', 'active', '${at}'); INSERT INTO sessions (id, account_id, method, created_at, last_seen_at, expires_at) VALUES ('${hash}', 'u-rival', 'passkey', '${at}', '${at}', '${later}')`], { cwd: APP, stdio: "pipe", env: { ...process.env, CI: "1" } }); |
| 192 | } |
| 193 | const vars = { PUBLIC_URL: base, MASTER_KEY: randomBytes(32).toString("base64"), SANDBOX_POOL_URL: `http://127.0.0.1:${poolPort}`, SANDBOX_POOL_SECRET: poolSecret, MODEL_UPSTREAM_ANTHROPIC: upstreamUrl, MODEL_UPSTREAM_OPENAI: upstreamUrl, MODEL_UPSTREAM_OPENROUTER: upstreamUrl, AGENT_POLL_MS: "500", TERMINAL_POLL_MS: "3000" }; |
| 194 | dev = spawn("pnpm", ["exec", "wrangler", "dev", "-e", "dev", "--ip", "127.0.0.1", "--port", String(port), "--inspector-port", String(await freePort()), "--persist-to", persist, ...Object.entries(vars).flatMap(([k, v]) => ["--var", `${k}:${v}`]), "--show-interactive-dev-session=false"], { |
| 195 | cwd: APP, |
| 196 | env: { ...process.env, CI: "1", WRANGLER_SEND_METRICS: "false" }, |
| 197 | stdio: ["ignore", "pipe", "pipe"], |
| 198 | detached: true, |
| 199 | }); |
| 200 | dev.stdout.on("data", (d) => (devLog += d)); |
| 201 | dev.stderr.on("data", (d) => (devLog += d)); |
| 202 | await until( |
| 203 | "wrangler dev", |
| 204 | async () => { |
| 205 | try { |
| 206 | return (await fetch(`${base}/healthz`)).ok; |
| 207 | } catch { |
| 208 | return false; |
| 209 | } |
| 210 | }, |
| 211 | 120_000, |
| 212 | ); |
| 213 | const res = await fetch(`${base}/api/v3/diffinery/tokens`, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ name: "e2e", scopes: ["admin"] }) }); |
| 214 | assert.equal(res.status, 201, await res.clone().text()); |
| 215 | token = (await res.json()).token; |
| 216 | }); |
| 217 | |
| 218 | after(() => { |
| 219 | for (const p of [dev, pool]) { |
| 220 | if (p?.pid) { |
| 221 | try { |
| 222 | process.kill(-p.pid, "SIGTERM"); |
| 223 | } catch {} |
| 224 | } |
| 225 | } |
| 226 | upstream?.close(); |
| 227 | if (process.env.E2E_KEEP !== "1") rmSync(top, { recursive: true, force: true, maxRetries: 5, retryDelay: 200 }); |
| 228 | }); |