| 1 | // End-to-end: a real `wrangler dev` Worker, the real git CLI and the real runner agent. |
| 2 | // |
| 3 | // pnpm --filter @diffinery/runner build && pnpm --filter @diffinery/app e2e |
| 4 | // |
| 5 | // Covers: token auth, repo creation, git push/clone/fetch (protocol v2), push |
| 6 | // protection, workflows triggered by push and pull_request and executed by a |
| 7 | // self-hosted runner (the same agent that runs in Cloudflare Containers), |
| 8 | // check runs, branch protection with required checks and human review for |
| 9 | // agent-authored PRs, merging, the REST contents API and the MCP server. |
| 10 | import { spawn, spawnSync, execFileSync } from "node:child_process"; |
| 11 | import { mkdtempSync, writeFileSync, mkdirSync, readFileSync, existsSync, rmSync } from "node:fs"; |
| 12 | import { tmpdir } from "node:os"; |
| 13 | import { join, dirname, resolve } from "node:path"; |
| 14 | import { fileURLToPath } from "node:url"; |
| 15 | import { createServer } from "node:net"; |
| 16 | import { test, before, after } from "node:test"; |
| 17 | import assert from "node:assert/strict"; |
| 18 | |
| 19 | const APP = resolve(dirname(fileURLToPath(import.meta.url)), "../.."); |
| 20 | const AGENT = resolve(APP, "../../containers/runner/dist/agent.mjs"); |
| 21 | const work = mkdtempSync(join(tmpdir(), "diffinery-e2e-")); |
| 22 | let base; |
| 23 | let dev; |
| 24 | let devLog = ""; |
| 25 | let token; |
| 26 | let agentToken; |
| 27 | |
| 28 | const freePort = () => |
| 29 | new Promise((res) => { |
| 30 | const s = createServer(); |
| 31 | s.listen(0, "127.0.0.1", () => { |
| 32 | const { port } = s.address(); |
| 33 | s.close(() => res(port)); |
| 34 | }); |
| 35 | }); |
| 36 | |
| 37 | async function api(method, path, body, tok = token) { |
| 38 | const res = await fetch(`${base}/api/v3${path}`, { |
| 39 | method, |
| 40 | headers: { authorization: `Bearer ${tok}`, "content-type": "application/json" }, |
| 41 | body: body === undefined ? undefined : JSON.stringify(body), |
| 42 | }); |
| 43 | const text = await res.text(); |
| 44 | let json = null; |
| 45 | try { |
| 46 | json = text ? JSON.parse(text) : null; |
| 47 | } catch { |
| 48 | json = text; |
| 49 | } |
| 50 | return { status: res.status, json }; |
| 51 | } |
| 52 | async function ok(method, path, body, tok) { |
| 53 | const r = await api(method, path, body, tok); |
| 54 | assert.ok(r.status >= 200 && r.status < 300, `${method} ${path} -> ${r.status} ${JSON.stringify(r.json)}`); |
| 55 | return r.json; |
| 56 | } |
| 57 | |
| 58 | function git(cwd, args, tok = token, opts = {}) { |
| 59 | const auth = Buffer.from(`x-token:${tok}`).toString("base64"); |
| 60 | return execFileSync("git", ["-c", `http.extraHeader=Authorization: Basic ${auth}`, "-c", "protocol.version=2", ...args], { |
| 61 | cwd, |
| 62 | encoding: "utf8", |
| 63 | stdio: ["ignore", "pipe", "pipe"], |
| 64 | env: { ...process.env, GIT_TERMINAL_PROMPT: "0", GIT_AUTHOR_NAME: "E2E", GIT_AUTHOR_EMAIL: "e2e@example.com", GIT_COMMITTER_NAME: "E2E", GIT_COMMITTER_EMAIL: "e2e@example.com" }, |
| 65 | ...opts, |
| 66 | }); |
| 67 | } |
| 68 | function gitFails(cwd, args, tok) { |
| 69 | try { |
| 70 | git(cwd, args, tok); |
| 71 | } catch (e) { |
| 72 | return `${e.stdout ?? ""}${e.stderr ?? ""}`; |
| 73 | } |
| 74 | assert.fail(`expected git ${args.join(" ")} to fail`); |
| 75 | } |
| 76 | |
| 77 | const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); |
| 78 | async function until(what, fn, timeoutMs = 60_000) { |
| 79 | const end = Date.now() + timeoutMs; |
| 80 | let last; |
| 81 | while (Date.now() < end) { |
| 82 | last = await fn(); |
| 83 | if (last) return last; |
| 84 | await sleep(500); |
| 85 | } |
| 86 | assert.fail(`timed out waiting for ${what}\n--- wrangler log tail ---\n${devLog.slice(-4000)}`); |
| 87 | } |
| 88 | |
| 89 | /** Claim one waiting self-hosted job and run it with the real agent. */ |
| 90 | async function runOneJob(owner, repo) { |
| 91 | const claim = await until("a job to claim", async () => { |
| 92 | const r = await api("POST", `/repos/${owner}/${repo}/actions/runners/claim`, { name: "e2e", labels: ["linux"] }); |
| 93 | return r.status === 200 ? r.json : null; |
| 94 | }); |
| 95 | const jobEnv = { DIFFINERY_API_URL: claim.api_url, DIFFINERY_JOB_ID: claim.job_id, DIFFINERY_JOB_TOKEN: claim.token }; |
| 96 | // E2E_RUNNER_IMAGE runs the job inside the real runner image instead of the local agent bundle. |
| 97 | const image = process.env.E2E_RUNNER_IMAGE; |
| 98 | const r = image |
| 99 | ? spawnSync("docker", ["run", "--rm", "--network", "host", ...Object.entries(jobEnv).flatMap(([k, v]) => ["-e", `${k}=${v}`]), image], { encoding: "utf8", timeout: 300_000 }) |
| 100 | : spawnSync(process.execPath, [AGENT], { encoding: "utf8", env: { ...process.env, ...jobEnv, DIFFINERY_WORK_ROOT: join(work, `job-${claim.job_id}`) }, timeout: 120_000 }); |
| 101 | const out = `${r.stdout}${r.stderr}`; |
| 102 | if (r.status !== 0) { |
| 103 | const log = await (await fetch(`${base}/api/v3/repos/${owner}/${repo}/actions/jobs/${claim.job_id}/logs`, { headers: { authorization: `Bearer ${token}` } })).text(); |
| 104 | const job = await api("GET", `/repos/${owner}/${repo}/actions/jobs/${claim.job_id}`); |
| 105 | assert.fail(`runner agent exited ${r.status}:\n${out}\n--- job ---\n${JSON.stringify(job.json, null, 2)}\n--- job log ---\n${log}`); |
| 106 | } |
| 107 | return { claim, out }; |
| 108 | } |
| 109 | |
| 110 | before(async () => { |
| 111 | assert.ok(existsSync(AGENT), `build the runner first: pnpm --filter @diffinery/runner build (missing ${AGENT})`); |
| 112 | const port = await freePort(); |
| 113 | base = `http://127.0.0.1:${port}`; |
| 114 | const persist = join(work, "state"); |
| 115 | execFileSync("pnpm", ["exec", "wrangler", "d1", "migrations", "apply", "diffinery-dev", "--local", "-e", "dev", "--persist-to", persist], { cwd: APP, stdio: "pipe", env: { ...process.env, CI: "1" } }); |
| 116 | // Until the web UI exists, run the same app through the API-only entry. |
| 117 | const entry = existsSync(join(APP, "src/web/routes.tsx")) ? [] : ["test/e2e/entry.ts"]; |
| 118 | dev = spawn("pnpm", ["exec", "wrangler", "dev", ...entry, "-e", "dev", "--ip", "127.0.0.1", "--port", String(port), "--persist-to", persist, "--var", `PUBLIC_URL:${base}`, "--show-interactive-dev-session=false"], { |
| 119 | cwd: APP, |
| 120 | env: { ...process.env, CI: "1", WRANGLER_SEND_METRICS: "false" }, |
| 121 | stdio: ["ignore", "pipe", "pipe"], |
| 122 | detached: true, |
| 123 | }); |
| 124 | dev.stdout.on("data", (d) => (devLog += d)); |
| 125 | dev.stderr.on("data", (d) => (devLog += d)); |
| 126 | await until("wrangler dev", async () => { |
| 127 | try { |
| 128 | return (await fetch(`${base}/healthz`)).ok; |
| 129 | } catch { |
| 130 | return false; |
| 131 | } |
| 132 | }, 120_000); |
| 133 | // DEV_AUTH signs requests without credentials in as dev@diffinery.local; mint a real token for everything else. |
| 134 | const res = await fetch(`${base}/api/v3/diffinery/tokens`, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ name: "e2e", scopes: ["admin"] }) }); |
| 135 | assert.equal(res.status, 201, await res.clone().text()); |
| 136 | token = (await res.json()).token; |
| 137 | }); |
| 138 | |
| 139 | after(() => { |
| 140 | if (dev?.pid) { |
| 141 | try { |
| 142 | process.kill(-dev.pid, "SIGTERM"); |
| 143 | } catch {} |
| 144 | } |
| 145 | if (process.env.E2E_KEEP !== "1") rmSync(work, { recursive: true, force: true }); |
| 146 | }); |
| 147 | |
| 148 | test("git hosting, Actions, protected PR flow and MCP", async () => { |
| 149 | const me = await ok("GET", "/user"); |
| 150 | const owner = me.login; |
| 151 | assert.ok(me.site_admin, "dev user is a bootstrap admin"); |
| 152 | |
| 153 | // ------------------------------------------------------------ repo + first push |
| 154 | const repo = await ok("POST", "/user/repos", { name: "demo", description: "e2e" }); |
| 155 | assert.equal(repo.full_name, `${owner}/demo`); |
| 156 | const src = join(work, "src"); |
| 157 | mkdirSync(join(src, ".github/workflows"), { recursive: true }); |
| 158 | writeFileSync(join(src, "README.md"), "# demo\n"); |
| 159 | writeFileSync(join(src, "app.txt"), "one\ntwo\nthree\n"); |
| 160 | writeFileSync( |
| 161 | join(src, ".github/workflows/ci.yml"), |
| 162 | `name: CI |
| 163 | on: |
| 164 | push: |
| 165 | branches: [main] |
| 166 | pull_request: |
| 167 | jobs: |
| 168 | build: |
| 169 | runs-on: [self-hosted, linux] |
| 170 | steps: |
| 171 | - uses: actions/checkout@v4 |
| 172 | - name: Check files |
| 173 | run: | |
| 174 | test -f README.md |
| 175 | echo "event=\${{ github.event_name }} sha=\${{ github.sha }}" |
| 176 | grep -q bad app.txt && { echo "found bad line"; exit 1; } || true |
| 177 | - id: out |
| 178 | run: echo "lines=$(wc -l < app.txt | tr -d ' ')" >> "$GITHUB_OUTPUT" |
| 179 | - run: echo "app.txt has \${{ steps.out.outputs.lines }} lines" |
| 180 | `, |
| 181 | ); |
| 182 | git(src, ["init", "-q", "-b", "main"]); |
| 183 | git(src, ["add", "-A"]); |
| 184 | git(src, ["commit", "-q", "-m", "initial"]); |
| 185 | git(src, ["remote", "add", "origin", `${base}/${owner}/demo.git`]); |
| 186 | git(src, ["push", "-q", "origin", "main"]); |
| 187 | const mainSha = git(src, ["rev-parse", "HEAD"]).trim(); |
| 188 | |
| 189 | const branch = await ok("GET", `/repos/${owner}/demo/branches/main`); |
| 190 | assert.equal(branch.commit.sha, mainSha); |
| 191 | |
| 192 | // ------------------------------------------------------------ the push ran CI on a self-hosted runner |
| 193 | const pushRun = await until("push run", async () => (await ok("GET", `/repos/${owner}/demo/actions/runs?event=push`)).workflow_runs[0]); |
| 194 | assert.equal(pushRun.head_sha, mainSha); |
| 195 | const job1 = await runOneJob(owner, "demo"); |
| 196 | const finished = await until("push run to complete", async () => { |
| 197 | const r = await ok("GET", `/repos/${owner}/demo/actions/runs/${pushRun.id}`); |
| 198 | return r.status === "completed" ? r : null; |
| 199 | }); |
| 200 | assert.equal(finished.conclusion, "success", job1.out); |
| 201 | const logs = await (await fetch(`${base}/api/v3/repos/${owner}/demo/actions/jobs/${job1.claim.job_id}/logs`, { headers: { authorization: `Bearer ${token}` } })).text(); |
| 202 | assert.match(logs, /event=push/); |
| 203 | assert.match(logs, /app\.txt has 3 lines/); |
| 204 | const checks = await ok("GET", `/repos/${owner}/demo/commits/${mainSha}/check-runs`); |
| 205 | assert.equal(checks.check_runs.find((c) => c.name === "CI / build")?.conclusion, "success"); |
| 206 | |
| 207 | // ------------------------------------------------------------ clone over protocol v2 |
| 208 | const clone = join(work, "clone"); |
| 209 | git(work, ["clone", "-q", `${base}/${owner}/demo.git`, clone]); |
| 210 | assert.equal(readFileSync(join(clone, "app.txt"), "utf8"), "one\ntwo\nthree\n"); |
| 211 | |
| 212 | // ------------------------------------------------------------ secrets are blocked on push |
| 213 | writeFileSync(join(clone, "config.env"), "AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPL3\n"); |
| 214 | git(clone, ["add", "-A"]); |
| 215 | git(clone, ["commit", "-q", "-m", "oops"]); |
| 216 | assert.match(gitFails(clone, ["push", "origin", "main"]), /secret|AWS/i); |
| 217 | git(clone, ["reset", "-q", "--hard", "origin/main"]); |
| 218 | |
| 219 | // ------------------------------------------------------------ protect main: PR + green CI + review |
| 220 | await ok("PUT", `/repos/${owner}/demo/branches/main/protection`, { |
| 221 | requirePullRequest: true, |
| 222 | requiredApprovals: 0, |
| 223 | dismissStaleReviews: true, |
| 224 | requiredChecks: ["CI / build"], |
| 225 | allowForcePush: false, |
| 226 | allowDeletion: false, |
| 227 | requireLinearHistory: false, |
| 228 | enforceAdmins: true, |
| 229 | agentsMayPush: false, |
| 230 | agentsMayMerge: false, |
| 231 | }); |
| 232 | writeFileSync(join(clone, "app.txt"), "one\ntwo\nthree\nfour\n"); |
| 233 | git(clone, ["commit", "-q", "-am", "direct push"]); |
| 234 | assert.match(gitFails(clone, ["push", "origin", "main"]), /pull request|protected/i); |
| 235 | git(clone, ["reset", "-q", "--hard", "origin/main"]); |
| 236 | |
| 237 | // ------------------------------------------------------------ an AI agent opens a PR |
| 238 | const created = await ok("POST", "/diffinery/agents", { login: "e2e-bot", name: "E2E Bot" }); |
| 239 | agentToken = created.token.token; |
| 240 | const agentClone = join(work, "agent"); |
| 241 | git(work, ["clone", "-q", `${base}/${owner}/demo.git`, agentClone], agentToken); |
| 242 | git(agentClone, ["checkout", "-q", "-b", "feature"], agentToken); |
| 243 | writeFileSync(join(agentClone, "app.txt"), "one\ntwo\nthree\nfour\n"); |
| 244 | git(agentClone, ["commit", "-q", "-am", "Add a fourth line"], agentToken); |
| 245 | assert.match(gitFails(agentClone, ["push", "origin", "HEAD:main"], agentToken), /agent|protected|pull request/i); |
| 246 | git(agentClone, ["push", "-q", "origin", "feature"], agentToken); |
| 247 | const pr = await ok("POST", `/repos/${owner}/demo/pulls`, { title: "Add a fourth line", head: "feature", base: "main", intent: "Extend app.txt for the e2e test" }, agentToken); |
| 248 | assert.equal(pr.number, 1); |
| 249 | |
| 250 | // The pull_request run executes; until then the PR is blocked on the required check. |
| 251 | let m = await ok("GET", `/repos/${owner}/demo/pulls/1/merge`); |
| 252 | assert.equal(m.mergeable, false); |
| 253 | await runOneJob(owner, "demo"); |
| 254 | await until("PR check to pass", async () => { |
| 255 | const r = await ok("GET", `/repos/${owner}/demo/pulls/1/merge`); |
| 256 | return r.checks?.length > 0 && r.checks.every((c) => c.state === "success") ? r : null; |
| 257 | }); |
| 258 | m = await ok("GET", `/repos/${owner}/demo/pulls/1/merge`); |
| 259 | assert.equal(m.mergeable, false, "agent-authored PRs need a human approval"); |
| 260 | assert.ok(JSON.stringify(m.blockers).match(/approv|human/i), JSON.stringify(m.blockers)); |
| 261 | |
| 262 | // The agent cannot approve or merge its own work. |
| 263 | const agentMerge = await api("PUT", `/repos/${owner}/demo/pulls/1/merge`, { merge_method: "squash" }, agentToken); |
| 264 | assert.ok([403, 405, 409].includes(agentMerge.status), JSON.stringify(agentMerge)); |
| 265 | |
| 266 | // Another agent's approval is advisory and does not unblock the merge. |
| 267 | const reviewer = (await ok("POST", "/diffinery/agents", { login: "e2e-reviewer" })).token.token; |
| 268 | await ok("POST", `/repos/${owner}/demo/pulls/1/reviews`, { event: "APPROVE", body: "bot says ok" }, reviewer); |
| 269 | assert.equal((await ok("GET", `/repos/${owner}/demo/pulls/1/merge`)).mergeable, false); |
| 270 | |
| 271 | // A human reviews (line comment + approval) and merges. |
| 272 | await ok("POST", `/repos/${owner}/demo/pulls/1/reviews`, { event: "APPROVE", body: "LGTM", comments: [{ path: "app.txt", line: 4, body: "nice line" }] }); |
| 273 | m = await ok("GET", `/repos/${owner}/demo/pulls/1/merge`); |
| 274 | assert.equal(m.mergeable, true, JSON.stringify(m)); |
| 275 | const merged = await ok("PUT", `/repos/${owner}/demo/pulls/1/merge`, { merge_method: "squash", delete_branch: true }); |
| 276 | assert.ok(merged.sha); |
| 277 | const prAfter = await ok("GET", `/repos/${owner}/demo/pulls/1`); |
| 278 | assert.equal(prAfter.merged, true); |
| 279 | |
| 280 | // ------------------------------------------------------------ fetch the merge, read through REST |
| 281 | git(clone, ["pull", "-q", "--ff-only", "origin", "main"]); |
| 282 | assert.equal(readFileSync(join(clone, "app.txt"), "utf8"), "one\ntwo\nthree\nfour\n"); |
| 283 | const content = await ok("GET", `/repos/${owner}/demo/contents/app.txt`); |
| 284 | assert.equal(Buffer.from(content.content, "base64").toString(), "one\ntwo\nthree\nfour\n"); |
| 285 | const branches = await ok("GET", `/repos/${owner}/demo/branches`); |
| 286 | assert.ok(!branches.some((b) => b.name === "feature"), "head branch deleted after merge"); |
| 287 | |
| 288 | // ------------------------------------------------------------ shallow clone, then deepen |
| 289 | const shallow = join(work, "shallow"); |
| 290 | git(work, ["clone", "-q", "--depth", "1", `${base}/${owner}/demo.git`, shallow]); |
| 291 | assert.equal(git(shallow, ["rev-list", "--count", "HEAD"]).trim(), "1"); |
| 292 | assert.equal(readFileSync(join(shallow, "app.txt"), "utf8"), "one\ntwo\nthree\nfour\n"); |
| 293 | git(shallow, ["fetch", "-q", "--unshallow"]); |
| 294 | assert.equal(git(shallow, ["rev-list", "--count", "HEAD"]).trim(), "2"); |
| 295 | git(shallow, ["fsck", "--no-progress"]); |
| 296 | |
| 297 | // ------------------------------------------------------------ MCP: the agent works without git |
| 298 | const rpc = async (method, params, id = 1) => { |
| 299 | const r = await fetch(`${base}/mcp`, { method: "POST", headers: { authorization: `Bearer ${agentToken}`, "content-type": "application/json", accept: "application/json, text/event-stream" }, body: JSON.stringify({ jsonrpc: "2.0", id, method, params }) }); |
| 300 | assert.equal(r.status, 200); |
| 301 | return (await r.json()).result; |
| 302 | }; |
| 303 | const init = await rpc("initialize", { protocolVersion: "2025-06-18", capabilities: {}, clientInfo: { name: "e2e", version: "1" } }); |
| 304 | assert.equal(init.serverInfo.name, "diffinery"); |
| 305 | const tools = await rpc("tools/list", {}); |
| 306 | assert.ok(tools.tools.some((t) => t.name === "commit_files")); |
| 307 | const file = await rpc("tools/call", { name: "get_file", arguments: { owner, repo: "demo", path: "README.md" } }); |
| 308 | assert.equal(file.isError, false); |
| 309 | assert.equal(file.structuredContent.content, "# demo\n"); |
| 310 | const commit = await rpc("tools/call", { name: "commit_files", arguments: { owner, repo: "demo", branch: "mcp-change", create_branch_from: "main", message: "Edit via MCP", files: [{ path: "docs/notes.md", content: "hello\n" }] } }); |
| 311 | assert.equal(commit.isError, false, JSON.stringify(commit)); |
| 312 | const blocked = await rpc("tools/call", { name: "commit_files", arguments: { owner, repo: "demo", branch: "main", message: "sneaky", files: [{ path: "x", content: "y" }] } }); |
| 313 | assert.equal(blocked.isError, true); |
| 314 | const opened = await rpc("tools/call", { name: "create_pull_request", arguments: { owner, repo: "demo", title: "Notes", head: "mcp-change", intent: "Add notes" } }); |
| 315 | assert.equal(opened.isError, false, JSON.stringify(opened)); |
| 316 | assert.equal(opened.structuredContent.number, 2); |
| 317 | |
| 318 | // ------------------------------------------------------------ web UI renders the same data (signed in via DEV_AUTH) |
| 319 | for (const path of ["/", `/${owner}`, `/${owner}/demo`, `/${owner}/demo/blob/main/app.txt`, `/${owner}/demo/commits/main`, `/${owner}/demo/pulls`, `/${owner}/demo/pull/1`, `/${owner}/demo/pull/1/files`, `/${owner}/demo/actions`, `/${owner}/demo/actions/runs/${pushRun.id}`, `/${owner}/demo/actions/runs/${pushRun.id}/job/${job1.claim.job_id}`, `/${owner}/demo/settings/branches`, "/settings/tokens"]) { |
| 320 | const res = await fetch(`${base}${path}`); |
| 321 | const html = await res.text(); |
| 322 | assert.equal(res.status, 200, `${path} -> ${res.status}\n${html.slice(0, 500)}`); |
| 323 | assert.match(res.headers.get("content-type") ?? "", /text\/html/, path); |
| 324 | } |
| 325 | assert.match(await (await fetch(`${base}/${owner}/demo/pull/1`)).text(), /Add a fourth line/); |
| 326 | const forged = await fetch(`${base}/${owner}/demo/issues/1/comment`, { method: "POST", headers: { "content-type": "application/x-www-form-urlencoded", origin: "https://evil.example", "sec-fetch-site": "cross-site" }, body: "body=hi" }); |
| 327 | assert.equal(forged.status, 403); |
| 328 | |
| 329 | // ------------------------------------------------------------ the CLI against the same server |
| 330 | const CLI = resolve(APP, "../cli/dist/diffinery.mjs"); |
| 331 | if (existsSync(CLI)) { |
| 332 | const cli = (...args) => execFileSync(process.execPath, [CLI, ...args], { encoding: "utf8", cwd: clone, env: { ...process.env, DIFFINERY_URL: base, DIFFINERY_TOKEN: token, NO_COLOR: "1", XDG_CONFIG_HOME: join(work, "cfg") } }); |
| 333 | assert.match(cli("whoami"), new RegExp(owner)); |
| 334 | const prs = JSON.parse(cli("pr", "list", "--state", "all", "--json")); |
| 335 | assert.ok(prs.some((p) => p.number === 1)); |
| 336 | assert.match(cli("workflow", "validate"), /ci\.yml/); |
| 337 | } |
| 338 | |
| 339 | // The commit made through MCP carries the agent's identity. |
| 340 | const agentCommit = await ok("GET", `/repos/${owner}/demo/commits/mcp-change`); |
| 341 | assert.match(JSON.stringify(agentCommit), /e2e-bot/); |
| 342 | }); |