| 1 | import { execFileSync } from "node:child_process"; |
| 2 | import { randomBytes } from "node:crypto"; |
| 3 | import { chmodSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs"; |
| 4 | import { tmpdir } from "node:os"; |
| 5 | import path from "node:path"; |
| 6 | import { describe, expect, it } from "vitest"; |
| 7 | import type { ImportSpec } from "@diffinery/shared"; |
| 8 | import { batchBySize, furthestWithin, importDestination, importRefKind, isSecretRejection, isSizeRejection, parsePorcelain, parseSecretFindings, sortRefsForImport, storageRejection } from "../src/import-plan.js"; |
| 9 | import { pushFailure, transfer } from "../src/importer.js"; |
| 10 | |
| 11 | describe("import chunk planning", () => { |
| 12 | it("finds the furthest commit within budget with few probes", async () => { |
| 13 | const sizes = [10, 25, 40, 70, 90, 140, 200]; |
| 14 | const probes: number[] = []; |
| 15 | const at = async (i: number) => (probes.push(i), sizes[i]!); |
| 16 | expect(await furthestWithin(sizes.length, at, 100)).toEqual({ index: 4, size: 90 }); |
| 17 | expect(probes.length).toBeLessThanOrEqual(4); |
| 18 | expect(await furthestWithin(sizes.length, at, 1000)).toEqual({ index: 6, size: 200 }); |
| 19 | expect(await furthestWithin(sizes.length, at, 5)).toBeNull(); |
| 20 | expect(await furthestWithin(0, at, 100)).toBeNull(); |
| 21 | // Exactly on the budget counts as fitting. |
| 22 | expect(await furthestWithin(sizes.length, at, 40)).toEqual({ index: 2, size: 40 }); |
| 23 | }); |
| 24 | |
| 25 | it("batches objects by size, keeping oversized ones alone", () => { |
| 26 | const items = [{ size: 30 }, { size: 30 }, { size: 50 }, { size: 500 }, { size: 10 }]; |
| 27 | expect(batchBySize(items, 100).map((b) => b.map((x) => x.size))).toEqual([[30, 30], [50], [500], [10]]); |
| 28 | expect(batchBySize([], 100)).toEqual([]); |
| 29 | }); |
| 30 | |
| 31 | it("orders refs: default branch, branches, tags, then pull heads by number", () => { |
| 32 | const refs = ["refs/pull/10/head", "refs/tags/v1", "refs/heads/zeta", "refs/pull/2/head", "refs/heads/main", "refs/heads/alpha"]; |
| 33 | expect(sortRefsForImport(refs, "main")).toEqual(["refs/heads/main", "refs/heads/alpha", "refs/heads/zeta", "refs/tags/v1", "refs/pull/2/head", "refs/pull/10/head"]); |
| 34 | expect(importRefKind("refs/pull/3/merge")).toBeNull(); |
| 35 | expect(importRefKind("refs/notes/commits")).toBeNull(); |
| 36 | expect(importRefKind("refs/pull/3/head")).toBe("pull"); |
| 37 | expect(importRefKind("refs/heads/gh-readonly-queue/main/pr-3-abc")).toBeNull(); |
| 38 | expect(importDestination("refs/pull/3/head")).toBe("refs/diffinery-import/pull/3"); |
| 39 | expect(importDestination("refs/heads/main")).toBe("refs/heads/main"); |
| 40 | }); |
| 41 | |
| 42 | it("reads push protection and size rejections from git output", () => { |
| 43 | const out = "remote: Push protection bypassed by alice: Imported from GitHub o/r\nremote: AWS access key in blob 0123456789\nremote: Private key in blob abcdef0123\n"; |
| 44 | expect(parseSecretFindings(out)).toEqual([ |
| 45 | { kind: "AWS access key", blob: "0123456789" }, |
| 46 | { kind: "Private key", blob: "abcdef0123" }, |
| 47 | ]); |
| 48 | expect(isSecretRejection("! [remote rejected] main -> main (secret detected (push protection))")).toBe(true); |
| 49 | expect(isSizeRejection("remote: push exceeds 100 MB; split it into smaller pushes")).toBe(true); |
| 50 | expect(isSizeRejection("error: RPC failed; HTTP 413 curl 22")).toBe(true); |
| 51 | expect(isSizeRejection("! [rejected] main (non-fast-forward)")).toBe(false); |
| 52 | expect(isSizeRejection("error: remote unpack failed: the push needs more memory than one push may use; split it into smaller pushes (Array buffer allocation failed)")).toBe(true); |
| 53 | expect(storageRejection("remote: Push refused: acme has reached its storage limit of 1000 bytes (git objects); delete some \nerror: failed to push some refs")).toBe("acme has reached its storage limit of 1000 bytes (git objects); delete some"); |
| 54 | expect(storageRejection("error: failed to push some refs")).toBeNull(); |
| 55 | expect(parsePorcelain("To x\n*\trefs/heads/a:refs/heads/a\t[new branch]\n!\trefs/heads/b:refs/heads/b\t[remote rejected] (invalid ref name)\nDone\n")).toEqual([ |
| 56 | { ref: "refs/heads/a", ok: true, reason: "[new branch]" }, |
| 57 | { ref: "refs/heads/b", ok: false, reason: "invalid ref name" }, |
| 58 | ]); |
| 59 | // A failed push says why in the server's words, not git's generic last line. |
| 60 | const generic = "error: failed to push some refs to 'https://diffinery.com/o/r.git'"; |
| 61 | expect(pushFailure({ stdout: "To x\n!\trefs/heads/b:refs/heads/b\t[remote rejected] (missing objects)\nDone\n", stderr: generic })).toBe("missing objects"); |
| 62 | const unpack = "error: remote unpack failed: pack checksum mismatch"; |
| 63 | expect(pushFailure({ stdout: "To x\n!\trefs/heads/b:refs/heads/b\t[remote rejected] (unpacker error)\nDone\n", stderr: `${unpack}\n${generic}` })).toBe("pack checksum mismatch"); |
| 64 | expect(pushFailure({ stdout: "", stderr: `remote: Push refused: bad tree\n${generic}` })).toBe("remote: Push refused: bad tree"); |
| 65 | expect(pushFailure({ stdout: "", stderr: generic })).toBe(generic); |
| 66 | }); |
| 67 | }); |
| 68 | |
| 69 | describe("import transfer (real git)", () => { |
| 70 | const env = { ...process.env, GIT_AUTHOR_NAME: "T", GIT_AUTHOR_EMAIL: "t@example.com", GIT_COMMITTER_NAME: "T", GIT_COMMITTER_EMAIL: "t@example.com", GIT_CONFIG_GLOBAL: "/dev/null", GIT_CONFIG_NOSYSTEM: "1" }; |
| 71 | const g = (cwd: string, ...args: string[]) => execFileSync("git", args, { cwd, env, stdio: "pipe" }).toString().trim(); |
| 72 | |
| 73 | // Real git with many pushes: well past vitest's 5 s default on a busy runner. |
| 74 | it("pushes history in chunks under a size limit the server enforces, then every ref, and cleans up", async () => { |
| 75 | const root = mkdtempSync(path.join(tmpdir(), "diffinery-import-")); |
| 76 | const work = path.join(root, "work"); |
| 77 | g(root, "init", "-q", "-b", "main", work); |
| 78 | for (let i = 0; i < 12; i++) { |
| 79 | writeFileSync(path.join(work, `f${i}.bin`), randomBytes(40 * 1024)); |
| 80 | g(work, "add", "-A"); |
| 81 | g(work, "commit", "-q", "-m", `c${i}`); |
| 82 | } |
| 83 | // A side branch merged back, and one commit too large for any single push. |
| 84 | g(work, "checkout", "-q", "-b", "side", "HEAD~6"); |
| 85 | for (let i = 0; i < 4; i++) { |
| 86 | writeFileSync(path.join(work, `s${i}.bin`), randomBytes(40 * 1024)); |
| 87 | g(work, "add", "-A"); |
| 88 | g(work, "commit", "-q", "-m", `s${i}`); |
| 89 | } |
| 90 | g(work, "checkout", "-q", "main"); |
| 91 | g(work, "merge", "-q", "--no-ff", "-m", "merge side", "side"); |
| 92 | for (let i = 0; i < 5; i++) writeFileSync(path.join(work, `big${i}.bin`), randomBytes(60 * 1024)); |
| 93 | g(work, "add", "-A"); |
| 94 | g(work, "commit", "-q", "-m", "big"); |
| 95 | g(work, "tag", "-a", "v1", "-m", "release 1"); |
| 96 | g(work, "tag", "light"); |
| 97 | const source = path.join(root, "source.git"); |
| 98 | g(root, "clone", "-q", "--bare", work, source); |
| 99 | g(source, "update-ref", "refs/pull/7/head", g(work, "rev-parse", "side")); |
| 100 | g(source, "update-ref", "refs/pull/7/merge", g(work, "rev-parse", "main")); |
| 101 | g(source, "update-ref", "refs/heads/gh-readonly-queue/main/pr-7-abc", g(work, "rev-parse", "main")); |
| 102 | |
| 103 | // The "server": a bare repository whose pre-receive hook refuses packs over 150 KB. |
| 104 | const target = path.join(root, "target.git"); |
| 105 | g(root, "init", "-q", "--bare", target); |
| 106 | g(target, "config", "receive.advertisePushOptions", "true"); |
| 107 | const hook = path.join(target, "hooks", "pre-receive"); |
| 108 | const sizes = path.join(root, "sizes.log"); |
| 109 | writeFileSync( |
| 110 | hook, |
| 111 | `#!/bin/sh |
| 112 | size=$(du -sb "$GIT_QUARANTINE_PATH" 2>/dev/null | cut -f1) |
| 113 | echo "$size $GIT_PUSH_OPTION_0" >> "${sizes}" |
| 114 | if [ "\${size:-0}" -gt 153600 ]; then echo "push exceeds 100 MB; split it into smaller pushes"; exit 1; fi |
| 115 | `, |
| 116 | ); |
| 117 | chmodSync(hook, 0o755); |
| 118 | |
| 119 | const spec: ImportSpec = { |
| 120 | importId: "t1", |
| 121 | source: { fullName: "octo/repo", cloneUrl: `file://${source}`, defaultBranch: "main" }, |
| 122 | githubToken: null, |
| 123 | target: { fullName: "me/repo", pushUrl: `file://${target}` }, |
| 124 | pushToken: "dfy_test", |
| 125 | lfs: false, |
| 126 | secrets: "bypass", |
| 127 | bypassReason: "Imported from GitHub octo/repo", |
| 128 | // Deliberately larger than what the server accepts: the agent must split further. |
| 129 | chunkBytes: 1024 * 1024, |
| 130 | }; |
| 131 | const phases: string[] = []; |
| 132 | const result = await transfer(spec, { workRoot: path.join(root, "agent"), progress: async (p) => void phases.push(p.phase), log: () => {} }); |
| 133 | expect(result.ok).toBe(true); |
| 134 | expect(result.warnings).toEqual([]); |
| 135 | expect(result).toMatchObject({ branches: 2, tags: 2, pullRefs: 1, defaultBranch: "main" }); |
| 136 | expect(result.chunks).toBeGreaterThan(3); |
| 137 | expect(phases).toContain("push"); |
| 138 | |
| 139 | const refsOf = (dir: string) => g(dir, "for-each-ref", "--format=%(refname) %(objectname)").split("\n").sort(); |
| 140 | // Pull request heads wait in the import namespace for the server; merge refs and merge queue branches are skipped. |
| 141 | const expected = refsOf(source) |
| 142 | .filter((l) => !l.startsWith("refs/pull/7/merge") && !l.startsWith("refs/heads/gh-readonly-queue/")) |
| 143 | .map((l) => l.replace("refs/pull/7/head", "refs/diffinery-import/pull/7")) |
| 144 | .sort(); |
| 145 | expect(refsOf(target)).toEqual(expected); |
| 146 | g(target, "fsck", "--no-progress"); |
| 147 | // Every accepted push carried the import option, and the oversized ones were refused, then split. |
| 148 | const log = readFileSync(sizes, "utf8").trim().split("\n"); |
| 149 | expect(log.every((l) => l.endsWith("diffinery-import"))).toBe(true); |
| 150 | expect(log.some((l) => Number(l.split(" ")[0]) > 153600)).toBe(true); |
| 151 | }, 30_000); |
| 152 | it("pushes every version of a large file first and whole, so the server never rebuilds one from a delta", async () => { |
| 153 | const root = mkdtempSync(path.join(tmpdir(), "diffinery-import-")); |
| 154 | const work = path.join(root, "work"); |
| 155 | g(root, "init", "-q", "-b", "main", work); |
| 156 | // A large text file in six versions (git stores them as deltas of each other), and small random files. |
| 157 | let lines = Array.from({ length: 6000 }, (_, i) => `line ${i} ${randomBytes(12).toString("hex")}`); |
| 158 | for (let v = 0; v < 6; v++) { |
| 159 | lines = lines.map((l, i) => (i % 500 === v ? `${l} v${v}` : l)); |
| 160 | writeFileSync(path.join(work, "data.txt"), lines.join("\n")); |
| 161 | writeFileSync(path.join(work, `small${v}.bin`), randomBytes(2048)); |
| 162 | g(work, "add", "-A"); |
| 163 | g(work, "commit", "-q", "-m", `v${v}`); |
| 164 | } |
| 165 | const source = path.join(root, "source.git"); |
| 166 | g(root, "clone", "-q", "--bare", work, source); |
| 167 | g(source, "repack", "-adq"); |
| 168 | const target = path.join(root, "target.git"); |
| 169 | g(root, "init", "-q", "--bare", target); |
| 170 | g(target, "config", "receive.advertisePushOptions", "true"); |
| 171 | // Pushes are kept as packs (not unpacked to loose objects), so the hook can look at them. |
| 172 | g(target, "config", "receive.unpackLimit", "1"); |
| 173 | // The "server" refuses a push that holds a file as a delta, as Diffinery does past its memory. |
| 174 | const hook = path.join(target, "hooks", "pre-receive"); |
| 175 | writeFileSync( |
| 176 | hook, |
| 177 | `#!/bin/sh |
| 178 | for idx in "$GIT_QUARANTINE_PATH"/pack/*.idx; do |
| 179 | [ -f "$idx" ] || continue |
| 180 | if git verify-pack -v "$idx" | awk '$2 == "blob" && NF >= 7 { found = 1 } END { exit !found }'; then |
| 181 | echo "pack needs more than 80 MiB of memory to unpack; split it into smaller pushes"; exit 1 |
| 182 | fi |
| 183 | done |
| 184 | `, |
| 185 | ); |
| 186 | chmodSync(hook, 0o755); |
| 187 | const spec: ImportSpec = { |
| 188 | importId: "t3", |
| 189 | source: { fullName: "octo/data", cloneUrl: `file://${source}`, defaultBranch: "main" }, |
| 190 | githubToken: null, |
| 191 | target: { fullName: "me/data", pushUrl: `file://${target}` }, |
| 192 | pushToken: "dfy_test", |
| 193 | lfs: false, |
| 194 | secrets: "bypass", |
| 195 | bypassReason: "Imported from GitHub octo/data", |
| 196 | chunkBytes: 1024 * 1024, |
| 197 | }; |
| 198 | const result = await transfer(spec, { workRoot: path.join(root, "agent"), largeBlobBytes: 64 * 1024, progress: async () => {}, log: () => {} }); |
| 199 | expect(result.ok).toBe(true); |
| 200 | expect(g(target, "rev-parse", "refs/heads/main")).toBe(g(work, "rev-parse", "HEAD")); |
| 201 | g(target, "fsck", "--no-progress"); |
| 202 | }); |
| 203 | |
| 204 | it("splits a refused batch of a large commit's files again, rather than retrying it whole", async () => { |
| 205 | const root = mkdtempSync(path.join(tmpdir(), "diffinery-import-")); |
| 206 | const work = path.join(root, "work"); |
| 207 | g(root, "init", "-q", "-b", "main", work); |
| 208 | // One commit over the budget, of files that each fit in a push the server takes, but not all together. |
| 209 | for (let i = 0; i < 8; i++) writeFileSync(path.join(work, `f${i}.bin`), randomBytes(40 * 1024)); |
| 210 | g(work, "add", "-A"); |
| 211 | g(work, "commit", "-q", "-m", "many files"); |
| 212 | const source = path.join(root, "source.git"); |
| 213 | g(root, "clone", "-q", "--bare", work, source); |
| 214 | const target = path.join(root, "target.git"); |
| 215 | g(root, "init", "-q", "--bare", target); |
| 216 | g(target, "config", "receive.advertisePushOptions", "true"); |
| 217 | // Every push is kept as the pack that came (git otherwise stores only the objects it lacks), so |
| 218 | // the commit itself must come without the files its batches already brought. |
| 219 | g(target, "config", "receive.unpackLimit", "1"); |
| 220 | const hook = path.join(target, "hooks", "pre-receive"); |
| 221 | writeFileSync( |
| 222 | hook, |
| 223 | `#!/bin/sh |
| 224 | size=$(du -sb "$GIT_QUARANTINE_PATH" 2>/dev/null | cut -f1) |
| 225 | if [ "\${size:-0}" -gt 153600 ]; then echo "pack takes more than 500 reads (or 1 bytes) of storage to unpack; split it into smaller pushes"; exit 1; fi |
| 226 | `, |
| 227 | ); |
| 228 | chmodSync(hook, 0o755); |
| 229 | const spec: ImportSpec = { |
| 230 | importId: "t2", |
| 231 | source: { fullName: "octo/many", cloneUrl: `file://${source}`, defaultBranch: "main" }, |
| 232 | githubToken: null, |
| 233 | target: { fullName: "me/many", pushUrl: `file://${target}` }, |
| 234 | pushToken: "dfy_test", |
| 235 | lfs: false, |
| 236 | secrets: "bypass", |
| 237 | bypassReason: "Imported from GitHub octo/many", |
| 238 | // The commit is batched at 256 KB (six files), which the server refuses. |
| 239 | chunkBytes: 256 * 1024, |
| 240 | }; |
| 241 | const result = await transfer(spec, { workRoot: path.join(root, "agent"), progress: async () => {}, log: () => {} }); |
| 242 | expect(result.ok).toBe(true); |
| 243 | expect(g(target, "rev-parse", "refs/heads/main")).toBe(g(work, "rev-parse", "HEAD")); |
| 244 | g(target, "fsck", "--no-progress"); |
| 245 | }, 30_000); |
| 246 | }); |