Diffinery

codician/diffinerypublic

Open-source, agent-first GitHub and GitHub Actions alternative that runs on your own Cloudflare account: git hosting, pull requests, code review and drop-in GitHub Actions YAML on Workers, Durable Objects, R2 and Containers.

End-to-end tests: large pushes side by side, and big git traffic past wrangler dev's proxy (#247)

* End-to-end tests: large pushes side by side, and big git traffic past wrangler's proxy
* End-to-end tests: a git command that doesn't finish in 10 minutes fails with the server's log
* End-to-end tests: real histories side by side again, with git past wrangler dev's proxy
* End-to-end tests: real histories two at a time, which fits a runner's memory
test-bot authored 1 parent be7e0c4commit 5355e600e326597a3759ab22ea964c099d00b6cfBrowse files
1 file changed+183−143
modifiedapps/diffinery/test/e2e/e2e.test.mjs+183−143View file
@@ -211,7 +211,7 @@
211211 process.kill(-child.pid, "SIGTERM");
212212 } catch {}
213213 };
214- return { base: url, direct: workerUrl(debugLogFile), token: (await res.json()).token, log, stop };
214+ return { base: url, direct: workerUrl(debugLogFile), workerUrl: () => workerUrl(debugLogFile), token: (await res.json()).token, log, stop };
215215 }
216216
217217 /**
@@ -2822,10 +2822,12 @@
28222822 });
28232823
28242824 /**
2825- * `git` without blocking this process, so the histories below run side by side; it rejects with
2826- * git's output and, on a server error, the tail of `log`. A `tok` of null sends no credentials (for GitHub).
2825+ * `git` without blocking this process, so the tests below run side by side; it rejects with git's
2826+ * output and, on a server error or after 10 minutes without finishing (the command is killed, so a
2827+ * stuck push fails the test instead of the whole job), the tail of `log` and what `more` says.
2828+ * A `tok` of null sends no credentials (for GitHub).
28272829 */
2828-function gitAsync(cwd, args, tok = token, log = devLog) {
2830+function gitAsync(cwd, args, tok = token, log = devLog, more = () => "") {
28292831 const auth = tok === null ? [] : ["-c", `http.extraHeader=Authorization: Basic ${Buffer.from(`x-token:${tok}`).toString("base64")}`];
28302832 return new Promise((done, fail) => {
28312833 const child = spawn("git", [...auth, "-c", "protocol.version=2", ...args], {
@@ -2835,24 +2837,30 @@
28352837 });
28362838 let stdout = "";
28372839 let stderr = "";
2840+ let timedOut = false;
2841+ const timer = setTimeout(() => {
2842+ timedOut = true;
2843+ child.kill("SIGKILL");
2844+ }, 600_000);
28382845 child.stdout.on("data", (d) => (stdout += d));
28392846 child.stderr.on("data", (d) => (stderr += d));
28402847 child.on("error", fail);
28412848 child.on("close", (status) => {
2849+ clearTimeout(timer);
28422850 if (status === 0) return done(stdout);
2843- let message = `git ${args.join(" ")} exited ${status}\n${stderr}`;
2844- if (/HTTP 5\d\d/.test(stderr)) message += `\n--- wrangler log tail ---\n${log().slice(-4000)}`;
2851+ let message = `git ${args.join(" ")} ${timedOut ? "didn't finish in 10 minutes" : `exited ${status}`}\n${stderr}`;
2852+ if (timedOut || /HTTP 5\d\d|Failed to connect/.test(stderr)) message += `\n${more()}\n--- wrangler log tail ---\n${log().slice(-4000)}`;
28452853 fail(Object.assign(new Error(message), { stdout, stderr }));
28462854 });
28472855 });
28482856 }
28492857
28502858 /**
2851- * A real project's history, pushed whole to a new repository on `server` and cloned back. Pinned
2852- * commits, so the packs are the same each run; cloned from GitHub without the Diffinery token.
2859+ * `ok`, `git` and `gitFails` against `server` (startServer's, or the shared one), with git that doesn't
2860+ * block this process: for the tests below that run side by side.
28532861 */
2854-async function pushHistory(server, name, upstream) {
2855- const call = async (method, path, body) => {
2862+function client(server) {
2863+ const ok = async (method, path, body) => {
28562864 const send = () => fetch(`${server.base}/api/v3${path}`, { method, headers: { authorization: `Bearer ${server.token}`, "content-type": "application/json" }, body: body === undefined ? undefined : JSON.stringify(body) });
28572865 // A GET goes again on a kept-alive connection the server closed meanwhile, as in api().
28582866 const r = await send().catch((e) => (method === "GET" ? sleep(1000).then(send) : Promise.reject(e)));
@@ -2860,23 +2868,57 @@
28602868 assert.ok(r.ok, `${method} ${path} -> ${r.status} ${text}`);
28612869 return JSON.parse(text);
28622870 };
2863- const g = (cwd, args) => gitAsync(cwd, args, server.token, server.log);
2864- const me = await call("GET", "/user");
2865- await call("POST", "/user/repos", { name });
2866- const remote = `${server.base}/${me.login}/${name}.git`;
2871+ const git = (cwd, args) => gitAsync(cwd, args, server.token, server.log, () => `the Worker was at ${server.direct}; wrangler dev now names ${server.workerUrl?.() ?? "(the shared server)"}`);
2872+ const gitFails = async (cwd, args) => {
2873+ try {
2874+ await git(cwd, args);
2875+ } catch (e) {
2876+ return `${e.stdout ?? ""}${e.stderr ?? ""}`;
2877+ }
2878+ assert.fail(`expected git ${args.join(" ")} to fail`);
2879+ };
2880+ // Git goes straight to the Worker: wrangler dev's own proxy drops large pushes and clones streaming
2881+ // through it (see workerUrl), which isn't Diffinery and doesn't exist in production.
2882+ assert.ok(server.direct, `the Worker's own address wasn't found in wrangler dev's debug log`);
2883+ return { ...server, gitBase: server.direct, ok, git, gitFails };
2884+}
2885+const shared = () => client({ base, direct, token, log: devLog });
2886+
2887+/** Runs `fn` with a client of a server of its own, stopped afterwards. */
2888+async function onOwnServer(name, fn, ready = Promise.resolve()) {
2889+ const [server] = await Promise.all([startServer(name), ready]);
2890+ try {
2891+ assert.ok(server.direct && (await fetch(`${server.direct}/healthz`)).ok, `the Worker's own address (${server.direct}) doesn't answer`);
2892+ return await fn(client(server));
2893+ } finally {
2894+ server.stop();
2895+ }
2896+}
2897+
2898+/**
2899+ * A real project's history, pushed whole to a new repository on `server` and cloned back. Pinned
2900+ * commits, so the packs are the same each run; cloned from GitHub without the Diffinery token.
2901+ */
2902+async function pushHistory({ gitBase, ok, git }, name, upstream) {
2903+ const me = await ok("GET", "/user");
2904+ await ok("POST", "/user/repos", { name });
2905+ const remote = `${gitBase}/${me.login}/${name}.git`;
28672906 // Test fixtures in these histories look like credentials (tornado has a test private key): pushed as an admin's bypass.
2868- await g(upstream, ["-c", "http.postBuffer=1048576", "push", "-q", "-o", "allow-secrets=e2e of a public history", remote, "main"]);
2907+ await git(upstream, ["-c", "http.postBuffer=1048576", "push", "-q", "-o", "allow-secrets=e2e of a public history", remote, "main"]);
28692908 const out = join(work, `${name}-clone`);
2870- await g(work, ["clone", "-q", "--bare", remote, out]);
2871- assert.equal((await g(out, ["rev-parse", "main"])).trim(), (await g(upstream, ["rev-parse", "main"])).trim());
2872- assert.equal((await g(out, ["rev-list", "--count", "main"])).trim(), (await g(upstream, ["rev-list", "--count", "main"])).trim());
2873- await g(out, ["fsck", "--full"]);
2909+ await git(work, ["clone", "-q", "--bare", remote, out]);
2910+ assert.equal((await git(out, ["rev-parse", "main"])).trim(), (await git(upstream, ["rev-parse", "main"])).trim());
2911+ assert.equal((await git(out, ["rev-list", "--count", "main"])).trim(), (await git(upstream, ["rev-list", "--count", "main"])).trim());
2912+ await git(out, ["fsck", "--full"]);
28742913 }
28752914
2876-// One at a time, each on a server of its own (the lockfile's on the shared one). Side by side they
2877-// overloaded wrangler dev's proxy (a 500 mid-push or mid-clone in about one run in five) until that
2878-// is proven fixed.
2879-describe("real histories: whole projects, one at a time", { concurrency: false }, () => {
2915+// Two at a time, each on a server of its own (the lockfile's on the shared one): one big push keeps
2916+// one server process busy, so on separate processes they use the runner's other cores; jquery's push
2917+// takes about as long as the other three's together. Not all four: a local server's memory grows to
2918+// 2 to 4 GB during such a push, and four at once ran a runner out of memory (the Workers runtime
2919+// crashed mid-push); two at a time peak about 1 GB above one at a time. Git goes past wrangler dev's
2920+// proxy (see client).
2921+describe("real histories: whole projects, side by side", { concurrency: 2 }, () => {
28802922 for (const [name, url, sha] of [
28812923 // 37 MB: staged in R2 and unpacked in pieces.
28822924 ["jquery", "https://github.com/jquery/jquery", "c2f27864e4ffc2d1409c289fcb9a42b6165dfeaf"],
@@ -2886,15 +2928,8 @@
28862928 ]) {
28872929 test(`real histories: ${name}'s whole history goes in one push and clones back intact`, async () => {
28882930 const upstream = join(work, `${name}-upstream.git`);
2889- const [server] = await Promise.all([
2890- startServer(name),
2891- gitAsync(work, ["clone", "-q", "--bare", "--no-tags", url, upstream], null).then(() => gitAsync(upstream, ["update-ref", "refs/heads/main", sha], null)),
2892- ]);
2893- try {
2894- await pushHistory(server, name, upstream);
2895- } finally {
2896- server.stop();
2897- }
2931+ const download = gitAsync(work, ["clone", "-q", "--bare", "--no-tags", url, upstream], null).then(() => gitAsync(upstream, ["update-ref", "refs/heads/main", sha], null));
2932+ await onOwnServer(name, (c) => pushHistory(c, name, upstream), download);
28982933 });
28992934 }
29002935
@@ -2918,122 +2953,127 @@
29182953 assert.ok(readFileSync(join(upstream, "package-lock.json")).length > 2_500_000);
29192954 const packBytes = Number(execFileSync("bash", ["-c", "git pack-objects --revs --stdout -q <<<main | wc -c"], { cwd: upstream, encoding: "utf8" }).trim());
29202955 assert.ok(packBytes < 16 * 1024 * 1024, `the pack is ${packBytes} bytes`);
2921- await pushHistory({ base, token, log: devLog }, "lockfile", upstream);
2956+ await pushHistory(shared(), "lockfile", upstream);
29222957 });
29232958 });
29242959
2925-test("large files: a 40 MB file is pushed and cloned intact, and push protection still reads big text files", async () => {
2926- const me = await ok("GET", "/user");
2927- await ok("POST", "/user/repos", { name: "bigfile" });
2928- const src = join(work, "bigfile");
2929- mkdirSync(src, { recursive: true });
2930- git(src, ["init", "-q", "-b", "main"]);
2931- // Random bytes don't compress: the push is over 16 MB, so it goes through R2 on its way in.
2932- const data = randomBytes(40 * 1024 * 1024);
2933- writeFileSync(join(src, "big.bin"), data);
2934- git(src, ["add", "-A"]);
2935- git(src, ["commit", "-q", "-m", "a large file"]);
2936- git(src, ["-c", "http.postBuffer=1048576", "push", "-q", `${base}/${me.login}/bigfile.git`, "main"], token, { maxBuffer: 16 * 1024 * 1024 });
2937- const out = join(work, "bigfile-clone");
2938- git(work, ["clone", "-q", `${base}/${me.login}/bigfile.git`, out]);
2939- assert.ok(readFileSync(join(out, "big.bin")).equals(data), "the cloned file is the pushed file");
2940- const wantBlob = git(src, ["rev-parse", "HEAD:big.bin"]).trim();
2941- assert.equal(git(out, ["rev-parse", "HEAD:big.bin"]).trim(), wantBlob);
2942- // A large text file (scanned slice by slice) with a key near its end is still refused.
2943- // diffinery:allow-secret (a fake key)
2944- writeFileSync(join(src, "dump.sql"), `${"-- row\n".repeat(1024 * 1024)}aws_key = AKIAIOSFODNN7EXAMPL3\n`);
2945- git(src, ["add", "-A"]);
2946- git(src, ["commit", "-q", "-m", "a large dump"]);
2947- assert.match(gitFails(src, ["push", "-q", `${base}/${me.login}/bigfile.git`, "main"]), /possible secrets detected/);
2948-});
2960+// Side by side, each on a server of its own, as the real histories above: about as long as the 40 MB
2961+// history's pushes alone instead of the sum.
2962+describe("large pushes: side by side, each on a server of its own", { concurrency: true }, () => {
2963+ test("large files: a 40 MB file is pushed and cloned intact, and push protection still reads big text files", () =>
2964+ onOwnServer("bigfile", async ({ gitBase, ok, git, gitFails }) => {
2965+ const me = await ok("GET", "/user");
2966+ await ok("POST", "/user/repos", { name: "bigfile" });
2967+ const src = join(work, "bigfile");
2968+ mkdirSync(src, { recursive: true });
2969+ await git(src, ["init", "-q", "-b", "main"]);
2970+ // Random bytes don't compress: the push is over 16 MB, so it goes through R2 on its way in.
2971+ const data = randomBytes(40 * 1024 * 1024);
2972+ writeFileSync(join(src, "big.bin"), data);
2973+ await git(src, ["add", "-A"]);
2974+ await git(src, ["commit", "-q", "-m", "a large file"]);
2975+ await git(src, ["-c", "http.postBuffer=1048576", "push", "-q", `${gitBase}/${me.login}/bigfile.git`, "main"]);
2976+ const out = join(work, "bigfile-clone");
2977+ await git(work, ["clone", "-q", `${gitBase}/${me.login}/bigfile.git`, out]);
2978+ assert.ok(readFileSync(join(out, "big.bin")).equals(data), "the cloned file is the pushed file");
2979+ const wantBlob = (await git(src, ["rev-parse", "HEAD:big.bin"])).trim();
2980+ assert.equal((await git(out, ["rev-parse", "HEAD:big.bin"])).trim(), wantBlob);
2981+ // A large text file (scanned slice by slice) with a key near its end is still refused.
2982+ // diffinery:allow-secret (a fake key)
2983+ writeFileSync(join(src, "dump.sql"), `${"-- row\n".repeat(1024 * 1024)}aws_key = AKIAIOSFODNN7EXAMPL3\n`);
2984+ await git(src, ["add", "-A"]);
2985+ await git(src, ["commit", "-q", "-m", "a large dump"]);
2986+ assert.match(await gitFails(src, ["push", "-q", `${gitBase}/${me.login}/bigfile.git`, "main"]), /possible secrets detected/);
2987+ }));
29492988
2950-test("large pushes: three large pushes into one repository at once all go in, and a fetch meanwhile works", async () => {
2951- const me = await ok("GET", "/user");
2952- await ok("POST", "/user/repos", { name: "pushes-at-once" });
2953- // Straight to the Worker: wrangler dev's own proxy, not Diffinery, drops pushes streaming through it at once (see workerUrl).
2954- assert.ok(direct && (await fetch(`${direct}/healthz`)).ok, `the Worker's own address wasn't found in wrangler dev's debug log (${direct})`);
2955- const remote = `${direct}/${me.login}/pushes-at-once.git`;
2956- const src = join(work, "pushes-at-once");
2957- mkdirSync(src, { recursive: true });
2958- git(src, ["init", "-q", "-b", "main"]);
2959- writeFileSync(join(src, "README.md"), "base\n");
2960- git(src, ["add", "-A"]);
2961- git(src, ["commit", "-q", "-m", "base"]);
2962- git(src, ["push", "-q", remote, "main"]);
2963- // Each branch: many medium files over a few commits, so unpacking holds a lot. Two packs stay under
2964- // 16 MB (handed over in memory), one goes over (staged in R2), as when several agents push at once.
2965- const sizes = { a: 180, b: 180, c: 420 };
2966- const tips = {};
2967- for (const [branch, count] of Object.entries(sizes)) {
2968- git(src, ["checkout", "-q", "-b", branch, "main"]);
2969- const files = Array.from({ length: count }, () => randomBytes(48 * 1024).toString("base64"));
2970- mkdirSync(join(src, branch), { recursive: true });
2971- for (let c = 0; c < 3; c++) {
2972- files.forEach((f, i) => writeFileSync(join(src, branch, `f${i}.txt`), `${f}\nrevision ${c}\n${randomBytes(1024).toString("hex")}\n`));
2973- git(src, ["add", "-A"]);
2974- git(src, ["commit", "-q", "-m", `${branch} ${c}`]);
2975- }
2976- tips[branch] = git(src, ["rev-parse", "HEAD"]).trim();
2977- }
2978- git(src, ["checkout", "-q", "main"]);
2979- const push = (branch) => gitAsync(src, ["-c", "http.postBuffer=1048576", "push", "-q", remote, branch]);
2980- const fetchWhile = gitAsync(work, ["clone", "-q", remote, join(work, "pushes-at-once-early")]);
2981- await Promise.all([push("a"), push("b"), push("c"), fetchWhile]);
2982- const branches = (await ok("GET", `/repos/${me.login}/pushes-at-once/branches`)).map((b) => b.name).sort();
2983- assert.deepEqual(branches, ["a", "b", "c", "main"]);
2984- const out = join(work, "pushes-at-once-clone");
2985- git(work, ["clone", "-q", "--mirror", remote, out]);
2986- for (const [branch, tip] of Object.entries(tips)) assert.equal(git(out, ["rev-parse", `refs/heads/${branch}`]).trim(), tip);
2987- git(out, ["fsck", "--full", "--strict"]);
2988-});
2989+ test("large pushes: three large pushes into one repository at once all go in, and a fetch meanwhile works", () =>
2990+ onOwnServer("pushes-at-once", async ({ gitBase, ok, git }) => {
2991+ const me = await ok("GET", "/user");
2992+ await ok("POST", "/user/repos", { name: "pushes-at-once" });
2993+ const remote = `${gitBase}/${me.login}/pushes-at-once.git`;
2994+ const src = join(work, "pushes-at-once");
2995+ mkdirSync(src, { recursive: true });
2996+ await git(src, ["init", "-q", "-b", "main"]);
2997+ writeFileSync(join(src, "README.md"), "base\n");
2998+ await git(src, ["add", "-A"]);
2999+ await git(src, ["commit", "-q", "-m", "base"]);
3000+ await git(src, ["push", "-q", remote, "main"]);
3001+ // Each branch: many medium files over a few commits, so unpacking holds a lot. Two packs stay under
3002+ // 16 MB (handed over in memory), one goes over (staged in R2), as when several agents push at once.
3003+ const sizes = { a: 180, b: 180, c: 420 };
3004+ const tips = {};
3005+ for (const [branch, count] of Object.entries(sizes)) {
3006+ await git(src, ["checkout", "-q", "-b", branch, "main"]);
3007+ const files = Array.from({ length: count }, () => randomBytes(48 * 1024).toString("base64"));
3008+ mkdirSync(join(src, branch), { recursive: true });
3009+ for (let c = 0; c < 3; c++) {
3010+ files.forEach((f, i) => writeFileSync(join(src, branch, `f${i}.txt`), `${f}\nrevision ${c}\n${randomBytes(1024).toString("hex")}\n`));
3011+ await git(src, ["add", "-A"]);
3012+ await git(src, ["commit", "-q", "-m", `${branch} ${c}`]);
3013+ }
3014+ tips[branch] = (await git(src, ["rev-parse", "HEAD"])).trim();
3015+ }
3016+ await git(src, ["checkout", "-q", "main"]);
3017+ const push = (branch) => git(src, ["-c", "http.postBuffer=1048576", "push", "-q", remote, branch]);
3018+ const fetchWhile = git(work, ["clone", "-q", remote, join(work, "pushes-at-once-early")]);
3019+ await Promise.all([push("a"), push("b"), push("c"), fetchWhile]);
3020+ const branches = (await ok("GET", `/repos/${me.login}/pushes-at-once/branches`)).map((b) => b.name).sort();
3021+ assert.deepEqual(branches, ["a", "b", "c", "main"]);
3022+ const out = join(work, "pushes-at-once-clone");
3023+ await git(work, ["clone", "-q", "--mirror", remote, out]);
3024+ for (const [branch, tip] of Object.entries(tips)) assert.equal((await git(out, ["rev-parse", `refs/heads/${branch}`])).trim(), tip);
3025+ await git(out, ["fsck", "--full", "--strict"]);
3026+ }));
29893027
2990-test("large pushes: a 40 MB history of many files goes in one push, read from R2 in ranges, and is still checked", async () => {
2991- const me = await ok("GET", "/user");
2992- await ok("POST", "/user/repos", { name: "bighistory" });
2993- const remote = `${base}/${me.login}/bighistory.git`;
2994- const src = join(work, "bighistory");
2995- mkdirSync(join(src, "files"), { recursive: true });
2996- git(src, ["init", "-q", "-b", "main"]);
2997- // Hundreds of medium files, each changed a little in later commits: a pack of deltas over 16 MB,
2998- // as a fork's pull request heads made one in jekyll/minima's import.
2999- const files = Array.from({ length: 400 }, () => randomBytes(96 * 1024).toString("base64"));
3000- for (let c = 0; c < 6; c++) {
3001- files.forEach((f, i) => writeFileSync(join(src, "files", `f${i}.txt`), `${f}\nrevision ${c} of file ${i}\n${randomBytes(2048).toString("hex")}\n`));
3002- git(src, ["add", "-A"]);
3003- git(src, ["commit", "-q", "-m", `revision ${c}`]);
3004- }
3005- const packBytes = Number(execFileSync("bash", ["-c", "git pack-objects --revs --stdout -q <<<main | wc -c"], { cwd: src, encoding: "utf8" }).trim());
3006- assert.ok(packBytes > 30 * 1024 * 1024, `the pack is ${packBytes} bytes`);
3007- git(src, ["-c", "http.postBuffer=1048576", "push", "-q", remote, "main"], token, { maxBuffer: 16 * 1024 * 1024 });
3008- const out = join(work, "bighistory-clone");
3009- git(work, ["clone", "-q", remote, out]);
3010- assert.equal(git(out, ["rev-parse", "HEAD"]).trim(), git(src, ["rev-parse", "HEAD"]).trim());
3011- git(out, ["fsck", "--full", "--strict"]);
3028+ test("large pushes: a 40 MB history of many files goes in one push, read from R2 in ranges, and is still checked", () =>
3029+ onOwnServer("bighistory", async ({ gitBase, ok, git, gitFails }) => {
3030+ const me = await ok("GET", "/user");
3031+ await ok("POST", "/user/repos", { name: "bighistory" });
3032+ const remote = `${gitBase}/${me.login}/bighistory.git`;
3033+ const src = join(work, "bighistory");
3034+ mkdirSync(join(src, "files"), { recursive: true });
3035+ await git(src, ["init", "-q", "-b", "main"]);
3036+ // Hundreds of medium files, each changed a little in later commits: a pack of deltas over 16 MB,
3037+ // as a fork's pull request heads made one in jekyll/minima's import.
3038+ const files = Array.from({ length: 400 }, () => randomBytes(96 * 1024).toString("base64"));
3039+ for (let c = 0; c < 6; c++) {
3040+ files.forEach((f, i) => writeFileSync(join(src, "files", `f${i}.txt`), `${f}\nrevision ${c} of file ${i}\n${randomBytes(2048).toString("hex")}\n`));
3041+ await git(src, ["add", "-A"]);
3042+ await git(src, ["commit", "-q", "-m", `revision ${c}`]);
3043+ }
3044+ const packBytes = Number(execFileSync("bash", ["-c", "git pack-objects --revs --stdout -q <<<main | wc -c"], { cwd: src, encoding: "utf8" }).trim());
3045+ assert.ok(packBytes > 30 * 1024 * 1024, `the pack is ${packBytes} bytes`);
3046+ await git(src, ["-c", "http.postBuffer=1048576", "push", "-q", remote, "main"]);
3047+ const out = join(work, "bighistory-clone");
3048+ await git(work, ["clone", "-q", remote, out]);
3049+ assert.equal((await git(out, ["rev-parse", "HEAD"])).trim(), (await git(src, ["rev-parse", "HEAD"])).trim());
3050+ await git(out, ["fsck", "--full", "--strict"]);
30123051
3013- // A second large push builds on objects the repository has (a thin pack, its bases read from storage).
3014- files.forEach((f, i) => writeFileSync(join(src, "files", `f${i}.txt`), `${f}\nrevision 6 of file ${i}\n${randomBytes(2048).toString("hex")}\n`));
3015- writeFileSync(join(src, "new.bin"), randomBytes(20 * 1024 * 1024));
3016- git(src, ["add", "-A"]);
3017- git(src, ["commit", "-q", "-m", "revision 6"]);
3018- git(src, ["-c", "http.postBuffer=1048576", "push", "-q", remote, "main"], token, { maxBuffer: 16 * 1024 * 1024 });
3019- git(out, ["pull", "-q", "--ff-only"]);
3020- assert.equal(git(out, ["rev-parse", "HEAD"]).trim(), git(src, ["rev-parse", "HEAD"]).trim());
3021- git(out, ["fsck", "--full", "--strict"]);
3052+ // A second large push builds on objects the repository has (a thin pack, its bases read from storage).
3053+ files.forEach((f, i) => writeFileSync(join(src, "files", `f${i}.txt`), `${f}\nrevision 6 of file ${i}\n${randomBytes(2048).toString("hex")}\n`));
3054+ writeFileSync(join(src, "new.bin"), randomBytes(20 * 1024 * 1024));
3055+ await git(src, ["add", "-A"]);
3056+ await git(src, ["commit", "-q", "-m", "revision 6"]);
3057+ await git(src, ["-c", "http.postBuffer=1048576", "push", "-q", remote, "main"]);
3058+ await git(out, ["pull", "-q", "--ff-only"]);
3059+ assert.equal((await git(out, ["rev-parse", "HEAD"])).trim(), (await git(src, ["rev-parse", "HEAD"])).trim());
3060+ await git(out, ["fsck", "--full", "--strict"]);
30223061
3023- // A large push is checked like a small one: a key in one small file, or an unsafe .gitmodules, refuses all of it.
3024- git(src, ["checkout", "-q", "-b", "leak"]);
3025- writeFileSync(join(src, "more.bin"), randomBytes(20 * 1024 * 1024));
3026- writeFileSync(join(src, "config.txt"), "aws_key = AKIAIOSFODNN7EXAMPL3\n"); // diffinery:allow-secret (a fake key; the pushed file doesn't carry the marker)
3027- git(src, ["add", "-A"]);
3028- git(src, ["commit", "-q", "-m", "a key"]);
3029- assert.match(gitFails(src, ["-c", "http.postBuffer=1048576", "push", "-q", remote, "leak"]), /possible secrets detected/);
3030- git(src, ["checkout", "-q", "main"]);
3031- git(src, ["checkout", "-q", "-b", "modules"]);
3032- writeFileSync(join(src, "more.bin"), randomBytes(20 * 1024 * 1024));
3033- writeFileSync(join(src, ".gitmodules"), '[submodule "x"]\n\tpath = x\n\turl = --upload-pack=touch /tmp/pwned\n');
3034- git(src, ["add", "-A"]);
3035- git(src, ["commit", "-q", "-m", "an unsafe submodule"]);
3036- assert.match(gitFails(src, ["-c", "http.postBuffer=1048576", "push", "-q", remote, "modules"]), /fsck error: \.gitmodules .*starts with '-'/);
3037- const branches = (await ok("GET", `/repos/${me.login}/bighistory/branches`)).map((b) => b.name);
3038- assert.deepEqual(branches, ["main"]);
3062+ // A large push is checked like a small one: a key in one small file, or an unsafe .gitmodules, refuses all of it.
3063+ await git(src, ["checkout", "-q", "-b", "leak"]);
3064+ writeFileSync(join(src, "more.bin"), randomBytes(20 * 1024 * 1024));
3065+ writeFileSync(join(src, "config.txt"), "aws_key = AKIAIOSFODNN7EXAMPL3\n"); // diffinery:allow-secret (a fake key; the pushed file doesn't carry the marker)
3066+ await git(src, ["add", "-A"]);
3067+ await git(src, ["commit", "-q", "-m", "a key"]);
3068+ assert.match(await gitFails(src, ["-c", "http.postBuffer=1048576", "push", "-q", remote, "leak"]), /possible secrets detected/);
3069+ await git(src, ["checkout", "-q", "main"]);
3070+ await git(src, ["checkout", "-q", "-b", "modules"]);
3071+ writeFileSync(join(src, "more.bin"), randomBytes(20 * 1024 * 1024));
3072+ writeFileSync(join(src, ".gitmodules"), '[submodule "x"]\n\tpath = x\n\turl = --upload-pack=touch /tmp/pwned\n');
3073+ await git(src, ["add", "-A"]);
3074+ await git(src, ["commit", "-q", "-m", "an unsafe submodule"]);
3075+ assert.match(await gitFails(src, ["-c", "http.postBuffer=1048576", "push", "-q", remote, "modules"]), /fsck error: \.gitmodules .*starts with '-'/);
3076+ const branches = (await ok("GET", `/repos/${me.login}/bighistory/branches`)).map((b) => b.name);
3077+ assert.deepEqual(branches, ["main"]);
3078+ }));
30393079 });