Packs: count every inflation of a large blob that deltas use as their base (#125)

#120 keeps blobs of 4 MiB or more compressed and inflates them again for
each delta against them, counting only the delta's result toward the
pack's unpack budget. A push of many one-byte deltas against one large
blob made the repository inflate it once per delta, without bound. Each
delta now counts the larger of its result and its large base.


Claude-Session: https://claude.ai/code/session_016yhgAu6oKfqes812Yd4qjo

Co-authored-by: Claude <noreply@anthropic.com>

Canberk Demirci authored 13 days ago· GitHub committed 13 days agosigned1 parent 1631fd0commit 7678c79e65219f7c414666a53114960cd14b931aBrowse files

2 files changed+43−1

modifiedpackages/git-core/src/pack.ts+6−1View file

modifiedpackages/git-core/test/bombs.test.ts+37−0View file

		@@ -151,8 +151,13 @@
151	151	next.push(raw);
152	152	continue;
153	153	}
	154	+ // A large blob kept compressed is inflated again for every delta against it. Count that work
	155	+ // (or the result, if larger), so many tiny deltas against one large blob can't make the server
	156	+ // inflate it without bound.
	157	+ const large = (base as PackedObject).deflated ? ((base as PackedObject).size ?? 0) : 0;
	158	+ if (unpacked + large > maxBytes) throw tooBig();
154	159	const content = applyDelta(base.content, raw.data, maxBytes - unpacked);
155		- unpacked += content.length;
	160	+ unpacked += Math.max(content.length, large);
156	161	await add(raw, base.type, content);
157	162	}
158	163	if (next.length === pending.length) {

		@@ -54,6 +54,43 @@
54	54	expect((await parsePack(p, undefined, { maxBytes: 2 * 1024 * 1024 })).length).toBe(2);
55	55	});
56	56
	57	+ it("counts every inflation of a large blob that tiny deltas use as their base", async () => {
	58	+ // One 2 MiB blob, kept compressed, then 20 one-byte deltas against it by offset.
	59	+ const blob = Buffer.alloc(2 * 1024 * 1024);
	60	+ const header = (type: number, size: number) => {
	61	+ const out: number[] = [];
	62	+ let c = (type << 4) \| (size & 15);
	63	+ size = Math.floor(size / 16);
	64	+ while (size > 0) {
	65	+ out.push(c \| 0x80);
	66	+ c = size & 0x7f;
	67	+ size = Math.floor(size / 128);
	68	+ }
	69	+ out.push(c);
	70	+ return out;
	71	+ };
	72	+ const ofs = (n: number) => {
	73	+ const out = [n & 127];
	74	+ while ((n = Math.floor(n / 128))) out.unshift(0x80 \| (--n & 127));
	75	+ return out;
	76	+ };
	77	+ const parts: number[] = [..."PACK"].map((c) => c.charCodeAt(0)), u32 = (n: number) => parts.push((n >>> 24) & 255, (n >>> 16) & 255, (n >>> 8) & 255, n & 255);
	78	+ u32(2);
	79	+ u32(21);
	80	+ const base = parts.length;
	81	+ parts.push(...header(3, blob.length), ...deflateSync(blob));
	82	+ const delta = Uint8Array.from([...varint(blob.length), ...varint(1), 0x91, 0, 1]);
	83	+ for (let i = 0; i < 20; i++) parts.push(...header(6, delta.length), ...ofs(parts.length - base), ...deflateSync(delta));
	84	+ const body = new Uint8Array(parts);
	85	+ const p = new Uint8Array(body.length + 20);
	86	+ p.set(body);
	87	+ p.set(await sha1(body), body.length);
	88	+ const opts = { maxBytes: 16 * 1024 * 1024, largeObjectBytes: 1024 * 1024 };
	89	+ await expect(parsePack(p, undefined, opts)).rejects.toThrow(/unpacks to more than/);
	90	+ // Kept whole, the blob is inflated once and the same pack fits.
	91	+ expect((await parsePack(p, undefined, { maxBytes: opts.maxBytes })).length).toBe(21);
	92	+ });
	93	+
57	94	it("checks a delta's result size before allocating it", () => {
58	95	const delta = Uint8Array.from([...varint(3), ...varint(2 ** 40), 0x91, 0, 3]);
59	96	expect(() => applyDelta(Uint8Array.from([1, 2, 3]), delta, 1024 * 1024)).toThrow(/larger than/);

codician/diffinerypublic

Packs: count every inflation of a large blob that deltas use as their base (#125)