Diffinery

codician/diffinerypublic

Open-source, agent-first GitHub and GitHub Actions alternative that runs on your own Cloudflare account: git hosting, pull requests, code review and drop-in GitHub Actions YAML on Workers, Durable Objects, R2 and Containers.

A push stores its commits after their files, parents first (#199)

* Store a push's commits after their files, parents first
test-bot authored 1 parent 12fae42commit 83f34e3261c0437fb78c65e88a7cffef848af2b3Browse files
2 files changed+183−6
modifiedapps/diffinery/src/git/push-contents.ts+82−4View file
@@ -1,4 +1,4 @@
1-import { StreamedPack, type ExternalLookup, type ObjectType, type PackSource, type PackedObject } from "@diffinery/git-core";
1+import { StreamedPack, fromUtf8, inflateSized, type ExternalLookup, type ObjectType, type PackSource, type PackedObject } from "@diffinery/git-core";
22 import { INLINE_LIMIT, type RepoObjectStore } from "../repo/objects.js";
33 import { fsckGitmodules, fsckObjects, fsckSize, fsckTree } from "./fsck.js";
44 import { parseReceivePackHead, type ReceiveRequest } from "./protocol.js";
@@ -21,6 +21,59 @@
2121
2222 const scanned = (type: string) => type === "blob" || type === "commit" || type === "tag";
2323
24+/**
25+ * Storing order. A push can be cut off while it is stored (a deploy, a limit), and git then pushes
26+ * again asking which commits the server has (push.negotiate): it won't send what such a commit
27+ * reaches. So files and trees are stored first, and commits after them, each after its parents,
28+ * then tags: a commit that is stored never lacks its files or history.
29+ */
30+const last = (type: string) => type === "commit" || type === "tag";
31+
32+/** A commit's parents, from its header. */
33+export function commitParents(content: Uint8Array): string[] {
34+ const end = headerEnd(content);
35+ const out: string[] = [];
36+ for (const line of fromUtf8(content.subarray(0, end)).split("\n")) {
37+ const m = /^parent ([0-9a-f]{40})$/.exec(line);
38+ if (m) out.push(m[1]!);
39+ }
40+ return out;
41+}
42+
43+/** Where a commit's header ends (the first empty line). */
44+function headerEnd(content: Uint8Array): number {
45+ for (let i = 0; i + 1 < content.length; i++) if (content[i] === 10 && content[i + 1] === 10) return i;
46+ return content.length;
47+}
48+
49+/** `items` reordered so each commit comes after those of its parents that are among them; tags last. */
50+export function historyOrder<T extends { oid: string; type: string }>(items: T[], parents: (t: T) => string[]): T[] {
51+ const commits = new Map(items.filter((t) => t.type === "commit").map((t) => [t.oid, t]));
52+ const out: T[] = [];
53+ const entered = new Set<string>();
54+ for (const start of commits.values()) {
55+ if (entered.has(start.oid)) continue;
56+ entered.add(start.oid);
57+ // Depth first without recursion (a history can be thousands of commits deep).
58+ const stack: Array<{ item: T; ps: string[]; i: number }> = [{ item: start, ps: parents(start), i: 0 }];
59+ while (stack.length) {
60+ const top = stack[stack.length - 1]!;
61+ if (top.i < top.ps.length) {
62+ const p = commits.get(top.ps[top.i++]!);
63+ if (p && !entered.has(p.oid)) {
64+ entered.add(p.oid);
65+ stack.push({ item: p, ps: parents(p), i: 0 });
66+ }
67+ continue;
68+ }
69+ out.push(top.item);
70+ stack.pop();
71+ }
72+ }
73+ for (const t of items) if (t.type !== "commit") out.push(t);
74+ return out;
75+}
76+
2477 /** A pack parsed whole in memory (pushes up to 16 MB). */
2578 export function packedContents(store: RepoObjectStore, objects: PackedObject[], bytes: number): PushContents {
2679 return {
@@ -30,13 +83,15 @@
3083 scan: (marked) => scanForSecrets(objects.filter((o) => scanned(o.type)), marked),
3184 anyNew: () => objects.some((o) => !store.has(o.oid)),
3285 write: async () => {
33- for (const o of objects) {
86+ const put = async (o: PackedObject) => {
3487 if (o.deflated) await store.writeDeflated(o.oid, o.type, o.size!, o.deflated);
3588 // An object that isn't a delta keeps the zlib data it came with: compressing it again cost
3689 // CPU on every push for the same bytes.
3790 else if (o.zlib) await store.writeDeflated(o.oid, o.type, o.content.length, o.zlib, o.content);
3891 else await store.writeKnown(o.oid, o.type, o.content);
39- }
92+ };
93+ for (const o of objects) if (!last(o.type)) await put(o);
94+ for (const o of historyOrder(objects.filter((o) => last(o.type)), (o) => commitParents(o.content))) await put(o);
4095 },
4196 };
4297 }
@@ -135,9 +190,25 @@
135190 },
136191 write: async () => {
137192 const p = pack!;
193+ // Commits and tags wait (see `last`): their parents are noted, and a rebuilt one kept.
194+ const later: Array<{ oid: string; type: string; parents: string[]; slice?: { at: number; length: number; size: number }; content?: Uint8Array }> = [];
195+ const readSlices = async (at: number, length: number) => {
196+ const out = new Uint8Array(length);
197+ let n = 0;
198+ for await (const s of p.slices(at, length)) {
199+ out.set(s, n);
200+ n += s.length;
201+ }
202+ return out;
203+ };
138204 // An object that isn't a delta is stored from its zlib data in the pack, as it is.
139205 for (const x of p.objects()) {
140206 if (!x.deflated || store.has(x.oid)) continue;
207+ if (last(x.type)) {
208+ const content = x.type === "commit" ? inflateSized(await readSlices(x.deflated.at, x.deflated.length), x.size) : null;
209+ later.push({ oid: x.oid, type: x.type, parents: content ? commitParents(content) : [], slice: { ...x.deflated, size: x.size } });
210+ continue;
211+ }
141212 await store.writeDeflatedSlices(x.oid, x.type as ObjectType, x.size, x.deflated.length, () => p.slices(x.deflated!.at, x.deflated!.length));
142213 }
143214 // Deltas are rebuilt exactly as the checks rebuilt them, so no limit can stop the push
@@ -145,8 +216,15 @@
145216 // it for nothing.
146217 await p.replayDeltas(async (entry, content) => {
147218 const oid = p.oidOf(entry);
148- if (!store.has(oid)) await store.writeKnown(oid, p.typeOf(entry), content, false, p.zlibSizeOf(entry));
219+ if (store.has(oid)) return;
220+ const type = p.typeOf(entry);
221+ if (last(type)) later.push({ oid, type, parents: type === "commit" ? commitParents(content) : [], content });
222+ else await store.writeKnown(oid, type, content, false, p.zlibSizeOf(entry));
149223 });
224+ for (const c of historyOrder(later, (c) => c.parents)) {
225+ if (c.content) await store.writeKnown(c.oid, c.type as ObjectType, c.content, false);
226+ else await store.writeDeflatedSlices(c.oid, c.type as ObjectType, c.slice!.size, c.slice!.length, () => p.slices(c.slice!.at, c.slice!.length));
227+ }
150228 },
151229 };
152230 }
modifiedapps/diffinery/test/unit/storage.test.ts+101−2View file
@@ -1,8 +1,12 @@
11 import { describe, expect, it } from "vitest";
22 import { DatabaseSync } from "node:sqlite";
33 import { randomBytes } from "node:crypto";
4-import { deflate, hashObject, inflate, utf8 } from "@diffinery/git-core";
5-import { repositoryBases } from "../../src/git/push-contents.js";
4+import { execFileSync } from "node:child_process";
5+import { mkdtempSync, writeFileSync } from "node:fs";
6+import { tmpdir } from "node:os";
7+import { join } from "node:path";
8+import { deflate, hashObject, inflate, parsePack, utf8 } from "@diffinery/git-core";
9+import { commitParents, historyOrder, memorySource, packedContents, repositoryBases, streamedContents } from "../../src/git/push-contents.js";
610 import { RepoObjectStore } from "../../src/repo/objects.js";
711 import { REPO_SCHEMA, migrateRepoSchema } from "../../src/repo/schema.js";
812
@@ -100,3 +104,98 @@
100104 expect(await lookup("0".repeat(40), cost)).toBeNull();
101105 });
102106 });
107+
108+const git = (cwd: string, ...args: string[]) =>
109+ execFileSync("git", args, { cwd, env: { ...process.env, GIT_AUTHOR_NAME: "T", GIT_AUTHOR_EMAIL: "t@x", GIT_COMMITTER_NAME: "T", GIT_COMMITTER_EMAIL: "t@x" } });
110+
111+/** A history with a merge and a tag, packed as git sends it: newest commits first, files last. */
112+function historyPack(): { pack: Uint8Array; commits: Map<string, string[]>; tag: string } {
113+ const dir = mkdtempSync(join(tmpdir(), "order-"));
114+ git(dir, "init", "-q", "-b", "main");
115+ const text = Array.from({ length: 300 }, (_, i) => `line ${i} of a file long enough for deltas`).join("\n");
116+ writeFileSync(join(dir, "a.txt"), text);
117+ git(dir, "add", "-A");
118+ git(dir, "commit", "-q", "-m", "one");
119+ git(dir, "checkout", "-q", "-b", "side");
120+ writeFileSync(join(dir, "b.txt"), "side\n");
121+ git(dir, "add", "-A");
122+ git(dir, "commit", "-q", "-m", "side");
123+ git(dir, "checkout", "-q", "main");
124+ for (let i = 0; i < 3; i++) {
125+ writeFileSync(join(dir, "a.txt"), text.replace(`line ${i * 50}`, `LINE ${i * 50}`));
126+ git(dir, "commit", "-qam", `main ${i}`);
127+ }
128+ git(dir, "merge", "-q", "--no-edit", "side");
129+ git(dir, "tag", "-a", "v1", "-m", "release");
130+ const commits = new Map<string, string[]>();
131+ for (const line of git(dir, "rev-list", "--parents", "--all").toString().trim().split("\n")) {
132+ const [oid, ...parents] = line.split(" ");
133+ commits.set(oid!, parents);
134+ }
135+ const tag = git(dir, "rev-parse", "v1").toString().trim();
136+ const pack = execFileSync("git", ["pack-objects", "--stdout", "--revs", "--all", "--delta-base-offset"], { cwd: dir, input: "" });
137+ return { pack: new Uint8Array(pack), commits, tag };
138+}
139+
140+/** The oids `store` is asked to store, in order. */
141+function recordWrites(store: RepoObjectStore): string[] {
142+ const order: string[] = [];
143+ for (const m of ["writeKnown", "writeDeflated", "writeDeflatedSlices"] as const) {
144+ const real = (store[m] as (...a: unknown[]) => Promise<void>).bind(store);
145+ (store as unknown as Record<string, unknown>)[m] = (oid: string, ...rest: unknown[]) => (order.push(oid), real(oid, ...rest));
146+ }
147+ return order;
148+}
149+
150+/** Every commit is stored after every file and tree, and after its parents; the tag last. */
151+function expectHistoryOrder(order: string[], commits: Map<string, string[]>, tag: string) {
152+ const at = new Map(order.map((oid, i) => [oid, i]));
153+ expect(new Set(order).size).toBe(order.length);
154+ const firstCommit = Math.min(...[...commits.keys()].map((c) => at.get(c)!));
155+ expect(order.slice(firstCommit).every((oid) => commits.has(oid) || oid === tag)).toBe(true);
156+ for (const [c, parents] of commits) for (const p of parents) expect(at.get(p)!).toBeLessThan(at.get(c)!);
157+ expect(at.get(tag)).toBe(order.length - 1);
158+}
159+
160+describe("the order a push is stored in", () => {
161+ it("reads a commit's parents from its header only", () => {
162+ const a = "a".repeat(40), b = "b".repeat(40), c = "c".repeat(40);
163+ expect(commitParents(utf8(`tree ${c}\nparent ${a}\nparent ${b}\nauthor T <t@x> 1 +0000\n\nparent ${c}\n`))).toEqual([a, b]);
164+ expect(commitParents(utf8(`tree ${c}\nauthor T <t@x> 1 +0000\n\nroot\n`))).toEqual([]);
165+ });
166+
167+ it("puts each commit after its parents however deep the history, and everything else after", () => {
168+ const n = 50_000;
169+ const oid = (i: number) => i.toString(16).padStart(40, "0");
170+ // Newest first, as a pack lists them, with a tree in between and a parent outside the push.
171+ const items = [{ oid: "t".repeat(40), type: "tag" }, ...Array.from({ length: n }, (_, k) => ({ oid: oid(n - k), type: "commit" }))];
172+ const parents = (t: { oid: string }) => { const i = parseInt(t.oid, 16); return [oid(i - 1)]; };
173+ const out = historyOrder(items, parents);
174+ expect(out.length).toBe(n + 1);
175+ expect(out.slice(0, n).map((t) => t.oid)).toEqual(Array.from({ length: n }, (_, k) => oid(k + 1)));
176+ expect(out[n]!.type).toBe("tag");
177+ });
178+
179+ it("stores a pack parsed in memory files and trees first, then commits after their parents, then tags", async () => {
180+ const { pack, commits, tag } = historyPack();
181+ const { store } = setup();
182+ const order = recordWrites(store);
183+ const objects = await parsePack(pack);
184+ // git sends the newest commits first: stored as they come, a cut-off push would leave commits without their files.
185+ expect(objects.findIndex((o) => o.type === "commit")).toBeLessThan(objects.findIndex((o) => o.type === "blob"));
186+ await packedContents(store, objects, pack.length).write();
187+ expect(order.length).toBe(objects.length);
188+ expectHistoryOrder(order, commits, tag);
189+ });
190+
191+ it("stores a pack unpacked in pieces in the same order", async () => {
192+ const { pack, commits, tag } = historyPack();
193+ const { store } = setup();
194+ const contents = await streamedContents(store, memorySource(pack), { largeObjectBytes: 4 * 1024 * 1024 });
195+ expect(contents.fsck()).toBeNull();
196+ const order = recordWrites(store);
197+ await contents.write();
198+ expectHistoryOrder(order, commits, tag);
199+ for (const c of commits.keys()) expect(store.has(c)).toBe(true);
200+ });
201+});