Diffinery

codician/diffinerypublic

Open-source, agent-first GitHub and GitHub Actions alternative that runs on your own Cloudflare account: git hosting, pull requests, code review and drop-in GitHub Actions YAML on Workers, Durable Objects, R2 and Containers.

Pushes over 16 MB are unpacked from R2 in ranges, within bounded memory (#157)

Large pushes are staged in R2 and unpacked in pieces with every object charged against the 80 MiB cap; small pushes that don't fit in memory fall back to the same path. Hashing and storing never copy a whole object, repository bases are size-checked before they are read, chains hold two versions at a time, reads are capped (jQuery's history takes 35), large text is secret-scanned in slices and trees over 8 MiB are refused. Real-history e2e for jQuery, tornado, flask and a lockfile chain.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01AizKXSsengXZRegJxRATkL
Canberk Demirci authored · GitHub committed signed1 parent 102662fcommit b52f9e75dc0f0f2bf3d264d176f2bb7d6e2fd9beBrowse files
24 files changed+2326−139
Files (24)
modified.github/workflows/import-campaign.yml+40−5View file
@@ -36,9 +36,13 @@
3636 type: boolean
3737 default: false
3838 rerun_failed:
39- description: Instead of starting a campaign, re-run the failed jobs of this Diffinery run of an imported repository ("<repository> <run id>") and print how they end
39+ description: Instead of starting a campaign, re-run the failed jobs of this Diffinery run of an imported repository ("<repository> <run id>"), or only the job whose name contains the rest ("<repository> <run id> <job name part>"), and print how they end
4040 type: string
4141 default: ""
42+ push_test:
43+ description: Instead of starting a campaign, push this ref of a GitHub repository ("<repository> <ref>") to a new private Diffinery repository in one push, and clone it back
44+ type: string
45+ default: ""
4246 inspect_run:
4347 description: Instead of starting a campaign, print the jobs of this Diffinery run of an imported repository ("<repository> [<run id>]", the newest run without an id), with each failed job's runner, steps and log
4448 type: string
@@ -49,7 +53,7 @@
4953
5054 # A peek or an inspection runs next to the campaign it looks at; campaigns queue behind each other.
5155 concurrency:
52- group: ${{ (inputs.peek || inputs.inspect_run != '') && 'import-campaign-peek' || 'import-campaign' }}
56+ group: ${{ (inputs.peek || inputs.inspect_run != '' || inputs.push_test != '') && 'import-campaign-peek' || 'import-campaign' }}
5357 cancel-in-progress: false
5458
5559 jobs:
@@ -65,7 +69,7 @@
6569 R: codician-team/diffinery
6670 steps:
6771 - name: Copy IMPORT_GITHUB_TOKEN to Diffinery
68- if: ${{ inputs.follow_run == '' && !inputs.peek && inputs.rerun_failed == '' && inputs.inspect_run == '' }}
72+ if: ${{ inputs.follow_run == '' && !inputs.peek && inputs.rerun_failed == '' && inputs.inspect_run == '' && inputs.push_test == '' }}
6973 env:
7074 IMPORT_GITHUB_TOKEN: ${{ secrets.IMPORT_GITHUB_TOKEN }}
7175 run: |
@@ -85,11 +89,34 @@
8589 PEEK: ${{ inputs.peek }}
8690 RERUN_FAILED: ${{ inputs.rerun_failed }}
8791 INSPECT_RUN: ${{ inputs.inspect_run }}
92+ PUSH_TEST: ${{ inputs.push_test }}
8893 run: |
8994 set -euo pipefail
9095 api() { curl -fsS --retry 3 -X "$1" "$DIFFINERY/api/v3$2" -H "Authorization: Bearer $TOKEN" -H "content-type: application/json" ${3:+--data "$3"}; }
96+ if [ -n "$PUSH_TEST" ]; then
97+ # One large push, as `git push` sends it: no import, no chunks.
98+ read -r src ref <<<"$PUSH_TEST"
99+ [[ "$src" =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ && "$ref" =~ ^refs/[A-Za-z0-9_./-]+$ ]] || { echo "::error::push_test takes \"<owner>/<repository> <ref>\""; exit 1; }
100+ owner=$(api GET /user | jq -r .login)
101+ name="campaign-push-$GITHUB_RUN_ID"
102+ api POST /user/repos "$(jq -nc --arg n "$name" '{name: $n, private: true}')" >/dev/null
103+ git init -q --bare one && cd one
104+ git fetch -q "https://github.com/$src" "+$ref:refs/heads/one-push"
105+ echo "Pack for $src $ref: $(git pack-objects --revs --stdout -q <<<refs/heads/one-push | wc -c) bytes, $(git rev-list --objects refs/heads/one-push | wc -l) objects"
106+ auth="Authorization: Basic $(printf 'x-token:%s' "$TOKEN" | base64 -w0)"
107+ started=$(date +%s)
108+ if git -c "http.extraHeader=$auth" -c http.postBuffer=1048576 push "$DIFFINERY/$owner/$name.git" refs/heads/one-push:refs/heads/main 2>&1 | tail -20; then :; fi
109+ echo "Push took $(( $(date +%s) - started )) s"
110+ want=$(git rev-parse refs/heads/one-push)
111+ got=$(git -c "http.extraHeader=$auth" ls-remote "$DIFFINERY/$owner/$name.git" refs/heads/main | cut -f1)
112+ echo "Diffinery has main at ${got:-nothing}; pushed $want" | tee -a "$GITHUB_STEP_SUMMARY"
113+ [ "$got" = "$want" ] || { echo "::error::the push did not land"; exit 1; }
114+ cd .. && git -c "http.extraHeader=$auth" clone -q --bare "$DIFFINERY/$owner/$name.git" back
115+ git -C back fsck --full --strict && echo "Cloned back and checked: $DIFFINERY/$owner/$name" | tee -a "$GITHUB_STEP_SUMMARY"
116+ exit 0
117+ fi
91118 if [ -n "$RERUN_FAILED$INSPECT_RUN" ]; then
92- read -r repo run <<<"${RERUN_FAILED:-$INSPECT_RUN}"
119+ read -r repo run only <<<"${RERUN_FAILED:-$INSPECT_RUN}"
93120 if [ -n "$INSPECT_RUN" ] && [ -z "$run" ]; then
94121 # No run id: list the repository's latest runs and inspect the newest.
95122 runs=$(api GET "/repos/$repo/actions/runs?per_page=10")
@@ -97,7 +124,15 @@
97124 run=$(jq -r '.workflow_runs[0].id // ""' <<<"$runs")
98125 fi
99126 [[ "$repo" =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ && "$run" =~ ^[A-Za-z0-9_-]+$ ]] || { echo "::error::rerun_failed takes \"<owner>/<repository> <run id>\", inspect_run \"<owner>/<repository> [<run id>]\""; exit 1; }
100- if [ -n "$RERUN_FAILED" ]; then
127+ if [ -n "$RERUN_FAILED" ] && [ -n "$only" ]; then
128+ # One job: the first whose name contains the rest of the input (e.g. "Node.js 16.x").
129+ job=$(api GET "/repos/$repo/actions/runs/$run/jobs?per_page=100" | jq -r --arg n "$only" '[.jobs[] | select(.name | contains($n))][0].id // ""')
130+ [ -n "$job" ] || { echo "::error::no job of run $run has \"$only\" in its name"; exit 1; }
131+ run=$(api POST "/repos/$repo/actions/jobs/$job/rerun" | jq -r --arg run "$run" '.run_id // $run')
132+ echo "Re-running job $job as $(api GET "/repos/$repo/actions/runs/$run" | jq -r .html_url)"
133+ sleep 30
134+ while [ "$(api GET "/repos/$repo/actions/runs/$run" | jq -r .status)" != completed ]; do sleep 30; done
135+ elif [ -n "$RERUN_FAILED" ]; then
101136 # Diffinery answers with the run that re-runs it ({run_id}); GitHub re-runs in place.
102137 run=$(api POST "/repos/$repo/actions/runs/$run/rerun-failed-jobs" | jq -r --arg run "$run" '.run_id // $run')
103138 echo "Re-running the failed jobs as $(api GET "/repos/$repo/actions/runs/$run" | jq -r .html_url)"
modifiedapps/diffinery/docs/reference/limits.md+4−2View file
@@ -134,8 +134,10 @@
134134 | Push size | 100 MB per push | The whole push request after decompression. A larger push is rejected with `push exceeds 100 MB; split it into smaller pushes`. Push in several steps (for example a range of commits at a time). |
135135 | Fetch request size | 16 MB | The request a client sends to fetch or clone (its wants and haves), after decompression. A larger one is refused with `413 fetch request exceeds 16 MB`. |
136136 | File size in a push | Up to the push size | A single file (blob) can be as large as fits in one push (GitHub allows files up to 100 MB). Files of 4 MiB or more are checked and stored from their compressed bytes, so they never need to fit in memory inflated. |
137-| Unpacked size of a push | 512 MiB | What a push's pack inflates to, deltas resolved. A pack that unpacks to more is refused, however small it is on the wire. |
138-| Memory to unpack a push | 80 MiB | What unpacking a push holds at once: its files under 4 MiB inflated, and about 768 bytes for each object. A push that needs more (for example well over 100,000 objects, or many files under 4 MiB that compress well) is refused with `pack needs more than 80 MiB of memory to unpack; split it into smaller pushes`. Deltas are resolved in one pass, however the pack orders them. |
137+| Unpacked size of a push | 512 MiB | What a push's pack inflates to, deltas resolved. A pack that unpacks to more is refused with `pack unpacks to more than 512 MiB; split it into smaller pushes`, however small it is on the wire: a long history of one large file can, since each version counts whole. |
138+| Tree size in a push | 8 MiB | One directory's listing (a tree object). A larger one is refused with `fsck error: tree <id>: larger than 8 MiB`. 100,000 files in one directory come to about 4 MiB. |
139+| Objects in a push unpacked in pieces | 1,000,000 | A push over 16 MB is unpacked from storage in pieces, and so is a smaller one whose objects don't all fit in memory at once (a long history, such as flask's or tornado's). Inflating its objects again to check and store them may take at most four times the unpacked size limit, and reading it from storage at most 500 requests, counting bases read from the repository and each object stored on its own (jQuery's 37 MB history takes about 50). Past any of these, it is refused with a message that ends `split it into smaller pushes`. Its deltas count toward the storage quota at their full size. |
140+| Memory to unpack a push | 80 MiB | What unpacking a push holds at once. Up to 16 MB: its files under 4 MiB inflated, and about 768 bytes for each object. Unpacked in pieces: 24 MiB of caches and scratch, up to 16 MiB of deltas kept to rebuild them, each object while it is in use (a file, a delta's base and its result, a base read from the repository), about 128 bytes for each object, and 640 more for each delta named by its base's id rather than its place in the pack (git sends those only for bases the repository already has). Two versions of a file over about 25 MB, which git stores as a delta, need more; a long history of one file holds two versions at a time. A base from the repository is checked before it is read. A push that needs more is refused with `pack needs more than 80 MiB of memory to unpack; split it into smaller pushes`. Deltas are resolved in one pass, however the pack orders them. |
139141 | Git protocol | Version 2 for fetch and clone | Clients older than git 2.26 must run `git config --global protocol.version 2`, or the server answers `Diffinery requires Git protocol v2`. |
140142 | Secret scanning coverage | Every new text file | Push protection scans every new text file in a push, whatever its size. The rejection message lists up to 10 findings. |
141143 | Linear history check | 1,000 new commits | A protected branch with "require linear history" checks up to 1,000 new commits in a push for merge commits. |
modifiedapps/diffinery/docs/repositories/git.md+4−4View file
@@ -217,7 +217,7 @@
217217
218218 1. **Authorization.** You must send a token: a push without credentials gets HTTP 401 (`authentication required`) with a `WWW-Authenticate` challenge, for public and private repositories alike, which makes Git ask for credentials and retry. Your account needs write access, your token the `write` or `admin` scope, and the repository must not be archived. Otherwise the server answers HTTP 403 (`write access required` when your role only allows reading, `this token needs the write scope` for a `read`-scoped token, or `repository is archived (read-only)`), or HTTP 404 if the account can't see the repository at all. Git shows these as `Authentication failed` or `The requested URL returned error: 403`.
219219 2. **Size check.** A push larger than 100 MB (after decompression), or one whose pack unpacks to more than 512 MiB, is refused. Git shows `remote unpack failed: push exceeds 100 MB; split it into smaller pushes`, or an HTTP error if the request is stopped before it reaches Diffinery.
220-3. **Unpacking.** The pack is parsed and every object checked. Unpacking may hold at most 80 MiB at once; a push that needs more is refused with `pack needs more than 80 MiB of memory to unpack; split it into smaller pushes`.
220+3. **Unpacking.** The pack is parsed and every object checked. A push over 16 MB is staged in storage as it arrives and unpacked from there in pieces, one object at a time, so its memory use depends on its largest files rather than on its size. A smaller push whose objects don't all fit in memory at once, such as a long history, is unpacked in pieces too. Either way, unpacking may hold at most 80 MiB at once; a push that needs more is refused with `pack needs more than 80 MiB of memory to unpack; split it into smaller pushes`.
221221 4. **Push protection.** New text files are scanned for credentials such as AWS keys, GitHub and Diffinery tokens, Slack and Stripe keys, Google, Anthropic and OpenAI API keys, and private keys. If any are found, the whole push is rejected, nothing from it is stored, and the output lists each finding:
222222
223223 ```text
@@ -337,10 +337,10 @@
337337 | Git asks for a username and password when you push to a public repository you cloned without credentials | Expected: anonymous pushes get `401`, so Git asks for credentials. | Enter any username and your token as the password, or set up a credential helper with `diffinery auth setup-git`. |
338338 | `The requested URL returned error: 403` on push from a workflow | The job's token has no `contents: write` (`Resource not accessible by integration`). | Add `permissions: contents: write` to the workflow or job, or have an admin change the repository's default workflow permissions. |
339339 | `[remote rejected] ... (refusing to allow a GitHub App to create or update workflow ...)` | A workflow's job token tried to push a change to a workflow file. | Push workflow changes with a personal or agent token instead. |
340-| `remote unpack failed: the push needs more memory than one push may use; split it into smaller pushes (...)` | Unpacking or storing this push ran out of the memory one push may use, even though it is under the size limits. Nothing from the push is kept. | Push history in steps, oldest first, as for the 100 MB limit below. The GitHub importer does this by itself. |
341-| `remote unpack failed: pack needs more than 80 MiB of memory to unpack; split it into smaller pushes` | Unpacking would hold more than 80 MiB at once: files under 4 MiB inflated, plus about 768 bytes per object. Very many objects (well over 100,000) or many small files that compress well hit it before the size limits. See [Limits](/docs/reference/limits). | Push history in steps, oldest first, as for the 100 MB limit below. The GitHub importer does this by itself. |
340+| `remote unpack failed: the push needs more memory than one push may use; split it into smaller pushes (...)` | Unpacking or storing this push ran out of the memory one push may use, even though it is under the size limits. Pushes over 16 MB are unpacked in pieces, so this now comes from a single very large file, or a delta against one. Nothing from the push is kept. | Push history in steps, oldest first, as for the 100 MB limit below. The GitHub importer does this by itself. |
341+| `remote unpack failed: pack needs more than 80 MiB of memory to unpack; split it into smaller pushes` | Unpacking would hold more than 80 MiB at once: files under 4 MiB inflated, plus about 768 bytes per object. Very many objects (well over 100,000) or many small files that compress well hit it before the size limits. A push unpacked in pieces (over 16 MB, or one that didn't fit) counts its caches, the files in use and about 128 bytes per object instead, so there it comes from very large files. See [Limits](/docs/reference/limits). | Push history in steps, oldest first, as for the 100 MB limit below. The GitHub importer does this by itself. |
342342 | `remote unpack failed: push exceeds 100 MB; split it into smaller pushes` | The pack for this push is larger than 100 MB. | Push history in steps, oldest first, for example `git push origin <older-commit>:refs/heads/main`, then later commits. Move large binaries to Git LFS. |
343-| `remote: Push refused: tree <id>: duplicate entry '<name>'` (or another reason), `remote unpack failed: fsck error: ...` and `[remote rejected] ... (fsck failed)` | A tree or `.gitmodules` file in the push fails git's `fsck` checks: an entry that is empty, `.`, `..`, a spelling of `.git`, contains `/` or appears twice, a `.gitmodules` that is a symbolic link, or a submodule whose name has `..` or whose `url` or `path` starts with `-`. Nothing from the push is kept. | Run `git fsck` locally to find the object, fix the tree or `.gitmodules` in a new commit (or rewrite the commit that added it), and push again. |
343+| `remote: Push refused: tree <id>: duplicate entry '<name>'` (or another reason), `remote unpack failed: fsck error: ...` and `[remote rejected] ... (fsck failed)` | A tree or `.gitmodules` file in the push fails git's `fsck` checks: an entry that is empty, `.`, `..`, a spelling of `.git`, contains `/` or appears twice, a tree over 8 MiB, a `.gitmodules` that is a symbolic link, or a submodule whose name has `..` or whose `url` or `path` starts with `-`. Nothing from the push is kept. | Run `git fsck` locally to find the object, fix the tree or `.gitmodules` in a new commit (or rewrite the commit that added it), and push again. |
344344 | `[remote rejected] ... (protected branch: required check '<name>' has not run on <sha>; ...)` | The branch requires status checks, and the commit you pushed hasn't passed them. | Push to another branch and open a pull request, or push a commit that already passed the checks. |
345345 | `[remote rejected] ... (secret detected (push protection))` | A pushed file contains something that looks like a credential. | Remove it from every commit in the push (`git commit --amend`, or `git rebase -i`), rotate the credential, and push again. See [Push protection](/docs/security/push-protection). |
346346 | `[remote rejected] ... (fetch first (stale info))` | The branch moved since you fetched. | `git pull --rebase` (or fetch and merge), then push. |
modifiedapps/diffinery/src/git/fsck.ts+47−25View file
@@ -1,4 +1,4 @@
1-import { fromUtf8, parseTree } from "@diffinery/git-core";
1+import { fromUtf8, toHex } from "@diffinery/git-core";
22 import { isDotGitName } from "../lib/tree-names.js";
33
44 /**
@@ -54,35 +54,57 @@
5454 const gitmodules = new Set<string>();
5555 for (const o of objects) {
5656 if (o.type !== "tree") continue;
57- let entries;
58- try {
59- entries = parseTree(o.content);
60- } catch {
61- return `tree ${o.oid}: malformed tree`;
62- }
63- const seen = new Set<string>();
64- for (const e of entries) {
65- const why =
66- !e.name ? "empty name"
67- : e.name === "." || e.name === ".." ? `'${e.name}' is not a valid name`
68- : e.name.includes("/") ? `'${e.name}' contains '/'`
69- : isDotGitName(e.name) ? `'${e.name}' is reserved ('.git')`
70- : seen.has(e.name) ? `duplicate entry '${e.name}'`
71- : null;
72- if (why) return `tree ${o.oid}: ${why}`;
73- seen.add(e.name);
74- if (isDotGitmodulesName(e.name)) {
75- if (e.mode === "120000") return `tree ${o.oid}: '.gitmodules' is a symbolic link`;
76- gitmodules.add(e.oid);
77- }
78- }
57+ const why = fsckTree(o.oid, o.content, gitmodules);
58+ if (why) return why;
7959 }
8060 if (gitmodules.size) {
8161 for (const o of objects) {
8262 if (o.type !== "blob" || !gitmodules.has(o.oid)) continue;
83- const why = unsafeGitmodules(fromUtf8(o.content));
84- if (why) return `.gitmodules ${o.oid}: ${why}`;
63+ const why = fsckGitmodules(o.oid, o.content);
64+ if (why) return why;
8565 }
8666 }
8767 return null;
8868 }
69+
70+/**
71+ * The largest tree a push may hold. Checking one keeps every name in it, at several times its
72+ * size; real trees are rarely more than a few MB (100,000 files in one directory is about 4 MB).
73+ */
74+export const MAX_TREE_BYTES = 8 * 1024 * 1024;
75+
76+/** Why one pushed tree can't be accepted, or null. The blobs it names `.gitmodules` go into `gitmodules`, for `fsckGitmodules`. */
77+export function fsckTree(oid: string, content: Uint8Array, gitmodules: Set<string>): string | null {
78+ if (content.length > MAX_TREE_BYTES) return `tree ${oid}: larger than ${MAX_TREE_BYTES / (1024 * 1024)} MiB`;
79+ // Read in place: only the names are kept, not an object for each entry.
80+ const seen = new Set<string>();
81+ for (let i = 0; i < content.length; ) {
82+ const sp = content.indexOf(0x20, i);
83+ const nul = sp < 0 ? -1 : content.indexOf(0x00, sp);
84+ if (sp < 0 || nul < 0 || nul + 21 > content.length) return `tree ${oid}: malformed tree`;
85+ const name = fromUtf8(content.subarray(sp + 1, nul));
86+ const i0 = i;
87+ const at = nul + 1;
88+ i = nul + 21;
89+ const why =
90+ !name ? "empty name"
91+ : name === "." || name === ".." ? `'${name}' is not a valid name`
92+ : name.includes("/") ? `'${name}' contains '/'`
93+ : isDotGitName(name) ? `'${name}' is reserved ('.git')`
94+ : seen.has(name) ? `duplicate entry '${name}'`
95+ : null;
96+ if (why) return `tree ${oid}: ${why}`;
97+ seen.add(name);
98+ if (isDotGitmodulesName(name)) {
99+ if (fromUtf8(content.subarray(i0, sp)) === "120000") return `tree ${oid}: '.gitmodules' is a symbolic link`;
100+ gitmodules.add(toHex(content.subarray(at, at + 20)));
101+ }
102+ }
103+ return null;
104+}
105+
106+/** Why a pushed blob that a tree names `.gitmodules` can't be accepted, or null. */
107+export function fsckGitmodules(oid: string, content: Uint8Array): string | null {
108+ const why = unsafeGitmodules(fromUtf8(content));
109+ return why ? `.gitmodules ${oid}: ${why}` : null;
110+}
modifiedapps/diffinery/src/git/protocol.ts+28−0View file
@@ -255,6 +255,34 @@
255255 return { commands, capabilities, pushOptions, pack };
256256 }
257257
258+/**
259+ * `parseReceivePack` for the start of a push body read on its own (a large push staged in R2): the
260+ * commands and push options, and the offset where the pack starts, or null when `head` ends before
261+ * them, so the caller reads more.
262+ */
263+export function parseReceivePackHead(head: Uint8Array): { req: ReceiveRequest; packOffset: number } | null {
264+ const complete = (start: number) => {
265+ try {
266+ const r = readPkts(head, start, true);
267+ return r.pkts.at(-1)?.type === "flush" ? r.offset : null;
268+ } catch (e) {
269+ if (/truncated pkt-line/.test((e as Error).message)) return null;
270+ throw e;
271+ }
272+ };
273+ const commandsEnd = complete(0);
274+ if (commandsEnd === null) return null;
275+ const req = parseReceivePack(head.subarray(0, commandsEnd));
276+ let packOffset = commandsEnd;
277+ if (req.capabilities.has("push-options")) {
278+ const optionsEnd = complete(commandsEnd);
279+ if (optionsEnd === null) return null;
280+ for (const p of readPkts(head, commandsEnd, true).pkts) if (p.type === "data") req.pushOptions.push(pktText(p));
281+ packOffset = optionsEnd;
282+ }
283+ return { req, packOffset };
284+}
285+
258286 export interface RefResult {
259287 ref: string;
260288 ok: boolean;
addedapps/diffinery/src/git/push-contents.ts+150−0View file
@@ -1,0 +1,150 @@
1+import { StreamedPack, type ExternalLookup, type ObjectType, type PackSource, type PackedObject } from "@diffinery/git-core";
2+import { INLINE_LIMIT, type RepoObjectStore } from "../repo/objects.js";
3+import { fsckGitmodules, fsckObjects, fsckTree } from "./fsck.js";
4+import { parseReceivePackHead, type ReceiveRequest } from "./protocol.js";
5+import { scanFeed, scanForSecrets, type SecretHit } from "./secret-scan.js";
6+
7+/**
8+ * What a push brings, as receive-pack checks and stores it: `fsck`, then the secret scan, then
9+ * `write` once every check passed (nothing is stored before, like git's quarantine).
10+ */
11+export interface PushContents {
12+ /** The pack's size, for the storage quota. */
13+ bytes: number;
14+ fsck(): string | null;
15+ /** Credentials found; opted-out ones go to `marked`. */
16+ scan(marked: SecretHit[]): SecretHit[];
17+ /** Whether the push stores anything the repository doesn't have. */
18+ anyNew(): boolean;
19+ write(): Promise<void>;
20+}
21+
22+const scanned = (type: string) => type === "blob" || type === "commit" || type === "tag";
23+
24+/** A pack parsed whole in memory (pushes up to 16 MB). */
25+export function packedContents(store: RepoObjectStore, objects: PackedObject[], bytes: number): PushContents {
26+ return {
27+ bytes,
28+ fsck: () => fsckObjects(objects),
29+ // The objects themselves, not copies: a large blob is scanned from its compressed bytes.
30+ scan: (marked) => scanForSecrets(objects.filter((o) => scanned(o.type)), marked),
31+ anyNew: () => objects.some((o) => !store.has(o.oid)),
32+ write: async () => {
33+ for (const o of objects) {
34+ if (o.deflated) await store.writeDeflated(o.oid, o.type, o.size!, o.deflated);
35+ else await store.writeKnown(o.oid, o.type, o.content);
36+ }
37+ },
38+ };
39+}
40+
41+/**
42+ * Bases a thin pack's deltas name, from the repository: what reading one takes is checked first,
43+ * so a small delta against a huge stored object is refused before it is inflated.
44+ */
45+export function repositoryBases(store: RepoObjectStore): ExternalLookup {
46+ return async (oid, maxBytes) => {
47+ const cost = store.readCost(oid);
48+ if (cost === null) return null;
49+ return cost > maxBytes ? cost : store.read(oid);
50+ };
51+}
52+
53+/** A pack already in memory, as a source to unpack in pieces. */
54+export function memorySource(pack: Uint8Array): PackSource {
55+ return { size: pack.length, read: async (offset, length) => pack.subarray(offset, offset + length) };
56+}
57+
58+/** Bytes of an R2 object from `start` on, read in ranges. */
59+function r2Source(bucket: R2Bucket, key: string, start: number, size: number): PackSource {
60+ return {
61+ size,
62+ read: async (offset, length) => {
63+ const o = await bucket.get(key, { range: { offset: start + offset, length } });
64+ if (!o) throw new Error("the push's data is missing; push again");
65+ return new Uint8Array(await o.arrayBuffer());
66+ },
67+ };
68+}
69+
70+/** Where a push staged in R2 has its commands, push options and pack. */
71+export async function readStagedHead(bucket: R2Bucket, key: string, size: number): Promise<{ req: ReceiveRequest; packOffset: number }> {
72+ // Commands for many refs can run long; read more until they end.
73+ for (let n = 64 * 1024; ; n *= 4) {
74+ const o = await bucket.get(key, { range: { offset: 0, length: Math.min(n, size) } });
75+ if (!o) throw new Error("the push's data is missing; push again");
76+ const head = parseReceivePackHead(new Uint8Array(await o.arrayBuffer()));
77+ if (head) return head;
78+ if (n >= size) throw new Error("truncated push: its commands never end");
79+ if (n >= 16 * 1024 * 1024) throw new Error("the push names too many refs; split it into smaller pushes");
80+ }
81+}
82+
83+class FsckRefusal extends Error {}
84+
85+/**
86+ * A pack staged in R2, unpacked without holding it whole (see StreamedPack): every object is
87+ * checked as it is resolved, and read again from R2 to be stored.
88+ */
89+export function stagedContents(store: RepoObjectStore, bucket: R2Bucket, key: string, packOffset: number, packBytes: number, o: { largeObjectBytes: number }): Promise<PushContents> {
90+ return streamedContents(store, r2Source(bucket, key, packOffset, packBytes), o);
91+}
92+
93+/** A pack unpacked from `src` in pieces, one object at a time (see StreamedPack). */
94+export async function streamedContents(store: RepoObjectStore, src: PackSource, o: { largeObjectBytes: number }): Promise<PushContents> {
95+ const packBytes = src.size;
96+ const hits: SecretHit[] = [];
97+ const opted: SecretHit[] = [];
98+ const gitmodules = new Set<string>();
99+ let refusal: string | null = null;
100+ let pack: StreamedPack | null = null;
101+ try {
102+ const p = (pack = await StreamedPack.open(src, { external: repositoryBases(store), largeObjectBytes: o.largeObjectBytes, writesOver: INLINE_LIMIT }, async (obj, content, feed) => {
103+ if (obj.type === "tree") {
104+ const why = fsckTree(obj.oid, content!, gitmodules);
105+ if (why) throw new FsckRefusal(why);
106+ } else if (scanned(obj.type)) {
107+ if (content) scanForSecrets([{ oid: obj.oid, content }], opted).forEach((h) => hits.push(h));
108+ else await scanFeed(obj.oid, feed!, hits, opted);
109+ }
110+ }));
111+ // Like fsckObjects: a `.gitmodules` blob in this push that a pushed tree names is checked too.
112+ for (const [oid, entry] of p.entriesOf(gitmodules)) {
113+ const why = fsckGitmodules(oid, await p.content(entry));
114+ if (why) throw new FsckRefusal(why);
115+ }
116+ } catch (e) {
117+ if (!(e instanceof FsckRefusal)) throw e;
118+ refusal = e.message;
119+ }
120+ const refused = refusal;
121+ const objects = () => (pack ? pack.objects() : []);
122+ return {
123+ // What storing adds, at most: a delta is stored compressed on its own, not as in the pack.
124+ bytes: pack ? (pack as StreamedPack).storedBytes : packBytes,
125+ fsck: () => refused,
126+ scan: (marked) => {
127+ marked.push(...opted);
128+ return hits;
129+ },
130+ anyNew: () => {
131+ for (const x of objects()) if (!store.has(x.oid)) return true;
132+ return false;
133+ },
134+ write: async () => {
135+ const p = pack!;
136+ // An object that isn't a delta is stored from its zlib data in the pack, as it is.
137+ for (const x of p.objects()) {
138+ if (!x.deflated || store.has(x.oid)) continue;
139+ await store.writeDeflatedSlices(x.oid, x.type as ObjectType, x.size, x.deflated.length, () => p.slices(x.deflated!.at, x.deflated!.length));
140+ }
141+ // Deltas are rebuilt exactly as the checks rebuilt them, so no limit can stop the push
142+ // halfway through being stored. Not kept in the store's memory cache: the push would fill
143+ // it for nothing.
144+ await p.replayDeltas(async (entry, content) => {
145+ const oid = p.oidOf(entry);
146+ if (!store.has(oid)) await store.writeKnown(oid, p.typeOf(entry), content, false, p.zlibSizeOf(entry));
147+ });
148+ },
149+ };
150+}
modifiedapps/diffinery/src/git/secret-scan.ts+43−16View file
@@ -40,6 +40,13 @@
4040 scanDeflated(b.oid, b.deflated, hits, marked);
4141 continue;
4242 }
43+ // Decoded whole, a large file would be a string of up to twice its size: scanned in slices instead.
44+ if (b.content.length > SCAN_WHOLE_MAX) {
45+ const s = chunkScanner(b.oid);
46+ for (let at = 0; at < b.content.length; at += SCAN_SLICE) s.push(b.content.subarray(at, at + SCAN_SLICE));
47+ s.end(hits, marked);
48+ continue;
49+ }
4350 const text = textOf(b.content);
4451 if (text === null) continue;
4552 const out = /diffinery:allow-secret/.test(text) ? marked : hits;
@@ -51,12 +58,30 @@
5158 /** Carried from one slice of a large blob to the next, longer than any pattern can match. */
5259 const OVERLAP = 1024;
5360
61+/** Content larger than this is decoded and scanned a slice at a time. */
62+const SCAN_WHOLE_MAX = 1024 * 1024;
63+const SCAN_SLICE = 256 * 1024;
64+
5465 /**
5566 * `scanForSecrets` for a large blob kept compressed: it is inflated and decoded slice by slice, never
5667 * whole, with the same rules (binary when it has a NUL byte, unless it starts with a UTF-16 byte
5768 * order mark; `diffinery:allow-secret` anywhere moves its findings to `marked`).
5869 */
5970 function scanDeflated(oid: string, deflated: Uint8Array, hits: SecretHit[], marked: SecretHit[]): void {
71+ const s = chunkScanner(oid);
72+ inflateEach(deflated, 0, s.push);
73+ s.end(hits, marked);
74+}
75+
76+/** A large blob's content handed over in slices (a pushed pack read in ranges). */
77+export async function scanFeed(oid: string, feed: (onChunk: (chunk: Uint8Array) => void) => Promise<void>, hits: SecretHit[], marked: SecretHit[]): Promise<void> {
78+ const s = chunkScanner(oid);
79+ await feed(s.push);
80+ s.end(hits, marked);
81+}
82+
83+/** The rules of `scanForSecrets`, for content that arrives in slices. */
84+function chunkScanner(oid: string): { push: (chunk: Uint8Array) => void; end: (hits: SecretHit[], marked: SecretHit[]) => void } {
6085 const found = new Set<string>();
6186 let decoder: TextDecoder | null = null;
6287 let utf8 = true;
@@ -69,22 +94,24 @@
6994 for (const p of PATTERNS) if (p.re.test(window)) found.add(p.kind);
7095 tail = window.slice(-OVERLAP);
7196 };
72- inflateEach(deflated, 0, (chunk) => {
73- if (binary) return;
74- if (!decoder) {
75- const bom = chunk[0] === 0xff && chunk[1] === 0xfe ? "utf-16le" : chunk[0] === 0xfe && chunk[1] === 0xff ? "utf-16be" : null;
76- utf8 = !bom;
77- decoder = new TextDecoder(bom ?? "utf-8");
78- }
79- if (utf8 && chunk.includes(0)) binary = true;
80- else scan(decoder.decode(chunk, { stream: true }));
81- });
82- // TypeScript can't see the assignment inside the callback.
83- const last = decoder as TextDecoder | null;
84- if (binary) return;
85- if (last) scan(last.decode());
86- const out = allowed ? marked : hits;
87- for (const p of PATTERNS) if (found.has(p.kind)) out.push({ oid, kind: p.kind });
97+ return {
98+ push: (chunk) => {
99+ if (binary) return;
100+ if (!decoder) {
101+ const bom = chunk[0] === 0xff && chunk[1] === 0xfe ? "utf-16le" : chunk[0] === 0xfe && chunk[1] === 0xff ? "utf-16be" : null;
102+ utf8 = !bom;
103+ decoder = new TextDecoder(bom ?? "utf-8");
104+ }
105+ if (utf8 && chunk.includes(0)) binary = true;
106+ else scan(decoder.decode(chunk, { stream: true }));
107+ },
108+ end: (hits, marked) => {
109+ if (binary) return;
110+ if (decoder) scan(decoder.decode());
111+ const out = allowed ? marked : hits;
112+ for (const p of PATTERNS) if (found.has(p.kind)) out.push({ oid, kind: p.kind });
113+ },
114+ };
88115 }
89116
90117 /** The reason given with `git push -o allow-secrets=<reason>`, if any. */
modifiedapps/diffinery/src/repo/RepoDO.ts+45−39View file
@@ -26,6 +26,7 @@
2626 readCommit,
2727 readTree,
2828 resolvePath,
29+ TooMuchMemory,
2930 serializeCommit,
3031 walkCommits,
3132 type Commit,
@@ -42,11 +43,12 @@
4243 handleUploadPackV2,
4344 parseReceivePack,
4445 reportStatus,
46+ type ReceiveRequest,
4547 unpackFailure,
4648 type RefEntry,
4749 type RefResult,
4850 } from "../git/protocol.js";
49-import { fsckObjects } from "../git/fsck.js";
51+import { memorySource, packedContents, readStagedHead, repositoryBases, stagedContents, streamedContents, type PushContents } from "../git/push-contents.js";
5052 import { scanForSecrets, secretBypassReason, type SecretHit } from "../git/secret-scan.js";
5153 import { GHOST, HttpError, isValidRefName, now } from "../lib/util.js";
5254 import { R2Sink } from "../lib/r2-sink.js";
@@ -547,38 +549,36 @@
547549 return readable;
548550 }
549551
550- /** A push the Worker staged in R2, read into one buffer of its exact size (and then deleted). */
551- private async readSpilledPush(b: SpilledBody): Promise<Uint8Array> {
552+ /** `moved`: the push came in under an old name (a transfer or rename), so say where the repository lives now, like GitHub. */
553+ async gitReceivePack(input: Uint8Array | SpilledBody, caller: Caller, o: { moved?: boolean } = {}): Promise<Uint8Array> {
554+ if (input instanceof Uint8Array) return this.receivePack(input, caller, o);
552555 try {
553- const obj = await this.env.BUCKET.get(b.spilled);
554- if (!obj) throw new Error("the push's data is missing; push again");
555- const out = new Uint8Array(obj.size);
556- let at = 0;
557- const reader = obj.body.getReader();
558- for (;;) {
559- const { done, value } = await reader.read();
560- if (done) break;
561- if (at + value.length > out.length) throw new Error("the push's data is longer than announced; push again");
562- out.set(value, at);
563- at += value.length;
564- }
565- if (at !== out.length) throw new Error("the push's data is shorter than announced; push again");
566- return out;
556+ return await this.receivePack(input, caller, o);
567557 } finally {
568- await this.env.BUCKET.delete(b.spilled).catch(() => {});
558+ await this.env.BUCKET.delete(input.spilled).catch(() => {});
569559 }
570560 }
571561
572- /** `moved`: the push came in under an old name (a transfer or rename), so say where the repository lives now, like GitHub. */
573- async gitReceivePack(input: Uint8Array | SpilledBody, caller: Caller, o: { moved?: boolean } = {}): Promise<Uint8Array> {
562+ private async receivePack(input: Uint8Array | SpilledBody, caller: Caller, o: { moved?: boolean }): Promise<Uint8Array> {
574563 const size = input instanceof Uint8Array ? input.length : input.size;
575564 if (size > MAX_PUSH_BYTES) {
576- if (!(input instanceof Uint8Array)) await this.env.BUCKET.delete(input.spilled).catch(() => {});
577565 const req = { commands: [], capabilities: new Set<string>(), pushOptions: [], pack: null };
578566 return reportStatus(req, "push exceeds 100 MB; split it into smaller pushes", []);
579567 }
580- const body = input instanceof Uint8Array ? input : await this.readSpilledPush(input);
581- const req = parseReceivePack(body);
568+ // A push the Worker staged in R2 (over 16 MB) is read in ranges, never whole: its objects are
569+ // unpacked one at a time (git/push-contents.ts), so its size doesn't decide the memory it takes.
570+ let req: ReceiveRequest;
571+ let packAt = 0;
572+ if (input instanceof Uint8Array) req = parseReceivePack(input);
573+ else {
574+ try {
575+ const head = await readStagedHead(this.env.BUCKET, input.spilled, input.size);
576+ req = head.req;
577+ packAt = head.packOffset;
578+ } catch (e) {
579+ return reportStatus({ commands: [], capabilities: new Set<string>(), pushOptions: [], pack: null }, unpackFailure(e), []);
580+ }
581+ }
582582 // A purge of unreachable objects that ran while this push was in flight may have deleted objects
583583 // the push counted on the repository having: refuse rather than point a ref at them.
584584 const epoch = this.purgeEpoch;
@@ -596,31 +596,40 @@
596596 let unpackError: string | null = null;
597597 // Like git's quarantine: nothing from the pack is stored until push protection passes,
598598 // so a blocked push leaves no objects behind for a retry to build on.
599- let objects: Awaited<ReturnType<typeof parsePack>> = [];
600- if (req.pack && req.pack.length > 0) {
601- try {
602- objects = await parsePack(req.pack, (oid) => this.store.read(oid), { largeObjectBytes: LARGE_OBJECT_BYTES });
603- } catch (e) {
604- unpackError = unpackFailure(e);
599+ let contents: PushContents = packedContents(this.store, [], 0);
600+ try {
601+ if (!(input instanceof Uint8Array)) {
602+ if (packAt < input.size) contents = await stagedContents(this.store, this.env.BUCKET, input.spilled, packAt, input.size - packAt, { largeObjectBytes: LARGE_OBJECT_BYTES });
603+ } else if (req.pack && req.pack.length > 0) {
604+ const pack = req.pack;
605+ contents = await parsePack(pack, repositoryBases(this.store), { largeObjectBytes: LARGE_OBJECT_BYTES }).then(
606+ (objects) => packedContents(this.store, objects, pack.length),
607+ // Parsed whole, a pack holds every object at once: one that can't, such as a long history
608+ // of a large file, is unpacked in pieces instead, holding only the objects in use.
609+ (e: unknown) => {
610+ if (!(e instanceof TooMuchMemory)) throw e;
611+ return streamedContents(this.store, memorySource(pack), { largeObjectBytes: LARGE_OBJECT_BYTES });
612+ },
613+ );
605614 }
615+ } catch (e) {
616+ unpackError = unpackFailure(e);
606617 }
607- // Commit and tag messages are scanned too, not only files.
608- // The objects themselves, not copies: a large blob is scanned from its compressed bytes.
609- const newBlobs = objects.filter((o) => o.type === "blob" || o.type === "commit" || o.type === "tag");
610618 const results: RefResult[] = [];
611619 if (unpackError) {
612620 for (const c of req.commands) results.push({ ref: c.ref, ok: false, reason: "unpacker error" });
613621 return reportStatus(req, unpackError, results);
614622 }
615623 // Like GitHub's receive.fsckObjects: no tree a git client would refuse, or that writes outside the work tree.
616- const bad = fsckObjects(objects);
624+ const bad = contents.fsck();
617625 if (bad) {
618626 for (const c of req.commands) results.push({ ref: c.ref, ok: false, reason: "fsck failed" });
619627 return reportStatus(req, `fsck error: ${bad}`, results, [`Push refused: ${bad}`]);
620628 }
621629
622630 const marked: SecretHit[] = [];
623- const leaks = scanForSecrets(newBlobs, marked);
631+ // Commit and tag messages are scanned too, not only files.
632+ const leaks = contents.scan(marked);
624633 // A file can opt out with `diffinery:allow-secret`; that it did is on the record.
625634 if (marked.length) this.emit({ type: "audit", action: "push.secret_marked", actor: caller, target: `${this.meta.owner}/${this.meta.name}`, detail: { kinds: [...new Set(marked.map((l) => l.kind))], blobs: marked.slice(0, 50).map((l) => l.oid), refs: req.commands.map((c) => c.ref) }, at: now() });
626635 if (leaks.length) {
@@ -646,17 +655,14 @@
646655 }
647656
648657 // Past the owner's storage quota nothing new is stored; a push that only deletes or moves refs still works.
649- const full = objects.some((o) => !this.store.has(o.oid)) ? await this.storageRefusal(req.pack?.length ?? 0) : null;
658+ const full = contents.anyNew() ? await this.storageRefusal(contents.bytes) : null;
650659 if (full) {
651660 for (const c of req.commands) results.push({ ref: c.ref, ok: false, reason: "storage limit reached" });
652661 return reportStatus(req, null, results, [...messages, `Push refused: ${full}`]);
653662 }
654663
655664 try {
656- for (const o of objects) {
657- if (o.deflated) await this.store.writeDeflated(o.oid, o.type, o.size!, o.deflated);
658- else await this.store.writeKnown(o.oid, o.type, o.content);
659- }
665+ await contents.write();
660666 } catch (e) {
661667 for (const c of req.commands) results.push({ ref: c.ref, ok: false, reason: "unpacker error" });
662668 return reportStatus(req, unpackFailure(e), results);
modifiedapps/diffinery/src/repo/actions.ts+3−1View file
@@ -2088,7 +2088,9 @@
20882088 await this.completeJob(s.id, { conclusion: "cancelled", outputs: {}, failure: { message: `Cancelled because ${job.name} failed (fail-fast).` } });
20892089 }
20902090 } else {
2091- const queued = siblings.find((s) => s.status === "queued");
2091+ // A leg that never started (max-parallel), not one back in line after getting no machine or
2092+ // being restarted: the run's workflow starts that one again (placementFailed, the safe restart).
2093+ const queued = siblings.find((s) => s.status === "queued" && !s.placement_tries && !s.restarts);
20922094 if (queued) await this.launch(queued.id);
20932095 }
20942096 // An ephemeral runner is done once its one job is.
modifiedapps/diffinery/src/repo/objects.ts+78−10View file
@@ -1,8 +1,14 @@
1-import { CODE_TYPES, TYPE_CODES, deflate, hashObject, inflate, type GitObject, type ObjectStore, type ObjectType } from "@diffinery/git-core";
1+import { CODE_TYPES, TYPE_CODES, deflate, deflateChunks, hashObject, inflateSized, type GitObject, type ObjectStore, type ObjectType } from "@diffinery/git-core";
22 import { sized } from "../lib/blob-store.js";
33
44 /** Objects larger than this (compressed) go to R2 instead of SQLite (rows are capped at 2 MB). */
5-const INLINE_LIMIT = 512 * 1024;
5+export const INLINE_LIMIT = 512 * 1024;
6+
7+/**
8+ * Objects larger than this are compressed a chunk at a time to be stored, never holding a second,
9+ * compressed copy whole: once to learn its length, then again as it is written.
10+ */
11+const WHOLE_DEFLATE_MAX = 1024 * 1024;
612 const CACHE_BYTES = 24 * 1024 * 1024;
713
814 export interface RawObject {
@@ -48,6 +54,12 @@
4854 }
4955 }
5056
57+ /** What reading an object holds at once, its zlib data and its content, or null when it isn't stored. */
58+ readCost(oid: string): number | null {
59+ const row = this.sql.exec<{ size: number; z: number }>("SELECT size, coalesce(stored, CASE WHEN r2 = 1 THEN size ELSE length(data) END) AS z FROM objects WHERE oid = ?", oid).toArray()[0];
60+ return row ? row.size + row.z : null;
61+ }
62+
5163 /** An object's inflated size without reading it, or null when it isn't stored. */
5264 size(oid: string): number | null {
5365 return this.sql.exec<{ size: number }>("SELECT size FROM objects WHERE oid = ?", oid).toArray()[0]?.size ?? null;
@@ -80,7 +92,7 @@
8092 }
8193 const raw = await this.readRaw(oid);
8294 if (!raw) return null;
83- const obj = { type: raw.type, content: inflate(raw.deflated) };
95+ const obj = { type: raw.type, content: inflateSized(raw.deflated, raw.size) };
8496 this.remember(oid, obj);
8597 return obj;
8698 }
@@ -91,14 +103,42 @@
91103 return oid;
92104 }
93105
94- /** Store an object whose oid is already verified. */
95- async writeKnown(oid: string, type: ObjectType, content: Uint8Array): Promise<void> {
106+ /**
107+ * Store an object whose oid is already verified. `cache`: keep it in memory too (a large push
108+ * doesn't, to stay small). `zlibSize`: what it compresses to, when already measured.
109+ */
110+ async writeKnown(oid: string, type: ObjectType, content: Uint8Array, cache = true, zlibSize?: number): Promise<void> {
96111 if (this.has(oid)) return;
97- const deflated = deflate(content);
98- const r2 = deflated.length > INLINE_LIMIT;
99- if (r2) await this.bucket.put(this.r2Key(oid), deflated);
100- this.insert(oid, type, content.length, deflated.length, r2 ? null : deflated);
101- this.remember(oid, { type, content });
112+ if (content.length <= WHOLE_DEFLATE_MAX) {
113+ const deflated = deflate(content);
114+ const r2 = deflated.length > INLINE_LIMIT;
115+ if (r2) await this.bucket.put(this.r2Key(oid), deflated);
116+ this.insert(oid, type, content.length, deflated.length, r2 ? null : deflated);
117+ } else {
118+ let length = zlibSize ?? 0;
119+ if (zlibSize === undefined) for (const c of deflateChunks(content)) length += c.length;
120+ if (length > INLINE_LIMIT) {
121+ const chunks = deflateChunks(content);
122+ const stream = new ReadableStream<Uint8Array>({
123+ pull(controller) {
124+ const next = chunks.next();
125+ if (next.done) controller.close();
126+ else controller.enqueue(next.value);
127+ },
128+ });
129+ await this.bucket.put(this.r2Key(oid), sized(stream, length));
130+ this.insert(oid, type, content.length, length, null);
131+ } else {
132+ const out = new Uint8Array(length);
133+ let at = 0;
134+ for (const c of deflateChunks(content)) {
135+ out.set(c, at);
136+ at += c.length;
137+ }
138+ this.insert(oid, type, content.length, length, out);
139+ }
140+ }
141+ if (cache) this.remember(oid, { type, content });
102142 }
103143
104144 /**
@@ -112,6 +152,34 @@
112152 this.insert(oid, type, size, deflated.length, r2 ? null : deflated.slice());
113153 }
114154
155+ /**
156+ * `writeDeflated` for zlib data that arrives in slices (a large blob in a pushed pack read from R2
157+ * in ranges): what is too big for SQLite goes to R2 as it arrives, never held whole.
158+ */
159+ async writeDeflatedSlices(oid: string, type: ObjectType, size: number, length: number, data: () => AsyncIterable<Uint8Array>): Promise<void> {
160+ if (this.has(oid)) return;
161+ if (length > INLINE_LIMIT) {
162+ const it = data()[Symbol.asyncIterator]();
163+ const stream = new ReadableStream<Uint8Array>({
164+ async pull(controller) {
165+ const next = await it.next();
166+ if (next.done) controller.close();
167+ else controller.enqueue(next.value.slice());
168+ },
169+ });
170+ await this.bucket.put(this.r2Key(oid), sized(stream, length));
171+ this.insert(oid, type, size, length, null);
172+ return;
173+ }
174+ const out = new Uint8Array(length);
175+ let at = 0;
176+ for await (const s of data()) {
177+ out.set(s, at);
178+ at += s.length;
179+ }
180+ this.insert(oid, type, size, length, out);
181+ }
182+
115183 /** Record an object (`data` null: it is in R2) and count its bytes, unless a concurrent write recorded it first. */
116184 private insert(oid: string, type: ObjectType, size: number, stored: number, data: Uint8Array | null): void {
117185 const added = this.sql.exec("INSERT OR IGNORE INTO objects (oid, type, size, data, r2, stored) VALUES (?, ?, ?, ?, ?, ?) RETURNING oid", oid, TYPE_CODES[type], size, data, data ? 0 : 1, stored).toArray().length;
modifiedapps/diffinery/test/e2e/e2e.test.mjs+148−21View file
@@ -9,9 +9,9 @@
99 // agent-authored PRs, merging, the REST contents API, the MCP server and
1010 // profile pictures (WASM image processing in the Worker).
1111 import { spawn, spawnSync, execFileSync } from "node:child_process";
12-import { mkdtempSync, writeFileSync, mkdirSync, readFileSync, existsSync, rmSync } from "node:fs";
12+import { mkdtempSync, writeFileSync, mkdirSync, readFileSync, existsSync, rmSync, openSync } from "node:fs";
1313 import { tmpdir } from "node:os";
14-import { randomBytes } from "node:crypto";
14+import { createHash, randomBytes } from "node:crypto";
1515 import { join, dirname, resolve } from "node:path";
1616 import { fileURLToPath } from "node:url";
1717 import { createServer } from "node:net";
@@ -25,7 +25,10 @@
2525 const work = mkdtempSync(join(tmpdir(), "diffinery-e2e-"));
2626 let base;
2727 let dev;
28-let devLog = "";
28+// The dev server writes to a file, not a pipe: a long synchronous git command keeps this process
29+// from reading a pipe, and a full one (retried webhooks log a stack each) stalls the server mid-push.
30+const devLogFile = join(work, "wrangler.log");
31+const devLog = () => (existsSync(devLogFile) ? readFileSync(devLogFile, "utf8") : "");
2932 let token;
3033 let agentToken;
3134
@@ -39,11 +42,24 @@
3942 });
4043
4144 async function api(method, path, body, tok = token) {
42- const res = await fetch(`${base}/api/v3${path}`, {
43- method,
44- headers: { ...(tok === null ? {} : { authorization: `Bearer ${tok}` }), "content-type": "application/json" },
45- body: body === undefined ? undefined : JSON.stringify(body),
46- });
45+ const send = () =>
46+ fetch(`${base}/api/v3${path}`, {
47+ method,
48+ headers: { ...(tok === null ? {} : { authorization: `Bearer ${tok}` }), "content-type": "application/json" },
49+ body: body === undefined ? undefined : JSON.stringify(body),
50+ });
51+ // After minutes of git traffic alone, the kept-alive connection from the last API call may have
52+ // been closed by the dev server just as it is reused ("other side closed"): a GET is sent again.
53+ let res;
54+ for (let i = 0; ; i++) {
55+ try {
56+ res = await send();
57+ break;
58+ } catch (e) {
59+ if (method !== "GET" || i >= 2) throw new Error(`${method} ${path}: ${e.message} (${e.cause?.code ?? e.cause})\n--- wrangler log tail ---\n${devLog().slice(-3000)}`);
60+ await sleep(1000);
61+ }
62+ }
4763 const text = await res.text();
4864 let json = null;
4965 try {
@@ -82,13 +98,19 @@
8298
8399 function git(cwd, args, tok = token, opts = {}) {
84100 const auth = Buffer.from(`x-token:${tok}`).toString("base64");
85- return execFileSync("git", ["-c", `http.extraHeader=Authorization: Basic ${auth}`, "-c", "protocol.version=2", ...args], {
86- cwd,
87- encoding: "utf8",
88- stdio: ["ignore", "pipe", "pipe"],
89- env: { ...process.env, GIT_TERMINAL_PROMPT: "0", GIT_AUTHOR_NAME: "E2E", GIT_AUTHOR_EMAIL: "e2e@example.com", GIT_COMMITTER_NAME: "E2E", GIT_COMMITTER_EMAIL: "e2e@example.com" },
90- ...opts,
91- });
101+ try {
102+ return execFileSync("git", ["-c", `http.extraHeader=Authorization: Basic ${auth}`, "-c", "protocol.version=2", ...args], {
103+ cwd,
104+ encoding: "utf8",
105+ stdio: ["ignore", "pipe", "pipe"],
106+ env: { ...process.env, GIT_TERMINAL_PROMPT: "0", GIT_AUTHOR_NAME: "E2E", GIT_AUTHOR_EMAIL: "e2e@example.com", GIT_COMMITTER_NAME: "E2E", GIT_COMMITTER_EMAIL: "e2e@example.com" },
107+ ...opts,
108+ });
109+ } catch (e) {
110+ // What the server said, for a git command that wasn't meant to fail (gitFails reads stderr, not this).
111+ if (/HTTP 5\d\d/.test(e.stderr ?? "")) e.message += `\n--- wrangler log tail ---\n${devLog().slice(-4000)}`;
112+ throw e;
113+ }
92114 }
93115 function gitFails(cwd, args, tok) {
94116 try {
@@ -108,7 +130,7 @@
108130 if (last) return last;
109131 await sleep(500);
110132 }
111- assert.fail(`timed out waiting for ${what}\n--- wrangler log tail ---\n${devLog.slice(-4000)}`);
133+ assert.fail(`timed out waiting for ${what}\n--- wrangler log tail ---\n${devLog().slice(-4000)}`);
112134 }
113135
114136 function spawnAsync(cmd, args, env, timeoutMs) {
@@ -146,7 +168,7 @@
146168 if (r.status !== 0 && !allowFailure) {
147169 const log = await fetch(`${base}/api/v3/repos/${owner}/${repo}/actions/jobs/${claim.job_id}/logs`, { headers: { authorization: `Bearer ${token}` } }).then((x) => x.text(), (e) => String(e));
148170 const job = await api("GET", `/repos/${owner}/${repo}/actions/jobs/${claim.job_id}`).catch((e) => ({ json: String(e) }));
149- assert.fail(`runner agent exited ${r.status}:\n${out}\n--- job ---\n${JSON.stringify(job.json, null, 2)}\n--- job log ---\n${log}\n--- wrangler log tail ---\n${devLog.slice(-3000)}`);
171+ assert.fail(`runner agent exited ${r.status}:\n${out}\n--- job ---\n${JSON.stringify(job.json, null, 2)}\n--- job log ---\n${log}\n--- wrangler log tail ---\n${devLog().slice(-3000)}`);
150172 }
151173 return { claim, out };
152174 }
@@ -159,14 +181,13 @@
159181 execFileSync("pnpm", ["exec", "wrangler", "d1", "migrations", "apply", "diffinery-dev", "--local", "-e", "dev", "--persist-to", persist], { cwd: APP, stdio: "pipe", env: { ...process.env, CI: "1" } });
160182 // Until the web UI exists, run the same app through the API-only entry.
161183 const entry = existsSync(join(APP, "src/web/routes.tsx")) ? [] : ["test/e2e/entry.ts"];
184+ const out = openSync(devLogFile, "a");
162185 dev = spawn("pnpm", ["exec", "wrangler", "dev", ...entry, "-e", "dev", "--ip", "127.0.0.1", "--port", String(port), "--inspector-port", String(await freePort()), "--persist-to", persist, "--var", `PUBLIC_URL:${base}`, "--var", `MASTER_KEY:${randomBytes(32).toString("base64")}`, "--var", "SCHEDULE_TICK_MINUTES:1", "--show-interactive-dev-session=false"], {
163186 cwd: APP,
164187 env: { ...process.env, CI: "1", WRANGLER_SEND_METRICS: "false" },
165- stdio: ["ignore", "pipe", "pipe"],
188+ stdio: ["ignore", out, out],
166189 detached: true,
167190 });
168- dev.stdout.on("data", (d) => (devLog += d));
169- dev.stderr.on("data", (d) => (devLog += d));
170191 await until("wrangler dev", async () => {
171192 try {
172193 return (await fetch(`${base}/healthz`)).ok;
@@ -2554,7 +2575,7 @@
25542575 // The LFS object is stored in Diffinery.
25552576 const lfsRes = await fetch(`${base}/${owner}/imported-app.git/info/lfs/objects/batch`, { method: "POST", headers: { authorization: `Bearer ${token}`, "content-type": "application/vnd.git-lfs+json" }, body: JSON.stringify({ operation: "download", objects: [{ oid: fx.asset.oid, size: fx.asset.data.length }] }) });
25562577 const lfsBody = await lfsRes.json();
2557- assert.ok(lfsBody.objects[0].actions?.download, `${JSON.stringify(lfsBody)}\n${JSON.stringify(done.warnings)}\n${agent.out}\n${devLog.split("\n").filter((l) => /lfs|rror/i.test(l)).slice(-40).join("\n")}`);
2578+ assert.ok(lfsBody.objects[0].actions?.download, `${JSON.stringify(lfsBody)}\n${JSON.stringify(done.warnings)}\n${agent.out}\n${devLog().split("\n").filter((l) => /lfs|rror/i.test(l)).slice(-40).join("\n")}`);
25582579 const blob = await fetch(lfsBody.objects[0].actions.download.href, { headers: { authorization: `Bearer ${token}` } });
25592580 assert.equal(Buffer.from(await blob.arrayBuffer()).toString(), fx.asset.data.toString());
25602581 // The push token is revoked.
@@ -2744,6 +2765,61 @@
27442765 git(src, ["push", "-q", `${base}/${tickRepo}.git`, "main"]);
27452766 });
27462767
2768+/**
2769+ * A real project's history, pushed whole to a new repository and cloned back. Pinned commits, so
2770+ * the packs are the same each run; cloned from GitHub without the Diffinery token.
2771+ */
2772+async function pushHistory(name, upstream) {
2773+ const me = await ok("GET", "/user");
2774+ await ok("POST", "/user/repos", { name });
2775+ const remote = `${base}/${me.login}/${name}.git`;
2776+ // Test fixtures in these histories look like credentials (tornado has a test private key): pushed as an admin's bypass.
2777+ git(upstream, ["-c", "http.postBuffer=1048576", "push", "-q", "-o", "allow-secrets=e2e of a public history", remote, "main"], token, { maxBuffer: 16 * 1024 * 1024 });
2778+ const out = join(work, `${name}-clone`);
2779+ git(work, ["clone", "-q", "--bare", remote, out]);
2780+ assert.equal(git(out, ["rev-parse", "main"]).trim(), git(upstream, ["rev-parse", "main"]).trim());
2781+ assert.equal(git(out, ["rev-list", "--count", "main"]).trim(), git(upstream, ["rev-list", "--count", "main"]).trim());
2782+ git(out, ["fsck", "--full"]);
2783+}
2784+
2785+for (const [name, url, sha] of [
2786+ // 37 MB: staged in R2 and unpacked in pieces.
2787+ ["jquery", "https://github.com/jquery/jquery", "c2f27864e4ffc2d1409c289fcb9a42b6165dfeaf"],
2788+ // About 12 MB each: too many objects for parsing whole in memory, so unpacked in pieces too.
2789+ ["tornado", "https://github.com/tornadoweb/tornado", "056cacaefc7de31bda28363378c4ffc86b97d7e4"],
2790+ ["flask", "https://github.com/pallets/flask", "d086db856be187255b8ec61ef409357393020f32"],
2791+]) {
2792+ test(`real histories: ${name}'s whole history goes in one push and clones back intact`, async () => {
2793+ const upstream = join(work, `${name}-upstream.git`);
2794+ execFileSync("git", ["clone", "-q", "--bare", "--no-tags", url, upstream], { stdio: ["ignore", "pipe", "pipe"], env: { ...process.env, GIT_TERMINAL_PROMPT: "0" } });
2795+ execFileSync("git", ["update-ref", "refs/heads/main", sha], { cwd: upstream });
2796+ await pushHistory(name, upstream);
2797+ });
2798+}
2799+
2800+test("real histories: a 2.6 MB lockfile's 60 versions, a chain of deltas, go in one small push", async () => {
2801+ const upstream = join(work, "lockfile-upstream");
2802+ mkdirSync(upstream, { recursive: true });
2803+ git(upstream, ["init", "-q", "-b", "main"]);
2804+ const versions = new Map();
2805+ const pkgs = Array.from({ length: 11_000 }, (_, i) => `pkg-${String(i).padStart(5, "0")}`);
2806+ pkgs.forEach((p, i) => versions.set(p, `1.${i % 51}.${i % 97}`));
2807+ for (let c = 0; c < 60; c++) {
2808+ for (let k = 0; k < 40; k++) versions.set(pkgs[(c * 7919 + k * 104729) % pkgs.length], `2.${c}.${k}`);
2809+ const body = pkgs.map((p) => {
2810+ const v = versions.get(p);
2811+ return ` "node_modules/${p}": {\n "version": "${v}",\n "resolved": "https://registry.npmjs.org/${p}/-/${p}-${v}.tgz",\n "integrity": "sha512-${createHash("sha512").update(p + v).digest("hex").slice(0, 64)}"\n },\n`;
2812+ });
2813+ writeFileSync(join(upstream, "package-lock.json"), `{\n "lockfileVersion": 3,\n "packages": {\n${body.join("")} }\n}\n`);
2814+ git(upstream, ["add", "-A"]);
2815+ git(upstream, ["commit", "-q", "-m", `lockfile ${c}`]);
2816+ }
2817+ assert.ok(readFileSync(join(upstream, "package-lock.json")).length > 2_500_000);
2818+ const packBytes = Number(execFileSync("bash", ["-c", "git pack-objects --revs --stdout -q <<<main | wc -c"], { cwd: upstream, encoding: "utf8" }).trim());
2819+ assert.ok(packBytes < 16 * 1024 * 1024, `the pack is ${packBytes} bytes`);
2820+ await pushHistory("lockfile", upstream);
2821+});
2822+
27472823 test("large files: a 40 MB file is pushed and cloned intact, and push protection still reads big text files", async () => {
27482824 const me = await ok("GET", "/user");
27492825 await ok("POST", "/user/repos", { name: "bigfile" });
@@ -2768,3 +2844,54 @@
27682844 git(src, ["commit", "-q", "-m", "a large dump"]);
27692845 assert.match(gitFails(src, ["push", "-q", `${base}/${me.login}/bigfile.git`, "main"]), /possible secrets detected/);
27702846 });
2847+
2848+test("large pushes: a 40 MB history of many files goes in one push, read from R2 in ranges, and is still checked", async () => {
2849+ const me = await ok("GET", "/user");
2850+ await ok("POST", "/user/repos", { name: "bighistory" });
2851+ const remote = `${base}/${me.login}/bighistory.git`;
2852+ const src = join(work, "bighistory");
2853+ mkdirSync(join(src, "files"), { recursive: true });
2854+ git(src, ["init", "-q", "-b", "main"]);
2855+ // Hundreds of medium files, each changed a little in later commits: a pack of deltas over 16 MB,
2856+ // as a fork's pull request heads made one in jekyll/minima's import.
2857+ const files = Array.from({ length: 400 }, () => randomBytes(96 * 1024).toString("base64"));
2858+ for (let c = 0; c < 6; c++) {
2859+ files.forEach((f, i) => writeFileSync(join(src, "files", `f${i}.txt`), `${f}\nrevision ${c} of file ${i}\n${randomBytes(2048).toString("hex")}\n`));
2860+ git(src, ["add", "-A"]);
2861+ git(src, ["commit", "-q", "-m", `revision ${c}`]);
2862+ }
2863+ const packBytes = Number(execFileSync("bash", ["-c", "git pack-objects --revs --stdout -q <<<main | wc -c"], { cwd: src, encoding: "utf8" }).trim());
2864+ assert.ok(packBytes > 30 * 1024 * 1024, `the pack is ${packBytes} bytes`);
2865+ git(src, ["-c", "http.postBuffer=1048576", "push", "-q", remote, "main"], token, { maxBuffer: 16 * 1024 * 1024 });
2866+ const out = join(work, "bighistory-clone");
2867+ git(work, ["clone", "-q", remote, out]);
2868+ assert.equal(git(out, ["rev-parse", "HEAD"]).trim(), git(src, ["rev-parse", "HEAD"]).trim());
2869+ git(out, ["fsck", "--full", "--strict"]);
2870+
2871+ // A second large push builds on objects the repository has (a thin pack, its bases read from storage).
2872+ files.forEach((f, i) => writeFileSync(join(src, "files", `f${i}.txt`), `${f}\nrevision 6 of file ${i}\n${randomBytes(2048).toString("hex")}\n`));
2873+ writeFileSync(join(src, "new.bin"), randomBytes(20 * 1024 * 1024));
2874+ git(src, ["add", "-A"]);
2875+ git(src, ["commit", "-q", "-m", "revision 6"]);
2876+ git(src, ["-c", "http.postBuffer=1048576", "push", "-q", remote, "main"], token, { maxBuffer: 16 * 1024 * 1024 });
2877+ git(out, ["pull", "-q", "--ff-only"]);
2878+ assert.equal(git(out, ["rev-parse", "HEAD"]).trim(), git(src, ["rev-parse", "HEAD"]).trim());
2879+ git(out, ["fsck", "--full", "--strict"]);
2880+
2881+ // A large push is checked like a small one: a key in one small file, or an unsafe .gitmodules, refuses all of it.
2882+ git(src, ["checkout", "-q", "-b", "leak"]);
2883+ writeFileSync(join(src, "more.bin"), randomBytes(20 * 1024 * 1024));
2884+ writeFileSync(join(src, "config.txt"), "aws_key = AKIAIOSFODNN7EXAMPL3\n"); // diffinery:allow-secret (a fake key; the pushed file doesn't carry the marker)
2885+ git(src, ["add", "-A"]);
2886+ git(src, ["commit", "-q", "-m", "a key"]);
2887+ assert.match(gitFails(src, ["-c", "http.postBuffer=1048576", "push", "-q", remote, "leak"]), /possible secrets detected/);
2888+ git(src, ["checkout", "-q", "main"]);
2889+ git(src, ["checkout", "-q", "-b", "modules"]);
2890+ writeFileSync(join(src, "more.bin"), randomBytes(20 * 1024 * 1024));
2891+ writeFileSync(join(src, ".gitmodules"), '[submodule "x"]\n\tpath = x\n\turl = --upload-pack=touch /tmp/pwned\n');
2892+ git(src, ["add", "-A"]);
2893+ git(src, ["commit", "-q", "-m", "an unsafe submodule"]);
2894+ assert.match(gitFails(src, ["-c", "http.postBuffer=1048576", "push", "-q", remote, "modules"]), /fsck error: \.gitmodules .*starts with '-'/);
2895+ const branches = (await ok("GET", `/repos/${me.login}/bighistory/branches`)).map((b) => b.name);
2896+ assert.deepEqual(branches, ["main"]);
2897+});
modifiedapps/diffinery/test/unit/actions-runtime.test.ts+58−0View file
@@ -770,6 +770,64 @@
770770 expect([1, 2, 3, 4, 5, 6].map(placementDelay)).toEqual([30, 60, 120, 240, 300, 300]);
771771 });
772772
773+ it("a matrix leg back in line after getting no machine waits out its pause when a sibling finishes", async () => {
774+ const sql = sqlStorage();
775+ sql.exec(REPO_SCHEMA);
776+ migrateRepoSchema(sql as unknown as SqlStorage);
777+ let full = true;
778+ const started: string[] = [];
779+ const events: { type: string; payload: Record<string, unknown> }[] = [];
780+ const repo = {
781+ sql,
782+ meta: { repoId: "r", owner: "acme", ownerId: "acme", name: "app", defaultBranch: "main" },
783+ upsertCheck: () => {},
784+ broadcast: () => {},
785+ deploys: { jobCompleted: () => {} },
786+ runtime: { sweepJob: async () => {} },
787+ } as unknown as RepoDO;
788+ const DB = testD1();
789+ DB.sqlite.prepare("INSERT INTO accounts (id, login, kind, status, created_at) VALUES ('acme', 'acme', 'org', 'active', ?)").run(new Date().toISOString());
790+ const env = {
791+ DB,
792+ PUBLIC_URL: "https://d.test",
793+ BUCKET: memoryBucket().bucket,
794+ RUNNER: {
795+ idFromName: (n: string) => n,
796+ get: (n: string) => ({
797+ startJob: async () => {
798+ started.push(n);
799+ if (full) throw new Error("there is no container instance that can be provided to this durable object");
800+ },
801+ stopJob: async () => {},
802+ }),
803+ },
804+ RUN_WORKFLOW: { get: async () => ({ sendEvent: async (e: { type: string; payload: Record<string, unknown> }) => void events.push(e) }) },
805+ } as unknown as Env;
806+ const actions = new Actions(repo, env);
807+ const a = actions as unknown as { launch(id: string): Promise<void>; completeJob(id: string, c: unknown): Promise<void> };
808+ const t = new Date().toISOString();
809+ sql.exec("INSERT INTO runs (id, run_number, workflow_path, workflow_name, event, ref, sha, actor, status, created_at, updated_at, event_payload) VALUES ('run1', 1, 'w.yml', 'w', 'push', 'refs/heads/main', 'abc', '{}', 'in_progress', ?, ?, '{}')", t, t);
810+ // Legs of one matrix: m1 and m2 started, m3 not yet (max-parallel).
811+ sql.exec("INSERT INTO jobs (id, run_id, job_key, name, status) VALUES ('m1', 'run1', 'm', 'm (1)', 'queued'), ('m2', 'run1', 'm', 'm (2)', 'queued'), ('m4', 'run1', 'm', 'm (4)', 'queued'), ('m3', 'run1', 'm', 'm (3)', 'queued')");
812+ // m4 is back in line after a safe restart (its runner was replaced before its steps ran).
813+ sql.exec("UPDATE jobs SET restarts = 1 WHERE id = 'm4'");
814+ const job = (id: string) => sql.exec("SELECT status, placement_tries FROM jobs WHERE id = ?", id).toArray()[0];
815+ await a.launch("m1");
816+ expect(job("m1")).toMatchObject({ status: "queued", placement_tries: 1 });
817+ full = false;
818+ await a.launch("m2");
819+ expect(job("m2")).toMatchObject({ status: "waiting" });
820+ started.length = 0;
821+
822+ // m2 finishing starts the leg that never started, not the ones back in line: those are started
823+ // again by the run's workflow, so each try still waits its turn (placementDelay).
824+ await a.completeJob("m2", { conclusion: "success", outputs: {} });
825+ expect(started).toEqual(["job-m3"]);
826+ expect(job("m1")).toMatchObject({ status: "queued", placement_tries: 1 });
827+ expect(job("m4")).toMatchObject({ status: "queued" });
828+ expect(events.filter((e) => e.type === "job-m1")).toHaveLength(1);
829+ });
830+
773831 it("a hosted runner on an image other than the deployed one is turned away and the job placed again, up to STALE_IMAGE_TRIES", async () => {
774832 const sql = sqlStorage();
775833 sql.exec(REPO_SCHEMA);
modifiedapps/diffinery/test/unit/fsck.test.ts+17−1View file
@@ -1,5 +1,5 @@
11 import { describe, expect, it } from "vitest";
2-import { fsckObjects, isDotGitmodulesName, unsafeGitmodules } from "../../src/git/fsck.js";
2+import { MAX_TREE_BYTES, fsckObjects, isDotGitmodulesName, unsafeGitmodules } from "../../src/git/fsck.js";
33
44 const enc = new TextEncoder();
55 const OID = "a".repeat(40);
@@ -56,4 +56,20 @@
5656 it("refuses malformed trees", () => {
5757 expect(fsckObjects([{ oid: "t".repeat(40), type: "tree", content: enc.encode("100644 broken") }])).toMatch(/malformed tree/);
5858 });
59+
60+ it("refuses a tree too large to check, before reading its names", () => {
61+ // Entries of 36 bytes each: "100644 f0000001\0" and a 20-byte oid.
62+ const many = (n: number) => {
63+ const content = new Uint8Array(n * 36);
64+ for (let i = 0; i < n; i++) content.set(enc.encode(`100644 f${String(i).padStart(7, "0")}\0`), i * 36);
65+ return { oid: "t".repeat(40), type: "tree", content };
66+ };
67+ const huge = many(Math.ceil((MAX_TREE_BYTES + 1) / 36));
68+ expect(huge.content.length).toBeGreaterThan(MAX_TREE_BYTES);
69+ expect(fsckObjects([huge])).toMatch(/larger than 8 MiB/);
70+ // A large but ordinary directory is fine, and a truncated last entry is malformed.
71+ const big = many(50_000);
72+ expect(fsckObjects([big])).toBeNull();
73+ expect(fsckObjects([{ ...big, content: big.content.subarray(0, big.content.length - 3) }])).toMatch(/malformed tree/);
74+ });
5975 });
addedapps/diffinery/test/unit/receive-head.test.ts+34−0View file
@@ -1,0 +1,34 @@
1+import { describe, expect, it } from "vitest";
2+import { FLUSH, concat, pkt, utf8 } from "@diffinery/git-core";
3+import { parseReceivePack, parseReceivePackHead } from "../../src/git/protocol.js";
4+
5+const old = "0".repeat(40);
6+const oid = "a".repeat(40);
7+const body = (options: string[] | null) =>
8+ concat([
9+ pkt(`${old} ${oid} refs/heads/main\0report-status${options ? " push-options" : ""} atomic\n`),
10+ pkt(`${old} ${oid} refs/heads/next\n`),
11+ FLUSH,
12+ ...(options ? [...options.map((o) => pkt(`${o}\n`)), FLUSH] : []),
13+ utf8("PACK...."),
14+ ]);
15+
16+describe("the start of a staged push", () => {
17+ it("gives the commands, push options and where the pack starts, as parseReceivePack does", () => {
18+ for (const options of [null, ["diffinery-import", "allow-secrets=a reason"]]) {
19+ const b = body(options);
20+ const head = parseReceivePackHead(b)!;
21+ const whole = parseReceivePack(b);
22+ expect(head.req.commands).toEqual(whole.commands);
23+ expect([...head.req.capabilities]).toEqual([...whole.capabilities]);
24+ expect(head.req.pushOptions).toEqual(whole.pushOptions);
25+ expect(b.subarray(head.packOffset)).toEqual(whole.pack);
26+ }
27+ });
28+ it("asks for more when the head is cut short, wherever it is cut", () => {
29+ const b = body(["diffinery-import"]);
30+ const full = parseReceivePackHead(b)!.packOffset;
31+ for (let n = 0; n < full; n++) expect(parseReceivePackHead(b.subarray(0, n))).toBeNull();
32+ expect(parseReceivePackHead(b.subarray(0, full))?.packOffset).toBe(full);
33+ });
34+});
modifiedapps/diffinery/test/unit/secret-scan.test.ts+26−1View file
@@ -1,4 +1,4 @@
1-import { describe, expect, it } from "vitest";
1+import { describe, expect, it, vi } from "vitest";
22 import { deflate } from "@diffinery/git-core";
33 import { scanForSecrets, secretBypassReason } from "../../src/git/secret-scan.js";
44
@@ -70,4 +70,29 @@
7070 expect(secretBypassReason(["ci.skip", "allow-secrets= test fixture "])).toBe("test fixture");
7171 expect(secretBypassReason([`allow-secrets=${"x".repeat(300)}`])).toHaveLength(200);
7272 });
73+
74+ it("scans a large file in slices, never as one string, with the same results", () => {
75+ const decode = vi.spyOn(TextDecoder.prototype, "decode");
76+ // 3 MiB of text with a key across the 256 KiB slice boundary, and a "€" so it isn't ASCII.
77+ const filler = "€ line of a large generated file\n".repeat(100_000);
78+ const key = " AKIAIOSFODNN7EXAMPL3 "; // diffinery:allow-secret (a fake key; the scanned text doesn't carry the marker)
79+ const at = 256 * 1024 - 8;
80+ const text = new TextEncoder().encode(filler);
81+ const big = new Uint8Array(text.length + key.length);
82+ big.set(text.subarray(0, at));
83+ big.set(new TextEncoder().encode(key), at);
84+ big.set(text.subarray(at), at + key.length);
85+ expect(big.length).toBeGreaterThan(3 * 1024 * 1024);
86+ expect(scanForSecrets([{ oid: "b".repeat(40), content: big }]).map((h) => h.kind)).toEqual(["AWS access key"]);
87+ for (const [input] of decode.mock.calls) if (input) expect((input as Uint8Array).byteLength).toBeLessThanOrEqual(256 * 1024);
88+ decode.mockRestore();
89+ // Opting out anywhere in it still counts, and a NUL byte still makes it binary.
90+ const marked: { oid: string; kind: string }[] = [];
91+ const optOut = new Uint8Array([...big, ...new TextEncoder().encode("\ndiffinery:allow-secret\n")]);
92+ expect(scanForSecrets([{ oid: "b".repeat(40), content: optOut }], marked)).toEqual([]);
93+ expect(marked.map((h) => h.kind)).toEqual(["AWS access key"]);
94+ const binary = big.slice();
95+ binary[2 * 1024 * 1024] = 0;
96+ expect(scanForSecrets([{ oid: "b".repeat(40), content: binary }])).toEqual([]);
97+ });
7398 });
modifiedapps/diffinery/test/unit/storage.test.ts+38−2View file
@@ -1,7 +1,8 @@
11 import { describe, expect, it } from "vitest";
22 import { DatabaseSync } from "node:sqlite";
33 import { randomBytes } from "node:crypto";
4-import { deflate, hashObject, utf8 } from "@diffinery/git-core";
4+import { deflate, hashObject, inflate, utf8 } from "@diffinery/git-core";
5+import { repositoryBases } from "../../src/git/push-contents.js";
56 import { RepoObjectStore } from "../../src/repo/objects.js";
67 import { REPO_SCHEMA, migrateRepoSchema } from "../../src/repo/schema.js";
78
@@ -23,7 +24,13 @@
2324 function setup() {
2425 const sql = sqlStorage();
2526 const r2 = new Map<string, Uint8Array>();
26- const bucket = { put: async (key: string, body: Uint8Array) => void r2.set(key, body) } as unknown as R2Bucket;
27+ const bucket = {
28+ put: async (key: string, body: Uint8Array | ReadableStream) => void r2.set(key, body instanceof ReadableStream ? new Uint8Array(await new Response(body).arrayBuffer()) : body),
29+ get: async (key: string) => {
30+ const v = r2.get(key);
31+ return v ? { arrayBuffer: async () => v.slice().buffer } : null;
32+ },
33+ } as unknown as R2Bucket;
2734 let added = 0;
2835 const store = new RepoObjectStore(sql, bucket, "r1", () => added++);
2936 return { sql, r2, store, added: () => added };
@@ -63,4 +70,33 @@
6370 store.recount();
6471 expect(store.sizeBytes()).toBe(4 + deflate(utf8("new\n")).length);
6572 });
73+
74+ it("compresses a large object a chunk at a time to store it, to the same zlib data", async () => {
75+ const { store, r2 } = setup();
76+ // Random: its zlib data goes to R2. Text: small enough compressed to stay in SQLite.
77+ const random = new Uint8Array(randomBytes(3 * 1024 * 1024));
78+ const text = utf8("a line of a large text file\n".repeat(120_000));
79+ for (const content of [random, text]) {
80+ const oid = await hashObject("blob", content);
81+ await store.writeKnown(oid, "blob", content, false);
82+ const raw = (await store.readRaw(oid))!;
83+ expect(Buffer.from(raw.deflated).equals(Buffer.from(deflate(content)))).toBe(true);
84+ expect(Buffer.from(inflate(raw.deflated)).equals(Buffer.from(content))).toBe(true);
85+ expect(store.readCost(oid)).toBe(content.length + raw.deflated.length);
86+ expect(Buffer.from((await store.read(oid))!.content).equals(Buffer.from(content))).toBe(true);
87+ }
88+ expect(r2.size).toBe(1);
89+ });
90+
91+ it("gives a thin pack's bases only when reading them fits, saying what they would take otherwise", async () => {
92+ const { store } = setup();
93+ const zeros = new Uint8Array(8 * 1024 * 1024);
94+ const oid = await store.write("blob", zeros);
95+ const lookup = repositoryBases(store);
96+ const cost = store.readCost(oid)!;
97+ expect(cost).toBeGreaterThan(zeros.length);
98+ expect(await lookup(oid, cost - 1)).toBe(cost);
99+ expect(((await lookup(oid, cost)) as { content: Uint8Array }).content.length).toBe(zeros.length);
100+ expect(await lookup("0".repeat(40), cost)).toBeNull();
101+ });
66102 });
modifiedpackages/git-core/src/delta.ts+19−0View file
@@ -1,3 +1,22 @@
1+/** The size a delta says its result has (after its base's size), read from its header. */
2+export function deltaResultSize(delta: Uint8Array): number {
3+ let i = 0;
4+ const varint = (): number => {
5+ let result = 0;
6+ let shift = 0;
7+ let c: number;
8+ do {
9+ if (i >= delta.length) throw new Error("truncated delta header");
10+ c = delta[i++]!;
11+ result += (c & 0x7f) * 2 ** shift;
12+ shift += 7;
13+ } while (c & 0x80);
14+ return result;
15+ };
16+ varint();
17+ return varint();
18+}
19+
120 /** Apply a git delta (as found in OFS_DELTA / REF_DELTA pack entries) to a base. */
221 export function applyDelta(base: Uint8Array, delta: Uint8Array, maxSize = Infinity): Uint8Array {
322 let i = 0;
modifiedpackages/git-core/src/index.ts+1−0View file
@@ -8,3 +8,4 @@
88 export * from "./store.js";
99 export * from "./diff.js";
1010 export * from "./memory-store.js";
11+export * from "./pack-stream.js";
modifiedpackages/git-core/src/objects.ts+10−2View file
@@ -1,5 +1,5 @@
11 import { concat, fromHex, fromUtf8, toHex, utf8 } from "./bytes.js";
2-import { sha1Hex } from "./sha1.js";
2+import { Sha1, sha1Hex } from "./sha1.js";
33
44 export type ObjectType = "commit" | "tree" | "blob" | "tag";
55
@@ -18,8 +18,16 @@
1818 return utf8(`${type} ${size}\0`);
1919 }
2020
21+/**
22+ * Content larger than this is hashed in pieces after its header, never copied: slower than
23+ * WebCrypto, which only hashes one whole buffer, but an object never needs twice its size to hash.
24+ */
25+export const HASH_COPY_MAX = 1024 * 1024;
26+
2127 export async function hashObject(type: ObjectType, content: Uint8Array): Promise<string> {
22- return sha1Hex(concat([objectHeader(type, content.length), content]));
28+ const header = objectHeader(type, content.length);
29+ if (content.length > HASH_COPY_MAX) return toHex(new Sha1().update(header).update(content).digest());
30+ return sha1Hex(concat([header, content]));
2331 }
2432
2533 export const isOid = (s: string): boolean => /^[0-9a-f]{40}$/.test(s);
addedpackages/git-core/src/pack-stream.ts+970−0View file
@@ -1,0 +1,970 @@
1+import pako from "pako";
2+import { equalBytes, fromHex, toHex } from "./bytes.js";
3+import { applyDelta, deltaResultSize } from "./delta.js";
4+import { CODE_TYPES, TYPE_CODES, hashObject, objectHeader, type GitObject, type ObjectType } from "./objects.js";
5+import { MAX_HELD_BYTES, MAX_UNPACKED_BYTES, tooBigUnpacked, tooMuchMemory, type ExternalLookup } from "./pack.js";
6+import { deflateChunks } from "./zlib.js";
7+import { Sha1 } from "./sha1.js";
8+
9+/**
10+ * Unpacking a pack that is never held whole: it is read in ranges (from R2, where a large push is
11+ * staged), indexed in one pass, and each object is inflated again, and its deltas applied again,
12+ * whenever it is needed, with a bounded cache of resolved objects.
13+ *
14+ * The same limits as `parsePack` apply: each object stops inflating at its declared size, and the
15+ * pack may unpack to at most `maxBytes` counted the same way (every entry's declared size, then
16+ * every delta's result, and a large base once for all its deltas), and deltas are resolved in one
17+ * pass from their bases, whatever order the pack lists them in. Inflating objects again is counted
18+ * separately, against `maxWork`.
19+ *
20+ * Everything it holds is charged against `maxHeldBytes` before it is allocated: the caches, about
21+ * ENTRY_OVERHEAD bytes for each object, the zlib data it keeps to inflate again, SCRATCH_BYTES for
22+ * hashing and storing an object, and every object while it is in use (a delta's instructions, its
23+ * base and its result, a base from the repository). Requests are counted and capped too: each R2
24+ * read, each base read from the repository and each object storing writes to R2 on its own is a
25+ * subrequest, and an invocation has a limited number.
26+ */
27+
28+export interface PackSource {
29+ /** The pack's length, its 20-byte checksum included. */
30+ readonly size: number;
31+ read(offset: number, length: number): Promise<Uint8Array>;
32+}
33+
34+export interface StreamedObject {
35+ oid: string;
36+ type: ObjectType;
37+ size: number;
38+}
39+
40+export interface UnpackOptions {
41+ external?: ExternalLookup;
42+ maxBytes?: number;
43+ /** Bytes inflated or rebuilt from deltas in all, re-reads included (default 4 × maxBytes). */
44+ maxWork?: number;
45+ /** Blobs at least this large are hashed and handed over as they inflate, never whole (like `parsePack`'s). */
46+ largeObjectBytes?: number;
47+ /** Objects a pack may hold. */
48+ maxObjects?: number;
49+ /** The pack is read in blocks of this size for its one sequential pass (and for storing it). */
50+ blockBytes?: number;
51+ maxBlocks?: number;
52+ /** Resolved objects kept for reuse as delta bases, in bytes. */
53+ cacheBytes?: number;
54+ /** Zlib data of deltas kept from the first pass, so they are inflated again without reading the source (default 16 MiB). */
55+ keepBytes?: number;
56+ /** Bytes read at once when an entry not kept is read again, so entries near each other take one read. */
57+ windowBytes?: number;
58+ /** What it may hold in all (default MAX_HELD_BYTES, as parsePack). */
59+ maxHeldBytes?: number;
60+ /**
61+ * Requests in all (default 500, well within an invocation's 1,000 subrequests): reads of the
62+ * source, storing's included, bases read from the repository, and objects stored on their own.
63+ */
64+ maxReads?: number;
65+ /** Bytes read from the source in all, storing included (default 6 × the pack's size + 64 MiB). */
66+ maxReadBytes?: number;
67+ /**
68+ * Storing writes an object whose zlib data is larger than this with a request of its own (to R2):
69+ * each one is counted with the reads. Default: none is.
70+ */
71+ writesOver?: number;
72+ /** Bytes of inflating between yields to the event loop (default 4 MiB), so a long unpack doesn't hold up everything else in the isolate. */
73+ yieldEvery?: number;
74+}
75+
76+/** Hands a large blob's content to `onChunk` slice by slice. */
77+export type ChunkFeed = (onChunk: (chunk: Uint8Array) => void) => Promise<void>;
78+
79+const OFS_DELTA = 6;
80+const REF_DELTA = 7;
81+export const MAX_STREAMED_OBJECTS = 1_000_000;
82+
83+/**
84+ * What one object costs to hold beyond its content: its slots in the index's typed arrays and its
85+ * oid, and, for a delta, its place in the lists of deltas waiting on a base and in the replay plan.
86+ * Measured at about 100 bytes in Node at the end of indexing 200,000 small objects, half of them
87+ * OFS_DELTAs.
88+ */
89+export const ENTRY_OVERHEAD = 128;
90+
91+/**
92+ * What a REF_DELTA costs besides: its base's oid, kept by entry and as the key it waits on. Measured
93+ * at about 510 bytes more than an OFS_DELTA.
94+ */
95+export const REF_DELTA_OVERHEAD = 640;
96+
97+/** What keeping one object in a cache costs beyond its bytes: a map entry, its key, and the Uint8Array. */
98+const CACHED_OVERHEAD = 192;
99+
100+/** Zlib data longer than this isn't kept from the first pass: it is read again when needed. */
101+const KEEP_ENTRY_MAX = 1024 * 1024;
102+
103+/** Kept zlib data is packed into buffers of this size; an entry over a quarter of it gets its own. */
104+const KEEP_CHUNK = 1024 * 1024;
105+
106+/**
107+ * Held throughout for the one object being hashed or stored: hashing copies one of up to
108+ * HASH_COPY_MAX bytes after its header (a larger one is hashed in pieces), and storing compresses
109+ * one, whole up to 1 MiB (its zlib chunks and their join), larger ones a chunk at a time.
110+ */
111+export const SCRATCH_BYTES = 3 * 1024 * 1024;
112+
113+const DEFAULTS = { blockBytes: 2 * 1024 * 1024, maxBlocks: 4, cacheBytes: 12 * 1024 * 1024, keepBytes: 16 * 1024 * 1024, windowBytes: 1024 * 1024, maxReads: 500 };
114+
115+/** The most zlib data `n` bytes compress to (zlib's deflateBound, with its 6 bytes of header and trailer). */
116+function deflateBound(n: number): number {
117+ return n + Math.floor(n / 4096) + Math.floor(n / 16384) + Math.floor(n / 33554432) + 13 + 6;
118+}
119+
120+/** Fixed-size blocks of the source, the most recently used kept: for reading it in order. */
121+class Blocks {
122+ private cache = new Map<number, Uint8Array>();
123+ constructor(
124+ private src: PackSource,
125+ private blockBytes: number,
126+ private maxBlocks: number,
127+ private onRead: (bytes: number) => void,
128+ ) {}
129+
130+ /** Forget the blocks kept, so what follows reads exactly as it would from the start. */
131+ clear(): void {
132+ this.cache.clear();
133+ }
134+
135+ async block(i: number): Promise<Uint8Array> {
136+ const hit = this.cache.get(i);
137+ if (hit) {
138+ this.cache.delete(i);
139+ this.cache.set(i, hit);
140+ return hit;
141+ }
142+ const start = i * this.blockBytes;
143+ const len = Math.min(this.blockBytes, this.src.size - start);
144+ if (len <= 0) throw new Error("read past the end of the pack");
145+ this.onRead(len);
146+ const data = await this.src.read(start, len);
147+ if (data.length !== len) throw new Error(`short read of the pack at ${start}: ${data.length} of ${len} bytes`);
148+ this.cache.set(i, data);
149+ while (this.cache.size > this.maxBlocks) this.cache.delete(this.cache.keys().next().value as number);
150+ return data;
151+ }
152+
153+ /** Views of the source from `offset` on, block by block, up to `end`. */
154+ async *views(offset: number, end = this.src.size): AsyncGenerator<Uint8Array> {
155+ let at = offset;
156+ while (at < end) {
157+ const i = Math.floor(at / this.blockBytes);
158+ const b = await this.block(i);
159+ const from = at - i * this.blockBytes;
160+ const to = Math.min(b.length, end - i * this.blockBytes);
161+ yield b.subarray(from, to);
162+ at += to - from;
163+ }
164+ }
165+
166+ async bytes(offset: number, n: number): Promise<Uint8Array> {
167+ const end = Math.min(offset + n, this.src.size);
168+ const out = new Uint8Array(end - offset);
169+ let at = 0;
170+ for await (const v of this.views(offset, end)) {
171+ out.set(v, at);
172+ at += v.length;
173+ }
174+ return out;
175+ }
176+}
177+
178+/**
179+ * An inflater for one zlib stream (zlib only: pako also takes gzip unless told not to, the zlib data
180+ * is stored as sent, and git can't read gzip) that stops at `expectedSize` and writes into one
181+ * buffer of that size, or hands its output to `onChunk`.
182+ */
183+function inflater(offset: number, expectedSize: number, onChunk?: (chunk: Uint8Array) => void) {
184+ const inflator = new pako.Inflate(onChunk ? { chunkSize: 256 * 1024, windowBits: 15 } : { windowBits: 15 }) as pako.Inflate & { ended: boolean };
185+ const out = onChunk ? null : new Uint8Array(expectedSize);
186+ let produced = 0;
187+ inflator.onData = (chunk) => {
188+ const c = chunk as Uint8Array;
189+ // A small zlib stream can't be made to inflate past the size the pack declared.
190+ if (produced + c.length > expectedSize) throw new Error(`object at ${offset} is larger than its declared size (${expectedSize} bytes)`);
191+ if (onChunk) onChunk(c);
192+ else out!.set(c, produced);
193+ produced += c.length;
194+ };
195+ return {
196+ push(v: Uint8Array) {
197+ inflator.push(v, false);
198+ if (inflator.err) throw new Error(`zlib error at ${offset}: ${inflator.msg}`);
199+ return inflator.ended;
200+ },
201+ /** Input pushed but not used: what follows the stream. */
202+ unused() {
203+ return (inflator as unknown as { strm: { avail_in: number } }).strm.avail_in;
204+ },
205+ finish(pushed: number) {
206+ if (!inflator.ended) throw new Error(`truncated zlib stream at offset ${offset}`);
207+ if (produced !== expectedSize) throw new Error(`size mismatch at ${offset}: ${produced} != ${expectedSize}`);
208+ return { consumed: pushed - this.unused(), out };
209+ },
210+ };
211+}
212+
213+/**
214+ * One zlib stream starting at `offset` in the source: its input length, and its output (whole, or
215+ * to `onChunk`). `onInput` gets exactly the stream's bytes, as they are read.
216+ */
217+async function inflateFrom(blocks: Blocks, offset: number, expectedSize: number, onChunk?: (chunk: Uint8Array) => void, onInput?: (bytes: Uint8Array) => void): Promise<{ consumed: number; out: Uint8Array | null }> {
218+ const z = inflater(offset, expectedSize, onChunk);
219+ let pushed = 0;
220+ for await (const v of blocks.views(offset)) {
221+ pushed += v.length;
222+ const ended = z.push(v);
223+ onInput?.(ended ? v.subarray(0, v.length - z.unused()) : v);
224+ if (ended) break;
225+ }
226+ return z.finish(pushed);
227+}
228+
229+/** LRU by entry (or by oid, for a base from outside the pack), bounded in bytes, each entry counted with what holding it costs. */
230+class Lru {
231+ private map = new Map<number | string, Uint8Array>();
232+ private bytes = 0;
233+ constructor(private max: number) {}
234+ get(i: number | string): Uint8Array | undefined {
235+ const hit = this.map.get(i);
236+ if (hit) {
237+ this.map.delete(i);
238+ this.map.set(i, hit);
239+ }
240+ return hit;
241+ }
242+ set(i: number | string, content: Uint8Array): void {
243+ if (content.length > this.max / 4 || this.map.has(i)) return;
244+ this.map.set(i, content);
245+ this.bytes += content.length + CACHED_OVERHEAD;
246+ while (this.bytes > this.max) {
247+ const [k, v] = this.map.entries().next().value as [number | string, Uint8Array];
248+ this.map.delete(k);
249+ this.bytes -= v.length + CACHED_OVERHEAD;
250+ }
251+ }
252+}
253+
254+export class StreamedPack {
255+ private blocks: Blocks;
256+ private cache: Lru;
257+ // The index, one slot per entry in pack order, in typed arrays: the pack says how many entries it has.
258+ private offsets = new Float64Array(0);
259+ /** Bytes from the entry's start to its zlib data. */
260+ private headLen = new Uint8Array(0);
261+ private zlen = new Float64Array(0);
262+ private codes = new Uint8Array(0);
263+ /** The size each entry's header declares: an object's, or a delta's instructions'. */
264+ private declared = new Float64Array(0);
265+ /** Each object's size (a delta's once it is resolved). */
266+ private sizes = new Float64Array(0);
267+ /** Each object's type code, once known. */
268+ private typeCodes = new Uint8Array(0);
269+ /** Each delta's base: an entry, or EXTERNAL for one from outside the pack. */
270+ private baseOf = new Int32Array(0);
271+ /** Each object's oid, 20 bytes apiece, once known. */
272+ private oidBytes = new Uint8Array(0);
273+ /**
274+ * Zlib data kept from the first pass: deltas', packed into a few large buffers (no per-entry
275+ * object). An entry's is at `keptOff` in buffer `keptChunk` (-1: not kept).
276+ */
277+ private keptChunks: Uint8Array[] = [];
278+ private keptChunk = new Int32Array(0);
279+ private keptOff = new Uint32Array(0);
280+ /** Where the next small entry goes in the last shared buffer. */
281+ private keptFill = { chunk: -1, at: 0 };
282+ /** The source last read again, from `start`. */
283+ private window: { start: number; data: Uint8Array } | null = null;
284+ private keptBytes = 0;
285+ /** A REF_DELTA's base oid, until its base is found in the pack (kept for one outside it). */
286+ private refBase = new Map<number, string>();
287+ /** Deltas waiting on a base in the pack, by its entry, or by oid (a REF_DELTA's). */
288+ private waitingOnEntry = new Map<number, number[]>();
289+ private waitingOnOid = new Map<string, number[]>();
290+ /** The zlib data's size of each delta result large enough that storing may write it on its own. */
291+ private zlibSizes = new Map<number, number>();
292+ /** Bases from outside the pack (a thin pack's), by oid: their type. */
293+ private externals = new Map<string, ObjectType>();
294+ /**
295+ * What resolving the deltas did, in order, for `replayDeltas` to do again exactly: a base (its
296+ * entry, or -1 - k for the k-th base from outside the pack), the number of deltas against it, then
297+ * those deltas.
298+ */
299+ private plan: number[] = [];
300+ private externalOids: string[] = [];
301+ private entries = 0;
302+ private unpacked = 0;
303+ private work = 0;
304+ /** What is held, as charged against `maxHeldBytes`, and the most it has been. */
305+ private held = 0;
306+ private peak = 0;
307+ private reads = 0;
308+ private readBytes = 0;
309+ /** What a read counts for: twice while resolving (replaying reads it again), and nothing while replaying. */
310+ private readWeight = 1;
311+ /** Storing's reads of the objects that aren't deltas, counted in advance. */
312+ private inSlices = false;
313+ /** Work, what is held, and reads once the pack was indexed, where resolving the deltas starts. */
314+ private afterIndex = { work: 0, held: 0 };
315+ private yieldedAt = 0;
316+ private readonly maxBytes: number;
317+ private readonly maxWork: number;
318+ private readonly maxHeld: number;
319+ private readonly maxReads: number;
320+ private readonly maxReadBytes: number;
321+ private readonly large: number;
322+ private readonly cacheBytes: number;
323+
324+ private constructor(
325+ private src: PackSource,
326+ private o: UnpackOptions,
327+ ) {
328+ this.blocks = new Blocks(src, o.blockBytes ?? DEFAULTS.blockBytes, o.maxBlocks ?? DEFAULTS.maxBlocks, (n) => {
329+ if (!this.inSlices) this.read_(n, this.readWeight);
330+ });
331+ this.cacheBytes = o.cacheBytes ?? DEFAULTS.cacheBytes;
332+ this.cache = new Lru(this.cacheBytes);
333+ this.maxBytes = o.maxBytes ?? MAX_UNPACKED_BYTES;
334+ this.maxWork = o.maxWork ?? 4 * this.maxBytes;
335+ this.maxHeld = o.maxHeldBytes ?? MAX_HELD_BYTES;
336+ this.maxReads = o.maxReads ?? DEFAULTS.maxReads;
337+ this.maxReadBytes = o.maxReadBytes ?? 6 * src.size + 64 * 1024 * 1024;
338+ this.large = o.largeObjectBytes ?? 4 * 1024 * 1024;
339+ }
340+
341+ /** The index, once it is known to fit. */
342+ private allocate(count: number) {
343+ this.offsets = new Float64Array(count);
344+ this.headLen = new Uint8Array(count);
345+ this.zlen = new Float64Array(count);
346+ this.codes = new Uint8Array(count);
347+ this.declared = new Float64Array(count);
348+ this.sizes = new Float64Array(count);
349+ this.typeCodes = new Uint8Array(count);
350+ this.baseOf = new Int32Array(count).fill(NONE);
351+ this.oidBytes = new Uint8Array(count * 20);
352+ this.keptChunk = new Int32Array(count).fill(-1);
353+ this.keptOff = new Uint32Array(count);
354+ }
355+
356+ get count(): number {
357+ return this.entries;
358+ }
359+
360+ /** Bytes inflated or rebuilt so far, as counted against `maxWork`. */
361+ get workDone(): number {
362+ return this.work;
363+ }
364+
365+ /** What is held now, as charged against `maxHeldBytes`. */
366+ get heldBytes(): number {
367+ return this.held;
368+ }
369+
370+ /** The most it has held at once, as charged against `maxHeldBytes`. */
371+ get peakHeldBytes(): number {
372+ return this.peak;
373+ }
374+
375+ /** Reads of the source so far, those storing will make counted in advance. */
376+ get readsDone(): number {
377+ return this.reads;
378+ }
379+
380+ /**
381+ * At most what storing the push adds: the zlib data of each object that isn't a delta, stored as
382+ * sent, and each delta's content, stored compressed on its own (counted uncompressed).
383+ */
384+ get storedBytes(): number {
385+ let n = 0;
386+ for (let i = 0; i < this.entries; i++) n += this.codes[i] === OFS_DELTA || this.codes[i] === REF_DELTA ? (this.zlibSizes.get(i) ?? this.sizes[i]!) : this.zlen[i]!;
387+ return n;
388+ }
389+
390+ /** A rebuilt delta's size compressed, when it was large enough to be measured (for storing it). */
391+ zlibSizeOf(entry: number): number | undefined {
392+ return this.zlibSizes.get(entry);
393+ }
394+
395+ /**
396+ * Index the pack and resolve every object, handing each one to `visit` once: with its content,
397+ * or, for a large blob, with a feed of its content in slices.
398+ */
399+ static async open(src: PackSource, o: UnpackOptions, visit: Visit): Promise<StreamedPack> {
400+ if (src.size < 32) throw new Error("not a packfile");
401+ const p = new StreamedPack(src, o);
402+ p.read_(12, 1);
403+ const head = await src.read(0, 12);
404+ if (head.length !== 12 || head[0] !== 0x50 || head[1] !== 0x41 || head[2] !== 0x43 || head[3] !== 0x4b) throw new Error("not a packfile");
405+ const view = new DataView(head.buffer, head.byteOffset, 12);
406+ const version = view.getUint32(4);
407+ if (version !== 2 && version !== 3) throw new Error(`unsupported pack version ${version}`);
408+ const count = view.getUint32(8);
409+ if (count > (o.maxObjects ?? MAX_STREAMED_OBJECTS)) throw new Error(`the pack has ${count} objects, more than one push may hold; split it into smaller pushes`);
410+ // An entry is at least a byte of header and a zlib stream (8 bytes): no more can fit.
411+ if (count > (src.size - 32) / 9) throw new Error(`truncated pack: ${src.size} bytes can't hold ${count} objects`);
412+ // Its caches, scratch and index are held throughout: refused up front if they can't fit.
413+ p.hold((o.blockBytes ?? DEFAULTS.blockBytes) * (o.maxBlocks ?? DEFAULTS.maxBlocks) + p.cacheBytes + (o.windowBytes ?? DEFAULTS.windowBytes) + SCRATCH_BYTES + count * ENTRY_OVERHEAD);
414+ p.allocate(count);
415+ // Storing reads the pack once more in order, for the objects that aren't deltas: counted now.
416+ p.reads += Math.ceil(src.size / (o.blockBytes ?? DEFAULTS.blockBytes));
417+ p.readBytes += src.size;
418+ p.checkReads();
419+ await p.index(count, visit);
420+ p.afterIndex = { work: p.work, held: p.held };
421+ p.readWeight = 2;
422+ await p.resolveDeltas(visit);
423+ p.readWeight = 1;
424+ return p;
425+ }
426+
427+ private tooBig() {
428+ return tooBigUnpacked(this.maxBytes);
429+ }
430+
431+ /** Let other work in the isolate run now and then: an unpack can take most of a minute of CPU. */
432+ async breathe(): Promise<void> {
433+ if (this.work - this.yieldedAt < (this.o.yieldEvery ?? 4 * 1024 * 1024)) return;
434+ this.yieldedAt = this.work;
435+ await new Promise((r) => setTimeout(r, 0));
436+ }
437+
438+ private count_(n: number) {
439+ this.work += n;
440+ if (this.work > this.maxWork) throw new Error(`pack takes more than ${this.maxWork} bytes of inflating to unpack; split it into smaller pushes`);
441+ }
442+
443+ private unpack_(n: number) {
444+ if (this.unpacked + n > this.maxBytes) throw this.tooBig();
445+ this.unpacked += n;
446+ }
447+
448+ private hold(n: number) {
449+ if (this.held + n > this.maxHeld) throw tooMuchMemory(this.maxHeld);
450+ this.held += n;
451+ if (this.held > this.peak) this.peak = this.held;
452+ }
453+
454+ private release(n: number) {
455+ this.held -= n;
456+ }
457+
458+ /** What is left to hold an object with now. */
459+ private room(): number {
460+ return this.maxHeld - this.held;
461+ }
462+
463+ private read_(bytes: number, weight: number) {
464+ this.reads += weight;
465+ this.readBytes += bytes * weight;
466+ this.checkReads();
467+ }
468+
469+ /** Storing will write an object of `zlibBytes` (at most) with a request of its own: counted now. */
470+ private willWrite(zlibBytes: number) {
471+ if (zlibBytes > (this.o.writesOver ?? Infinity)) this.read_(0, 1);
472+ }
473+
474+ private checkReads() {
475+ if (this.reads > this.maxReads || this.readBytes > this.maxReadBytes) throw new Error(`pack takes more than ${this.maxReads} reads (or ${this.maxReadBytes} bytes) of storage to unpack; split it into smaller pushes`);
476+ }
477+
478+ private setOid(i: number, oid: string) {
479+ this.oidBytes.set(fromHex(oid), i * 20);
480+ }
481+
482+ private dataAt(i: number): number {
483+ return this.offsets[i]! + this.headLen[i]!;
484+ }
485+
486+ /** Keep entry `i`'s zlib data from the first pass, if it is small and there is room. */
487+ private async keep(i: number): Promise<void> {
488+ const len = this.zlen[i]!;
489+ if (len > KEEP_ENTRY_MAX) return;
490+ const f = this.keptFill;
491+ if (len > KEEP_CHUNK / 4 || f.chunk < 0 || f.at + len > KEEP_CHUNK) {
492+ // A new buffer: of its own for a large entry, else shared by the small ones that follow.
493+ const size = len > KEEP_CHUNK / 4 ? len : KEEP_CHUNK;
494+ if (this.keptBytes + size > (this.o.keepBytes ?? DEFAULTS.keepBytes) || size + CACHED_OVERHEAD > this.room()) return;
495+ this.hold(size + CACHED_OVERHEAD);
496+ this.keptBytes += size + CACHED_OVERHEAD;
497+ this.keptChunks.push(new Uint8Array(size));
498+ if (size === KEEP_CHUNK) this.keptFill = { chunk: this.keptChunks.length - 1, at: 0 };
499+ }
500+ const own = len > KEEP_CHUNK / 4;
501+ const c = own ? this.keptChunks.length - 1 : this.keptFill.chunk;
502+ const at = own ? 0 : this.keptFill.at;
503+ let w = at;
504+ for await (const v of this.blocks.views(this.dataAt(i), this.dataAt(i) + len)) {
505+ this.keptChunks[c]!.set(v, w);
506+ w += v.length;
507+ }
508+ if (!own) this.keptFill.at += len;
509+ this.keptChunk[i] = c;
510+ this.keptOff[i] = at;
511+ }
512+
513+ /** Entry `i`'s zlib data as the first pass kept it, or null. */
514+ private keptData(i: number): Uint8Array | null {
515+ const c = this.keptChunk[i]!;
516+ return c < 0 ? null : this.keptChunks[c]!.subarray(this.keptOff[i]!, this.keptOff[i]! + this.zlen[i]!);
517+ }
518+
519+ private async index(count: number, visit: Visit): Promise<void> {
520+ const size = this.src.size;
521+ // Hashed as it is read, so nothing is read twice.
522+ const checksum = new Sha1().update(await this.blocks.bytes(0, 12));
523+ const hash = (bytes: Uint8Array) => void checksum.update(bytes);
524+ let pos = 12;
525+ for (let n = 0; n < count; n++) {
526+ const offset = pos;
527+ // An entry's header is at most 10 bytes of size, then 20 of base oid (or 10 of base offset).
528+ const h = await this.blocks.bytes(pos, 32);
529+ let k = 0;
530+ let c = h[k++]!;
531+ const code = (c >> 4) & 7;
532+ let objSize = c & 15;
533+ let shift = 4;
534+ while (c & 0x80) {
535+ if (k >= 11) throw new Error(`bad object header at ${offset}`);
536+ c = h[k++]!;
537+ objSize += (c & 0x7f) * 2 ** shift;
538+ shift += 7;
539+ }
540+ let baseEntry = NONE;
541+ let baseOid: string | undefined;
542+ if (code === OFS_DELTA) {
543+ c = h[k++]!;
544+ let ofs = c & 0x7f;
545+ while (c & 0x80) {
546+ if (k >= 22) throw new Error(`bad delta base offset at ${offset}`);
547+ c = h[k++]!;
548+ ofs = (ofs + 1) * 128 + (c & 0x7f);
549+ }
550+ const base = offset - ofs;
551+ if (base < 12 || base >= offset) throw new Error(`bad delta base offset at ${offset}`);
552+ baseEntry = this.entryAt(base, n);
553+ } else if (code === REF_DELTA) {
554+ if (k + 20 > h.length) throw new Error(`truncated pack at ${offset}`);
555+ baseOid = toHex(h.subarray(k, k + 20));
556+ k += 20;
557+ } else if (!CODE_TYPES[code]) {
558+ throw new Error(`bad object type ${code} at ${offset}`);
559+ }
560+ checksum.update(h.subarray(0, k));
561+ pos += k;
562+ this.unpack_(objSize);
563+ const type = CODE_TYPES[code];
564+ let consumed: number;
565+ if (type === "blob" && objSize >= this.large) {
566+ // Hashed as it inflates, never held whole.
567+ const sha = new Sha1().update(objectHeader("blob", objSize));
568+ const r = await inflateFrom(this.blocks, pos, objSize, (chunk) => sha.update(chunk), hash);
569+ this.count_(objSize);
570+ consumed = r.consumed;
571+ const oid = toHex(sha.digest());
572+ this.setOid(n, oid);
573+ this.willWrite(consumed);
574+ const at = pos;
575+ await visit({ oid, type, size: objSize }, null, (onChunk) => this.feed(at, objSize, onChunk));
576+ } else if (type) {
577+ // Held from before it is inflated (at its declared size) until it has been visited.
578+ this.hold(objSize);
579+ const r = await inflateFrom(this.blocks, pos, objSize, undefined, hash);
580+ this.count_(objSize);
581+ consumed = r.consumed;
582+ const oid = await hashObject(type, r.out!);
583+ this.setOid(n, oid);
584+ this.willWrite(consumed);
585+ await visit({ oid, type, size: objSize }, r.out!, null);
586+ this.release(objSize);
587+ } else {
588+ // A delta: read through to find where it ends; it is inflated again when resolved.
589+ const r = await inflateFrom(this.blocks, pos, objSize, () => {}, hash);
590+ this.count_(objSize);
591+ consumed = r.consumed;
592+ if (baseOid !== undefined) {
593+ this.hold(REF_DELTA_OVERHEAD);
594+ this.refBase.set(n, baseOid);
595+ waitOn(this.waitingOnOid, baseOid, n);
596+ } else waitOn(this.waitingOnEntry, baseEntry, n);
597+ }
598+ this.offsets[n] = offset;
599+ this.headLen[n] = k;
600+ this.zlen[n] = consumed;
601+ this.codes[n] = code;
602+ this.declared[n] = objSize;
603+ this.sizes[n] = type ? objSize : 0;
604+ this.typeCodes[n] = type ? code : 0;
605+ this.baseOf[n] = baseEntry;
606+ this.entries = n + 1;
607+ // A delta is inflated again: kept while it is at hand, if there is room.
608+ if (!type) await this.keep(n);
609+ pos += consumed;
610+ if (pos > size - 20) throw new Error(`truncated pack at ${offset}`);
611+ await this.breathe();
612+ }
613+ if (pos !== size - 20) throw new Error("trailing garbage in pack");
614+ const trailer = await this.blocks.bytes(size - 20, 20);
615+ if (!equalBytes(trailer, checksum.digest())) throw new Error("pack checksum mismatch");
616+ }
617+
618+ /** Entry number of the entry starting at `offset`, among the first `n` (offsets ascend). */
619+ private entryAt(offset: number, n: number): number {
620+ let lo = 0;
621+ let hi = n - 1;
622+ while (lo <= hi) {
623+ const mid = (lo + hi) >> 1;
624+ const v = this.offsets[mid]!;
625+ if (v === offset) return mid;
626+ if (v < offset) lo = mid + 1;
627+ else hi = mid - 1;
628+ }
629+ throw new Error(`no object starts at delta base offset ${offset}`);
630+ }
631+
632+ /**
633+ * One pass: each object, once known, has the deltas waiting on it applied, depth first, as git's
634+ * index-pack does: a delta's own deltas are rebuilt from it while it is at hand. The objects that
635+ * aren't deltas start it in pack order, so those read again are read in order. What is left
636+ * waits on bases from outside the pack, each read once.
637+ */
638+ private async resolveDeltas(visit: Visit): Promise<void> {
639+ this.window = null;
640+ this.blocks.clear();
641+ const onKid = (kid: number, content: Uint8Array) => this.visitResolved(kid, content, visit);
642+ const takeKids = (kid: number) => this.takeWaiting(kid);
643+ for (let i = 0; i < this.entries; i++) {
644+ if (this.codes[i] === OFS_DELTA || this.codes[i] === REF_DELTA) continue;
645+ const kids = this.takeWaiting(i);
646+ if (!kids) continue;
647+ // A large blob is inflated once for all its deltas: counted once, as parsePack does.
648+ if (this.codes[i] === TYPE_CODES.blob && this.sizes[i]! >= this.large) this.unpack_(this.sizes[i]!);
649+ await this.descend({ base: await this.content_(i) }, i, this.typeCodes[i]!, kids, onKid, takeKids);
650+ }
651+ for (const oid of [...this.waitingOnOid.keys()]) {
652+ if (!this.waitingOnOid.has(oid) || !this.o.external) continue;
653+ const start = await this.externalStart(oid);
654+ if (!start) continue;
655+ const kids = this.waitingOnOid.get(oid)!;
656+ this.waitingOnOid.delete(oid);
657+ this.planGroup(-1 - this.externalOids.length, kids);
658+ this.externalOids.push(oid);
659+ this.unpack_(start.first.base!.length);
660+ await this.descend(start.first, EXTERNAL, start.typeCode, kids, onKid, takeKids);
661+ }
662+ const unresolved = [...this.waitingOnOid.keys(), ...[...this.waitingOnEntry.keys()].map((e) => `@${this.offsets[e]}`)];
663+ if (unresolved.length) throw new Error(`unresolvable delta base(s): ${unresolved.join(", ")}`);
664+ }
665+
666+ /** The deltas waiting on entry `b`, now known, taken (and recorded for the replay); null if none. */
667+ private takeWaiting(b: number): number[] | null {
668+ const byEntry = this.waitingOnEntry.get(b);
669+ const oid = this.oidOf(b);
670+ const byOid = this.waitingOnOid.get(oid);
671+ if (!byEntry && !byOid) return null;
672+ this.waitingOnEntry.delete(b);
673+ this.waitingOnOid.delete(oid);
674+ const kids = byEntry && byOid ? [...byEntry, ...byOid] : (byEntry ?? byOid)!;
675+ this.planGroup(b, kids);
676+ return kids;
677+ }
678+
679+ private planGroup(b: number, kids: number[]) {
680+ this.plan.push(b, kids.length);
681+ for (const k of kids) this.plan.push(k);
682+ }
683+
684+ private async visitResolved(kid: number, content: Uint8Array, visit: Visit): Promise<void> {
685+ const type = CODE_TYPES[this.typeCodes[kid]!]!;
686+ const oid = await hashObject(type, content);
687+ this.setOid(kid, oid);
688+ // One that may be stored on its own is compressed now to learn its size, which storing reuses.
689+ let zlib = deflateBound(content.length);
690+ if (zlib > (this.o.writesOver ?? Infinity)) {
691+ zlib = 0;
692+ for (const c of deflateChunks(content)) zlib += c.length;
693+ this.zlibSizes.set(kid, zlib);
694+ }
695+ this.willWrite(zlib);
696+ await visit({ oid, type, size: content.length }, content, null);
697+ }
698+
699+ /**
700+ * Rebuild each of `kids` from `base` (held, released here), handing each on, and each one's own
701+ * deltas from it while it is held. The same steps when resolving and when replaying, so both do
702+ * the same work, hold the same and read the same.
703+ */
704+ private async descend(
705+ first: { base: Uint8Array | null },
706+ b: number,
707+ typeCode: number,
708+ kids: number[],
709+ onKid: (kid: number, content: Uint8Array) => Promise<void>,
710+ takeKids: (kid: number) => number[] | null,
711+ ): Promise<void> {
712+ // The stack is the only reference to the first base (the caller's is dropped), so letting go of it frees it.
713+ const stack = [{ base: first.base!, b, kids, k: 0 }];
714+ first.base = null;
715+ while (stack.length) {
716+ const top = stack[stack.length - 1]!;
717+ const kid = top.kids[top.k++]!;
718+ const content = await this.applyOne(kid, top.base, this.maxBytes - this.unpacked);
719+ if (top.b !== EXTERNAL) this.refBase.delete(kid);
720+ this.baseOf[kid] = top.b;
721+ this.typeCodes[kid] = typeCode;
722+ this.sizes[kid] = content.length;
723+ this.unpack_(content.length);
724+ this.cache.set(kid, content);
725+ await onKid(kid, content);
726+ await this.breathe();
727+ // A base is let go once its last delta is rebuilt, before that delta's own deltas: a long
728+ // chain of versions of one file holds two of them at a time, not all of them.
729+ if (top.k === top.kids.length) {
730+ this.release(top.base.length);
731+ top.base = EMPTY;
732+ stack.pop();
733+ }
734+ const next = takeKids(kid);
735+ if (next) stack.push({ base: content, b: kid, kids: next, k: 0 });
736+ else this.release(content.length);
737+ }
738+ }
739+
740+ /** Delta `i`'s instructions, inflated again and applied to `base`: its result, held. */
741+ private async applyOne(i: number, base: Uint8Array, limit: number): Promise<Uint8Array> {
742+ const delta = await this.inflateAgain(i);
743+ const size = deltaResultSize(delta);
744+ // The pack's limit first, with parsePack's message; then memory, before the result is allocated.
745+ if (size > limit) throw this.tooBig();
746+ this.hold(size);
747+ const content = applyDelta(base, delta, size);
748+ this.release(delta.length);
749+ this.count_(content.length);
750+ return content;
751+ }
752+
753+ /**
754+ * Entry `i`'s zlib data inflated again, held. An object that isn't a delta is read through the
755+ * blocks: resolving starts from those in pack order, so they are read in order. A delta's comes
756+ * from what the first pass kept, or is read again.
757+ */
758+ private async inflateAgain(i: number): Promise<Uint8Array> {
759+ const declared = this.declared[i]!;
760+ const len = this.zlen[i]!;
761+ this.hold(declared);
762+ if (this.codes[i] !== OFS_DELTA && this.codes[i] !== REF_DELTA) {
763+ const { out } = await inflateFrom(this.blocks, this.dataAt(i), declared);
764+ this.count_(declared);
765+ return out!;
766+ }
767+ const z = this.keptData(i) ?? (await this.reread(i));
768+ const data = z instanceof Uint8Array ? z : z.data;
769+ const inf = inflater(this.offsets[i]!, declared);
770+ inf.push(data);
771+ const { out } = inf.finish(len);
772+ if (!(z instanceof Uint8Array)) this.release(z.held);
773+ this.count_(declared);
774+ return out!;
775+ }
776+
777+ /**
778+ * Entry `i`'s zlib data read again: from the window of the source last read, or a new window
779+ * from it on (exactly its range, held while in use, if larger than a window).
780+ */
781+ private async reread(i: number): Promise<Uint8Array | { data: Uint8Array; held: number }> {
782+ const at = this.dataAt(i);
783+ const len = this.zlen[i]!;
784+ const w = this.window;
785+ if (w && at >= w.start && at + len <= w.start + w.data.length) return w.data.subarray(at - w.start, at - w.start + len);
786+ const windowBytes = this.o.windowBytes ?? DEFAULTS.windowBytes;
787+ const n = len > windowBytes ? len : Math.max(len, Math.min(windowBytes, this.src.size - 20 - at));
788+ if (len > windowBytes) this.hold(len);
789+ this.read_(n, this.readWeight);
790+ const data = await this.src.read(at, n);
791+ if (data.length !== n) throw new Error(`short read of the pack at ${at}: ${data.length} of ${n} bytes`);
792+ if (len > windowBytes) return { data, held: len };
793+ this.window = { start: at, data };
794+ return data.subarray(0, len);
795+ }
796+
797+ private async feed(at: number, size: number, onChunk: (chunk: Uint8Array) => void): Promise<void> {
798+ this.count_(size);
799+ await inflateFrom(this.blocks, at, size, onChunk);
800+ }
801+
802+ /** Entry `i`'s content, held by the caller until it releases it (a cache spares most of the work). */
803+ private async content_(i: number, depth = 0): Promise<Uint8Array> {
804+ const hit = this.cache.get(i);
805+ if (hit) {
806+ this.hold(hit.length);
807+ return hit;
808+ }
809+ if (depth > 10_000) throw new Error("delta chain too deep");
810+ const code = this.codes[i]!;
811+ let content: Uint8Array;
812+ if (code === OFS_DELTA || code === REF_DELTA) {
813+ const b = this.baseOf[i]!;
814+ let base: Uint8Array;
815+ if (b >= 0) base = await this.content_(b, depth + 1);
816+ else {
817+ const oid = this.refBase.get(i);
818+ const ext = oid === undefined ? null : await this.externalBase(oid);
819+ if (!ext) throw new Error(`delta base of the object at ${this.offsets[i]} is missing`);
820+ base = ext.content;
821+ }
822+ content = await this.applyOne(i, base, this.maxBytes);
823+ this.release(base.length);
824+ } else content = await this.inflateAgain(i);
825+ this.cache.set(i, content);
826+ return content;
827+ }
828+
829+ /** Entry `i`'s content: inflated, and rebuilt from its delta chain, again (a cache spares most of it). */
830+ async content(i: number): Promise<Uint8Array> {
831+ const c = await this.content_(i);
832+ this.release(c.length);
833+ return c;
834+ }
835+
836+ /**
837+ * A base from outside the pack, read (and inflated) by `external`, held by the caller until it
838+ * releases it: kept in the cache like a resolved object, and each read counted against
839+ * `maxWork`, so many tiny deltas against one large stored object can't make the server read it
840+ * again without bound.
841+ */
842+ private async externalBase(oid: string): Promise<GitObject | null> {
843+ const type = this.externals.get(oid);
844+ const hit = type && this.cache.get(oid);
845+ if (hit) {
846+ this.hold(hit.length);
847+ return { type, content: hit };
848+ }
849+ // Its size is checked before it is read: a small delta can name a huge stored object.
850+ const base = (await this.o.external?.(oid, this.room())) ?? null;
851+ if (base === null) return null;
852+ if (typeof base === "number") throw tooMuchMemory(this.maxHeld);
853+ this.read_(0, this.readWeight);
854+ this.hold(base.content.length);
855+ this.count_(base.content.length);
856+ this.externals.set(oid, base.type);
857+ this.cache.set(oid, base.content);
858+ return base;
859+ }
860+
861+ /** A base from outside the pack, to descend from: its content in a holder `descend` empties, and its type. */
862+ private async externalStart(oid: string): Promise<{ first: { base: Uint8Array | null }; typeCode: number } | null> {
863+ const ext = await this.externalBase(oid);
864+ return ext && { first: { base: ext.content }, typeCode: TYPE_CODES[ext.type] };
865+ }
866+
867+ /**
868+ * Every delta's content again, for storing: rebuilt in the order resolving them took and from
869+ * the same empty caches, so it does exactly the work, holds exactly as much, reads exactly the
870+ * same and reaches exactly the chain depth that already passed in `open`. Nothing in it can go
871+ * over a limit `open` didn't, so a push can't be refused partway through being stored.
872+ */
873+ async replayDeltas(each: (entry: number, content: Uint8Array) => Promise<void>): Promise<void> {
874+ this.cache = new Lru(this.cacheBytes);
875+ this.window = null;
876+ this.blocks.clear();
877+ const after = { work: this.work, held: this.held, unpacked: this.unpacked };
878+ this.work = this.afterIndex.work;
879+ this.held = this.afterIndex.held;
880+ // Counted from here again, or it wouldn't yield until the work passed where `open` left off.
881+ this.yieldedAt = this.work;
882+ // Its reads were counted in `open`, and the limit on what the pack unpacks to checked there.
883+ this.readWeight = 0;
884+ this.unpacked = 0;
885+ const plan = this.plan;
886+ let at = 0;
887+ const next = () => {
888+ const b = plan[at]!;
889+ const kids = plan.slice(at + 2, at + 2 + plan[at + 1]!);
890+ at += 2 + kids.length;
891+ return { b, kids };
892+ };
893+ // A delta's own group comes right after it in the plan, as `open` took it.
894+ const takeKids = (kid: number) => (at < plan.length && plan[at] === kid ? next().kids : null);
895+ while (at < plan.length) {
896+ const { b, kids } = next();
897+ if (b >= 0) await this.descend({ base: await this.content_(b) }, b, this.typeCodes[b]!, kids, each, takeKids);
898+ else {
899+ const oid = this.externalOids[-1 - b]!;
900+ const start = await this.externalStart(oid);
901+ if (!start) throw new Error(`delta base ${oid} is missing`);
902+ await this.descend(start.first, EXTERNAL, start.typeCode, kids, each, takeKids);
903+ }
904+ }
905+ this.readWeight = 1;
906+ this.work = after.work;
907+ this.held = after.held;
908+ this.unpacked = after.unpacked;
909+ }
910+
911+ /**
912+ * Every object in pack order, for storing. One that isn't a delta comes with its zlib data's place
913+ * in the pack (checked to inflate to exactly its content), to be stored as it is: neither inflated
914+ * nor compressed again.
915+ */
916+ *objects(): Generator<StreamedObject & { entry: number; deflated?: { at: number; length: number } }> {
917+ for (let i = 0; i < this.entries; i++) {
918+ const o = { oid: this.oidOf(i), type: this.typeOf(i), size: this.sizes[i]!, entry: i };
919+ if (this.codes[i] !== OFS_DELTA && this.codes[i] !== REF_DELTA) yield { ...o, deflated: { at: this.dataAt(i), length: this.zlen[i]! } };
920+ else yield o;
921+ }
922+ }
923+
924+ /** `length` bytes of the pack from `at`, slice by slice (in order: these reads were counted in `open`). */
925+ async *slices(at: number, length: number): AsyncGenerator<Uint8Array> {
926+ for (const views = this.blocks.views(at, at + length); ; ) {
927+ this.inSlices = true;
928+ let next: IteratorResult<Uint8Array>;
929+ try {
930+ next = await views.next();
931+ } finally {
932+ this.inSlices = false;
933+ }
934+ if (next.done) return;
935+ yield next.value;
936+ }
937+ }
938+
939+ oidOf(i: number): string {
940+ return toHex(this.oidBytes.subarray(i * 20, i * 20 + 20));
941+ }
942+
943+ typeOf(i: number): ObjectType {
944+ return CODE_TYPES[this.typeCodes[i]!]!;
945+ }
946+
947+ /** The pushed objects with these oids, as entry numbers (one pass over the index). */
948+ entriesOf(oids: ReadonlySet<string>): Map<string, number> {
949+ const found = new Map<string, number>();
950+ if (!oids.size) return found;
951+ for (let i = 0; i < this.entries; i++) {
952+ const oid = this.oidOf(i);
953+ if (oids.has(oid) && !found.has(oid)) found.set(oid, i);
954+ }
955+ return found;
956+ }
957+}
958+
959+type Visit = (obj: StreamedObject, content: Uint8Array | null, feed: ChunkFeed | null) => void | Promise<void>;
960+
961+const NONE = -1;
962+const EMPTY = new Uint8Array(0);
963+/** A delta's base from outside the pack. */
964+const EXTERNAL = -2;
965+
966+function waitOn<K>(m: Map<K, number[]>, k: K, i: number) {
967+ const list = m.get(k);
968+ if (list) list.push(i);
969+ else m.set(k, [i]);
970+}
modifiedpackages/git-core/src/pack.ts+31−6View file
@@ -1,5 +1,5 @@
11 import { ByteWriter, equalBytes, fromUtf8, toHex } from "./bytes.js";
2-import { applyDelta } from "./delta.js";
2+import { applyDelta, deltaResultSize } from "./delta.js";
33 import { CODE_TYPES, TYPE_CODES, hashObject, objectHeader, type GitObject, type ObjectType } from "./objects.js";
44 import { Sha1, sha1 } from "./sha1.js";
55 import { inflate, inflateAt, inflateEach } from "./zlib.js";
@@ -34,8 +34,28 @@
3434 data: Uint8Array;
3535 }
3636
37-export type ExternalLookup = (oid: string) => Promise<GitObject | null>;
37+/**
38+ * A REF_DELTA base from the repository (a thin pack's), or null when it has no such object. Given
39+ * what the unpack may still hold, it checks what reading the object would take before reading it,
40+ * and returns that number of bytes instead when it is more, so a small delta against a huge stored
41+ * object is refused before the object is inflated.
42+ */
43+export type ExternalLookup = (oid: string, maxBytes: number) => Promise<GitObject | number | null>;
3844
45+/** What unpacking refuses when the pack inflates to more than `maxBytes` in all. */
46+export function tooBigUnpacked(maxBytes: number): Error {
47+ const mib = maxBytes / (1024 * 1024);
48+ return new Error(`pack unpacks to more than ${Number.isInteger(mib) ? `${mib} MiB` : `${maxBytes} bytes`}; split it into smaller pushes`);
49+}
50+
51+/** Unpacking would hold more than it may. */
52+export class TooMuchMemory extends Error {}
53+
54+/** What unpacking refuses when it would hold more than `maxHeld` bytes. */
55+export function tooMuchMemory(maxHeld: number): TooMuchMemory {
56+ return new TooMuchMemory(`pack needs more than ${Math.floor(maxHeld / (1024 * 1024))} MiB of memory to unpack; split it into smaller pushes`);
57+}
58+
3959 /** What a pack may unpack to by default (object contents and delta instructions together). */
4060 export const MAX_UNPACKED_BYTES = 512 * 1024 * 1024;
4161
@@ -88,11 +108,11 @@
88108 let unpacked = 0;
89109 let held = 0;
90110 const work = (n: number) => {
91- if (unpacked + n > maxBytes) throw new Error(`pack unpacks to more than ${maxBytes} bytes`);
111+ if (unpacked + n > maxBytes) throw tooBigUnpacked(maxBytes);
92112 unpacked += n;
93113 };
94114 const hold = (n: number) => {
95- if (held + n > maxHeld) throw new Error(`pack needs more than ${Math.floor(maxHeld / (1024 * 1024))} MiB of memory to unpack; split it into smaller pushes`);
115+ if (held + n > maxHeld) throw tooMuchMemory(maxHeld);
96116 held += n;
97117 };
98118
@@ -180,6 +200,10 @@
180200 }
181201 const content = base.content;
182202 for (const d of [...(byOffset ?? []), ...(byOid ?? [])]) {
203+ // A result the pack's limit allows but memory can't hold is refused as memory, before it is allocated.
204+ const size = deltaResultSize(d.data);
205+ if (size > maxBytes - unpacked) throw tooBigUnpacked(maxBytes);
206+ if (size > maxHeld - held + d.data.length) throw tooMuchMemory(maxHeld);
183207 const result = applyDelta(content, d.data, Math.min(maxBytes - unpacked, maxHeld - held + d.data.length));
184208 work(result.length);
185209 held -= d.data.length;
@@ -199,8 +223,9 @@
199223 if (external) {
200224 for (const oid of [...childrenByOid.keys()]) {
201225 if (!childrenByOid.has(oid)) continue;
202- const base = await external(oid);
203- if (!base) continue;
226+ const base = await external(oid, maxHeld - held);
227+ if (base === null) continue;
228+ if (typeof base === "number") throw tooMuchMemory(maxHeld);
204229 work(base.content.length);
205230 hold(base.content.length);
206231 await resolveDeltasOf({ oid, ...base }, -1);
modifiedpackages/git-core/src/zlib.ts+45−3View file
@@ -5,17 +5,59 @@
55 return pako.deflate(data);
66 }
77
8+/**
9+ * The same zlib data as `deflate`, piece by piece as it is produced, so compressing a large object
10+ * holds it and a few small chunks, not a second copy of it compressed. Each chunk is new.
11+ */
12+export function* deflateChunks(data: Uint8Array): Generator<Uint8Array> {
13+ const d = new pako.Deflate();
14+ let out: Uint8Array[] = [];
15+ d.onData = (chunk) => void out.push(chunk as Uint8Array);
16+ const step = 256 * 1024;
17+ for (let at = 0; ; at += step) {
18+ const last = at + step >= data.length;
19+ d.push(data.subarray(at, Math.min(data.length, at + step)), last);
20+ if (d.err) throw new Error(`zlib error: ${d.msg}`);
21+ yield* out;
22+ out = [];
23+ if (last) return;
24+ }
25+}
26+
27+// zlib only: pako also takes gzip unless `windowBits` is given, and git can't read gzip.
28+const ZLIB = { windowBits: 15 };
29+
830 export function inflate(data: Uint8Array): Uint8Array {
9- return pako.inflate(data);
31+ return pako.inflate(data, ZLIB);
1032 }
1133
1234 /**
35+ * Inflate zlib data known to hold exactly `size` bytes, into one buffer of that size: no chunks
36+ * joined at the end, so it never needs twice the object.
37+ */
38+export function inflateSized(data: Uint8Array, size: number): Uint8Array {
39+ const inflator = new pako.Inflate(ZLIB) as pako.Inflate & { ended: boolean };
40+ const out = new Uint8Array(size);
41+ let produced = 0;
42+ inflator.onData = (chunk) => {
43+ const c = chunk as Uint8Array;
44+ if (produced + c.length > size) throw new Error(`object is larger than its recorded size (${size} bytes)`);
45+ out.set(c, produced);
46+ produced += c.length;
47+ };
48+ inflator.push(data, true);
49+ if (inflator.err) throw new Error(`zlib error: ${inflator.msg}`);
50+ if (!inflator.ended || produced !== size) throw new Error(`object inflated to ${produced} bytes, not its recorded ${size}`);
51+ return out;
52+}
53+
54+/**
1355 * Inflate one zlib stream that starts at `offset` inside `data`, and report how
1456 * many input bytes it consumed. Pack files concatenate zlib streams without
1557 * lengths, so the consumed count is how we find the next entry.
1658 */
1759 export function inflateAt(data: Uint8Array, offset: number, expectedSize?: number): { out: Uint8Array; consumed: number } {
18- const inflator = new pako.Inflate() as pako.Inflate & { ended: boolean };
60+ const inflator = new pako.Inflate(ZLIB) as pako.Inflate & { ended: boolean };
1961 if (expectedSize !== undefined) {
2062 // Stop as soon as the output passes the size the pack declared, so a small zlib stream
2163 // can't be made to inflate to gigabytes.
@@ -49,7 +91,7 @@
4991 * bytes produced.
5092 */
5193 export function inflateEach(data: Uint8Array, offset: number, onChunk: (chunk: Uint8Array) => void, expectedSize?: number): { consumed: number; produced: number } {
52- const inflator = new pako.Inflate({ chunkSize: 256 * 1024 }) as pako.Inflate & { ended: boolean };
94+ const inflator = new pako.Inflate({ ...ZLIB, chunkSize: 256 * 1024 }) as pako.Inflate & { ended: boolean };
5395 let produced = 0;
5496 inflator.onData = (chunk) => {
5597 const c = chunk as Uint8Array;
modifiedpackages/git-core/test/pack-limits.test.ts+1−1View file
@@ -89,7 +89,7 @@
8989 const base = Buffer.from("base");
9090 const delta = Uint8Array.from([...varint(base.length), ...varint(300 * 1024 * 1024), 4, ...base]);
9191 const p = await pack([blob(base), refDelta(await hashObject("blob", base), delta)]);
92- await expect(parsePack(p)).rejects.toThrow(/delta result of \d+ bytes is larger/);
92+ await expect(parsePack(p)).rejects.toThrow(/pack needs more than 80 MiB of memory/);
9393 });
9494
9595 it("counts a thin pack's bases from the repository, once each", async () => {
addedpackages/git-core/test/pack-unpack-stream.test.ts+486−0View file
@@ -1,0 +1,486 @@
1+import { execFileSync } from "node:child_process";
2+import { randomBytes } from "node:crypto";
3+import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
4+import { tmpdir } from "node:os";
5+import { join } from "node:path";
6+import { createHash } from "node:crypto";
7+import { deflateSync, gzipSync, inflateSync } from "node:zlib";
8+import { setFlagsFromString } from "node:v8";
9+import { runInNewContext } from "node:vm";
10+import { afterAll, describe, expect, it, vi } from "vitest";
11+import { ENTRY_OVERHEAD, HASH_COPY_MAX, REF_DELTA_OVERHEAD, SCRATCH_BYTES, StreamedPack, TooMuchMemory, deflate, deflateChunks, hashObject, parsePack, writePack, type ChunkFeed, type GitObject, type PackSource, type StreamedObject } from "../src/index.js";
12+
13+const dir = mkdtempSync(join(tmpdir(), "pack-stream-"));
14+afterAll(() => rmSync(dir, { recursive: true, force: true }));
15+const git = (args: string[], input?: string) => execFileSync("git", args, { cwd: dir, input, maxBuffer: 1 << 28, env: { ...process.env, GIT_AUTHOR_NAME: "a", GIT_AUTHOR_EMAIL: "a@x", GIT_COMMITTER_NAME: "a", GIT_COMMITTER_EMAIL: "a@x" } });
16+
17+// A history whose files change a little each commit, so the pack has delta chains, plus a large blob.
18+git(["init", "-q", "-b", "main"]);
19+let text = Array.from({ length: 400 }, (_, i) => `line ${i} of a file that keeps changing\n`).join("");
20+// Random bytes, so the pack itself is several blocks long.
21+const large = new Uint8Array(randomBytes(5 * 1024 * 1024));
22+for (let c = 0; c < 12; c++) {
23+ text = text.replace(`line ${c * 3}`, `changed ${c}`) + `appended ${c}\n`;
24+ writeFileSync(join(dir, "a.txt"), text);
25+ writeFileSync(join(dir, "b.txt"), text.split("").reverse().join(""));
26+ if (c === 5) writeFileSync(join(dir, "big.bin"), large);
27+ git(["add", "-A"]);
28+ git(["commit", "-q", "-m", `c${c}`]);
29+}
30+const pack = (args: string[], revs: string) => new Uint8Array(git(["pack-objects", "--stdout", "-q", ...args], revs));
31+const full = pack(["--revs"], "main\n");
32+const thin = pack(["--revs", "--thin"], "main\n^main~4\n");
33+
34+/** Reads count how many bytes were fetched, to show the pack is read in ranges. */
35+function source(data: Uint8Array, reads: { bytes: number; calls: number } = { bytes: 0, calls: 0 }): PackSource {
36+ return {
37+ size: data.length,
38+ read: async (offset, length) => {
39+ reads.calls++;
40+ reads.bytes += length;
41+ return data.slice(offset, offset + length);
42+ },
43+ };
44+}
45+
46+/** A pack of deltas, each against the base with this oid (from outside the pack): `deltas[i]` is its instructions. */
47+function refDeltaPack(baseOid: string, deltas: Uint8Array[]): Uint8Array {
48+ const parts: Buffer[] = [Buffer.from("PACK"), Buffer.from([0, 0, 0, 2]), Buffer.alloc(4)];
49+ parts[2]!.writeUInt32BE(deltas.length);
50+ for (const d of deltas) {
51+ const head: number[] = [];
52+ let size = d.length;
53+ let c = (7 << 4) | (size & 15);
54+ size = Math.floor(size / 16);
55+ while (size > 0) {
56+ head.push(c | 0x80);
57+ c = size & 0x7f;
58+ size = Math.floor(size / 128);
59+ }
60+ head.push(c);
61+ parts.push(Buffer.from(head), Buffer.from(baseOid, "hex"), deflateSync(d));
62+ }
63+ const body = Buffer.concat(parts);
64+ return new Uint8Array(Buffer.concat([body, createHash("sha1").update(body).digest()]));
65+}
66+
67+/** REF_DELTAs, each against the oid given, then one blob. */
68+function refDeltaChain(deltas: { base: string; delta: Uint8Array }[], blob: Uint8Array): Uint8Array {
69+ const parts: Buffer[] = [Buffer.from("PACK"), Buffer.from([0, 0, 0, 2]), Buffer.alloc(4)];
70+ parts[2]!.writeUInt32BE(deltas.length + 1);
71+ const header = (code: number, size: number) => {
72+ const head: number[] = [];
73+ let c = (code << 4) | (size & 15);
74+ size = Math.floor(size / 16);
75+ while (size > 0) {
76+ head.push(c | 0x80);
77+ c = size & 0x7f;
78+ size = Math.floor(size / 128);
79+ }
80+ head.push(c);
81+ return Buffer.from(head);
82+ };
83+ for (const d of deltas) parts.push(header(7, d.delta.length), Buffer.from(d.base, "hex"), deflateSync(d.delta));
84+ parts.push(header(3, blob.length), deflateSync(blob));
85+ const body = Buffer.concat(parts);
86+ return new Uint8Array(Buffer.concat([body, createHash("sha1").update(body).digest()]));
87+}
88+
89+const varint = (n: number) => {
90+ const out: number[] = [];
91+ do {
92+ let b = n & 0x7f;
93+ n = Math.floor(n / 128);
94+ if (n > 0) b |= 0x80;
95+ out.push(b);
96+ } while (n > 0);
97+ return out;
98+};
99+
100+/** A delta that copies one byte of the base at `at`, then inserts `tag`. */
101+function tinyDelta(baseSize: number, at: number, tag: number): Uint8Array {
102+ return new Uint8Array([...varint(baseSize), ...varint(2), 0x80 | 0x0f | 0x10, at & 0xff, (at >> 8) & 0xff, (at >> 16) & 0xff, (at >> 24) & 0xff, 1, 1, tag]);
103+}
104+
105+async function unpack(data: Uint8Array, opts: Parameters<typeof StreamedPack.open>[1] = {}) {
106+ const seen = new Map<string, { type: string; content: Uint8Array }>();
107+ const visit = async (o: StreamedObject, content: Uint8Array | null, feed: ChunkFeed | null) => {
108+ let bytes = content;
109+ if (!bytes) {
110+ const parts: Uint8Array[] = [];
111+ await feed!((c) => parts.push(c.slice()));
112+ bytes = new Uint8Array(parts.reduce((n, p) => n + p.length, 0));
113+ let at = 0;
114+ for (const p of parts) bytes.set(p, (at += p.length) - p.length);
115+ }
116+ expect(bytes.length).toBe(o.size);
117+ seen.set(o.oid, { type: o.type, content: bytes });
118+ };
119+ const p = await StreamedPack.open(source(data), { largeObjectBytes: 4 * 1024 * 1024, blockBytes: 64 * 1024, maxBlocks: 4, cacheBytes: 256 * 1024, ...opts }, visit);
120+ return { p, seen };
121+}
122+
123+describe("StreamedPack", () => {
124+ it("finds the same objects, with the same contents, as parsePack", async () => {
125+ const want = await parsePack(full);
126+ const { p, seen } = await unpack(full);
127+ expect(p.count).toBe(want.length);
128+ expect(seen.size).toBe(new Set(want.map((o) => o.oid)).size);
129+ for (const o of want) {
130+ expect(seen.get(o.oid)?.type).toBe(o.type);
131+ expect(Buffer.from(seen.get(o.oid)!.content).equals(Buffer.from(o.content))).toBe(true);
132+ }
133+ // Every object comes back again for storing: a delta's content rebuilt, any other as its zlib data's place in the pack.
134+ const listed = [...p.objects()];
135+ expect(listed.map((o) => o.oid).sort()).toEqual(want.map((o) => o.oid).sort());
136+ const big = listed.find((o) => o.size === large.length)!;
137+ expect(big.deflated).toBeDefined();
138+ const deltas = listed.filter((x) => !x.deflated);
139+ expect(deltas.length).toBeGreaterThan(0);
140+ for (const o of [...deltas.slice(0, 25), ...listed.filter((x) => x.deflated && x !== big).slice(0, 25)]) {
141+ expect(Buffer.from(await p.content(o.entry)).equals(Buffer.from(seen.get(o.oid)!.content))).toBe(true);
142+ }
143+ for (const o of listed.filter((x) => x.deflated && x !== big).slice(0, 25)) {
144+ const parts: Uint8Array[] = [];
145+ for await (const s of p.slices(o.deflated!.at, o.deflated!.length)) parts.push(s.slice());
146+ expect(Buffer.from(inflateSync(Buffer.concat(parts))).equals(Buffer.from(seen.get(o.oid)!.content))).toBe(true);
147+ }
148+ let zlib = 0;
149+ for await (const s of p.slices(big.deflated!.at, big.deflated!.length)) zlib += s.length;
150+ expect(zlib).toBe(big.deflated!.length);
151+ });
152+
153+ it("resolves a thin pack's bases from outside the pack", async () => {
154+ const stored = new Map<string, GitObject>();
155+ for (const o of await parsePack(full)) stored.set(o.oid, { type: o.type, content: o.content });
156+ const external = async (oid: string) => stored.get(oid) ?? null;
157+ const want = await parsePack(thin, external);
158+ const { seen } = await unpack(thin, { external });
159+ expect(seen.size).toBe(want.length);
160+ for (const o of want) expect(Buffer.from(seen.get(o.oid)!.content).equals(Buffer.from(o.content))).toBe(true);
161+ await expect(unpack(thin)).rejects.toThrow(/unresolvable delta base/);
162+ });
163+
164+ it("reads the pack in ranges and keeps the #125 budget", async () => {
165+ const reads = { bytes: 0, calls: 0 };
166+ await StreamedPack.open(source(full, reads), { blockBytes: 64 * 1024, maxBlocks: 2 }, () => {});
167+ expect(reads.calls).toBeGreaterThan(1);
168+ // The same limit parsePack applies, counted the same way (deltas are resolved in another order,
169+ // so the one that goes over can differ).
170+ const over = /delta result of \d+ bytes is larger than the [\d.]+ bytes allowed|pack unpacks to more than/;
171+ await expect(parsePack(full, undefined, { maxBytes: 5.2 * 1024 * 1024, largeObjectBytes: 4 * 1024 * 1024 })).rejects.toThrow(over);
172+ await expect(unpack(full, { maxBytes: 5.2 * 1024 * 1024 })).rejects.toThrow(over);
173+ await expect(unpack(full, { maxWork: 1024 * 1024 })).rejects.toThrow(/split it into smaller pushes/);
174+ await expect(unpack(full, { maxObjects: 3 })).rejects.toThrow(/split it into smaller pushes/);
175+ });
176+
177+ it("refuses a corrupt pack", async () => {
178+ const bad = full.slice();
179+ bad[bad.length - 1]! ^= 1;
180+ await expect(unpack(bad)).rejects.toThrow(/checksum mismatch/);
181+ const longer = new Uint8Array(full.length + 5);
182+ longer.set(full.subarray(0, full.length - 20));
183+ longer.set(full.subarray(full.length - 20), full.length + 5 - 20);
184+ await expect(unpack(longer)).rejects.toThrow(/trailing garbage|zlib|bad object|size mismatch|truncated/);
185+ await expect(unpack(new TextEncoder().encode("NOPE".padEnd(40, "x")))).rejects.toThrow(/not a packfile/);
186+ });
187+
188+ it("refuses an object compressed with gzip instead of zlib", async () => {
189+ // git can't read gzip, and the object would be stored as sent. pako takes gzip unless told not
190+ // to, and only stumbles on what follows the stream; so these end exactly where a read does.
191+ const content = new TextEncoder().encode("hello\n".repeat(100));
192+ const gzipped = new Uint8Array(gzipSync(content));
193+ const gz = await writePack([{ type: "blob", size: content.length, deflated: gzipped }]);
194+ const blockBytes = 14 + gzipped.length;
195+ await expect(unpack(gz, { blockBytes })).rejects.toThrow(/zlib error/);
196+ await expect(unpack(gz, { blockBytes, largeObjectBytes: 16 })).rejects.toThrow(/zlib error/);
197+ // In memory, a large blob is inflated a MiB at a time.
198+ let n = 1024 * 1024 - 200;
199+ while (gzipSync(new Uint8Array(n), { level: 0 }).length < 1024 * 1024) n++;
200+ const big = new Uint8Array(randomBytes(n));
201+ const bigGz = new Uint8Array(gzipSync(big, { level: 0 }));
202+ expect(bigGz.length).toBe(1024 * 1024);
203+ const bigPack = await writePack([{ type: "blob", size: big.length, deflated: bigGz }]);
204+ await expect(parsePack(bigPack, undefined, { largeObjectBytes: 16 })).rejects.toThrow(/zlib error/);
205+ await expect(parsePack(gz)).rejects.toThrow(/zlib error/);
206+ // The same objects in zlib come in.
207+ const zl = await writePack([{ type: "blob", size: content.length, deflated: new Uint8Array(deflateSync(content)) }]);
208+ expect((await unpack(zl)).seen.size).toBe(1);
209+ expect((await unpack(zl, { largeObjectBytes: 16 })).seen.size).toBe(1);
210+ expect(await parsePack(await writePack([{ type: "blob", size: big.length, deflated: new Uint8Array(deflateSync(big)) }]), undefined, { largeObjectBytes: 16 })).toHaveLength(1);
211+ });
212+
213+ it("reads a base from outside the pack once for all its deltas, and counts it", async () => {
214+ const base = new Uint8Array(randomBytes(1024 * 1024));
215+ const oid = await hashObject("blob", base);
216+ let reads = 0;
217+ const external = async (o: string) => (o === oid ? (reads++, { type: "blob" as const, content: base }) : null);
218+ const many = refDeltaPack(oid, Array.from({ length: 200 }, (_, i) => tinyDelta(base.length, i * 4099, i)));
219+ // However small the cache, tiny deltas can't make the server read it again and again.
220+ for (const cacheBytes of [8 * 1024 * 1024, 64 * 1024]) {
221+ reads = 0;
222+ const { p, seen } = await unpack(many, { external, cacheBytes });
223+ expect(seen.size).toBe(200);
224+ expect(reads).toBe(1);
225+ expect(p.workDone).toBeLessThan(base.length + 64 * 1024);
226+ }
227+ // It counts toward what the pack unpacks to, as in parsePack.
228+ await expect(unpack(many, { external, maxBytes: base.length })).rejects.toThrow(/unpacks to more than/);
229+ await expect(parsePack(many, external, { maxBytes: base.length })).rejects.toThrow(/unpacks to more than/);
230+ });
231+
232+ it("resolves a long delta chain in one pass", { timeout: 60_000 }, async () => {
233+ // Each delta's base is listed after it: resolving round by round took a round, and a pass over
234+ // every delta left, per link.
235+ const n = 20_000;
236+ const blob = new TextEncoder().encode("x".repeat(64));
237+ const contents = [blob];
238+ for (let i = 1; i <= n; i++) contents.push(new Uint8Array([...blob, i & 0xff, i >> 8]));
239+ const oids = await Promise.all(contents.map((c) => hashObject("blob", c)));
240+ // Copy the 64 bytes the chain shares, then insert the two that number it.
241+ const deltas = contents.slice(1).map((c, i) => ({ base: oids[i]!, delta: new Uint8Array([...varint(contents[i]!.length), ...varint(c.length), 0x90, 64, 2, c[64]!, c[65]!]) }));
242+ const pack = refDeltaChain(deltas.reverse(), blob);
243+ const t = Date.now();
244+ const seen = new Map<string, Uint8Array>();
245+ await StreamedPack.open(source(pack), {}, (o, content) => void seen.set(o.oid, content!));
246+ // Round by round, this took most of a minute.
247+ expect(Date.now() - t).toBeLessThan(20_000);
248+ expect(seen.size).toBe(n + 1);
249+ expect(Buffer.from(seen.get(oids[n]!)!).equals(Buffer.from(contents[n]!))).toBe(true);
250+ });
251+
252+ it("refuses a pack whose index alone wouldn't fit in memory", async () => {
253+ // The caches unpack() uses (4 blocks of 64 KiB and 256 KiB), then ENTRY_OVERHEAD for each object.
254+ const caches = 4 * 64 * 1024 + 256 * 1024;
255+ const count = new DataView(full.buffer, full.byteOffset).getUint32(8);
256+ await expect(unpack(full, { maxHeldBytes: caches + (count - 1) * ENTRY_OVERHEAD })).rejects.toThrow(/pack needs more than 0 MiB of memory to unpack; split it into smaller pushes/);
257+ // A REF_DELTA costs more (git pack-objects writes them unless asked for OFS_DELTAs): charged as it is read.
258+ await expect(unpack(full, { maxHeldBytes: caches + count * ENTRY_OVERHEAD })).rejects.toThrow(/split it into smaller pushes/);
259+ // Then the objects themselves while they are in use, and the zlib data kept to inflate again.
260+ expect((await unpack(full, { maxHeldBytes: caches + count * (ENTRY_OVERHEAD + REF_DELTA_OVERHEAD) + 16 * 1024 * 1024 })).seen.size).toBeGreaterThan(0);
261+ });
262+
263+ it("charges every object it holds, so one huge object is refused rather than run the isolate out of memory", async () => {
264+ const base = new Uint8Array(randomBytes(64 * 1024));
265+ const oid = await hashObject("blob", base);
266+ // One delta that copies the 64 KiB base 6,000 times over: a 375 MiB blob from a 50 KB pack.
267+ const ops: number[] = [];
268+ for (let k = 0; k < 6000; k++) ops.push(0x80 | 0x20, 0x00); // copy 0x0000 -> 64 KiB (size 0 means 0x10000)
269+ const result = 6000 * 64 * 1024;
270+ const bomb = refDeltaChain([{ base: oid, delta: new Uint8Array([...varint(base.length), ...varint(result), ...ops]) }], base);
271+ // A tree of zeros: 120 MiB from about 120 KB.
272+ const zeros = await writePack([{ type: "tree", size: 120 * 1024 * 1024, deflated: new Uint8Array(deflateSync(new Uint8Array(120 * 1024 * 1024))) }]);
273+ for (const pack of [bomb, zeros]) {
274+ const before = process.memoryUsage().arrayBuffers;
275+ let peak = before;
276+ const t = setInterval(() => (peak = Math.max(peak, process.memoryUsage().arrayBuffers)), 1);
277+ await expect(StreamedPack.open(source(pack), {}, () => {})).rejects.toThrow(/pack needs more than 80 MiB of memory to unpack; split it into smaller pushes/);
278+ clearInterval(t);
279+ expect(peak - before).toBeLessThan(64 * 1024 * 1024);
280+ await expect(parsePack(pack)).rejects.toThrow(/split it into smaller pushes|larger than/);
281+ }
282+ });
283+
284+ it("reads each entry's own range again, keeps small ones from the first pass, and counts its reads", async () => {
285+ const reads = { bytes: 0, calls: 0 };
286+ const kept = await StreamedPack.open(source(full, reads), { blockBytes: 64 * 1024 }, () => {});
287+ const keptCalls = reads.calls;
288+ // Without keeping anything, every delta is read again, twice counted (storing replays it), each its own range.
289+ const bare = { bytes: 0, calls: 0 };
290+ const p = await StreamedPack.open(source(full, bare), { blockBytes: 64 * 1024, keepBytes: 0 }, () => {});
291+ expect(bare.calls).toBeGreaterThan(keptCalls);
292+ expect(bare.bytes).toBeLessThan(2 * full.length);
293+ expect(p.readsDone).toBeGreaterThan(kept.readsDone);
294+ await expect(StreamedPack.open(source(full), { blockBytes: 64 * 1024, keepBytes: 0, maxReads: p.readsDone - 1 }, () => {})).rejects.toThrow(/reads .* of storage to unpack; split it into smaller pushes/);
295+ // Replaying reads only what was counted.
296+ const counted = p.readsDone;
297+ await p.replayDeltas(async () => {});
298+ expect(p.readsDone).toBe(counted);
299+ });
300+
301+ it("says what storing it adds: each delta's content, not its few bytes in the pack", async () => {
302+ const base = new Uint8Array(randomBytes(64 * 1024));
303+ const oid = await hashObject("blob", base);
304+ // Each copies the base and adds one byte: 200 different 64 KiB blobs from a pack of a few KB.
305+ const deltas = Array.from({ length: 200 }, (_, k) => ({ base: oid, delta: new Uint8Array([...varint(base.length), ...varint(base.length + 1), 0x80 | 0x20, 0x00, 1, k]) }));
306+ const pack = refDeltaChain(deltas, base);
307+ const p = await StreamedPack.open(source(pack), {}, () => {});
308+ expect(pack.length).toBeLessThan(200 * 1024);
309+ expect(p.storedBytes).toBeGreaterThanOrEqual(200 * (64 * 1024 + 1));
310+ });
311+
312+ it("rebuilds the deltas for storing with exactly the work that already passed", async () => {
313+ const stored = new Map<string, GitObject>();
314+ for (const o of await parsePack(full)) stored.set(o.oid, { type: o.type, content: o.content });
315+ const external = async (oid: string) => stored.get(oid) ?? null;
316+ for (const [data, opts] of [
317+ [full, {}],
318+ [thin, { external }],
319+ ] as const) {
320+ const unlimited = await unpack(data, opts);
321+ // At the tightest limit that lets the pack in, storing it can't go over: nothing is refused halfway through.
322+ const { p, seen } = await unpack(data, { ...opts, maxWork: unlimited.p.workDone, yieldEvery: 4096 });
323+ const done = p.workDone;
324+ const rebuilt: string[] = [];
325+ await p.replayDeltas(async (entry, content) => {
326+ expect(Buffer.from(content).equals(Buffer.from(seen.get(p.oidOf(entry))!.content))).toBe(true);
327+ rebuilt.push(p.oidOf(entry));
328+ });
329+ expect(p.workDone).toBe(done);
330+ // It lets other work in the isolate run as it goes, as open does.
331+ const timers = vi.spyOn(globalThis, "setTimeout");
332+ await p.replayDeltas(async () => {});
333+ expect(timers.mock.calls.length).toBeGreaterThan(1);
334+ timers.mockRestore();
335+ const deltas = [...p.objects()].filter((x) => !x.deflated);
336+ expect(rebuilt.length).toBe(deltas.length);
337+ expect(new Set(rebuilt)).toEqual(new Set(deltas.map((x) => x.oid)));
338+ }
339+ });
340+
341+ it("hashes and compresses a large object without a second copy of it", async () => {
342+ const content = new Uint8Array(randomBytes(8 * 1024 * 1024));
343+ const want = createHash("sha1").update(`blob ${content.length}\0`).update(content).digest("hex");
344+ const digest = vi.spyOn(crypto.subtle, "digest");
345+ expect(await hashObject("blob", content)).toBe(want);
346+ // WebCrypto only hashes one whole buffer, so the header and content would be joined first.
347+ for (const [, data] of digest.mock.calls) expect((data as Uint8Array).byteLength).toBeLessThanOrEqual(HASH_COPY_MAX + 64);
348+ digest.mockRestore();
349+ expect(await hashObject("blob", content.subarray(0, 1000))).toBe(createHash("sha1").update("blob 1000\0").update(content.subarray(0, 1000)).digest("hex"));
350+ // The same zlib data as deflate, in chunks that are each far smaller than the object.
351+ const chunks = [...deflateChunks(content)];
352+ expect(Math.max(...chunks.map((c) => c.length))).toBeLessThanOrEqual(64 * 1024);
353+ expect(Buffer.concat(chunks).equals(Buffer.from(deflate(content)))).toBe(true);
354+ // A 50 MiB tree from a 51 KB pack is hashed where it lies: what it holds stays under the cap with the tree in it.
355+ const tree = await writePack([{ type: "tree", size: 50 * 1024 * 1024, deflated: new Uint8Array(deflateSync(new Uint8Array(50 * 1024 * 1024))) }]);
356+ const p = await StreamedPack.open(source(tree), {}, () => {});
357+ expect(p.peakHeldBytes).toBeLessThan(80 * 1024 * 1024);
358+ expect(p.peakHeldBytes).toBeGreaterThan(50 * 1024 * 1024 + SCRATCH_BYTES);
359+ });
360+
361+ it("checks what a base from the repository takes before reading it", async () => {
362+ // A 400 MiB blob of zeros is about 400 KB stored; a thin pack's tiny delta against it would inflate it whole.
363+ const oid = "1".repeat(40);
364+ const asked: number[] = [];
365+ let read = false;
366+ const external = async (o: string, maxBytes: number) => {
367+ if (o !== oid) return null;
368+ asked.push(maxBytes);
369+ if (400 * 1024 * 1024 > maxBytes) return 400 * 1024 * 1024;
370+ read = true;
371+ return { type: "blob" as const, content: new Uint8Array(0) };
372+ };
373+ const thinPack = refDeltaPack(oid, [tinyDelta(400 * 1024 * 1024, 0, 1)]);
374+ await expect(StreamedPack.open(source(thinPack), { external }, () => {})).rejects.toThrow(TooMuchMemory);
375+ await expect(parsePack(thinPack, external)).rejects.toThrow(/pack needs more than 80 MiB of memory to unpack; split it into smaller pushes/);
376+ expect(read).toBe(false);
377+ expect(asked).toHaveLength(2);
378+ for (const n of asked) expect(n).toBeLessThan(80 * 1024 * 1024);
379+ });
380+
381+ it("rebuilds a long history of one large file holding two versions at a time", { timeout: 60_000 }, async () => {
382+ // A 2.6 MiB lockfile over 60 commits, each version a delta against the one before, as git packs it.
383+ let version = new Uint8Array(Buffer.from(Array.from({ length: 110_000 }, (_, i) => `"pkg-${i}": "1.0.${i % 97}",\n`).join("").slice(0, 2_600_000)));
384+ const blob = version;
385+ const deltas: { base: string; delta: Uint8Array }[] = [];
386+ for (let k = 0; k < 60; k++) {
387+ const n = version.length;
388+ const half = n >> 1;
389+ const copy = (off: number, len: number) => [0x80 | 0x0f | 0x70, off & 0xff, (off >> 8) & 0xff, (off >> 16) & 0xff, (off >>> 24) & 0xff, len & 0xff, (len >> 8) & 0xff, (len >> 16) & 0xff];
390+ const delta = new Uint8Array([...varint(n), ...varint(n + 1), ...copy(0, half), 1, k, ...copy(half, n - half)]);
391+ const next = new Uint8Array(n + 1);
392+ next.set(version.subarray(0, half));
393+ next[half] = k;
394+ next.set(version.subarray(half), half + 1);
395+ deltas.push({ base: await hashObject("blob", version), delta });
396+ version = next;
397+ }
398+ const chain = refDeltaChain(deltas.reverse(), blob);
399+ const last = await hashObject("blob", version);
400+ let seen = false;
401+ const p = await StreamedPack.open(source(chain), {}, (o, content) => {
402+ if (o.oid === last) seen = Buffer.from(content!).equals(Buffer.from(version));
403+ });
404+ expect(seen).toBe(true);
405+ // Its caches and scratch, then the version being rebuilt from and the one being rebuilt.
406+ expect(p.peakHeldBytes).toBeLessThan(24 * 1024 * 1024 + 3 * version.length + 1024 * 1024);
407+ let stored = 0;
408+ await p.replayDeltas(async () => void stored++);
409+ expect(stored).toBe(60);
410+ // Parsed whole it holds every version: refused as memory, which a push takes as its cue to unpack it in pieces.
411+ await expect(parsePack(chain)).rejects.toThrow(TooMuchMemory);
412+ });
413+
414+ it("frees a chain's first version once the next is rebuilt from it", { timeout: 60_000 }, async () => {
415+ setFlagsFromString("--expose-gc");
416+ const gc = runInNewContext("gc") as () => void;
417+ // A 26 MiB text file, then three versions of it, each a small delta against the one before.
418+ const n = 26 * 1024 * 1024;
419+ let version = new Uint8Array(Buffer.from("a line of a large text file\n".repeat(Math.ceil(n / 28)).slice(0, n)));
420+ const blob = version;
421+ const deltas: { base: string; delta: Uint8Array }[] = [];
422+ for (let k = 0; k < 3; k++) {
423+ const len = version.length;
424+ // Copies of at most 8 MiB (a copy's size has three bytes), then the byte that's new.
425+ const ops: number[] = [];
426+ for (let off = 0; off < len; off += 8 << 20) {
427+ const m = Math.min(8 << 20, len - off);
428+ ops.push(0x80 | 0x0f | 0x70, off & 0xff, (off >> 8) & 0xff, (off >> 16) & 0xff, (off >>> 24) & 0xff, m & 0xff, (m >> 8) & 0xff, (m >> 16) & 0xff);
429+ }
430+ const delta = new Uint8Array([...varint(len), ...varint(len + 1), ...ops, 1, k]);
431+ const next = new Uint8Array(len + 1);
432+ next.set(version);
433+ next[len] = k;
434+ deltas.push({ base: await hashObject("blob", version), delta });
435+ version = next;
436+ }
437+ const chain = new Uint8Array(refDeltaChain(deltas.reverse(), blob));
438+ const last = await hashObject("blob", version);
439+ version = new Uint8Array(0);
440+ let atLast = 0;
441+ await StreamedPack.open(source(chain), {}, (o) => {
442+ if (o.oid !== last) return;
443+ gc();
444+ atLast = process.memoryUsage().arrayBuffers;
445+ });
446+ // Measured against what is left once the unpack is gone.
447+ gc();
448+ const live = atLast - process.memoryUsage().arrayBuffers;
449+ // The version it was rebuilt from and itself, and what the caches hold: not the first version as well.
450+ expect(live).toBeGreaterThan(2 * n);
451+ expect(live).toBeLessThan(2 * n + 8 * 1024 * 1024);
452+ });
453+
454+ it("measures what a large rebuilt delta compresses to, for counting and storing it", async () => {
455+ // 40 versions of a 600 KB file that compresses well: each one stored inline, so no write of its own is counted.
456+ let version = new Uint8Array(Buffer.from(Array.from({ length: 30_000 }, (_, i) => `"pkg-${i}": "1.0.${i % 7}",\n`).join("").slice(0, 600_000)));
457+ const blob = version;
458+ const deltas: { base: string; delta: Uint8Array }[] = [];
459+ const versions: Uint8Array[] = [];
460+ for (let k = 0; k < 40; k++) {
461+ const len = version.length;
462+ const delta = new Uint8Array([...varint(len), ...varint(len + 1), 0x80 | 0x70, len & 0xff, (len >> 8) & 0xff, (len >> 16) & 0xff, 1, k]);
463+ const next = new Uint8Array(len + 1);
464+ next.set(version);
465+ next[len] = k;
466+ deltas.push({ base: await hashObject("blob", version), delta });
467+ versions.push(next);
468+ version = next;
469+ }
470+ const chain = refDeltaChain(deltas.reverse(), blob);
471+ const counted = await StreamedPack.open(source(chain), {}, () => {});
472+ const p = await StreamedPack.open(source(chain), { writesOver: 512 * 1024 }, () => {});
473+ expect(p.readsDone).toBe(counted.readsDone);
474+ const byOid = new Map(await Promise.all(versions.map(async (v) => [await hashObject("blob", v), v] as const)));
475+ let measured = 0;
476+ for (const x of p.objects()) {
477+ if (x.deflated) continue;
478+ expect(p.zlibSizeOf(x.entry)).toBe(deflate(byOid.get(x.oid)!).length);
479+ measured++;
480+ }
481+ expect(measured).toBe(40);
482+ // A pack that unpacks to more than it may says so for the whole push, not for one object.
483+ await expect(StreamedPack.open(source(chain), { maxBytes: 4 * 1024 * 1024 }, () => {})).rejects.toThrow("pack unpacks to more than 4 MiB; split it into smaller pushes");
484+ await expect(parsePack(chain, undefined, { maxBytes: 4 * 1024 * 1024 })).rejects.toThrow("pack unpacks to more than 4 MiB; split it into smaller pushes");
485+ });
486+});