Bound git requests, diffs and clones; signed-out callers can't probe private repositories (#59)
Fetch requests are capped at 16 MB and pushes at 100 MB after decompression, packs at 512 MiB unpacked, and objects can't inflate past their declared size. Diffs cap Myers at 1,000 changed lines per file (larger changes show as one replaced block) and share a budget per request; oversized three-way merges are conflicts. Clones stream the pack one object at a time. Signed out, a private repository and a missing one get the same answer everywhere. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_016yhgAu6oKfqes812Yd4qjo
Canberk Demirci authored · GitHub committed signed1 parent acdea76commit 811c98d9ffc6f7c7a2a60e6aab3a88b08889c59dBrowse files
17 files changed+620−50
Files (17)
- apps/diffinery/docs/reference/limits.md+5−0
- apps/diffinery/docs/reference/rest-api.md+1−1
- apps/diffinery/docs/repositories/overview.md+1−1
- apps/diffinery/src/git/protocol.ts+61−15
- apps/diffinery/src/http/common.ts+10−9
- apps/diffinery/src/http/git.ts+32−4
- apps/diffinery/src/repo/RepoDO.ts+18−7
- apps/diffinery/test/e2e/security.test.mjs+62−1
- apps/diffinery/test/unit/upload-pack.test.ts+58−0
- packages/git-core/src/delta.ts+2−1
- packages/git-core/src/diff.ts+61−8
- packages/git-core/src/pack.ts+40−3
- packages/git-core/src/sha1.ts+83−0
- packages/git-core/src/zlib.ts+11−0
- packages/git-core/test/bombs.test.ts+61−0
- packages/git-core/test/diff-budget.test.ts+63−0
- packages/git-core/test/pack-stream.test.ts+51−0
modifiedapps/diffinery/docs/reference/limits.md+5−0View file
| @@ -121,6 +121,8 @@ | ||
| 121 | 121 | | Limit | Value | Applies to | |
| 122 | 122 | | --- | --- | --- | |
| 123 | 123 | | Push size | 100 MB per push | The whole push request after decompression. A larger push is rejected with `push exceeds 100 MB; split it into smaller pushes`. Push in several steps (for example a range of commits at a time). | |
| 124 | +| Fetch request size | 16 MB | The request a client sends to fetch or clone (its wants and haves), after decompression. A larger one is refused with `413 fetch request exceeds 16 MB`. | | |
| 125 | +| Unpacked size of a push | 512 MiB | What a push's pack inflates to, deltas resolved. A pack that unpacks to more is refused, however small it is on the wire. | | |
| 124 | 126 | | Git protocol | Version 2 for fetch and clone | Clients older than git 2.26 must run `git config --global protocol.version 2`, or the server answers `Diffinery requires Git protocol v2`. | |
| 125 | 127 | | Secret scanning coverage | Every new text file | Push protection scans every new text file in a push, whatever its size. The rejection message lists up to 10 findings. | |
| 126 | 128 | | Linear history check | 1,000 new commits | A protected branch with "require linear history" checks up to 1,000 new commits in a push for merge commits. | |
| @@ -139,6 +141,9 @@ | ||
| 139 | 141 | | --- | --- | --- | |
| 140 | 142 | | Files in a diff | 300 | Commit, compare and pull request diffs (web UI, REST and MCP) include the first 300 changed files and mark the diff as truncated. | |
| 141 | 143 | | Size of one file's diff | 1 MiB (old and new version together) | Larger files show no line diff ("This diff is too large to display."). | |
| 144 | +| Content diffed per request | 16 MiB | One commit, comparison or pull request diffs at most 16 MiB of file content; later files are listed without a line diff. | | |
| 145 | +| Changed lines diffed exactly | 1,000 per file, about 2,000 per request | A file with more changed lines, or one diffed after the request's budget is spent, is shown as one block: the lines both versions share at the start and end stay as context, and everything between is removed and added. The line counts stay right; the diff is just not minimal. | | |
| 146 | +| Automatic line merges | 2,000 changed lines per side | Merging a pull request, a file both sides changed by more than this is a conflict to resolve yourself. | | |
| 142 | 147 | | Commits listed in a comparison or pull request | 250 | `GET /repos/{owner}/{repo}/compare/{base}...{head}`, `GET .../pulls/{n}/commits` and the matching web pages. | |
| 143 | 148 | | Viewing or editing a file in the browser | 1 MiB | Larger files show "This file is too large to display" with a **View raw** link, and can't be edited in the browser. READMEs over 1 MiB are not rendered. | |
| 144 | 149 | | Markdown source | 512 KiB | Longer Markdown (READMEs, issue and pull request bodies, comments) is shown as plain preformatted text. | |
modifiedapps/diffinery/docs/reference/rest-api.md+1−1View file
| @@ -147,7 +147,7 @@ | ||
| 147 | 147 | | --- | --- | --- | |
| 148 | 148 | | `401` | `invalid, expired or revoked token` | The `dfy_` token is unknown, past its expiry date, revoked, or its account is disabled or not yet active. Create a new token. | |
| 149 | 149 | | `401` | `unrecognised token; create one under Settings → Access tokens` | The credential does not start with `dfy_`. GitHub tokens (`ghp_...`) do not work here. | |
| 150 | -| `401` | `authentication required` | You called a private resource anonymously, or tried to change something in a public repository without a token. Add the `Authorization` header. | | |
| 150 | +| `401` | `authentication required` | You called a private resource anonymously, or tried to change something in a repository without a token. Add the `Authorization` header. Reading a private repository anonymously gets `404`, the same as one that doesn't exist. | | |
| 151 | 151 | | `401` | `malformed basic credentials` | The `Basic` value isn't valid base64. Encode `user:token` with base64. | |
| 152 | 152 | | `401` | `sign in required` | The endpoint needs an account (for example `GET /user`) and you sent no credentials. | |
| 153 | 153 | | `403` | `your account is waiting for an invite` | Your account exists but no one has invited you yet. Ask an administrator for an invite. | |
modifiedapps/diffinery/docs/repositories/overview.md+1−1View file
| @@ -116,7 +116,7 @@ | ||
| 116 | 116 | |
| 117 | 117 | Writing to a public repository still needs a signed-in account (or a token) with a role that allows it. People who aren't signed in can read a public repository but can't change anything in it: opening an issue, commenting, reacting, watching and pushing all need an account, and the API answers such requests with `401 authentication required`. An anonymous `git push` gets a `401` with a `WWW-Authenticate` challenge, so Git asks for your credentials (or uses its credential helper) and tries again with them. A `read`-scoped token can read too, but every write with it gets `403 this token needs the write scope`. Only public repositories are indexed by search engines; private pages carry a `noindex` tag. |
| 118 | 118 | |
| 119 | -For people without access, a private repository looks like it does not exist: the web UI, Git and the API answer `404` (`repository not found`) rather than `403`, so the name is not revealed. Signed-out requests get `401` (`authentication required`) instead, so Git knows to ask for credentials. | |
| 119 | +For people without access, a private repository looks like it does not exist: the web UI, Git and the API answer `404` (`repository not found`) rather than `403`, so the name is not revealed. Signed-out requests can't tell a private repository from a missing one either: Git, Git LFS and the web UI answer both with `401` (`authentication required`), so Git asks for credentials and the web UI offers sign-in, and the REST API answers both with `404`, as GitHub does. A signed-out write gets `401` whether or not the repository exists. | |
| 120 | 120 | |
| 121 | 121 | Change visibility on the **General** settings page or with `PATCH /api/v3/repos/{owner}/{repo}` and `{"visibility": "public"}`. As on GitHub, `{"private": false}` also makes a repository public and `{"private": true}` makes it private. `visibility` accepts only `public` and `private` here too: any other value gets the same `422 visibility must be public or private` as when creating a repository. |
| 122 | 122 |
modifiedapps/diffinery/src/git/protocol.ts+61−15View file
| @@ -1,4 +1,4 @@ | ||
| 1 | -import { ByteWriter, FLUSH, DELIM, ZERO_OID, fromUtf8, planFetch, parseTag, pkt, pktText, readPkts, sideband, writePack, utf8, type ObjectReader } from "@diffinery/git-core"; | |
| 1 | +import { ByteWriter, FLUSH, DELIM, ZERO_OID, fromUtf8, planFetch, parseTag, pkt, pktText, readPkts, sideband, packStream, utf8, type ObjectReader } from "@diffinery/git-core"; | |
| 2 | 2 | import type { RawObject } from "../repo/objects.js"; |
| 3 | 3 | |
| 4 | 4 | export const AGENT = "diffinery/0.1"; |
| @@ -70,10 +70,13 @@ | ||
| 70 | 70 | return req; |
| 71 | 71 | } |
| 72 | 72 | |
| 73 | -export async function handleUploadPackV2(body: Uint8Array, ctx: UploadContext): Promise<Uint8Array> { | |
| 73 | +/** Where a response goes, piece by piece; each write resolves once the client has taken it. */ | |
| 74 | +export type Sink = (bytes: Uint8Array) => Promise<void>; | |
| 75 | + | |
| 76 | +export async function handleUploadPackV2(body: Uint8Array, ctx: UploadContext, sink: Sink): Promise<void> { | |
| 74 | 77 | const req = parseV2(body); |
| 75 | - if (req.command === "ls-refs") return lsRefs(req, ctx); | |
| 76 | - if (req.command === "fetch") return fetchCommand(req, ctx); | |
| 78 | + if (req.command === "ls-refs") return sink(await lsRefs(req, ctx)); | |
| 79 | + if (req.command === "fetch") return fetchCommand(req, ctx, sink); | |
| 77 | 80 | throw new Error(`unsupported command: ${req.command || "(none)"}`); |
| 78 | 81 | } |
| 79 | 82 | |
| @@ -102,7 +105,7 @@ | ||
| 102 | 105 | return w.bytes(); |
| 103 | 106 | } |
| 104 | 107 | |
| 105 | -async function fetchCommand(req: V2Request, ctx: UploadContext): Promise<Uint8Array> { | |
| 108 | +async function fetchCommand(req: V2Request, ctx: UploadContext, sink: Sink): Promise<void> { | |
| 106 | 109 | const wants: string[] = []; |
| 107 | 110 | const haves: string[] = []; |
| 108 | 111 | let done = false; |
| @@ -139,7 +142,7 @@ | ||
| 139 | 142 | if (waitForDone) { |
| 140 | 143 | // Negotiation only (git push with push.negotiate): no "ready" and no pack until "done". |
| 141 | 144 | out.write(FLUSH); |
| 142 | - return out.bytes(); | |
| 145 | + return sink(out.bytes()); | |
| 143 | 146 | } |
| 144 | 147 | out.write(pkt("ready\n")); |
| 145 | 148 | out.write(DELIM); |
| @@ -154,18 +157,61 @@ | ||
| 154 | 157 | out.write(pkt("packfile\n")); |
| 155 | 158 | const oids = plan.oids; |
| 156 | 159 | if (!noProgress) sideband(2, utf8(`Enumerating objects: ${oids.length}, done.\n`), out); |
| 157 | - const entries = []; | |
| 158 | - for (const oid of oids) { | |
| 159 | - const raw = await ctx.store.readRaw(oid); | |
| 160 | - if (!raw) throw new Error(`missing object ${oid}`); | |
| 161 | - entries.push({ type: raw.type, size: raw.size, deflated: raw.deflated }); | |
| 160 | + await sink(out.bytes()); | |
| 161 | + | |
| 162 | + // The pack goes out as it is read, in side-band packets, so memory holds one object at a time and | |
| 163 | + // a clone of a large repository can't exhaust the Durable Object. | |
| 164 | + async function* entries() { | |
| 165 | + for (const oid of oids) { | |
| 166 | + const raw = await ctx.store.readRaw(oid); | |
| 167 | + if (!raw) throw new Error(`missing object ${oid}`); | |
| 168 | + yield { type: raw.type, size: raw.size, deflated: raw.deflated }; | |
| 169 | + } | |
| 162 | 170 | } |
| 163 | - sideband(1, await writePack(entries), out); | |
| 164 | - if (!noProgress) sideband(2, utf8(`Total ${oids.length} (delta 0), reused ${oids.length} (delta 0)\n`), out); | |
| 165 | - out.write(FLUSH); | |
| 166 | - return out.bytes(); | |
| 171 | + const band = new SidebandBuffer(sink); | |
| 172 | + try { | |
| 173 | + for await (const part of packStream(oids.length, entries())) await band.write(part); | |
| 174 | + await band.flush(); | |
| 175 | + } catch (e) { | |
| 176 | + // Mid-pack, the only way to tell the client is the error band. | |
| 177 | + const w = new ByteWriter(); | |
| 178 | + sideband(3, utf8(`${(e as Error).message}\n`), w); | |
| 179 | + await sink(w.bytes()); | |
| 180 | + throw e; | |
| 181 | + } | |
| 182 | + const end = new ByteWriter(); | |
| 183 | + if (!noProgress) sideband(2, utf8(`Total ${oids.length} (delta 0), reused ${oids.length} (delta 0)\n`), end); | |
| 184 | + end.write(FLUSH); | |
| 185 | + await sink(end.bytes()); | |
| 167 | 186 | } |
| 168 | 187 | |
| 188 | +/** Gathers pack bytes into full side-band-64k packets (65515 bytes of data each). */ | |
| 189 | +class SidebandBuffer { | |
| 190 | + private static readonly MAX = 65515; | |
| 191 | + private buf = new Uint8Array(SidebandBuffer.MAX); | |
| 192 | + private used = 0; | |
| 193 | + constructor(private sink: Sink) {} | |
| 194 | + | |
| 195 | + async write(data: Uint8Array): Promise<void> { | |
| 196 | + let i = 0; | |
| 197 | + while (i < data.length) { | |
| 198 | + const take = Math.min(SidebandBuffer.MAX - this.used, data.length - i); | |
| 199 | + this.buf.set(data.subarray(i, i + take), this.used); | |
| 200 | + this.used += take; | |
| 201 | + i += take; | |
| 202 | + if (this.used === SidebandBuffer.MAX) await this.flush(); | |
| 203 | + } | |
| 204 | + } | |
| 205 | + | |
| 206 | + async flush(): Promise<void> { | |
| 207 | + if (!this.used) return; | |
| 208 | + const w = new ByteWriter(); | |
| 209 | + sideband(1, this.buf.subarray(0, this.used), w); | |
| 210 | + this.used = 0; | |
| 211 | + await this.sink(w.bytes()); | |
| 212 | + } | |
| 213 | +} | |
| 214 | + | |
| 169 | 215 | // ------------------------------------------------------------------ receive-pack |
| 170 | 216 | |
| 171 | 217 | export interface RefCommand { |
modifiedapps/diffinery/src/http/common.ts+10−9View file
| @@ -21,7 +21,7 @@ | ||
| 21 | 21 | |
| 22 | 22 | /** |
| 23 | 23 | * Load a repository and the caller's role, enforcing a minimum role (404 hides |
| 24 | - * private repos). Anonymous callers get read access to public repositories but | |
| 24 | + * private repos; signed-out callers get the same answer for private and missing ones). Anonymous callers get read access to public repositories but | |
| 25 | 25 | * may only read: any other method needs a signed-in caller, as on GitHub, |
| 26 | 26 | * unless the route is a read that happens to use POST (git fetch, LFS |
| 27 | 27 | * download) and says so. Writes also need a token with the write scope. |
| @@ -41,18 +41,19 @@ | ||
| 41 | 41 | */ |
| 42 | 42 | export async function loadRepo(c: AppContext, owner: string, name: string, min: Role = "read", opts: { readOnlyPost?: boolean; whileArchived?: boolean; whileImporting?: boolean; follow?: boolean } = {}): Promise<{ repo: Repo; caller: Caller; stub: ReturnType<typeof repoStub>; moved: boolean }> { |
| 43 | 43 | const dir = c.get("dir"); |
| 44 | - let repo = await dir.repo(owner, name.replace(/\.git$/, "")); | |
| 45 | - const moved = !repo; | |
| 46 | - repo ??= await dir.movedRepo(owner, name); | |
| 47 | - if (!repo) throw new HttpError(404, "repository not found"); | |
| 48 | 44 | const anonymous = c.get("principal").type === "anonymous"; |
| 49 | 45 | const reading = c.req.method === "GET" || c.req.method === "HEAD" || (opts.readOnlyPost && c.req.method === "POST") || c.get("readOnly") === true; |
| 46 | + // Signed-out callers can't tell a private repository from one that doesn't exist: writes are | |
| 47 | + // refused before the lookup, and reads of either get the same answer. That is a 404 from the REST | |
| 48 | + // API, as on GitHub, and a 401 everywhere else, so git asks for credentials and the web offers sign-in. | |
| 50 | 49 | if (anonymous && !reading) throw new HttpError(401, "authentication required"); |
| 50 | + const hidden = () => (anonymous && !c.req.path.startsWith("/api/") ? new HttpError(401, "authentication required") : new HttpError(404, "repository not found")); | |
| 51 | + let repo = await dir.repo(owner, name.replace(/\.git$/, "")); | |
| 52 | + const moved = !repo; | |
| 53 | + repo ??= await dir.movedRepo(owner, name); | |
| 54 | + if (!repo) throw hidden(); | |
| 51 | 55 | const caller = await callerFor(c, repo); |
| 52 | - if (!caller) { | |
| 53 | - if (anonymous) throw new HttpError(401, "authentication required"); | |
| 54 | - throw new HttpError(404, "repository not found"); | |
| 55 | - } | |
| 56 | + if (!caller) throw hidden(); | |
| 56 | 57 | if (moved && !opts.follow) { |
| 57 | 58 | const url = new URL(c.req.url); |
| 58 | 59 | const path = movedPath(url.pathname, { owner, name }, { owner: repo.owner_login, name: repo.name }); |
modifiedapps/diffinery/src/http/git.ts+32−4View file
| @@ -34,7 +34,7 @@ | ||
| 34 | 34 | |
| 35 | 35 | git.post("/:owner/:repo/git-upload-pack", async (c) => { |
| 36 | 36 | const { stub } = await loadRepo(c, c.req.param("owner"), c.req.param("repo"), "read", { readOnlyPost: true, follow: true }); |
| 37 | - const body = await readBody(c.req.raw); | |
| 37 | + const body = await readBody(c.req.raw, MAX_UPLOAD_PACK_REQUEST, "fetch request exceeds 16 MB"); | |
| 38 | 38 | const out = await stub.gitUploadPack(body); |
| 39 | 39 | return new Response(out, { headers: { "content-type": "application/x-git-upload-pack-result", ...noCache } }); |
| 40 | 40 | }); |
| @@ -45,17 +45,45 @@ | ||
| 45 | 45 | // A GitHub import's own token, while that import is running and filling this repository, may push in import mode. |
| 46 | 46 | const p = c.get("principal"); |
| 47 | 47 | if (p.type === "account" && p.importId && p.tokenId && (await importPushAllowed(c.env, p.importId, repo.id, p.tokenId))) caller.importId = p.importId; |
| 48 | - const body = await readBody(c.req.raw); | |
| 48 | + const body = await readBody(c.req.raw, MAX_RECEIVE_PACK_REQUEST, "push exceeds 100 MB; split it into smaller pushes"); | |
| 49 | 49 | const out = await stub.gitReceivePack(body, caller, { moved }); |
| 50 | 50 | c.executionCtx.waitUntil(c.get("dir").touchPushed(repo.id)); |
| 51 | 51 | return new Response(out, { headers: { "content-type": "application/x-git-receive-pack-result", ...noCache } }); |
| 52 | 52 | }); |
| 53 | 53 | |
| 54 | -async function readBody(req: Request): Promise<Uint8Array> { | |
| 54 | +/** A fetch's wants and haves; git's own negotiation stays far below this. */ | |
| 55 | +const MAX_UPLOAD_PACK_REQUEST = 16 * 1024 * 1024; | |
| 56 | +/** The repository refuses pushes over 100 MB with a clear message; this stops anything far beyond. */ | |
| 57 | +const MAX_RECEIVE_PACK_REQUEST = 101 * 1024 * 1024; | |
| 58 | + | |
| 59 | +/** | |
| 60 | + * The request body, gunzipped when the client compressed it. Reading stops at `max` bytes | |
| 61 | + * (after decompression), so neither a huge body nor a gzip bomb is ever held in memory. | |
| 62 | + */ | |
| 63 | +async function readBody(req: Request, max: number, tooLarge: string): Promise<Uint8Array> { | |
| 55 | 64 | let stream = req.body; |
| 56 | 65 | if (!stream) return new Uint8Array(0); |
| 57 | 66 | if (req.headers.get("content-encoding") === "gzip") stream = stream.pipeThrough(new DecompressionStream("gzip")); |
| 58 | - return new Uint8Array(await new Response(stream).arrayBuffer()); | |
| 67 | + const chunks: Uint8Array[] = []; | |
| 68 | + let total = 0; | |
| 69 | + const reader = stream.getReader(); | |
| 70 | + for (;;) { | |
| 71 | + const { done, value } = await reader.read(); | |
| 72 | + if (done) break; | |
| 73 | + total += value.length; | |
| 74 | + if (total > max) { | |
| 75 | + await reader.cancel().catch(() => {}); | |
| 76 | + throw new HttpError(413, tooLarge); | |
| 77 | + } | |
| 78 | + chunks.push(value); | |
| 79 | + } | |
| 80 | + const out = new Uint8Array(total); | |
| 81 | + let at = 0; | |
| 82 | + for (const ch of chunks) { | |
| 83 | + out.set(ch, at); | |
| 84 | + at += ch.length; | |
| 85 | + } | |
| 86 | + return out; | |
| 59 | 87 | } |
| 60 | 88 | |
| 61 | 89 | // ---------------------------------------------------------------- Git LFS |
modifiedapps/diffinery/src/repo/RepoDO.ts+18−7View file
| @@ -5,6 +5,7 @@ | ||
| 5 | 5 | EMPTY_TREE_OID, |
| 6 | 6 | ZERO_OID, |
| 7 | 7 | applyChanges, |
| 8 | + DiffBudget, | |
| 8 | 9 | diffTrees, |
| 9 | 10 | fileDiff, |
| 10 | 11 | formatUnified, |
| @@ -262,12 +263,21 @@ | ||
| 262 | 263 | return advertiseReceivePack(this.refs().filter((r) => !r.name.startsWith("refs/pull/"))); |
| 263 | 264 | } |
| 264 | 265 | |
| 265 | - async gitUploadPack(body: Uint8Array): Promise<Uint8Array> { | |
| 266 | - try { | |
| 267 | - return await handleUploadPackV2(body, { store: this.store, refs: this.refs(), headTarget: this.headRef }); | |
| 268 | - } catch (e) { | |
| 269 | - return errorPacket((e as Error).message); | |
| 270 | - } | |
| 266 | + /** The response streams: the pack is read and sent one object at a time, as the client takes it. */ | |
| 267 | + async gitUploadPack(body: Uint8Array): Promise<ReadableStream<Uint8Array>> { | |
| 268 | + const { readable, writable } = new IdentityTransformStream(); | |
| 269 | + const writer = writable.getWriter(); | |
| 270 | + let started = false; | |
| 271 | + const sink = (b: Uint8Array) => ((started = true), writer.write(b)); | |
| 272 | + const ctx = { store: this.store, refs: this.refs(), headTarget: this.headRef }; | |
| 273 | + void handleUploadPackV2(body, ctx, sink) | |
| 274 | + .catch((e) => (started ? undefined : writer.write(errorPacket((e as Error).message)))) | |
| 275 | + .then( | |
| 276 | + () => writer.close(), | |
| 277 | + () => writer.abort(), | |
| 278 | + ) | |
| 279 | + .catch(() => {}); // the client went away | |
| 280 | + return readable; | |
| 271 | 281 | } |
| 272 | 282 | |
| 273 | 283 | /** `moved`: the push came in under an old name (a transfer or rename), so say where the repository lives now, like GitHub. */ |
| @@ -526,8 +536,9 @@ | ||
| 526 | 536 | const changes = await diffTrees(this.store, baseTree, head.tree); |
| 527 | 537 | const max = opts.maxFiles ?? 300; |
| 528 | 538 | const files = []; |
| 539 | + const budget = new DiffBudget(); | |
| 529 | 540 | for (const ch of changes.slice(0, max)) { |
| 530 | - const d = await fileDiff(this.store, ch); | |
| 541 | + const d = await fileDiff(this.store, ch, 3, budget); | |
| 531 | 542 | files.push({ ...d, patch: opts.patch === false ? undefined : formatUnified(d) }); |
| 532 | 543 | } |
| 533 | 544 | return { |
modifiedapps/diffinery/test/e2e/security.test.mjs+62−1View file
| @@ -23,6 +23,7 @@ | ||
| 23 | 23 | import { spawn, spawnSync, execFileSync } from "node:child_process"; |
| 24 | 24 | import { createHash, randomBytes } from "node:crypto"; |
| 25 | 25 | import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; |
| 26 | +import { gzipSync } from "node:zlib"; | |
| 26 | 27 | import { tmpdir } from "node:os"; |
| 27 | 28 | import { join, dirname, resolve } from "node:path"; |
| 28 | 29 | import { fileURLToPath } from "node:url"; |
| @@ -786,7 +787,7 @@ | ||
| 786 | 787 | await status(404, "GET", R, undefined, T.sam); |
| 787 | 788 | await status(404, "GET", `${R}/issues`, undefined, T.sam); |
| 788 | 789 | await status(404, "POST", `${R}/issues`, { title: "probe" }, T.sam); |
| 789 | - await status(401, "GET", R, undefined, null); | |
| 790 | + await status(404, "GET", R, undefined, null); | |
| 790 | 791 | assert.deepEqual(await api("GET", R, undefined, T.sam), await api("GET", `/repos/${OWNER}/never-existed`, undefined, T.sam)); |
| 791 | 792 | |
| 792 | 793 | // ------------------------------------------------------------ web pages: 301 to the same page, query kept |
| @@ -1435,3 +1436,63 @@ | ||
| 1435 | 1436 | await status(204, "DELETE", `/diffinery/tokens/${minted.id}`); |
| 1436 | 1437 | await until("the revoke in the audit log", async () => auditLog("token.revoke").some((e) => e.target === minted.id && JSON.parse(e.detail ?? "{}").via === "api")); |
| 1437 | 1438 | }); |
| 1439 | + | |
| 1440 | +test("security: git request bodies are bounded, compressed or not, before anything reads them", async () => { | |
| 1441 | + await repoWithReadme("bounded"); | |
| 1442 | + await ok("PATCH", `/repos/${OWNER}/bounded`, { visibility: "public" }); | |
| 1443 | + const url = `${base}/${OWNER}/bounded.git/git-upload-pack`; | |
| 1444 | + const headers = { "content-type": "application/x-git-upload-pack-request" }; | |
| 1445 | + // 64 MiB of zeros in about 64 KB of gzip, sent without signing in. | |
| 1446 | + const bomb = gzipSync(Buffer.alloc(64 * 1024 * 1024)); | |
| 1447 | + assert.ok(bomb.length < 1024 * 1024); | |
| 1448 | + const r = await fetch(url, { method: "POST", body: bomb, headers: { ...headers, "content-encoding": "gzip" } }); | |
| 1449 | + assert.equal(r.status, 413, await r.text()); | |
| 1450 | + const plain = await fetch(url, { method: "POST", body: Buffer.alloc(17 * 1024 * 1024, 48), headers }); | |
| 1451 | + assert.equal(plain.status, 413); | |
| 1452 | + // A real fetch still works. | |
| 1453 | + execFileSync("git", ["-c", "protocol.version=2", "clone", "-q", `${base}/${OWNER}/bounded.git`, join(work, "bounded")], { stdio: "pipe" }); | |
| 1454 | +}); | |
| 1455 | + | |
| 1456 | +test("security: a file rewritten from scratch is diffed quickly, even for anonymous readers", async () => { | |
| 1457 | + const R = `/repos/${OWNER}/rewrite`; | |
| 1458 | + await repoWithReadme("rewrite", { private: false }); | |
| 1459 | + // 2 x 20k lines with nothing in common, Myers' worst case: unbounded, one diff of it took minutes. | |
| 1460 | + const lines = (tag) => Array.from({ length: 20_000 }, (_, i) => `${tag} ${i}\n`).join(""); | |
| 1461 | + const first = await ok("POST", `${R}/diffinery/commits`, { branch: "main", message: "first", files: [{ path: "data.txt", content: lines("old") }] }); | |
| 1462 | + const second = await ok("POST", `${R}/diffinery/commits`, { branch: "main", message: "rewrite", files: [{ path: "data.txt", content: lines("new") }] }); | |
| 1463 | + const sha = second.sha ?? second.commit?.sha; | |
| 1464 | + const t = Date.now(); | |
| 1465 | + const commit = await ok("GET", `${R}/commits/${sha}`, undefined, null); | |
| 1466 | + assert.ok(Date.now() - t < 15_000, `the diff took ${Date.now() - t} ms`); | |
| 1467 | + const file = commit.files.find((f) => f.filename === "data.txt"); | |
| 1468 | + assert.deepEqual([file.additions, file.deletions], [20_000, 20_000]); | |
| 1469 | + const compared = await ok("GET", `${R}/compare/${first.sha ?? first.commit?.sha}...${sha}`, undefined, null); | |
| 1470 | + assert.equal(compared.files.find((f) => f.filename === "data.txt").additions, 20_000); | |
| 1471 | +}); | |
| 1472 | + | |
| 1473 | +test("security: signed out, a private repository and a missing one look the same everywhere", async () => { | |
| 1474 | + await repoWithReadme("secret-plans"); | |
| 1475 | + const probe = async (name) => { | |
| 1476 | + const raw = async (method, path, init = {}) => { | |
| 1477 | + const r = await fetch(`${base}${path}`, { method, redirect: "manual", ...init }); | |
| 1478 | + return [r.status, (r.headers.get("location") ?? "").replace(name, "NAME"), (await r.text()).replaceAll(name, "NAME")]; | |
| 1479 | + }; | |
| 1480 | + return { | |
| 1481 | + read: await raw("GET", `/api/v3/repos/${OWNER}/${name}`), | |
| 1482 | + issues: await raw("GET", `/api/v3/repos/${OWNER}/${name}/issues`), | |
| 1483 | + write: await raw("POST", `/api/v3/repos/${OWNER}/${name}/issues`, { headers: { "content-type": "application/json" }, body: JSON.stringify({ title: "probe" }) }), | |
| 1484 | + git: await raw("GET", `/${OWNER}/${name}.git/info/refs?service=git-upload-pack`, { headers: { "git-protocol": "version=2" } }), | |
| 1485 | + fetch: await raw("POST", `/${OWNER}/${name}.git/git-upload-pack`, { headers: { "git-protocol": "version=2", "content-type": "application/x-git-upload-pack-request" }, body: "0014command=ls-refs\n0000" }), | |
| 1486 | + lfs: await raw("POST", `/${OWNER}/${name}.git/info/lfs/objects/batch`, { headers: { accept: "application/vnd.git-lfs+json", "content-type": "application/vnd.git-lfs+json" }, body: JSON.stringify({ operation: "download", objects: [] }) }), | |
| 1487 | + page: await raw("GET", `/${OWNER}/${name}`), | |
| 1488 | + blob: await raw("GET", `/${OWNER}/${name}/blob/main/README.md`), | |
| 1489 | + }; | |
| 1490 | + }; | |
| 1491 | + const hidden = await probe("secret-plans"); | |
| 1492 | + assert.deepEqual(hidden, await probe("never-existed")); | |
| 1493 | + assert.equal(hidden.read[0], 404); | |
| 1494 | + assert.equal(hidden.write[0], 401); | |
| 1495 | + assert.equal(hidden.git[0], 401); | |
| 1496 | + assert.equal(hidden.page[0], 302); | |
| 1497 | + assert.match(hidden.page[1], /\/login\?next=/); | |
| 1498 | +}); |
addedapps/diffinery/test/unit/upload-pack.test.ts+58−0View file
| @@ -1,0 +1,58 @@ | ||
| 1 | +import { describe, expect, it } from "vitest"; | |
| 2 | +import { DELIM, FLUSH, MemoryStore, pkt, deflate, parsePack, readPkts, serializeCommit, serializeTree, utf8, concat, type ObjectType } from "@diffinery/git-core"; | |
| 3 | +import { handleUploadPackV2 } from "../../src/git/protocol.js"; | |
| 4 | + | |
| 5 | +const BLOB = 200 * 1024; | |
| 6 | + | |
| 7 | +/** A repository with one commit of `n` incompressible blobs, whose raw reads are counted. */ | |
| 8 | +async function repo(n: number) { | |
| 9 | + const mem = new MemoryStore(); | |
| 10 | + const blobs: string[] = []; | |
| 11 | + for (let i = 0; i < n; i++) { | |
| 12 | + const content = new Uint8Array(BLOB); | |
| 13 | + for (let j = 0; j < BLOB; j += 65536) crypto.getRandomValues(content.subarray(j, j + 65536)); | |
| 14 | + blobs.push(await mem.write("blob", content)); | |
| 15 | + } | |
| 16 | + const tree = await mem.write("tree", serializeTree(blobs.map((oid, i) => ({ name: `f${i}`, mode: "100644", oid })))); | |
| 17 | + const sig = { name: "T", email: "t@example.com", timestamp: 1, tz: "+0000" }; | |
| 18 | + const commit = await mem.write("commit", serializeCommit({ tree, parents: [], author: sig, committer: sig, message: "m\n" })); | |
| 19 | + let reads = 0; | |
| 20 | + const store = Object.assign(mem, { | |
| 21 | + has: (oid: string) => mem.objects.has(oid), | |
| 22 | + async readRaw(oid: string) { | |
| 23 | + const o = mem.objects.get(oid); | |
| 24 | + if (!o) return null; | |
| 25 | + reads++; | |
| 26 | + return { type: o.type as ObjectType, size: o.content.length, deflated: deflate(o.content) }; | |
| 27 | + }, | |
| 28 | + }); | |
| 29 | + return { store, commit, reads: () => reads }; | |
| 30 | +} | |
| 31 | + | |
| 32 | +const fetchRequest = (want: string) => concat([pkt("command=fetch\n"), DELIM, pkt(`want ${want}\n`), pkt("done\n"), FLUSH]); | |
| 33 | + | |
| 34 | +describe("upload-pack streams the pack", () => { | |
| 35 | + it("sends each object as it is read, in side-band packets, never the whole pack at once", async () => { | |
| 36 | + const { store, commit, reads } = await repo(40); | |
| 37 | + const chunks: Uint8Array[] = []; | |
| 38 | + let sent = 0; | |
| 39 | + let mostAhead = 0; | |
| 40 | + await handleUploadPackV2(fetchRequest(commit), { store, refs: [{ name: "refs/heads/main", oid: commit }], headTarget: "refs/heads/main" }, async (b) => { | |
| 41 | + expect(b.length).toBeLessThanOrEqual(65520 + 1024); | |
| 42 | + chunks.push(b); | |
| 43 | + sent += b.length; | |
| 44 | + // Objects read but not yet handed to the client: one, plus the side-band packet being filled. | |
| 45 | + mostAhead = Math.max(mostAhead, (reads() - 2) * BLOB - sent); | |
| 46 | + }); | |
| 47 | + expect(mostAhead).toBeLessThan(2 * BLOB); | |
| 48 | + // What the client got is a complete, valid pack of all 42 objects. | |
| 49 | + const pack: Uint8Array[] = []; | |
| 50 | + let inPack = false; | |
| 51 | + for (const p of readPkts(concat(chunks)).pkts) { | |
| 52 | + if (p.type !== "data") continue; | |
| 53 | + if (!inPack) inPack = new TextDecoder().decode(p.data) === "packfile\n"; | |
| 54 | + else if (p.data[0] === 1) pack.push(p.data.subarray(1)); | |
| 55 | + } | |
| 56 | + expect(await parsePack(concat(pack))).toHaveLength(42); | |
| 57 | + }); | |
| 58 | +}); |
modifiedpackages/git-core/src/delta.ts+2−1View file
| @@ -1,5 +1,5 @@ | ||
| 1 | 1 | /** Apply a git delta (as found in OFS_DELTA / REF_DELTA pack entries) to a base. */ |
| 2 | -export function applyDelta(base: Uint8Array, delta: Uint8Array): Uint8Array { | |
| 2 | +export function applyDelta(base: Uint8Array, delta: Uint8Array, maxSize = Infinity): Uint8Array { | |
| 3 | 3 | let i = 0; |
| 4 | 4 | const varint = (): number => { |
| 5 | 5 | let result = 0; |
| @@ -15,6 +15,7 @@ | ||
| 15 | 15 | const srcSize = varint(); |
| 16 | 16 | if (srcSize !== base.length) throw new Error(`delta base size mismatch: ${srcSize} != ${base.length}`); |
| 17 | 17 | const dstSize = varint(); |
| 18 | + if (dstSize > maxSize) throw new Error(`delta result of ${dstSize} bytes is larger than the ${maxSize} bytes allowed`); | |
| 18 | 19 | const out = new Uint8Array(dstSize); |
| 19 | 20 | let o = 0; |
| 20 | 21 | while (i < delta.length) { |
modifiedpackages/git-core/src/diff.ts+61−8View file
| @@ -66,24 +66,73 @@ | ||
| 66 | 66 | |
| 67 | 67 | const MAX_DIFF_BYTES = 1024 * 1024; |
| 68 | 68 | |
| 69 | -export async function fileDiff(store: ObjectReader, change: TreeChange, context = 3): Promise<FileDiff> { | |
| 69 | +/** | |
| 70 | + * Myers' diff costs about the square of the number of changed lines, so a file pair with nothing in | |
| 71 | + * common could hold a repository's Durable Object for minutes. Past this many changed lines the | |
| 72 | + * change is shown as one replaced block instead, which costs one pass over the lines. | |
| 73 | + */ | |
| 74 | +export const MAX_EDIT_LENGTH = 1000; | |
| 75 | + | |
| 76 | +/** Work left for one diff request (a commit, a comparison or a pull request), shared by its files. */ | |
| 77 | +export class DiffBudget { | |
| 78 | + constructor( | |
| 79 | + /** Bytes of file content still to diff; later files are listed as too large. */ | |
| 80 | + public bytes = 16 * 1024 * 1024, | |
| 81 | + /** Myers' work left, in squared changed lines. */ | |
| 82 | + public work = 4 * MAX_EDIT_LENGTH * MAX_EDIT_LENGTH, | |
| 83 | + ) {} | |
| 84 | +} | |
| 85 | + | |
| 86 | +export async function fileDiff(store: ObjectReader, change: TreeChange, context = 3, budget = new DiffBudget()): Promise<FileDiff> { | |
| 70 | 87 | const oldC = change.oldOid ? (await mustRead(store, change.oldOid)).content : new Uint8Array(0); |
| 71 | 88 | const newC = change.newOid ? (await mustRead(store, change.newOid)).content : new Uint8Array(0); |
| 72 | 89 | if (isBinary(oldC) || isBinary(newC)) return { ...change, binary: true, additions: 0, deletions: 0, hunks: [] }; |
| 73 | - if (oldC.length + newC.length > MAX_DIFF_BYTES) return { ...change, binary: false, additions: 0, deletions: 0, hunks: [], truncated: true }; | |
| 74 | - const patch = structuredPatch(change.path, change.path, fromUtf8(oldC), fromUtf8(newC), "", "", { context }); | |
| 90 | + const size = oldC.length + newC.length; | |
| 91 | + if (size > MAX_DIFF_BYTES || size > budget.bytes) return { ...change, binary: false, additions: 0, deletions: 0, hunks: [], truncated: true }; | |
| 92 | + budget.bytes -= size; | |
| 93 | + const oldS = fromUtf8(oldC); | |
| 94 | + const newS = fromUtf8(newC); | |
| 95 | + const maxEditLength = Math.min(MAX_EDIT_LENGTH, Math.floor(Math.sqrt(Math.max(0, budget.work)))); | |
| 96 | + const patch = structuredPatch(change.path, change.path, oldS, newS, "", "", { context, maxEditLength }); | |
| 97 | + const hunks: Hunk[] = patch ? patch.hunks.map((h) => ({ oldStart: h.oldStart, oldLines: h.oldLines, newStart: h.newStart, newLines: h.newLines, lines: h.lines })) : replacedBlock(oldS, newS, context); | |
| 75 | 98 | let additions = 0; |
| 76 | 99 | let deletions = 0; |
| 77 | - const hunks = patch.hunks.map((h) => { | |
| 100 | + for (const h of hunks) { | |
| 78 | 101 | for (const l of h.lines) { |
| 79 | 102 | if (l.startsWith("+")) additions++; |
| 80 | 103 | else if (l.startsWith("-")) deletions++; |
| 81 | 104 | } |
| 82 | - return { oldStart: h.oldStart, oldLines: h.oldLines, newStart: h.newStart, newLines: h.newLines, lines: h.lines }; | |
| 83 | - }); | |
| 105 | + } | |
| 106 | + budget.work -= patch ? (additions + deletions) ** 2 : maxEditLength ** 2; | |
| 84 | 107 | return { ...change, binary: false, additions, deletions, hunks }; |
| 85 | 108 | } |
| 86 | 109 | |
| 110 | +/** | |
| 111 | + * The cheap diff for a change too large for Myers: the lines the two sides share at the start and the | |
| 112 | + * end stay as context, and everything between is removed and added as one block. | |
| 113 | + */ | |
| 114 | +function replacedBlock(oldS: string, newS: string, context: number): Hunk[] { | |
| 115 | + const a = splitLines(oldS); | |
| 116 | + const b = splitLines(newS); | |
| 117 | + let p = 0; | |
| 118 | + while (p < a.length && p < b.length && a[p] === b[p]) p++; | |
| 119 | + let s = 0; | |
| 120 | + while (s < a.length - p && s < b.length - p && a[a.length - 1 - s] === b[b.length - 1 - s]) s++; | |
| 121 | + if (p === a.length && p === b.length) return []; | |
| 122 | + const from = Math.max(0, p - context); | |
| 123 | + const after = Math.min(s, context); | |
| 124 | + const lines: string[] = []; | |
| 125 | + const push = (mark: string, line: string) => { | |
| 126 | + lines.push(mark + (line.endsWith("\n") ? line.slice(0, -1) : line)); | |
| 127 | + if (!line.endsWith("\n")) lines.push("\\ No newline at end of file"); | |
| 128 | + }; | |
| 129 | + for (let i = from; i < p; i++) push(" ", a[i]!); | |
| 130 | + for (let i = p; i < a.length - s; i++) push("-", a[i]!); | |
| 131 | + for (let i = p; i < b.length - s; i++) push("+", b[i]!); | |
| 132 | + for (let i = a.length - s; i < a.length - s + after; i++) push(" ", a[i]!); | |
| 133 | + return [{ oldStart: from + 1, oldLines: a.length - s + after - from, newStart: from + 1, newLines: b.length - s + after - from, lines }]; | |
| 134 | +} | |
| 135 | + | |
| 87 | 136 | export function formatUnified(d: FileDiff): string { |
| 88 | 137 | const a = d.status === "added" ? "/dev/null" : `a/${d.path}`; |
| 89 | 138 | const b = d.status === "deleted" ? "/dev/null" : `b/${d.path}`; |
| @@ -100,11 +149,13 @@ | ||
| 100 | 149 | lines: string[]; |
| 101 | 150 | } |
| 102 | 151 | |
| 103 | -function edits(base: string[], other: string[]): Edit[] { | |
| 152 | +function edits(base: string[], other: string[]): Edit[] | null { | |
| 153 | + const parts = diffArrays(base, other, { maxEditLength: 2 * MAX_EDIT_LENGTH }); | |
| 154 | + if (!parts) return null; | |
| 104 | 155 | const out: Edit[] = []; |
| 105 | 156 | let bi = 0; |
| 106 | 157 | let cur: Edit | null = null; |
| 107 | - for (const part of diffArrays(base, other)) { | |
| 158 | + for (const part of parts) { | |
| 108 | 159 | if (part.removed) { |
| 109 | 160 | cur ??= { start: bi, end: bi, lines: [] }; |
| 110 | 161 | cur.end += part.count ?? part.value.length; |
| @@ -132,6 +183,8 @@ | ||
| 132 | 183 | const b = splitLines(base); |
| 133 | 184 | const ea = edits(b, splitLines(ours)); |
| 134 | 185 | const eb = edits(b, splitLines(theirs)); |
| 186 | + // A side too different from the base to diff cheaply is left for a person to merge. | |
| 187 | + if (!ea || !eb) return null; | |
| 135 | 188 | const out: string[] = []; |
| 136 | 189 | let pos = 0; |
| 137 | 190 | let i = 0; |
modifiedpackages/git-core/src/pack.ts+40−3View file
| @@ -1,7 +1,7 @@ | ||
| 1 | 1 | import { ByteWriter, equalBytes, fromUtf8, toHex } from "./bytes.js"; |
| 2 | 2 | import { applyDelta } from "./delta.js"; |
| 3 | 3 | import { CODE_TYPES, TYPE_CODES, hashObject, type GitObject, type ObjectType } from "./objects.js"; |
| 4 | -import { sha1 } from "./sha1.js"; | |
| 4 | +import { Sha1, sha1 } from "./sha1.js"; | |
| 5 | 5 | import { inflateAt } from "./zlib.js"; |
| 6 | 6 | |
| 7 | 7 | const OFS_DELTA = 6; |
| @@ -21,11 +21,16 @@ | ||
| 21 | 21 | |
| 22 | 22 | export type ExternalLookup = (oid: string) => Promise<GitObject | null>; |
| 23 | 23 | |
| 24 | +/** What a pack may unpack to by default (object contents and delta instructions together). */ | |
| 25 | +export const MAX_UNPACKED_BYTES = 512 * 1024 * 1024; | |
| 26 | + | |
| 24 | 27 | /** |
| 25 | 28 | * Parse a packfile and fully resolve every object (including deltas). |
| 26 | 29 | * `external` resolves REF_DELTA bases that are not inside the pack (thin packs). |
| 30 | + * Packs come from clients, so unpacking is bounded: each object stops inflating at its | |
| 31 | + * declared size, and the whole pack may unpack to at most `maxBytes`. | |
| 27 | 32 | */ |
| 28 | -export async function parsePack(pack: Uint8Array, external?: ExternalLookup): Promise<PackedObject[]> { | |
| 33 | +export async function parsePack(pack: Uint8Array, external?: ExternalLookup, opts: { maxBytes?: number } = {}): Promise<PackedObject[]> { | |
| 29 | 34 | if (pack.length < 32 || fromUtf8(pack.subarray(0, 4)) !== "PACK") throw new Error("not a packfile"); |
| 30 | 35 | const view = new DataView(pack.buffer, pack.byteOffset, pack.byteLength); |
| 31 | 36 | const version = view.getUint32(4); |
| @@ -36,6 +41,10 @@ | ||
| 36 | 41 | const digest = await sha1(pack.subarray(0, pack.length - 20)); |
| 37 | 42 | if (!equalBytes(trailer, digest)) throw new Error("pack checksum mismatch"); |
| 38 | 43 | |
| 44 | + const maxBytes = opts.maxBytes ?? MAX_UNPACKED_BYTES; | |
| 45 | + let unpacked = 0; | |
| 46 | + const tooBig = () => new Error(`pack unpacks to more than ${maxBytes} bytes`); | |
| 47 | + | |
| 39 | 48 | const raws: RawEntry[] = []; |
| 40 | 49 | let pos = 12; |
| 41 | 50 | for (let n = 0; n < count; n++) { |
| @@ -64,8 +73,10 @@ | ||
| 64 | 73 | } else if (!CODE_TYPES[typeCode]) { |
| 65 | 74 | throw new Error(`bad object type ${typeCode} at ${offset}`); |
| 66 | 75 | } |
| 76 | + if (unpacked + size > maxBytes) throw tooBig(); | |
| 67 | 77 | const { out, consumed } = inflateAt(pack, pos, size); |
| 68 | 78 | if (out.length !== size) throw new Error(`size mismatch at ${offset}: ${out.length} != ${size}`); |
| 79 | + unpacked += size; | |
| 69 | 80 | entry.data = out; |
| 70 | 81 | pos += consumed; |
| 71 | 82 | raws.push(entry); |
| @@ -100,7 +111,9 @@ | ||
| 100 | 111 | next.push(raw); |
| 101 | 112 | continue; |
| 102 | 113 | } |
| 103 | - await add(raw, base.type, applyDelta(base.content, raw.data)); | |
| 114 | + const content = applyDelta(base.content, raw.data, maxBytes - unpacked); | |
| 115 | + unpacked += content.length; | |
| 116 | + await add(raw, base.type, content); | |
| 104 | 117 | } |
| 105 | 118 | if (next.length === pending.length) { |
| 106 | 119 | // No progress: remaining bases must come from outside the pack. |
| @@ -140,6 +153,30 @@ | ||
| 140 | 153 | w.byte(c); |
| 141 | 154 | } |
| 142 | 155 | |
| 156 | +/** | |
| 157 | + * The same packfile as `writePack`, produced piece by piece as `entries` yields them, so a clone of a | |
| 158 | + * large repository never holds the whole pack: only the entry being written. | |
| 159 | + */ | |
| 160 | +export async function* packStream(count: number, entries: AsyncIterable<PackEntryInput>): AsyncGenerator<Uint8Array> { | |
| 161 | + const hash = new Sha1(); | |
| 162 | + const head = new ByteWriter(); | |
| 163 | + head.write(new TextEncoder().encode("PACK")); | |
| 164 | + head.uint32(2); | |
| 165 | + head.uint32(count); | |
| 166 | + let written = 0; | |
| 167 | + const out = (b: Uint8Array) => (hash.update(b), b); | |
| 168 | + yield out(head.bytes()); | |
| 169 | + for await (const e of entries) { | |
| 170 | + if (++written > count) throw new Error(`pack has more than the ${count} objects it announced`); | |
| 171 | + const h = new ByteWriter(); | |
| 172 | + writeEntryHeader(h, TYPE_CODES[e.type], e.size); | |
| 173 | + yield out(h.bytes()); | |
| 174 | + yield out(e.deflated); | |
| 175 | + } | |
| 176 | + if (written !== count) throw new Error(`pack announced ${count} objects but has ${written}`); | |
| 177 | + yield hash.digest(); | |
| 178 | +} | |
| 179 | + | |
| 143 | 180 | /** Build a version 2 packfile of undeltified objects from pre-deflated content. */ |
| 144 | 181 | export async function writePack(entries: readonly PackEntryInput[]): Promise<Uint8Array> { |
| 145 | 182 | const w = new ByteWriter(); |
modifiedpackages/git-core/src/sha1.ts+83−0View file
| @@ -13,3 +13,86 @@ | ||
| 13 | 13 | export async function sha256Hex(data: Uint8Array): Promise<string> { |
| 14 | 14 | return toHex(new Uint8Array(await crypto.subtle.digest("SHA-256", data as BufferSource))); |
| 15 | 15 | } |
| 16 | + | |
| 17 | +/** | |
| 18 | + * Incremental SHA-1, for checksums over data that is never held whole (a pack streamed to a client). | |
| 19 | + * WebCrypto only hashes a complete buffer. | |
| 20 | + */ | |
| 21 | +export class Sha1 { | |
| 22 | + private h = new Uint32Array([0x67452301, 0xefcdab89, 0x98badcfe, 0x10325476, 0xc3d2e1f0]); | |
| 23 | + private block = new Uint8Array(64); | |
| 24 | + private used = 0; | |
| 25 | + private total = 0; | |
| 26 | + private w = new Uint32Array(80); | |
| 27 | + | |
| 28 | + update(data: Uint8Array): this { | |
| 29 | + this.total += data.length; | |
| 30 | + let i = 0; | |
| 31 | + if (this.used) { | |
| 32 | + const take = Math.min(64 - this.used, data.length); | |
| 33 | + this.block.set(data.subarray(0, take), this.used); | |
| 34 | + this.used += take; | |
| 35 | + i = take; | |
| 36 | + if (this.used < 64) return this; | |
| 37 | + this.compress(this.block, 0); | |
| 38 | + this.used = 0; | |
| 39 | + } | |
| 40 | + for (; i + 64 <= data.length; i += 64) this.compress(data, i); | |
| 41 | + if (i < data.length) { | |
| 42 | + this.block.set(data.subarray(i), 0); | |
| 43 | + this.used = data.length - i; | |
| 44 | + } | |
| 45 | + return this; | |
| 46 | + } | |
| 47 | + | |
| 48 | + digest(): Uint8Array { | |
| 49 | + const bits = this.total * 8; | |
| 50 | + const pad = new Uint8Array((this.used < 56 ? 56 : 120) - this.used + 8); | |
| 51 | + pad[0] = 0x80; | |
| 52 | + const n = pad.length; | |
| 53 | + pad[n - 8] = Math.floor(bits / 2 ** 56) & 0xff; | |
| 54 | + pad[n - 7] = Math.floor(bits / 2 ** 48) & 0xff; | |
| 55 | + pad[n - 6] = Math.floor(bits / 2 ** 40) & 0xff; | |
| 56 | + pad[n - 5] = Math.floor(bits / 2 ** 32) & 0xff; | |
| 57 | + pad[n - 4] = (bits >>> 24) & 0xff; | |
| 58 | + pad[n - 3] = (bits >>> 16) & 0xff; | |
| 59 | + pad[n - 2] = (bits >>> 8) & 0xff; | |
| 60 | + pad[n - 1] = bits & 0xff; | |
| 61 | + const total = this.total; | |
| 62 | + this.update(pad); | |
| 63 | + this.total = total; | |
| 64 | + const out = new Uint8Array(20); | |
| 65 | + for (let j = 0; j < 5; j++) { | |
| 66 | + out[j * 4] = this.h[j]! >>> 24; | |
| 67 | + out[j * 4 + 1] = (this.h[j]! >>> 16) & 0xff; | |
| 68 | + out[j * 4 + 2] = (this.h[j]! >>> 8) & 0xff; | |
| 69 | + out[j * 4 + 3] = this.h[j]! & 0xff; | |
| 70 | + } | |
| 71 | + return out; | |
| 72 | + } | |
| 73 | + | |
| 74 | + private compress(b: Uint8Array, o: number) { | |
| 75 | + const w = this.w; | |
| 76 | + for (let t = 0; t < 16; t++) w[t] = (b[o + t * 4]! << 24) | (b[o + t * 4 + 1]! << 16) | (b[o + t * 4 + 2]! << 8) | b[o + t * 4 + 3]!; | |
| 77 | + for (let t = 16; t < 80; t++) { | |
| 78 | + const x = w[t - 3]! ^ w[t - 8]! ^ w[t - 14]! ^ w[t - 16]!; | |
| 79 | + w[t] = (x << 1) | (x >>> 31); | |
| 80 | + } | |
| 81 | + let [a, bb, c, d, e] = this.h as unknown as [number, number, number, number, number]; | |
| 82 | + for (let t = 0; t < 80; t++) { | |
| 83 | + const f = t < 20 ? (bb & c) | (~bb & d) : t < 40 ? bb ^ c ^ d : t < 60 ? (bb & c) | (bb & d) | (c & d) : bb ^ c ^ d; | |
| 84 | + const k = t < 20 ? 0x5a827999 : t < 40 ? 0x6ed9eba1 : t < 60 ? 0x8f1bbcdc : 0xca62c1d6; | |
| 85 | + const tmp = (((a << 5) | (a >>> 27)) + f + e + k + w[t]!) | 0; | |
| 86 | + e = d; | |
| 87 | + d = c; | |
| 88 | + c = (bb << 30) | (bb >>> 2); | |
| 89 | + bb = a; | |
| 90 | + a = tmp; | |
| 91 | + } | |
| 92 | + this.h[0] = (this.h[0]! + a) | 0; | |
| 93 | + this.h[1] = (this.h[1]! + bb) | 0; | |
| 94 | + this.h[2] = (this.h[2]! + c) | 0; | |
| 95 | + this.h[3] = (this.h[3]! + d) | 0; | |
| 96 | + this.h[4] = (this.h[4]! + e) | 0; | |
| 97 | + } | |
| 98 | +} |
modifiedpackages/git-core/src/zlib.ts+11−0View file
| @@ -16,6 +16,17 @@ | ||
| 16 | 16 | */ |
| 17 | 17 | export function inflateAt(data: Uint8Array, offset: number, expectedSize?: number): { out: Uint8Array; consumed: number } { |
| 18 | 18 | const inflator = new pako.Inflate() as pako.Inflate & { ended: boolean }; |
| 19 | + if (expectedSize !== undefined) { | |
| 20 | + // Stop as soon as the output passes the size the pack declared, so a small zlib stream | |
| 21 | + // can't be made to inflate to gigabytes. | |
| 22 | + const keep = inflator.onData.bind(inflator); | |
| 23 | + let produced = 0; | |
| 24 | + inflator.onData = (chunk) => { | |
| 25 | + produced += (chunk as Uint8Array).length; | |
| 26 | + if (produced > expectedSize) throw new Error(`object at ${offset} is larger than its declared size (${expectedSize} bytes)`); | |
| 27 | + keep(chunk); | |
| 28 | + }; | |
| 29 | + } | |
| 19 | 30 | // Feed in bounded windows so a huge trailing buffer is not copied repeatedly. |
| 20 | 31 | const window = Math.max(64 * 1024, expectedSize ? expectedSize + 1024 : 0); |
| 21 | 32 | let pos = offset; |
addedpackages/git-core/test/bombs.test.ts+61−0View file
| @@ -1,0 +1,61 @@ | ||
| 1 | +import { deflateSync } from "node:zlib"; | |
| 2 | +import { describe, expect, it } from "vitest"; | |
| 3 | +import { applyDelta, parsePack, sha1 } from "../src/index.js"; | |
| 4 | + | |
| 5 | +/** A pack from raw entries: [type code, declared size, body (already deflated, with any delta base prefix)]. */ | |
| 6 | +async function pack(entries: Array<[number, number, Uint8Array]>): Promise<Uint8Array> { | |
| 7 | + const parts: number[] = [..."PACK"].map((c) => c.charCodeAt(0)); | |
| 8 | + const u32 = (n: number) => parts.push((n >>> 24) & 255, (n >>> 16) & 255, (n >>> 8) & 255, n & 255); | |
| 9 | + u32(2); | |
| 10 | + u32(entries.length); | |
| 11 | + for (const [type, size, body] of entries) { | |
| 12 | + let s = size; | |
| 13 | + let c = (type << 4) | (s & 15); | |
| 14 | + s = Math.floor(s / 16); | |
| 15 | + while (s > 0) { | |
| 16 | + parts.push(c | 0x80); | |
| 17 | + c = s & 0x7f; | |
| 18 | + s = Math.floor(s / 128); | |
| 19 | + } | |
| 20 | + parts.push(c); | |
| 21 | + parts.push(...body); | |
| 22 | + } | |
| 23 | + const body = new Uint8Array(parts); | |
| 24 | + const out = new Uint8Array(body.length + 20); | |
| 25 | + out.set(body); | |
| 26 | + out.set(await sha1(body), body.length); | |
| 27 | + return out; | |
| 28 | +} | |
| 29 | +const varint = (n: number) => { | |
| 30 | + const out: number[] = []; | |
| 31 | + do { | |
| 32 | + let b = n % 128; | |
| 33 | + n = Math.floor(n / 128); | |
| 34 | + if (n > 0) b |= 0x80; | |
| 35 | + out.push(b); | |
| 36 | + } while (n > 0); | |
| 37 | + return out; | |
| 38 | +}; | |
| 39 | + | |
| 40 | +describe("untrusted packs can't make the server inflate or allocate without bound", () => { | |
| 41 | + it("stops inflating an object at its declared size", async () => { | |
| 42 | + // Declares 10 bytes, inflates to 64 MiB. | |
| 43 | + const bomb = deflateSync(Buffer.alloc(64 * 1024 * 1024)); | |
| 44 | + await expect(parsePack(await pack([[3, 10, bomb]]))).rejects.toThrow(/larger than its declared size/); | |
| 45 | + }); | |
| 46 | + | |
| 47 | + it("refuses a pack whose objects unpack past the budget", async () => { | |
| 48 | + const blob = Buffer.alloc(600 * 1024, 120); | |
| 49 | + const p = await pack([ | |
| 50 | + [3, blob.length, deflateSync(blob)], | |
| 51 | + [3, blob.length, deflateSync(Buffer.concat([blob.subarray(1), Buffer.from("y")]))], | |
| 52 | + ]); | |
| 53 | + await expect(parsePack(p, undefined, { maxBytes: 1024 * 1024 })).rejects.toThrow(/unpacks to more than/); | |
| 54 | + expect((await parsePack(p, undefined, { maxBytes: 2 * 1024 * 1024 })).length).toBe(2); | |
| 55 | + }); | |
| 56 | + | |
| 57 | + it("checks a delta's result size before allocating it", () => { | |
| 58 | + const delta = Uint8Array.from([...varint(3), ...varint(2 ** 40), 0x91, 0, 3]); | |
| 59 | + expect(() => applyDelta(Uint8Array.from([1, 2, 3]), delta, 1024 * 1024)).toThrow(/larger than/); | |
| 60 | + }); | |
| 61 | +}); |
addedpackages/git-core/test/diff-budget.test.ts+63−0View file
| @@ -1,0 +1,63 @@ | ||
| 1 | +import { structuredPatch } from "diff"; | |
| 2 | +import { describe, expect, it } from "vitest"; | |
| 3 | +import { DiffBudget, MAX_EDIT_LENGTH, MemoryStore, fileDiff, mergeText, utf8 } from "../src/index.js"; | |
| 4 | + | |
| 5 | +// Two files with no line in common: Myers' worst case, quadratic in their length. | |
| 6 | +const unrelated = (n: number) => [Array.from({ length: n }, (_, i) => `old ${i}\n`).join(""), Array.from({ length: n }, (_, i) => `new ${i}\n`).join("")]; | |
| 7 | + | |
| 8 | +async function diffOf(a: string, b: string, budget?: DiffBudget) { | |
| 9 | + const store = new MemoryStore(); | |
| 10 | + const oldOid = await store.write("blob", utf8(a)); | |
| 11 | + const newOid = await store.write("blob", utf8(b)); | |
| 12 | + return fileDiff(store, { path: "f.txt", status: "modified", oldOid, newOid }, 3, budget); | |
| 13 | +} | |
| 14 | + | |
| 15 | +describe("diffs are bounded", () => { | |
| 16 | + it("a file rewritten from scratch is diffed in one pass, as one replaced block", async () => { | |
| 17 | + // 2 x 40k lines (about 0.8 MB): unbounded, this takes minutes. | |
| 18 | + const [a, b] = unrelated(40_000); | |
| 19 | + const t = Date.now(); | |
| 20 | + const d = await diffOf(`same 1\nsame 2\n${a}same 3\n`, `same 1\nsame 2\n${b}same 3\n`); | |
| 21 | + expect(Date.now() - t).toBeLessThan(5_000); | |
| 22 | + expect(d.truncated).toBeUndefined(); | |
| 23 | + expect([d.additions, d.deletions]).toEqual([40_000, 40_000]); | |
| 24 | + expect(d.hunks).toHaveLength(1); | |
| 25 | + const h = d.hunks[0]!; | |
| 26 | + expect([h.oldStart, h.oldLines, h.newStart, h.newLines]).toEqual([1, 40_003, 1, 40_003]); | |
| 27 | + expect(h.lines.slice(0, 3)).toEqual([" same 1", " same 2", "-old 0"]); | |
| 28 | + expect(h.lines.at(-1)).toBe(" same 3"); | |
| 29 | + }); | |
| 30 | + | |
| 31 | + it("the cheap diff matches Myers' line counts and marks a missing final newline", async () => { | |
| 32 | + const d = await diffOf("x\n", "y", new DiffBudget(undefined, 0)); | |
| 33 | + expect(d.hunks).toEqual([{ oldStart: 1, oldLines: 1, newStart: 1, newLines: 1, lines: ["-x", "+y", "\\ No newline at end of file"] }]); | |
| 34 | + const added = await diffOf("", "a\nb\n", new DiffBudget(undefined, 0)); | |
| 35 | + expect(added.hunks).toEqual(structuredPatch("f", "f", "", "a\nb\n", "", "", { context: 3 }).hunks.map(({ oldStart, oldLines, newStart, newLines, lines }) => ({ oldStart, oldLines, newStart, newLines, lines }))); | |
| 36 | + }); | |
| 37 | + | |
| 38 | + it("small changes still get Myers' minimal diff", async () => { | |
| 39 | + const base = Array.from({ length: 200 }, (_, i) => `line ${i}\n`); | |
| 40 | + const changed = [...base]; | |
| 41 | + changed[10] = "changed 10\n"; | |
| 42 | + changed[150] = "changed 150\n"; | |
| 43 | + const d = await diffOf(base.join(""), changed.join("")); | |
| 44 | + expect([d.additions, d.deletions, d.hunks.length]).toEqual([2, 2, 2]); | |
| 45 | + }); | |
| 46 | + | |
| 47 | + it("one request shares a budget: after it is spent, files are still listed but not diffed", async () => { | |
| 48 | + const budget = new DiffBudget(100, MAX_EDIT_LENGTH ** 2); | |
| 49 | + const first = await diffOf("a\n".repeat(20), "b\n".repeat(20), budget); | |
| 50 | + expect(first.truncated).toBeUndefined(); | |
| 51 | + expect(budget.bytes).toBe(20); | |
| 52 | + const second = await diffOf("c\n".repeat(20), "d\n".repeat(20), budget); | |
| 53 | + expect(second.truncated).toBe(true); | |
| 54 | + }); | |
| 55 | + | |
| 56 | + it("a three-way merge too large to compute is a conflict, quickly", () => { | |
| 57 | + const [a, b] = unrelated(30_000); | |
| 58 | + const t = Date.now(); | |
| 59 | + expect(mergeText(`top\n${a}`, `top\n${b}`, `changed\n${a}`)).toBeNull(); | |
| 60 | + expect(Date.now() - t).toBeLessThan(5_000); | |
| 61 | + expect(mergeText("1\n2\n3\n", "1\nx\n3\n", "1\n2\n3\n4\n")).toBe("1\nx\n3\n4\n"); | |
| 62 | + }); | |
| 63 | +}); |
addedpackages/git-core/test/pack-stream.test.ts+51−0View file
| @@ -1,0 +1,51 @@ | ||
| 1 | +import { createHash } from "node:crypto"; | |
| 2 | +import { describe, expect, it } from "vitest"; | |
| 3 | +import { Sha1, concat, deflate, packStream, parsePack, utf8, writePack, type PackEntryInput } from "../src/index.js"; | |
| 4 | + | |
| 5 | +describe("streamed packs", () => { | |
| 6 | + it("incremental SHA-1 matches Node's for any split of the input", () => { | |
| 7 | + const data = new Uint8Array(1000).map((_, i) => (i * 31 + 7) & 0xff); | |
| 8 | + for (const len of [0, 1, 55, 56, 63, 64, 65, 119, 120, 1000]) { | |
| 9 | + const want = createHash("sha1").update(data.subarray(0, len)).digest(); | |
| 10 | + for (const step of [1, 7, 64, 1000]) { | |
| 11 | + const h = new Sha1(); | |
| 12 | + for (let i = 0; i < len; i += step) h.update(data.subarray(i, Math.min(len, i + step))); | |
| 13 | + expect(Buffer.from(h.digest()).equals(want)).toBe(true); | |
| 14 | + } | |
| 15 | + } | |
| 16 | + }); | |
| 17 | + | |
| 18 | + it("is byte for byte the pack writePack builds, and reads one entry at a time", async () => { | |
| 19 | + const entries: PackEntryInput[] = Array.from({ length: 50 }, (_, i) => { | |
| 20 | + const content = utf8(`blob ${i}\n`.repeat(i * 100 + 1)); | |
| 21 | + return { type: "blob", size: content.length, deflated: deflate(content) }; | |
| 22 | + }); | |
| 23 | + let pulled = 0; | |
| 24 | + async function* source() { | |
| 25 | + for (const e of entries) { | |
| 26 | + pulled++; | |
| 27 | + yield e; | |
| 28 | + } | |
| 29 | + } | |
| 30 | + const parts: Uint8Array[] = []; | |
| 31 | + let mostAhead = 0; | |
| 32 | + for await (const part of packStream(entries.length, source())) { | |
| 33 | + parts.push(part); | |
| 34 | + // Each entry is two parts (its header, then its data): the stream never runs ahead of its reader. | |
| 35 | + mostAhead = Math.max(mostAhead, pulled - Math.floor(parts.length / 2)); | |
| 36 | + } | |
| 37 | + expect(mostAhead).toBeLessThanOrEqual(1); | |
| 38 | + const streamed = concat(parts); | |
| 39 | + expect(streamed).toEqual(await writePack(entries)); | |
| 40 | + expect(await parsePack(streamed)).toHaveLength(50); | |
| 41 | + }); | |
| 42 | + | |
| 43 | + it("refuses a source that doesn't match the announced count", async () => { | |
| 44 | + const one: PackEntryInput = { type: "blob", size: 1, deflated: deflate(utf8("x")) }; | |
| 45 | + const run = async (n: number, items: PackEntryInput[]) => { | |
| 46 | + for await (const _ of packStream(n, (async function* () { yield* items; })())) void _; | |
| 47 | + }; | |
| 48 | + await expect(run(2, [one])).rejects.toThrow(/announced 2 objects but has 1/); | |
| 49 | + await expect(run(1, [one, one])).rejects.toThrow(/more than the 1 objects/); | |
| 50 | + }); | |
| 51 | +}); |