| 1 | import { ByteWriter, equalBytes, fromUtf8, toHex } from "./bytes.js"; |
| 2 | import { applyDelta, deltaResultSize } from "./delta.js"; |
| 3 | import { CODE_TYPES, TYPE_CODES, hashObject, objectHeader, type GitObject, type ObjectType } from "./objects.js"; |
| 4 | import { Sha1, sha1 } from "./sha1.js"; |
| 5 | import { inflate, inflateAt, inflateEach } from "./zlib.js"; |
| 6 | |
| 7 | const OFS_DELTA = 6; |
| 8 | const REF_DELTA = 7; |
| 9 | |
| 10 | export interface PackedObject extends GitObject { |
| 11 | oid: string; |
| 12 | /** |
| 13 | * Set on a large blob parsed with `largeObjectBytes`: its zlib data as it sits in the pack (a view, |
| 14 | * not a copy) and its size. Its `content` is then inflated afresh on every read, so callers that |
| 15 | * can work from `deflated` should. |
| 16 | */ |
| 17 | deflated?: Uint8Array; |
| 18 | size?: number; |
| 19 | /** |
| 20 | * Set on an object that isn't a delta (and not a large blob, which has `deflated`): its zlib data as |
| 21 | * it sits in the pack (a view, not a copy), which a store that keeps objects zlib-compressed can |
| 22 | * keep as is instead of compressing `content` again. |
| 23 | */ |
| 24 | zlib?: Uint8Array; |
| 25 | } |
| 26 | |
| 27 | /** A large blob kept compressed: `content` inflates it when something needs the bytes. */ |
| 28 | function largeBlob(oid: string, size: number, deflated: Uint8Array): PackedObject { |
| 29 | const obj = { oid, type: "blob" as const, size, deflated } as PackedObject; |
| 30 | Object.defineProperty(obj, "content", { enumerable: true, get: () => inflate(deflated) }); |
| 31 | return obj; |
| 32 | } |
| 33 | |
| 34 | /** Packs bigger than this get their checksum computed in slices, so the pack is never copied for hashing. */ |
| 35 | const SLICED_CHECKSUM_BYTES = 16 * 1024 * 1024; |
| 36 | |
| 37 | /** A delta entry waiting for its base: its instructions (inflated), freed once applied. */ |
| 38 | interface DeltaEntry { |
| 39 | offset: number; |
| 40 | data: Uint8Array; |
| 41 | } |
| 42 | |
| 43 | /** |
| 44 | * A REF_DELTA base from the repository (a thin pack's), or null when it has no such object. Given |
| 45 | * what the unpack may still hold, it checks what reading the object would take before reading it, |
| 46 | * and returns that number of bytes instead when it is more, so a small delta against a huge stored |
| 47 | * object is refused before the object is inflated. |
| 48 | */ |
| 49 | export type ExternalLookup = (oid: string, maxBytes: number) => Promise<GitObject | number | null>; |
| 50 | |
| 51 | /** What unpacking refuses when the pack inflates to more than `maxBytes` in all. */ |
| 52 | export function tooBigUnpacked(maxBytes: number): Error { |
| 53 | const mib = maxBytes / (1024 * 1024); |
| 54 | return new Error(`pack unpacks to more than ${Number.isInteger(mib) ? `${mib} MiB` : `${maxBytes} bytes`}; split it into smaller pushes`); |
| 55 | } |
| 56 | |
| 57 | /** Unpacking would hold more than it may. */ |
| 58 | export class TooMuchMemory extends Error {} |
| 59 | |
| 60 | /** What unpacking refuses when it would hold more than `maxHeld` bytes. */ |
| 61 | export function tooMuchMemory(maxHeld: number): TooMuchMemory { |
| 62 | return new TooMuchMemory(`pack needs more than ${Math.floor(maxHeld / (1024 * 1024))} MiB of memory to unpack; split it into smaller pushes`); |
| 63 | } |
| 64 | |
| 65 | /** What a pack may unpack to by default (object contents and delta instructions together). */ |
| 66 | export const MAX_UNPACKED_BYTES = 512 * 1024 * 1024; |
| 67 | |
| 68 | /** |
| 69 | * What unpacking a pack may hold in memory by default: the contents it returns (large blobs kept |
| 70 | * compressed aside), delta instructions until they are applied, a base while its deltas are applied, |
| 71 | * and OBJECT_OVERHEAD for each object. A repository's Durable Object has 128 MB, the pack included. |
| 72 | */ |
| 73 | export const MAX_HELD_BYTES = 80 * 1024 * 1024; |
| 74 | |
| 75 | /** |
| 76 | * Roughly what one object costs to hold beyond its content: the object and its oid, the Uint8Array |
| 77 | * and its buffer, and its entries in the indexes below. Measured at about 740 bytes in Node for |
| 78 | * 100,000 three-byte blobs. |
| 79 | */ |
| 80 | const OBJECT_OVERHEAD = 768; |
| 81 | |
| 82 | const EMPTY = new Uint8Array(0); |
| 83 | |
| 84 | /** |
| 85 | * Parse a packfile and fully resolve every object (including deltas). |
| 86 | * `external` resolves REF_DELTA bases that are not inside the pack (thin packs). |
| 87 | * Packs come from clients, so unpacking is bounded: each object stops inflating at its |
| 88 | * declared size, the whole pack may unpack to at most `maxBytes`, and what it holds in memory |
| 89 | * may come to at most `maxHeldBytes`. Deltas are resolved in one pass from their bases. |
| 90 | */ |
| 91 | export async function parsePack( |
| 92 | pack: Uint8Array, |
| 93 | external?: ExternalLookup, |
| 94 | opts: { maxBytes?: number; largeObjectBytes?: number; maxHeldBytes?: number } = {}, |
| 95 | ): Promise<PackedObject[]> { |
| 96 | if (pack.length < 32 || fromUtf8(pack.subarray(0, 4)) !== "PACK") throw new Error("not a packfile"); |
| 97 | const view = new DataView(pack.buffer, pack.byteOffset, pack.byteLength); |
| 98 | const version = view.getUint32(4); |
| 99 | if (version !== 2 && version !== 3) throw new Error(`unsupported pack version ${version}`); |
| 100 | const count = view.getUint32(8); |
| 101 | |
| 102 | const trailer = pack.subarray(pack.length - 20); |
| 103 | let digest: Uint8Array; |
| 104 | if (pack.length > SLICED_CHECKSUM_BYTES) { |
| 105 | const h = new Sha1(); |
| 106 | for (let i = 0; i < pack.length - 20; i += 1024 * 1024) h.update(pack.subarray(i, Math.min(i + 1024 * 1024, pack.length - 20))); |
| 107 | digest = h.digest(); |
| 108 | } else digest = await sha1(pack.subarray(0, pack.length - 20)); |
| 109 | if (!equalBytes(trailer, digest)) throw new Error("pack checksum mismatch"); |
| 110 | |
| 111 | const maxBytes = opts.maxBytes ?? MAX_UNPACKED_BYTES; |
| 112 | const maxHeld = opts.maxHeldBytes ?? MAX_HELD_BYTES; |
| 113 | const large = opts.largeObjectBytes ?? Infinity; |
| 114 | let unpacked = 0; |
| 115 | let held = 0; |
| 116 | const work = (n: number) => { |
| 117 | if (unpacked + n > maxBytes) throw tooBigUnpacked(maxBytes); |
| 118 | unpacked += n; |
| 119 | }; |
| 120 | const hold = (n: number) => { |
| 121 | if (held + n > maxHeld) throw tooMuchMemory(maxHeld); |
| 122 | held += n; |
| 123 | }; |
| 124 | |
| 125 | const results: PackedObject[] = []; |
| 126 | /** Each result's offset in the pack, to find the OFS_DELTAs against it. */ |
| 127 | const offsets: number[] = []; |
| 128 | const childrenByOffset = new Map<number, DeltaEntry[]>(); |
| 129 | const childrenByOid = new Map<string, DeltaEntry[]>(); |
| 130 | const waitFor = <K>(m: Map<K, DeltaEntry[]>, k: K, d: DeltaEntry) => { |
| 131 | const list = m.get(k); |
| 132 | if (list) list.push(d); |
| 133 | else m.set(k, [d]); |
| 134 | }; |
| 135 | const add = (offset: number, obj: PackedObject) => { |
| 136 | results.push(obj); |
| 137 | offsets.push(offset); |
| 138 | }; |
| 139 | |
| 140 | let pos = 12; |
| 141 | for (let n = 0; n < count; n++) { |
| 142 | const offset = pos; |
| 143 | let c = pack[pos++]!; |
| 144 | const typeCode = (c >> 4) & 7; |
| 145 | let size = c & 15; |
| 146 | let shift = 4; |
| 147 | while (c & 0x80) { |
| 148 | c = pack[pos++]!; |
| 149 | size += (c & 0x7f) * 2 ** shift; |
| 150 | shift += 7; |
| 151 | } |
| 152 | let baseOffset: number | undefined; |
| 153 | let baseOid: string | undefined; |
| 154 | if (typeCode === OFS_DELTA) { |
| 155 | c = pack[pos++]!; |
| 156 | let ofs = c & 0x7f; |
| 157 | while (c & 0x80) { |
| 158 | c = pack[pos++]!; |
| 159 | ofs = (ofs + 1) * 128 + (c & 0x7f); |
| 160 | } |
| 161 | baseOffset = offset - ofs; |
| 162 | } else if (typeCode === REF_DELTA) { |
| 163 | baseOid = toHex(pack.subarray(pos, pos + 20)); |
| 164 | pos += 20; |
| 165 | } else if (!CODE_TYPES[typeCode]) { |
| 166 | throw new Error(`bad object type ${typeCode} at ${offset}`); |
| 167 | } |
| 168 | if (typeCode === TYPE_CODES.blob && size >= large) { |
| 169 | // Hash it as it inflates and keep only the compressed bytes, so memory never holds it whole. |
| 170 | work(size); |
| 171 | hold(OBJECT_OVERHEAD); |
| 172 | const h = new Sha1().update(objectHeader("blob", size)); |
| 173 | const { consumed, produced } = inflateEach(pack, pos, (chunk) => h.update(chunk), size); |
| 174 | if (produced !== size) throw new Error(`size mismatch at ${offset}: ${produced} != ${size}`); |
| 175 | add(offset, largeBlob(toHex(h.digest()), size, pack.subarray(pos, pos + consumed))); |
| 176 | pos += consumed; |
| 177 | continue; |
| 178 | } |
| 179 | // Reserved at the declared size before inflating, so a declared size the budget can't hold is refused up front. |
| 180 | work(size); |
| 181 | hold(size + OBJECT_OVERHEAD); |
| 182 | const { out, consumed } = inflateAt(pack, pos, size); |
| 183 | if (out.length !== size) throw new Error(`size mismatch at ${offset}: ${out.length} != ${size}`); |
| 184 | const zlib = pack.subarray(pos, pos + consumed); |
| 185 | pos += consumed; |
| 186 | if (baseOffset !== undefined) waitFor(childrenByOffset, baseOffset, { offset, data: out }); |
| 187 | else if (baseOid !== undefined) waitFor(childrenByOid, baseOid, { offset, data: out }); |
| 188 | else { |
| 189 | const type = CODE_TYPES[typeCode]!; |
| 190 | add(offset, { oid: await hashObject(type, out), type, content: out, zlib }); |
| 191 | } |
| 192 | } |
| 193 | if (pos !== pack.length - 20) throw new Error("trailing garbage in pack"); |
| 194 | |
| 195 | /** Applies every delta waiting for `base` (in the pack at `offset`, or -1 when it came from the repository). */ |
| 196 | const resolveDeltasOf = async (base: GitObject & { oid: string; deflated?: Uint8Array; size?: number }, offset: number) => { |
| 197 | const byOffset = offset >= 0 ? childrenByOffset.get(offset) : undefined; |
| 198 | const byOid = childrenByOid.get(base.oid); |
| 199 | if (!byOffset && !byOid) return; |
| 200 | if (byOffset) childrenByOffset.delete(offset); |
| 201 | if (byOid) childrenByOid.delete(base.oid); |
| 202 | // A large blob kept compressed is inflated once for all its deltas, and held while they are applied. |
| 203 | const inflated = base.deflated ? (base.size ?? 0) : 0; |
| 204 | if (inflated) { |
| 205 | work(inflated); |
| 206 | hold(inflated); |
| 207 | } |
| 208 | const content = base.content; |
| 209 | for (const d of [...(byOffset ?? []), ...(byOid ?? [])]) { |
| 210 | // A result the pack's limit allows but memory can't hold is refused as memory, before it is allocated. |
| 211 | const size = deltaResultSize(d.data); |
| 212 | if (size > maxBytes - unpacked) throw tooBigUnpacked(maxBytes); |
| 213 | if (size > maxHeld - held + d.data.length) throw tooMuchMemory(maxHeld); |
| 214 | const result = applyDelta(content, d.data, Math.min(maxBytes - unpacked, maxHeld - held + d.data.length)); |
| 215 | work(result.length); |
| 216 | held -= d.data.length; |
| 217 | hold(result.length); |
| 218 | d.data = EMPTY; |
| 219 | add(d.offset, { oid: await hashObject(base.type, result), type: base.type, content: result }); |
| 220 | } |
| 221 | held -= inflated; |
| 222 | }; |
| 223 | // Each object, as it is added, has its deltas applied: one pass, whatever order the pack lists them in. |
| 224 | let next = 0; |
| 225 | const drain = async () => { |
| 226 | for (; next < results.length; next++) await resolveDeltasOf(results[next]!, offsets[next]!); |
| 227 | }; |
| 228 | await drain(); |
| 229 | // REF_DELTA bases that aren't in the pack (thin packs) come from the repository, each read once. |
| 230 | if (external) { |
| 231 | for (const oid of [...childrenByOid.keys()]) { |
| 232 | if (!childrenByOid.has(oid)) continue; |
| 233 | const base = await external(oid, maxHeld - held); |
| 234 | if (base === null) continue; |
| 235 | if (typeof base === "number") throw tooMuchMemory(maxHeld); |
| 236 | work(base.content.length); |
| 237 | hold(base.content.length); |
| 238 | await resolveDeltasOf({ oid, ...base }, -1); |
| 239 | held -= base.content.length; |
| 240 | await drain(); |
| 241 | } |
| 242 | } |
| 243 | const unresolved = [...childrenByOid.keys(), ...[...childrenByOffset.keys()].map((o) => `@${o}`)]; |
| 244 | if (unresolved.length) throw new Error(`unresolvable delta base(s): ${unresolved.join(", ")}`); |
| 245 | return results; |
| 246 | } |
| 247 | |
| 248 | export interface PackEntryInput { |
| 249 | type: ObjectType; |
| 250 | /** Uncompressed size of the object content. */ |
| 251 | size: number; |
| 252 | /** zlib-deflated object content. */ |
| 253 | deflated: Uint8Array; |
| 254 | } |
| 255 | |
| 256 | function writeEntryHeader(w: ByteWriter, typeCode: number, size: number) { |
| 257 | let c = (typeCode << 4) | (size & 15); |
| 258 | size = Math.floor(size / 16); |
| 259 | while (size > 0) { |
| 260 | w.byte(c | 0x80); |
| 261 | c = size & 0x7f; |
| 262 | size = Math.floor(size / 128); |
| 263 | } |
| 264 | w.byte(c); |
| 265 | } |
| 266 | |
| 267 | /** |
| 268 | * The same packfile as `writePack`, produced piece by piece as `entries` yields them, so a clone of a |
| 269 | * large repository never holds the whole pack: only the entry being written. |
| 270 | */ |
| 271 | export async function* packStream(count: number, entries: AsyncIterable<PackEntryInput>): AsyncGenerator<Uint8Array> { |
| 272 | const hash = new Sha1(); |
| 273 | const head = new ByteWriter(); |
| 274 | head.write(new TextEncoder().encode("PACK")); |
| 275 | head.uint32(2); |
| 276 | head.uint32(count); |
| 277 | let written = 0; |
| 278 | const out = (b: Uint8Array) => (hash.update(b), b); |
| 279 | yield out(head.bytes()); |
| 280 | for await (const e of entries) { |
| 281 | if (++written > count) throw new Error(`pack has more than the ${count} objects it announced`); |
| 282 | const h = new ByteWriter(); |
| 283 | writeEntryHeader(h, TYPE_CODES[e.type], e.size); |
| 284 | yield out(h.bytes()); |
| 285 | yield out(e.deflated); |
| 286 | } |
| 287 | if (written !== count) throw new Error(`pack announced ${count} objects but has ${written}`); |
| 288 | yield hash.digest(); |
| 289 | } |
| 290 | |
| 291 | /** Build a version 2 packfile of undeltified objects from pre-deflated content. */ |
| 292 | export async function writePack(entries: readonly PackEntryInput[]): Promise<Uint8Array> { |
| 293 | const w = new ByteWriter(); |
| 294 | w.write(new TextEncoder().encode("PACK")); |
| 295 | w.uint32(2); |
| 296 | w.uint32(entries.length); |
| 297 | for (const e of entries) { |
| 298 | writeEntryHeader(w, TYPE_CODES[e.type], e.size); |
| 299 | w.write(e.deflated); |
| 300 | } |
| 301 | const body = w.bytes(); |
| 302 | w.write(await sha1(body)); |
| 303 | return w.bytes(); |
| 304 | } |