| | @@ -29,12 +29,21 @@ |
| 29 | 29 | workRoot: string; |
| 30 | 30 | /** Overrides the spec's push size budget (tests use a small one to exercise chunking). */ |
| 31 | 31 | chunkBytes?: number; |
| 32 | + /** Overrides LARGE_BLOB_BYTES (tests use a small one). */ |
| 33 | + largeBlobBytes?: number; |
| 32 | 34 | log?: (line: string) => void; |
| 33 | 35 | } |
| 34 | 36 | |
| 35 | 37 | /** Chunk refs; only these are deleted at the end (pull request heads stay until the server has created refs/pull/N/head). */ |
| 36 | 38 | const TEMP_PREFIX = `${IMPORT_REF_PREFIX}tmp/`; |
| 37 | 39 | const MIN_BUDGET = 64 * 1024; |
| 40 | +/** |
| 41 | + * Files this large are pushed first, each version whole: sent as a delta, the server would hold the |
| 42 | + * version it is based on and the one it rebuilds at once, which it can't for files past about 30 MB |
| 43 | + * (a 32 MB file with a few versions stopped tactfolio's import). Whole, a large file is hashed and |
| 44 | + * stored as it streams in, up to the 100 MB a push may carry. |
| 45 | + */ |
| 46 | +const LARGE_BLOB_BYTES = 16 * 1024 * 1024; |
| 38 | 47 | const REF_BATCH = 200; |
| 39 | 48 | |
| 40 | 49 | export class ImportError extends Error {} |
| | @@ -99,7 +108,7 @@ |
| 99 | 108 | }; |
| 100 | 109 | let result: ImportGitResult; |
| 101 | 110 | try { |
| 102 | | - result = await transfer(spec, { workRoot: o.workRoot, chunkBytes: o.chunkBytes, progress, log }); |
| 111 | + result = await transfer(spec, { workRoot: o.workRoot, chunkBytes: o.chunkBytes, largeBlobBytes: o.largeBlobBytes, progress, log }); |
| 103 | 112 | } catch (e) { |
| 104 | 113 | result = emptyResult(); |
| 105 | 114 | result.ok = false; |
| | @@ -126,6 +135,8 @@ |
| 126 | 135 | export interface TransferOptions { |
| 127 | 136 | workRoot: string; |
| 128 | 137 | chunkBytes?: number; |
| 138 | + /** Overrides LARGE_BLOB_BYTES (tests use a small one). */ |
| 139 | + largeBlobBytes?: number; |
| 129 | 140 | progress: (p: ImportProgress, force?: boolean) => Promise<void>; |
| 130 | 141 | log: (line: string) => void; |
| 131 | 142 | } |
| | @@ -205,6 +216,7 @@ |
| 205 | 216 | const pushOpts = ["-o", "diffinery-import", ...(spec.secrets === "bypass" ? ["-o", `allow-secrets=${spec.bypassReason}`] : [])]; |
| 206 | 217 | const pusher = new ChunkPusher(git, spec, pushOpts, o, result); |
| 207 | 218 | await pusher.loadRemoteState(); |
| 219 | + await pusher.pushLargeBlobs(ordered.map((n) => refs.get(n)!.commit).filter((c): c is string => !!c), o.largeBlobBytes ?? LARGE_BLOB_BYTES); |
| 208 | 220 | for (const name of ordered) { |
| 209 | 221 | const ref = refs.get(name)!; |
| 210 | 222 | if (ref.commit) await pusher.ensurePushed(ref.commit, name); |
| | @@ -330,7 +342,8 @@ |
| 330 | 342 | const present = await this.git.run(["cat-file", "--batch-check=%(objectname) %(objecttype)"], { input: `${oids.join("\n")}\n` }); |
| 331 | 343 | for (const line of present.stdout.split("\n")) { |
| 332 | 344 | const [oid, type] = line.split(" "); |
| 333 | | - if (type === "commit") this.addHave(oid!); |
| 345 | + // Chunks of history are commits; batches of files are trees (see pushBlobs). |
| 346 | + if (type === "commit" || type === "tree") this.addHave(oid!); |
| 334 | 347 | } |
| 335 | 348 | for (const line of r.stdout.split("\n")) { |
| 336 | 349 | const ref = line.split("\t")[1]; |
| | @@ -386,7 +399,28 @@ |
| 386 | 399 | } |
| 387 | 400 | } |
| 388 | 401 | |
| 389 | | - /** Upload the blobs `commit` adds through throwaway commits of flat trees, so the commit itself stays small. */ |
| 402 | + /** |
| 403 | + * Every blob of `minBytes` or more in the history of `tips` that the server doesn't have yet, pushed |
| 404 | + * before the history and with no deltas (see LARGE_BLOB_BYTES). The history's pushes then leave |
| 405 | + * them out, as the server has them under a temporary ref. |
| 406 | + */ |
| 407 | + async pushLargeBlobs(tips: string[], minBytes: number): Promise<void> { |
| 408 | + if (!tips.length) return; |
| 409 | + // Blobs of minBytes or more are the ones the filter leaves out, printed with a leading "~". |
| 410 | + const listed = await this.git.lines(["rev-list", "--objects", `--filter=blob:limit=${minBytes}`, "--filter-print-omitted", "--stdin"], `${[...new Set(tips)].join("\n")}\n${this.notHaves()}`); |
| 411 | + const omitted = listed.filter((l) => l.startsWith("~")).map((l) => l.slice(1).split(" ")[0]!); |
| 412 | + if (!omitted.length) return; |
| 413 | + const check = await this.git.lines(["cat-file", "--batch-check=%(objectname) %(objecttype) %(objectsize)"], `${omitted.join("\n")}\n`); |
| 414 | + const large = check |
| 415 | + .map((l) => l.split(" ")) |
| 416 | + .filter((p) => p[1] === "blob") |
| 417 | + .map((p) => ({ oid: p[0]!, size: Number(p[2]) })); |
| 418 | + if (!large.length) return; |
| 419 | + this.o.log(`${large.length} version(s) of large files (${fmtBytes(minBytes)} or more) go first, whole`); |
| 420 | + await this.pushBlobs(large, "large files", true); |
| 421 | + } |
| 422 | + |
| 423 | + /** Upload the blobs `commit` adds in batches (see pushBlobs), so the commit itself stays small. */ |
| 390 | 424 | private async pushObjectBatches(commit: string, label: string): Promise<void> { |
| 391 | 425 | const listed = await this.git.lines(["rev-list", "--objects", "--stdin"], `${commit}\n${this.notHaves()}`); |
| 392 | 426 | const oids = listed.map((l) => l.split(" ")[0]!); |
| | @@ -395,25 +429,33 @@ |
| 395 | 429 | .map((l) => l.split(" ")) |
| 396 | 430 | .filter((p) => p[1] === "blob") |
| 397 | 431 | .map((p) => ({ oid: p[0]!, size: Number(p[2]) })); |
| 398 | | - // Batched at the budget as it stands: when the server refuses a batch the budget shrinks, and |
| 399 | | - // what is left is batched again at the new one. A lone object is retried as is (it can't be |
| 400 | | - // split further), until the budget can't shrink any more. |
| 401 | | - for (let left = blobs, n = 1; left.length; n++) { |
| 432 | + await this.pushBlobs(blobs, label, false); |
| 433 | + } |
| 434 | + |
| 435 | + /** |
| 436 | + * Blobs pushed in batches, each a flat tree under a temporary ref. A tree, not a commit holding |
| 437 | + * it: git leaves out of a later push what a ref on the server reaches, but the files of an |
| 438 | + * unrelated commit it would send again. Batched at the budget as it stands: when the server |
| 439 | + * refuses a batch the budget shrinks, and what is left is batched again at the new one. A lone |
| 440 | + * object is retried as is (it can't be split further), until the budget can't shrink any more. |
| 441 | + * `whole`: without deltas. |
| 442 | + */ |
| 443 | + private async pushBlobs(blobs: Array<{ oid: string; size: number }>, label: string, whole: boolean): Promise<void> { |
| 444 | + for (let left = blobs; left.length; ) { |
| 402 | 445 | const batch = batchBySize(left, this.budget)[0]!; |
| 403 | 446 | const tree = (await this.git.run(["mktree"], { input: batch.map((b, j) => `100644 blob ${b.oid}\tobject-${j}\n`).join("") })).stdout.trim(); |
| 404 | | - const carrier = (await this.git.run(["commit-tree", tree, "-m", `Diffinery import: objects for ${commit.slice(0, 12)} (${n})`])).stdout.trim(); |
| 405 | 447 | const size = batch.reduce((s, b) => s + b.size, 0); |
| 406 | | - if (await this.pushTemp(carrier, size, label)) left = left.slice(batch.length); |
| 448 | + if (await this.pushTemp(tree, size, label, whole)) left = left.slice(batch.length); |
| 407 | 449 | } |
| 408 | 450 | } |
| 409 | 451 | |
| 410 | 452 | /** Push one chunk to a temporary ref. Returns false when the budget was lowered and the caller should re-plan. */ |
| 411 | | - private async pushTemp(oid: string, size: number, label: string): Promise<boolean> { |
| 453 | + private async pushTemp(oid: string, size: number, label: string, whole = false): Promise<boolean> { |
| 412 | 454 | if (this.haveSet.has(oid)) return true; |
| 413 | 455 | const ref = `${TEMP_PREFIX}c${++this.temp}`; |
| 414 | 456 | const n = this.result.chunks + 1; |
| 415 | 457 | await this.o.progress({ phase: "push", message: `Pushing history of ${label.replace(/^refs\/(heads|tags)\//, "")}: chunk ${n} (${fmtBytes(size)})`, chunks: this.result.chunks, bytes: this.result.bytes }); |
| 416 | | - const out = await this.push([`${oid}:${ref}`]); |
| 458 | + const out = await this.push([`${oid}:${ref}`], whole); |
| 417 | 459 | if (out.code === 0 && !parsePorcelain(out.stdout).some((s) => !s.ok)) { |
| 418 | 460 | this.tempRefs.push(ref); |
| 419 | 461 | this.addHave(oid); |
| | @@ -455,8 +497,10 @@ |
| 455 | 497 | } |
| 456 | 498 | |
| 457 | 499 | /** `git push` to Diffinery in import mode; collects push protection findings from the output. */ |
| 458 | | - async push(refspecs: string[]): Promise<{ code: number; stdout: string; stderr: string }> { |
| 459 | | - const out = await this.git.run(["push", "--porcelain", "--no-verify", ...this.pushOpts, this.spec.target.pushUrl, ...refspecs], { allowFail: true }); |
| 500 | + async push(refspecs: string[], whole = false): Promise<{ code: number; stdout: string; stderr: string }> { |
| 501 | + // `whole`: no deltas at all, neither new ones (pack.window=0) nor those the mirror already |
| 502 | + // stores (pack.depth=0), so none against what the server has either. |
| 503 | + const out = await this.git.run([...(whole ? ["-c", "pack.window=0", "-c", "pack.depth=0"] : []), "push", "--porcelain", "--no-verify", ...this.pushOpts, this.spec.target.pushUrl, ...refspecs], { allowFail: true }); |
| 460 | 504 | if (/Push protection bypassed/i.test(out.stderr)) for (const f of parseSecretFindings(out.stderr)) this.findings.set(`${f.kind}:${f.blob}`, f); |
| 461 | 505 | return out; |
| 462 | 506 | } |