| 1 | /** |
| 2 | * Import mode: copy a repository's Git data from GitHub into Diffinery. |
| 3 | * |
| 4 | * Workers can't run git and a push request is capped (Diffinery refuses pushes |
| 5 | * over 100 MB, and a repository's Durable Object has 128 MB of memory), so this |
| 6 | * runs in a container: `git clone --mirror` from GitHub, then push history in |
| 7 | * chunks that stay well under the cap (first-parent walks to temporary refs, |
| 8 | * merged side branches first, and object batches for single huge commits), then |
| 9 | * every branch and tag, pull request heads to `refs/diffinery-import/pull/N` |
| 10 | * (the server creates `refs/pull/N/head` from the imported pull request and |
| 11 | * then drops these), then Git LFS objects. |
| 12 | * |
| 13 | * Tokens never appear in argv or logs: git reads them from GIT_CONFIG_* environment |
| 14 | * entries scoped to one host each. Pushes carry `-o diffinery-import`, which the |
| 15 | * server honours only for this import's own token, so they trigger no workflows, |
| 16 | * webhooks or notifications. |
| 17 | */ |
| 18 | import { mkdir, readdir, rm, stat } from "node:fs/promises"; |
| 19 | import path from "node:path"; |
| 20 | import type { ImportGitResult, ImportProgress, ImportSecretFinding, ImportSpec } from "@diffinery/shared"; |
| 21 | import { RunnerApi } from "./api.js"; |
| 22 | import { IMPORT_REF_PREFIX, batchBySize, furthestWithin, importDestination, importRefKind, isSecretRejection, isSizeRejection, parsePorcelain, parseSecretFindings, sortRefsForImport, storageRejection } from "./import-plan.js"; |
| 23 | import { exec } from "./util.js"; |
| 24 | |
| 25 | export interface ImportAgentOptions { |
| 26 | apiUrl: string; |
| 27 | importId: string; |
| 28 | token: string; |
| 29 | workRoot: string; |
| 30 | /** Overrides the spec's push size budget (tests use a small one to exercise chunking). */ |
| 31 | chunkBytes?: number; |
| 32 | /** Overrides LARGE_BLOB_BYTES (tests use a small one). */ |
| 33 | largeBlobBytes?: number; |
| 34 | log?: (line: string) => void; |
| 35 | } |
| 36 | |
| 37 | /** Chunk refs; only these are deleted at the end (pull request heads stay until the server has created refs/pull/N/head). */ |
| 38 | const TEMP_PREFIX = `${IMPORT_REF_PREFIX}tmp/`; |
| 39 | const MIN_BUDGET = 64 * 1024; |
| 40 | /** |
| 41 | * Files this large are pushed first, each version whole: sent as a delta, the server would hold the |
| 42 | * version it is based on and the one it rebuilds at once, which it can't for files past about 30 MB |
| 43 | * (a 32 MB file with a few versions stopped tactfolio's import). Whole, a large file is hashed and |
| 44 | * stored as it streams in, up to the 100 MB a push may carry. |
| 45 | */ |
| 46 | const LARGE_BLOB_BYTES = 16 * 1024 * 1024; |
| 47 | const REF_BATCH = 200; |
| 48 | |
| 49 | export class ImportError extends Error {} |
| 50 | |
| 51 | const errMsg = (e: unknown) => (e instanceof Error ? e.message : String(e)); |
| 52 | |
| 53 | /** `http.<origin>/.extraHeader` entry: the credential goes to that host only. */ |
| 54 | function authEntry(url: string, user: string, password: string): [string, string] { |
| 55 | return [`http.${new URL(url).origin}/.extraHeader`, `Authorization: Basic ${Buffer.from(`${user}:${password}`).toString("base64")}`]; |
| 56 | } |
| 57 | |
| 58 | function withConfig(env: NodeJS.ProcessEnv, entries: Array<[string, string]>): NodeJS.ProcessEnv { |
| 59 | const out = { ...env }; |
| 60 | let n = Number(out.GIT_CONFIG_COUNT ?? 0); |
| 61 | if (!Number.isInteger(n) || n < 0) n = 0; |
| 62 | for (const [k, v] of entries) { |
| 63 | out[`GIT_CONFIG_KEY_${n}`] = k; |
| 64 | out[`GIT_CONFIG_VALUE_${n}`] = v; |
| 65 | n++; |
| 66 | } |
| 67 | out.GIT_CONFIG_COUNT = String(n); |
| 68 | return out; |
| 69 | } |
| 70 | |
| 71 | /** Removes anything that looks like a credential from text we report. */ |
| 72 | function redact(s: string, secrets: string[]): string { |
| 73 | let out = s; |
| 74 | for (const x of secrets) if (x) out = out.split(x).join("***"); |
| 75 | return out.replace(/(Authorization:\s*\w+\s+)\S+/gi, "$1***"); |
| 76 | } |
| 77 | |
| 78 | class GitRunner { |
| 79 | constructor( |
| 80 | readonly dir: string, |
| 81 | private readonly env: NodeJS.ProcessEnv, |
| 82 | private readonly secrets: string[], |
| 83 | ) {} |
| 84 | |
| 85 | async run(args: string[], o: { input?: string; allowFail?: boolean; cwd?: string; maxCapture?: number } = {}): Promise<{ code: number; stdout: string; stderr: string }> { |
| 86 | const r = await exec("git", args, { cwd: o.cwd ?? this.dir, env: this.env, input: o.input, capture: true, maxCapture: o.maxCapture ?? 256 * 1024 * 1024 }); |
| 87 | const res = { code: r.exitCode, stdout: r.stdout, stderr: redact(r.stderr, this.secrets) }; |
| 88 | if (r.exitCode !== 0 && !o.allowFail) throw new ImportError(`git ${args[0]} failed: ${res.stderr.trim().split("\n").slice(-3).join(" ").slice(0, 500)}`); |
| 89 | return res; |
| 90 | } |
| 91 | |
| 92 | async lines(args: string[], input?: string): Promise<string[]> { |
| 93 | return (await this.run(args, { input })).stdout.split("\n").filter(Boolean); |
| 94 | } |
| 95 | } |
| 96 | |
| 97 | /** Fetch the spec, transfer everything, and report the result. Returns the result it reported. */ |
| 98 | export async function runImportAgent(o: ImportAgentOptions): Promise<ImportGitResult> { |
| 99 | const log = o.log ?? ((l: string) => process.stdout.write(`[diffinery-import] ${l}\n`)); |
| 100 | const api = new RunnerApi(`${o.apiUrl.replace(/\/$/, "")}/api/v3/diffinery/imports/${encodeURIComponent(o.importId)}/agent`, o.token); |
| 101 | const spec = (await (await api.request("/spec", { retries: 8 })).json()) as ImportSpec; |
| 102 | let lastProgress = 0; |
| 103 | const progress = async (p: ImportProgress, force = false) => { |
| 104 | log(p.message); |
| 105 | if (!force && Date.now() - lastProgress < 2000) return; |
| 106 | lastProgress = Date.now(); |
| 107 | await api.request("/progress", { method: "POST", json: p, retries: 2 }).catch((e) => log(`progress report failed: ${errMsg(e)}`)); |
| 108 | }; |
| 109 | let result: ImportGitResult; |
| 110 | try { |
| 111 | result = await transfer(spec, { workRoot: o.workRoot, chunkBytes: o.chunkBytes, largeBlobBytes: o.largeBlobBytes, progress, log }); |
| 112 | } catch (e) { |
| 113 | result = emptyResult(); |
| 114 | result.ok = false; |
| 115 | result.error = redact(errMsg(e), [spec.githubToken ?? "", spec.pushToken]); |
| 116 | if (e instanceof SecretsFound) result.secretFindings = e.findings; |
| 117 | } |
| 118 | await api.request("/complete", { method: "POST", json: result, retries: 10 }); |
| 119 | return result; |
| 120 | } |
| 121 | |
| 122 | class SecretsFound extends ImportError { |
| 123 | constructor( |
| 124 | message: string, |
| 125 | readonly findings: ImportSecretFinding[], |
| 126 | ) { |
| 127 | super(message); |
| 128 | } |
| 129 | } |
| 130 | |
| 131 | function emptyResult(): ImportGitResult { |
| 132 | return { ok: true, branches: 0, tags: 0, pullRefs: 0, chunks: 0, bytes: 0, lfsObjects: 0, lfsBytes: 0, defaultBranch: null, secretFindings: [], warnings: [] }; |
| 133 | } |
| 134 | |
| 135 | export interface TransferOptions { |
| 136 | workRoot: string; |
| 137 | chunkBytes?: number; |
| 138 | /** Overrides LARGE_BLOB_BYTES (tests use a small one). */ |
| 139 | largeBlobBytes?: number; |
| 140 | progress: (p: ImportProgress, force?: boolean) => Promise<void>; |
| 141 | log: (line: string) => void; |
| 142 | } |
| 143 | |
| 144 | /** The whole Git transfer for one spec. Exported for tests. */ |
| 145 | export async function transfer(spec: ImportSpec, o: TransferOptions): Promise<ImportGitResult> { |
| 146 | const result = emptyResult(); |
| 147 | const secrets = [spec.githubToken ?? "", spec.pushToken]; |
| 148 | const root = path.join(o.workRoot, `import-${spec.importId.replace(/[^\w-]/g, "")}`); |
| 149 | await rm(root, { recursive: true, force: true }); |
| 150 | await mkdir(root, { recursive: true }); |
| 151 | const mirror = path.join(root, "mirror.git"); |
| 152 | // The clone URL must be https (a loopback host or a file:// path is allowed for local test |
| 153 | // doubles), so the token never travels in the clear and never reaches a non-GitHub host. In |
| 154 | // production the orchestrator always supplies GitHub's https clone URL. |
| 155 | const localDouble = spec.source.cloneUrl.startsWith("file://"); |
| 156 | const cloneHost = localDouble ? "" : new URL(spec.source.cloneUrl).hostname.toLowerCase(); |
| 157 | const loopback = cloneHost === "localhost" || cloneHost === "127.0.0.1" || cloneHost === "::1"; |
| 158 | if (!spec.source.cloneUrl.startsWith("https://") && !loopback && !localDouble) throw new Error("the clone URL must use https"); |
| 159 | // Pin the LFS endpoint to each operation's own host so a repository's .lfsconfig cannot redirect |
| 160 | // LFS traffic (and a token) to another host: the source host for the fetch, the target host for |
| 161 | // the push. A command-line `-c lfs.url=` overrides .lfsconfig, so we set it per command rather |
| 162 | // than globally (a global lfs.url would also misdirect the push to Diffinery back at the source). |
| 163 | const sourceLfsUrl = `${spec.source.cloneUrl.replace(/\.git$/, "")}.git/info/lfs`; |
| 164 | const targetLfsUrl = `${spec.target.pushUrl.replace(/\.git$/, "")}.git/info/lfs`; |
| 165 | const entries: Array<[string, string]> = [ |
| 166 | ["credential.helper", ""], |
| 167 | ["protocol.version", "2"], |
| 168 | // Never follow an HTTP redirect: a redirect could carry the credential to another host. |
| 169 | ["http.followRedirects", "false"], |
| 170 | ["lfs.locksverify", "false"], |
| 171 | ["lfs.concurrenttransfers", "4"], |
| 172 | ["core.askPass", "true"], |
| 173 | ]; |
| 174 | if (spec.githubToken) entries.push(authEntry(spec.source.cloneUrl, "x-access-token", spec.githubToken)); |
| 175 | entries.push(authEntry(spec.target.pushUrl, "x-token", spec.pushToken)); |
| 176 | const env = withConfig( |
| 177 | { |
| 178 | ...process.env, |
| 179 | GIT_TERMINAL_PROMPT: "0", |
| 180 | GIT_LFS_SKIP_SMUDGE: "1", |
| 181 | GIT_AUTHOR_NAME: "Diffinery import", |
| 182 | GIT_AUTHOR_EMAIL: "import@diffinery.invalid", |
| 183 | GIT_COMMITTER_NAME: "Diffinery import", |
| 184 | GIT_COMMITTER_EMAIL: "import@diffinery.invalid", |
| 185 | }, |
| 186 | entries, |
| 187 | ); |
| 188 | const git = new GitRunner(mirror, env, secrets); |
| 189 | try { |
| 190 | // ------------------------------------------------------------ clone |
| 191 | await o.progress({ phase: "clone", message: `Cloning ${spec.source.fullName}` }, true); |
| 192 | await git.run(["clone", "--mirror", "--quiet", spec.source.cloneUrl, mirror], { cwd: root }); |
| 193 | const refLines = await git.lines(["for-each-ref", "--format=%(objectname) %(objecttype) %(*objectname) %(*objecttype) %(refname)"]); |
| 194 | const refs = new Map<string, { oid: string; commit: string | null }>(); |
| 195 | for (const line of refLines) { |
| 196 | const [oid, type, peeled, peeledType, ...rest] = line.split(" "); |
| 197 | const name = rest.join(" "); |
| 198 | if (!importRefKind(name)) continue; |
| 199 | const commit = type === "commit" ? oid! : peeledType === "commit" ? peeled! : null; |
| 200 | refs.set(name, { oid: oid!, commit }); |
| 201 | } |
| 202 | const ordered = sortRefsForImport([...refs.keys()], spec.source.defaultBranch); |
| 203 | if (!ordered.length) { |
| 204 | await o.progress({ phase: "done", message: "The GitHub repository is empty" }, true); |
| 205 | return result; |
| 206 | } |
| 207 | |
| 208 | // ------------------------------------------------------------ LFS objects from GitHub |
| 209 | if (spec.lfs) { |
| 210 | await o.progress({ phase: "lfs-fetch", message: "Fetching Git LFS objects" }, true); |
| 211 | const f = await git.run(["-c", `lfs.url=${sourceLfsUrl}`, "lfs", "fetch", "--all", "origin"], { allowFail: true }); |
| 212 | if (f.code !== 0) result.warnings.push(`Some Git LFS objects could not be fetched from GitHub: ${lastLine(f.stderr)}`); |
| 213 | } |
| 214 | |
| 215 | // ------------------------------------------------------------ history, in chunks |
| 216 | const pushOpts = ["-o", "diffinery-import", ...(spec.secrets === "bypass" ? ["-o", `allow-secrets=${spec.bypassReason}`] : [])]; |
| 217 | const pusher = new ChunkPusher(git, spec, pushOpts, o, result); |
| 218 | await pusher.loadRemoteState(); |
| 219 | await pusher.pushLargeBlobs(ordered.map((n) => refs.get(n)!.commit).filter((c): c is string => !!c), o.largeBlobBytes ?? LARGE_BLOB_BYTES); |
| 220 | for (const name of ordered) { |
| 221 | const ref = refs.get(name)!; |
| 222 | if (ref.commit) await pusher.ensurePushed(ref.commit, name); |
| 223 | } |
| 224 | |
| 225 | // ------------------------------------------------------------ the refs themselves |
| 226 | await o.progress({ phase: "refs", message: `Creating ${ordered.length} branches, tags and pull request refs`, chunks: result.chunks, bytes: result.bytes }, true); |
| 227 | for (let i = 0; i < ordered.length; i += REF_BATCH) { |
| 228 | const batch = ordered.slice(i, i + REF_BATCH); |
| 229 | const out = await pusher.push(batch.map((r) => `+${r}:${importDestination(r)}`)); |
| 230 | const status = new Map(parsePorcelain(out.stdout).map((s) => [s.ref, s])); |
| 231 | for (const r of batch) { |
| 232 | const s = status.get(importDestination(r)); |
| 233 | if (out.code !== 0 && !s) throw new ImportError(`pushing refs failed: ${pushFailure(out)}`); |
| 234 | if (s && !s.ok) { |
| 235 | result.warnings.push(`${r} was not imported: ${s.reason}`); |
| 236 | continue; |
| 237 | } |
| 238 | const kind = importRefKind(r); |
| 239 | if (kind === "branch") result.branches++; |
| 240 | else if (kind === "tag") result.tags++; |
| 241 | else if (kind === "pull") result.pullRefs++; |
| 242 | } |
| 243 | } |
| 244 | if (refs.has(`refs/heads/${spec.source.defaultBranch}`)) result.defaultBranch = spec.source.defaultBranch; |
| 245 | |
| 246 | // ------------------------------------------------------------ LFS objects to Diffinery |
| 247 | if (spec.lfs) { |
| 248 | const { count, bytes } = await lfsStore(mirror); |
| 249 | if (count > 0) { |
| 250 | await o.progress({ phase: "lfs-push", message: `Uploading ${count} Git LFS objects` }, true); |
| 251 | const p = await git.run(["-c", `lfs.url=${targetLfsUrl}`, "lfs", "push", "--all", spec.target.pushUrl], { allowFail: true }); |
| 252 | if (p.code !== 0) result.warnings.push(`Some Git LFS objects could not be uploaded: ${lastLine(p.stderr)}`); |
| 253 | result.lfsObjects = count; |
| 254 | result.lfsBytes = bytes; |
| 255 | } |
| 256 | } |
| 257 | |
| 258 | // ------------------------------------------------------------ cleanup |
| 259 | await o.progress({ phase: "cleanup", message: "Removing temporary refs" }, true); |
| 260 | await pusher.deleteTempRefs(); |
| 261 | result.secretFindings = await pusher.findingsWithPaths(); |
| 262 | await o.progress({ phase: "done", message: `Imported ${result.branches} branches, ${result.tags} tags and ${result.pullRefs} pull request refs`, chunks: result.chunks, bytes: result.bytes }, true); |
| 263 | return result; |
| 264 | } finally { |
| 265 | await rm(root, { recursive: true, force: true }).catch(() => {}); |
| 266 | } |
| 267 | } |
| 268 | |
| 269 | const lastLine = (s: string) => |
| 270 | s |
| 271 | .trim() |
| 272 | .split("\n") |
| 273 | .filter((l) => l.trim()) |
| 274 | .slice(-1)[0] |
| 275 | ?.slice(0, 300) ?? ""; |
| 276 | |
| 277 | /** |
| 278 | * Why a push failed, in the server's words: why it couldn't unpack the push, else the reason on a |
| 279 | * rejected ref (`--porcelain` puts it on stdout), else the last line of stderr other than git's |
| 280 | * generic "failed to push some refs". |
| 281 | */ |
| 282 | export const pushFailure = (out: { stdout: string; stderr: string }) => |
| 283 | /remote unpack failed: (.+)/.exec(out.stderr)?.[1]?.trim().slice(0, 300) || |
| 284 | parsePorcelain(out.stdout).find((s) => !s.ok)?.reason.slice(0, 300) || |
| 285 | lastLine(out.stderr.split("\n").filter((l) => !/^error: failed to push some refs/.test(l)).join("\n")) || |
| 286 | lastLine(out.stderr) || |
| 287 | lastLine(out.stdout); |
| 288 | |
| 289 | /** Count and size of the LFS objects in a bare repository's local store. */ |
| 290 | async function lfsStore(gitDir: string): Promise<{ count: number; bytes: number }> { |
| 291 | let count = 0; |
| 292 | let bytes = 0; |
| 293 | const walk = async (dir: string): Promise<void> => { |
| 294 | let entries; |
| 295 | try { |
| 296 | entries = await readdir(dir, { withFileTypes: true }); |
| 297 | } catch { |
| 298 | return; |
| 299 | } |
| 300 | for (const e of entries) { |
| 301 | const p = path.join(dir, e.name); |
| 302 | if (e.isDirectory()) await walk(p); |
| 303 | else if (/^[0-9a-f]{64}$/.test(e.name)) { |
| 304 | count++; |
| 305 | bytes += (await stat(p)).size; |
| 306 | } |
| 307 | } |
| 308 | }; |
| 309 | await walk(path.join(gitDir, "lfs", "objects")); |
| 310 | return { count, bytes }; |
| 311 | } |
| 312 | |
| 313 | /** |
| 314 | * Pushes history so that every request stays under the budget. The server keeps |
| 315 | * what it has under temporary refs, so later pushes (and a restarted import) only |
| 316 | * send what is missing. |
| 317 | */ |
| 318 | class ChunkPusher { |
| 319 | private haves: string[] = []; |
| 320 | private haveSet = new Set<string>(); |
| 321 | private budget: number; |
| 322 | private temp = 0; |
| 323 | private tempRefs: string[] = []; |
| 324 | private findings = new Map<string, ImportSecretFinding>(); |
| 325 | |
| 326 | constructor( |
| 327 | private readonly git: GitRunner, |
| 328 | private readonly spec: ImportSpec, |
| 329 | private readonly pushOpts: string[], |
| 330 | private readonly o: TransferOptions, |
| 331 | private readonly result: ImportGitResult, |
| 332 | ) { |
| 333 | this.budget = Math.max(MIN_BUDGET, o.chunkBytes ?? spec.chunkBytes); |
| 334 | } |
| 335 | |
| 336 | /** Objects the server already has (a resumed import) count as pushed. */ |
| 337 | async loadRemoteState(): Promise<void> { |
| 338 | const r = await this.git.run(["ls-remote", this.spec.target.pushUrl], { allowFail: true }); |
| 339 | if (r.code !== 0) throw new ImportError(`cannot reach the Diffinery repository: ${lastLine(r.stderr)}`); |
| 340 | const oids = [...new Set(r.stdout.split("\n").map((l) => l.split("\t")[0]!).filter((x) => /^[0-9a-f]{40}$/.test(x)))]; |
| 341 | if (!oids.length) return; |
| 342 | const present = await this.git.run(["cat-file", "--batch-check=%(objectname) %(objecttype)"], { input: `${oids.join("\n")}\n` }); |
| 343 | for (const line of present.stdout.split("\n")) { |
| 344 | const [oid, type] = line.split(" "); |
| 345 | // Chunks of history are commits; batches of files are trees (see pushBlobs). |
| 346 | if (type === "commit" || type === "tree") this.addHave(oid!); |
| 347 | } |
| 348 | for (const line of r.stdout.split("\n")) { |
| 349 | const ref = line.split("\t")[1]; |
| 350 | if (ref?.startsWith(TEMP_PREFIX)) this.tempRefs.push(ref); |
| 351 | } |
| 352 | this.temp = this.tempRefs.length; |
| 353 | } |
| 354 | |
| 355 | private addHave(oid: string): void { |
| 356 | if (this.haveSet.has(oid)) return; |
| 357 | this.haveSet.add(oid); |
| 358 | this.haves.push(oid); |
| 359 | } |
| 360 | |
| 361 | private notHaves(): string { |
| 362 | return this.haves.map((h) => `^${h}\n`).join(""); |
| 363 | } |
| 364 | |
| 365 | /** Bytes on disk of everything reachable from `tip` that the server doesn't have. */ |
| 366 | async sizeOf(tip: string): Promise<number> { |
| 367 | const r = await this.git.run(["rev-list", "--objects", "--disk-usage", "--stdin"], { input: `${tip}\n${this.notHaves()}` }); |
| 368 | return Number(r.stdout.trim()) || 0; |
| 369 | } |
| 370 | |
| 371 | /** Make sure the server has `tip` and everything it needs. */ |
| 372 | async ensurePushed(tip: string, label: string): Promise<void> { |
| 373 | if (this.haveSet.has(tip)) return; |
| 374 | let chain = await this.git.lines(["rev-list", "--first-parent", "--reverse", "--stdin"], `${tip}\n${this.notHaves()}`); |
| 375 | while (chain.length) { |
| 376 | const sizes = new Map<number, number>(); |
| 377 | const best = await furthestWithin( |
| 378 | chain.length, |
| 379 | async (i) => { |
| 380 | const s = await this.sizeOf(chain[i]!); |
| 381 | sizes.set(i, s); |
| 382 | return s; |
| 383 | }, |
| 384 | this.budget, |
| 385 | ); |
| 386 | if (best) { |
| 387 | if (!(await this.pushTemp(chain[best.index]!, best.size, label))) continue; |
| 388 | chain = chain.slice(best.index + 1); |
| 389 | continue; |
| 390 | } |
| 391 | // One first-parent step is over budget: push the side branches it merges first, |
| 392 | // then, if it still is, its new objects in batches. |
| 393 | const commit = chain[0]!; |
| 394 | const parents = (await this.git.lines(["rev-list", "--parents", "-n", "1", commit]))[0]!.split(" ").slice(2); |
| 395 | for (const p of parents) await this.ensurePushed(p, label); |
| 396 | if ((await this.sizeOf(commit)) > this.budget) await this.pushObjectBatches(commit, label); |
| 397 | if (!(await this.pushTemp(commit, await this.sizeOf(commit), label))) continue; |
| 398 | chain = chain.slice(1); |
| 399 | } |
| 400 | } |
| 401 | |
| 402 | /** |
| 403 | * Every blob of `minBytes` or more in the history of `tips` that the server doesn't have yet, pushed |
| 404 | * before the history and with no deltas (see LARGE_BLOB_BYTES). The history's pushes then leave |
| 405 | * them out, as the server has them under a temporary ref. |
| 406 | */ |
| 407 | async pushLargeBlobs(tips: string[], minBytes: number): Promise<void> { |
| 408 | if (!tips.length) return; |
| 409 | // Blobs of minBytes or more are the ones the filter leaves out, printed with a leading "~". |
| 410 | const listed = await this.git.lines(["rev-list", "--objects", `--filter=blob:limit=${minBytes}`, "--filter-print-omitted", "--stdin"], `${[...new Set(tips)].join("\n")}\n${this.notHaves()}`); |
| 411 | const omitted = listed.filter((l) => l.startsWith("~")).map((l) => l.slice(1).split(" ")[0]!); |
| 412 | if (!omitted.length) return; |
| 413 | const check = await this.git.lines(["cat-file", "--batch-check=%(objectname) %(objecttype) %(objectsize)"], `${omitted.join("\n")}\n`); |
| 414 | const large = check |
| 415 | .map((l) => l.split(" ")) |
| 416 | .filter((p) => p[1] === "blob") |
| 417 | .map((p) => ({ oid: p[0]!, size: Number(p[2]) })); |
| 418 | if (!large.length) return; |
| 419 | this.o.log(`${large.length} version(s) of large files (${fmtBytes(minBytes)} or more) go first, whole`); |
| 420 | await this.pushBlobs(large, "large files", true); |
| 421 | } |
| 422 | |
| 423 | /** Upload the blobs `commit` adds in batches (see pushBlobs), so the commit itself stays small. */ |
| 424 | private async pushObjectBatches(commit: string, label: string): Promise<void> { |
| 425 | const listed = await this.git.lines(["rev-list", "--objects", "--stdin"], `${commit}\n${this.notHaves()}`); |
| 426 | const oids = listed.map((l) => l.split(" ")[0]!); |
| 427 | const check = await this.git.lines(["cat-file", "--batch-check=%(objectname) %(objecttype) %(objectsize:disk)"], `${oids.join("\n")}\n`); |
| 428 | const blobs = check |
| 429 | .map((l) => l.split(" ")) |
| 430 | .filter((p) => p[1] === "blob") |
| 431 | .map((p) => ({ oid: p[0]!, size: Number(p[2]) })); |
| 432 | await this.pushBlobs(blobs, label, false); |
| 433 | } |
| 434 | |
| 435 | /** |
| 436 | * Blobs pushed in batches, each a flat tree under a temporary ref. A tree, not a commit holding |
| 437 | * it: git leaves out of a later push what a ref on the server reaches, but the files of an |
| 438 | * unrelated commit it would send again. Batched at the budget as it stands: when the server |
| 439 | * refuses a batch the budget shrinks, and what is left is batched again at the new one. A lone |
| 440 | * object is retried as is (it can't be split further), until the budget can't shrink any more. |
| 441 | * `whole`: without deltas. |
| 442 | */ |
| 443 | private async pushBlobs(blobs: Array<{ oid: string; size: number }>, label: string, whole: boolean): Promise<void> { |
| 444 | for (let left = blobs; left.length; ) { |
| 445 | const batch = batchBySize(left, this.budget)[0]!; |
| 446 | const tree = (await this.git.run(["mktree"], { input: batch.map((b, j) => `100644 blob ${b.oid}\tobject-${j}\n`).join("") })).stdout.trim(); |
| 447 | const size = batch.reduce((s, b) => s + b.size, 0); |
| 448 | if (await this.pushTemp(tree, size, label, whole)) left = left.slice(batch.length); |
| 449 | } |
| 450 | } |
| 451 | |
| 452 | /** Push one chunk to a temporary ref. Returns false when the budget was lowered and the caller should re-plan. */ |
| 453 | private async pushTemp(oid: string, size: number, label: string, whole = false): Promise<boolean> { |
| 454 | if (this.haveSet.has(oid)) return true; |
| 455 | const ref = `${TEMP_PREFIX}c${++this.temp}`; |
| 456 | const n = this.result.chunks + 1; |
| 457 | await this.o.progress({ phase: "push", message: `Pushing history of ${label.replace(/^refs\/(heads|tags)\//, "")}: chunk ${n} (${fmtBytes(size)})`, chunks: this.result.chunks, bytes: this.result.bytes }); |
| 458 | const out = await this.push([`${oid}:${ref}`], whole); |
| 459 | if (out.code === 0 && !parsePorcelain(out.stdout).some((s) => !s.ok)) { |
| 460 | this.tempRefs.push(ref); |
| 461 | this.addHave(oid); |
| 462 | this.result.chunks++; |
| 463 | this.result.bytes += size; |
| 464 | return true; |
| 465 | } |
| 466 | const text = `${out.stdout}\n${out.stderr}`; |
| 467 | if (isSecretRejection(text)) { |
| 468 | for (const f of parseSecretFindings(out.stderr)) this.findings.set(`${f.kind}:${f.blob}`, f); |
| 469 | const found = await this.findingsWithPaths(); |
| 470 | const what = found.map((f) => `${f.kind} in ${f.path ?? `blob ${f.blob}`}`).join("; "); |
| 471 | throw new SecretsFound( |
| 472 | this.spec.secrets === "bypass" |
| 473 | ? `Push protection found possible secrets in the repository's history (${what}), and only a human admin of the target repository can import them with a bypass.` |
| 474 | : `Push protection found possible secrets in the repository's history (${what}). The import was stopped as requested; rotate the secrets, or import again with the bypass option.`, |
| 475 | found, |
| 476 | ); |
| 477 | } |
| 478 | // Past the owner's storage quota nothing more can be stored: say so rather than "push failed". |
| 479 | const full = storageRejection(text); |
| 480 | if (full) throw new ImportError(full); |
| 481 | if (isSizeRejection(text)) { |
| 482 | if (this.budget / 2 >= MIN_BUDGET) { |
| 483 | this.budget = Math.floor(this.budget / 2); |
| 484 | this.o.log(`push of ${fmtBytes(size)} was rejected (${lastLine(out.stderr)}); retrying with chunks of ${fmtBytes(this.budget)}`); |
| 485 | this.temp--; |
| 486 | return false; |
| 487 | } |
| 488 | // Chunking can no longer help. Say what the server answered: the object may be under the |
| 489 | // 100 MB push limit and still be refused (or the server may be failing for another reason). |
| 490 | const said = pushFailure(out); |
| 491 | if (size > this.budget) { |
| 492 | throw new ImportError(`a single Git object of ${fmtBytes(size)} could not be pushed even on its own (${said}). Remove the large file from the repository's history (e.g. with git filter-repo) and import again.`); |
| 493 | } |
| 494 | throw new ImportError(`pushes kept failing even in chunks of ${fmtBytes(this.budget)} (${said})`); |
| 495 | } |
| 496 | throw new ImportError(`push failed: ${pushFailure(out)}`); |
| 497 | } |
| 498 | |
| 499 | /** `git push` to Diffinery in import mode; collects push protection findings from the output. */ |
| 500 | async push(refspecs: string[], whole = false): Promise<{ code: number; stdout: string; stderr: string }> { |
| 501 | // `whole`: no deltas at all, neither new ones (pack.window=0) nor those the mirror already |
| 502 | // stores (pack.depth=0), so none against what the server has either. |
| 503 | const out = await this.git.run([...(whole ? ["-c", "pack.window=0", "-c", "pack.depth=0"] : []), "push", "--porcelain", "--no-verify", ...this.pushOpts, this.spec.target.pushUrl, ...refspecs], { allowFail: true }); |
| 504 | if (/Push protection bypassed/i.test(out.stderr)) for (const f of parseSecretFindings(out.stderr)) this.findings.set(`${f.kind}:${f.blob}`, f); |
| 505 | return out; |
| 506 | } |
| 507 | |
| 508 | async deleteTempRefs(): Promise<void> { |
| 509 | for (let i = 0; i < this.tempRefs.length; i += REF_BATCH) { |
| 510 | const batch = this.tempRefs.slice(i, i + REF_BATCH); |
| 511 | const out = await this.push(batch.map((r) => `:${r}`)); |
| 512 | if (out.code !== 0) this.result.warnings.push(`Some temporary import refs could not be removed: ${lastLine(out.stderr)}`); |
| 513 | } |
| 514 | this.tempRefs = []; |
| 515 | } |
| 516 | |
| 517 | /** Findings with a path where each blob appears, so people know what to rotate. */ |
| 518 | async findingsWithPaths(): Promise<ImportSecretFinding[]> { |
| 519 | const out: ImportSecretFinding[] = []; |
| 520 | for (const f of [...this.findings.values()].slice(0, 100)) { |
| 521 | const full = (await this.git.run(["rev-parse", "--verify", "--quiet", `${f.blob}^{blob}`], { allowFail: true })).stdout.trim() || f.blob; |
| 522 | let where: string | undefined = f.path; |
| 523 | if (!where && /^[0-9a-f]{40}$/.test(full)) { |
| 524 | const raw = await this.git.run(["log", "--all", "-n", "1", `--find-object=${full}`, "--raw", "--no-abbrev", "--format="], { allowFail: true }); |
| 525 | const line = raw.stdout.split("\n").find((l) => l.includes(full)); |
| 526 | where = line?.split("\t").slice(-1)[0]; |
| 527 | } |
| 528 | out.push({ kind: f.kind, blob: full, path: where }); |
| 529 | } |
| 530 | return out; |
| 531 | } |
| 532 | } |
| 533 | |
| 534 | function fmtBytes(n: number): string { |
| 535 | if (n < 1024) return `${n} B`; |
| 536 | if (n < 1024 * 1024) return `${(n / 1024).toFixed(1)} KB`; |
| 537 | return `${(n / 1024 / 1024).toFixed(1)} MB`; |
| 538 | } |