| 1 | /** |
| 2 | * Import mode: copy a repository's Git data from GitHub into Diffinery. |
| 3 | * |
| 4 | * Workers can't run git and a push request is capped (Diffinery refuses pushes |
| 5 | * over 100 MB, and a repository's Durable Object has 128 MB of memory), so this |
| 6 | * runs in a container: `git clone --mirror` from GitHub, then push history in |
| 7 | * chunks that stay well under the cap (first-parent walks to temporary refs, |
| 8 | * merged side branches first, and object batches for single huge commits), then |
| 9 | * every branch and tag, pull request heads to `refs/diffinery-import/pull/N` |
| 10 | * (the server creates `refs/pull/N/head` from the imported pull request and |
| 11 | * then drops these), then Git LFS objects. |
| 12 | * |
| 13 | * Tokens never appear in argv or logs: git reads them from GIT_CONFIG_* environment |
| 14 | * entries scoped to one host each. Pushes carry `-o diffinery-import`, which the |
| 15 | * server honours only for this import's own token, so they trigger no workflows, |
| 16 | * webhooks or notifications. |
| 17 | */ |
| 18 | import { mkdir, readdir, rm, stat } from "node:fs/promises"; |
| 19 | import path from "node:path"; |
| 20 | import type { ImportGitResult, ImportProgress, ImportSecretFinding, ImportSpec } from "@diffinery/shared"; |
| 21 | import { RunnerApi } from "./api.js"; |
| 22 | import { IMPORT_REF_PREFIX, batchBySize, furthestWithin, importDestination, importRefKind, isSecretRejection, isSizeRejection, parsePorcelain, parseSecretFindings, sortRefsForImport, storageRejection } from "./import-plan.js"; |
| 23 | import { exec } from "./util.js"; |
| 24 | |
| 25 | export interface ImportAgentOptions { |
| 26 | apiUrl: string; |
| 27 | importId: string; |
| 28 | token: string; |
| 29 | workRoot: string; |
| 30 | /** Overrides the spec's push size budget (tests use a small one to exercise chunking). */ |
| 31 | chunkBytes?: number; |
| 32 | log?: (line: string) => void; |
| 33 | } |
| 34 | |
| 35 | /** Chunk refs; only these are deleted at the end (pull request heads stay until the server has created refs/pull/N/head). */ |
| 36 | const TEMP_PREFIX = `${IMPORT_REF_PREFIX}tmp/`; |
| 37 | const MIN_BUDGET = 64 * 1024; |
| 38 | const REF_BATCH = 200; |
| 39 | |
| 40 | export class ImportError extends Error {} |
| 41 | |
| 42 | const errMsg = (e: unknown) => (e instanceof Error ? e.message : String(e)); |
| 43 | |
| 44 | /** `http.<origin>/.extraHeader` entry: the credential goes to that host only. */ |
| 45 | function authEntry(url: string, user: string, password: string): [string, string] { |
| 46 | return [`http.${new URL(url).origin}/.extraHeader`, `Authorization: Basic ${Buffer.from(`${user}:${password}`).toString("base64")}`]; |
| 47 | } |
| 48 | |
| 49 | function withConfig(env: NodeJS.ProcessEnv, entries: Array<[string, string]>): NodeJS.ProcessEnv { |
| 50 | const out = { ...env }; |
| 51 | let n = Number(out.GIT_CONFIG_COUNT ?? 0); |
| 52 | if (!Number.isInteger(n) || n < 0) n = 0; |
| 53 | for (const [k, v] of entries) { |
| 54 | out[`GIT_CONFIG_KEY_${n}`] = k; |
| 55 | out[`GIT_CONFIG_VALUE_${n}`] = v; |
| 56 | n++; |
| 57 | } |
| 58 | out.GIT_CONFIG_COUNT = String(n); |
| 59 | return out; |
| 60 | } |
| 61 | |
| 62 | /** Removes anything that looks like a credential from text we report. */ |
| 63 | function redact(s: string, secrets: string[]): string { |
| 64 | let out = s; |
| 65 | for (const x of secrets) if (x) out = out.split(x).join("***"); |
| 66 | return out.replace(/(Authorization:\s*\w+\s+)\S+/gi, "$1***"); |
| 67 | } |
| 68 | |
| 69 | class GitRunner { |
| 70 | constructor( |
| 71 | readonly dir: string, |
| 72 | private readonly env: NodeJS.ProcessEnv, |
| 73 | private readonly secrets: string[], |
| 74 | ) {} |
| 75 | |
| 76 | async run(args: string[], o: { input?: string; allowFail?: boolean; cwd?: string; maxCapture?: number } = {}): Promise<{ code: number; stdout: string; stderr: string }> { |
| 77 | const r = await exec("git", args, { cwd: o.cwd ?? this.dir, env: this.env, input: o.input, capture: true, maxCapture: o.maxCapture ?? 256 * 1024 * 1024 }); |
| 78 | const res = { code: r.exitCode, stdout: r.stdout, stderr: redact(r.stderr, this.secrets) }; |
| 79 | if (r.exitCode !== 0 && !o.allowFail) throw new ImportError(`git ${args[0]} failed: ${res.stderr.trim().split("\n").slice(-3).join(" ").slice(0, 500)}`); |
| 80 | return res; |
| 81 | } |
| 82 | |
| 83 | async lines(args: string[], input?: string): Promise<string[]> { |
| 84 | return (await this.run(args, { input })).stdout.split("\n").filter(Boolean); |
| 85 | } |
| 86 | } |
| 87 | |
| 88 | /** Fetch the spec, transfer everything, and report the result. Returns the result it reported. */ |
| 89 | export async function runImportAgent(o: ImportAgentOptions): Promise<ImportGitResult> { |
| 90 | const log = o.log ?? ((l: string) => process.stdout.write(`[diffinery-import] ${l}\n`)); |
| 91 | const api = new RunnerApi(`${o.apiUrl.replace(/\/$/, "")}/api/v3/diffinery/imports/${encodeURIComponent(o.importId)}/agent`, o.token); |
| 92 | const spec = (await (await api.request("/spec", { retries: 8 })).json()) as ImportSpec; |
| 93 | let lastProgress = 0; |
| 94 | const progress = async (p: ImportProgress, force = false) => { |
| 95 | log(p.message); |
| 96 | if (!force && Date.now() - lastProgress < 2000) return; |
| 97 | lastProgress = Date.now(); |
| 98 | await api.request("/progress", { method: "POST", json: p, retries: 2 }).catch((e) => log(`progress report failed: ${errMsg(e)}`)); |
| 99 | }; |
| 100 | let result: ImportGitResult; |
| 101 | try { |
| 102 | result = await transfer(spec, { workRoot: o.workRoot, chunkBytes: o.chunkBytes, progress, log }); |
| 103 | } catch (e) { |
| 104 | result = emptyResult(); |
| 105 | result.ok = false; |
| 106 | result.error = redact(errMsg(e), [spec.githubToken ?? "", spec.pushToken]); |
| 107 | if (e instanceof SecretsFound) result.secretFindings = e.findings; |
| 108 | } |
| 109 | await api.request("/complete", { method: "POST", json: result, retries: 10 }); |
| 110 | return result; |
| 111 | } |
| 112 | |
| 113 | class SecretsFound extends ImportError { |
| 114 | constructor( |
| 115 | message: string, |
| 116 | readonly findings: ImportSecretFinding[], |
| 117 | ) { |
| 118 | super(message); |
| 119 | } |
| 120 | } |
| 121 | |
| 122 | function emptyResult(): ImportGitResult { |
| 123 | return { ok: true, branches: 0, tags: 0, pullRefs: 0, chunks: 0, bytes: 0, lfsObjects: 0, lfsBytes: 0, defaultBranch: null, secretFindings: [], warnings: [] }; |
| 124 | } |
| 125 | |
| 126 | export interface TransferOptions { |
| 127 | workRoot: string; |
| 128 | chunkBytes?: number; |
| 129 | progress: (p: ImportProgress, force?: boolean) => Promise<void>; |
| 130 | log: (line: string) => void; |
| 131 | } |
| 132 | |
| 133 | /** The whole Git transfer for one spec. Exported for tests. */ |
| 134 | export async function transfer(spec: ImportSpec, o: TransferOptions): Promise<ImportGitResult> { |
| 135 | const result = emptyResult(); |
| 136 | const secrets = [spec.githubToken ?? "", spec.pushToken]; |
| 137 | const root = path.join(o.workRoot, `import-${spec.importId.replace(/[^\w-]/g, "")}`); |
| 138 | await rm(root, { recursive: true, force: true }); |
| 139 | await mkdir(root, { recursive: true }); |
| 140 | const mirror = path.join(root, "mirror.git"); |
| 141 | // The clone URL must be https (a loopback host or a file:// path is allowed for local test |
| 142 | // doubles), so the token never travels in the clear and never reaches a non-GitHub host. In |
| 143 | // production the orchestrator always supplies GitHub's https clone URL. |
| 144 | const localDouble = spec.source.cloneUrl.startsWith("file://"); |
| 145 | const cloneHost = localDouble ? "" : new URL(spec.source.cloneUrl).hostname.toLowerCase(); |
| 146 | const loopback = cloneHost === "localhost" || cloneHost === "127.0.0.1" || cloneHost === "::1"; |
| 147 | if (!spec.source.cloneUrl.startsWith("https://") && !loopback && !localDouble) throw new Error("the clone URL must use https"); |
| 148 | // Pin the LFS endpoint to each operation's own host so a repository's .lfsconfig cannot redirect |
| 149 | // LFS traffic (and a token) to another host: the source host for the fetch, the target host for |
| 150 | // the push. A command-line `-c lfs.url=` overrides .lfsconfig, so we set it per command rather |
| 151 | // than globally (a global lfs.url would also misdirect the push to Diffinery back at the source). |
| 152 | const sourceLfsUrl = `${spec.source.cloneUrl.replace(/\.git$/, "")}.git/info/lfs`; |
| 153 | const targetLfsUrl = `${spec.target.pushUrl.replace(/\.git$/, "")}.git/info/lfs`; |
| 154 | const entries: Array<[string, string]> = [ |
| 155 | ["credential.helper", ""], |
| 156 | ["protocol.version", "2"], |
| 157 | // Never follow an HTTP redirect: a redirect could carry the credential to another host. |
| 158 | ["http.followRedirects", "false"], |
| 159 | ["lfs.locksverify", "false"], |
| 160 | ["lfs.concurrenttransfers", "4"], |
| 161 | ["core.askPass", "true"], |
| 162 | ]; |
| 163 | if (spec.githubToken) entries.push(authEntry(spec.source.cloneUrl, "x-access-token", spec.githubToken)); |
| 164 | entries.push(authEntry(spec.target.pushUrl, "x-token", spec.pushToken)); |
| 165 | const env = withConfig( |
| 166 | { |
| 167 | ...process.env, |
| 168 | GIT_TERMINAL_PROMPT: "0", |
| 169 | GIT_LFS_SKIP_SMUDGE: "1", |
| 170 | GIT_AUTHOR_NAME: "Diffinery import", |
| 171 | GIT_AUTHOR_EMAIL: "import@diffinery.invalid", |
| 172 | GIT_COMMITTER_NAME: "Diffinery import", |
| 173 | GIT_COMMITTER_EMAIL: "import@diffinery.invalid", |
| 174 | }, |
| 175 | entries, |
| 176 | ); |
| 177 | const git = new GitRunner(mirror, env, secrets); |
| 178 | try { |
| 179 | // ------------------------------------------------------------ clone |
| 180 | await o.progress({ phase: "clone", message: `Cloning ${spec.source.fullName}` }, true); |
| 181 | await git.run(["clone", "--mirror", "--quiet", spec.source.cloneUrl, mirror], { cwd: root }); |
| 182 | const refLines = await git.lines(["for-each-ref", "--format=%(objectname) %(objecttype) %(*objectname) %(*objecttype) %(refname)"]); |
| 183 | const refs = new Map<string, { oid: string; commit: string | null }>(); |
| 184 | for (const line of refLines) { |
| 185 | const [oid, type, peeled, peeledType, ...rest] = line.split(" "); |
| 186 | const name = rest.join(" "); |
| 187 | if (!importRefKind(name)) continue; |
| 188 | const commit = type === "commit" ? oid! : peeledType === "commit" ? peeled! : null; |
| 189 | refs.set(name, { oid: oid!, commit }); |
| 190 | } |
| 191 | const ordered = sortRefsForImport([...refs.keys()], spec.source.defaultBranch); |
| 192 | if (!ordered.length) { |
| 193 | await o.progress({ phase: "done", message: "The GitHub repository is empty" }, true); |
| 194 | return result; |
| 195 | } |
| 196 | |
| 197 | // ------------------------------------------------------------ LFS objects from GitHub |
| 198 | if (spec.lfs) { |
| 199 | await o.progress({ phase: "lfs-fetch", message: "Fetching Git LFS objects" }, true); |
| 200 | const f = await git.run(["-c", `lfs.url=${sourceLfsUrl}`, "lfs", "fetch", "--all", "origin"], { allowFail: true }); |
| 201 | if (f.code !== 0) result.warnings.push(`Some Git LFS objects could not be fetched from GitHub: ${lastLine(f.stderr)}`); |
| 202 | } |
| 203 | |
| 204 | // ------------------------------------------------------------ history, in chunks |
| 205 | const pushOpts = ["-o", "diffinery-import", ...(spec.secrets === "bypass" ? ["-o", `allow-secrets=${spec.bypassReason}`] : [])]; |
| 206 | const pusher = new ChunkPusher(git, spec, pushOpts, o, result); |
| 207 | await pusher.loadRemoteState(); |
| 208 | for (const name of ordered) { |
| 209 | const ref = refs.get(name)!; |
| 210 | if (ref.commit) await pusher.ensurePushed(ref.commit, name); |
| 211 | } |
| 212 | |
| 213 | // ------------------------------------------------------------ the refs themselves |
| 214 | await o.progress({ phase: "refs", message: `Creating ${ordered.length} branches, tags and pull request refs`, chunks: result.chunks, bytes: result.bytes }, true); |
| 215 | for (let i = 0; i < ordered.length; i += REF_BATCH) { |
| 216 | const batch = ordered.slice(i, i + REF_BATCH); |
| 217 | const out = await pusher.push(batch.map((r) => `+${r}:${importDestination(r)}`)); |
| 218 | const status = new Map(parsePorcelain(out.stdout).map((s) => [s.ref, s])); |
| 219 | for (const r of batch) { |
| 220 | const s = status.get(importDestination(r)); |
| 221 | if (out.code !== 0 && !s) throw new ImportError(`pushing refs failed: ${pushFailure(out)}`); |
| 222 | if (s && !s.ok) { |
| 223 | result.warnings.push(`${r} was not imported: ${s.reason}`); |
| 224 | continue; |
| 225 | } |
| 226 | const kind = importRefKind(r); |
| 227 | if (kind === "branch") result.branches++; |
| 228 | else if (kind === "tag") result.tags++; |
| 229 | else if (kind === "pull") result.pullRefs++; |
| 230 | } |
| 231 | } |
| 232 | if (refs.has(`refs/heads/${spec.source.defaultBranch}`)) result.defaultBranch = spec.source.defaultBranch; |
| 233 | |
| 234 | // ------------------------------------------------------------ LFS objects to Diffinery |
| 235 | if (spec.lfs) { |
| 236 | const { count, bytes } = await lfsStore(mirror); |
| 237 | if (count > 0) { |
| 238 | await o.progress({ phase: "lfs-push", message: `Uploading ${count} Git LFS objects` }, true); |
| 239 | const p = await git.run(["-c", `lfs.url=${targetLfsUrl}`, "lfs", "push", "--all", spec.target.pushUrl], { allowFail: true }); |
| 240 | if (p.code !== 0) result.warnings.push(`Some Git LFS objects could not be uploaded: ${lastLine(p.stderr)}`); |
| 241 | result.lfsObjects = count; |
| 242 | result.lfsBytes = bytes; |
| 243 | } |
| 244 | } |
| 245 | |
| 246 | // ------------------------------------------------------------ cleanup |
| 247 | await o.progress({ phase: "cleanup", message: "Removing temporary refs" }, true); |
| 248 | await pusher.deleteTempRefs(); |
| 249 | result.secretFindings = await pusher.findingsWithPaths(); |
| 250 | await o.progress({ phase: "done", message: `Imported ${result.branches} branches, ${result.tags} tags and ${result.pullRefs} pull request refs`, chunks: result.chunks, bytes: result.bytes }, true); |
| 251 | return result; |
| 252 | } finally { |
| 253 | await rm(root, { recursive: true, force: true }).catch(() => {}); |
| 254 | } |
| 255 | } |
| 256 | |
| 257 | const lastLine = (s: string) => |
| 258 | s |
| 259 | .trim() |
| 260 | .split("\n") |
| 261 | .filter((l) => l.trim()) |
| 262 | .slice(-1)[0] |
| 263 | ?.slice(0, 300) ?? ""; |
| 264 | |
| 265 | /** |
| 266 | * Why a push failed, in the server's words: why it couldn't unpack the push, else the reason on a |
| 267 | * rejected ref (`--porcelain` puts it on stdout), else the last line of stderr other than git's |
| 268 | * generic "failed to push some refs". |
| 269 | */ |
| 270 | export const pushFailure = (out: { stdout: string; stderr: string }) => |
| 271 | /remote unpack failed: (.+)/.exec(out.stderr)?.[1]?.trim().slice(0, 300) || |
| 272 | parsePorcelain(out.stdout).find((s) => !s.ok)?.reason.slice(0, 300) || |
| 273 | lastLine(out.stderr.split("\n").filter((l) => !/^error: failed to push some refs/.test(l)).join("\n")) || |
| 274 | lastLine(out.stderr) || |
| 275 | lastLine(out.stdout); |
| 276 | |
| 277 | /** Count and size of the LFS objects in a bare repository's local store. */ |
| 278 | async function lfsStore(gitDir: string): Promise<{ count: number; bytes: number }> { |
| 279 | let count = 0; |
| 280 | let bytes = 0; |
| 281 | const walk = async (dir: string): Promise<void> => { |
| 282 | let entries; |
| 283 | try { |
| 284 | entries = await readdir(dir, { withFileTypes: true }); |
| 285 | } catch { |
| 286 | return; |
| 287 | } |
| 288 | for (const e of entries) { |
| 289 | const p = path.join(dir, e.name); |
| 290 | if (e.isDirectory()) await walk(p); |
| 291 | else if (/^[0-9a-f]{64}$/.test(e.name)) { |
| 292 | count++; |
| 293 | bytes += (await stat(p)).size; |
| 294 | } |
| 295 | } |
| 296 | }; |
| 297 | await walk(path.join(gitDir, "lfs", "objects")); |
| 298 | return { count, bytes }; |
| 299 | } |
| 300 | |
| 301 | /** |
| 302 | * Pushes history so that every request stays under the budget. The server keeps |
| 303 | * what it has under temporary refs, so later pushes (and a restarted import) only |
| 304 | * send what is missing. |
| 305 | */ |
| 306 | class ChunkPusher { |
| 307 | private haves: string[] = []; |
| 308 | private haveSet = new Set<string>(); |
| 309 | private budget: number; |
| 310 | private temp = 0; |
| 311 | private tempRefs: string[] = []; |
| 312 | private findings = new Map<string, ImportSecretFinding>(); |
| 313 | |
| 314 | constructor( |
| 315 | private readonly git: GitRunner, |
| 316 | private readonly spec: ImportSpec, |
| 317 | private readonly pushOpts: string[], |
| 318 | private readonly o: TransferOptions, |
| 319 | private readonly result: ImportGitResult, |
| 320 | ) { |
| 321 | this.budget = Math.max(MIN_BUDGET, o.chunkBytes ?? spec.chunkBytes); |
| 322 | } |
| 323 | |
| 324 | /** Objects the server already has (a resumed import) count as pushed. */ |
| 325 | async loadRemoteState(): Promise<void> { |
| 326 | const r = await this.git.run(["ls-remote", this.spec.target.pushUrl], { allowFail: true }); |
| 327 | if (r.code !== 0) throw new ImportError(`cannot reach the Diffinery repository: ${lastLine(r.stderr)}`); |
| 328 | const oids = [...new Set(r.stdout.split("\n").map((l) => l.split("\t")[0]!).filter((x) => /^[0-9a-f]{40}$/.test(x)))]; |
| 329 | if (!oids.length) return; |
| 330 | const present = await this.git.run(["cat-file", "--batch-check=%(objectname) %(objecttype)"], { input: `${oids.join("\n")}\n` }); |
| 331 | for (const line of present.stdout.split("\n")) { |
| 332 | const [oid, type] = line.split(" "); |
| 333 | if (type === "commit") this.addHave(oid!); |
| 334 | } |
| 335 | for (const line of r.stdout.split("\n")) { |
| 336 | const ref = line.split("\t")[1]; |
| 337 | if (ref?.startsWith(TEMP_PREFIX)) this.tempRefs.push(ref); |
| 338 | } |
| 339 | this.temp = this.tempRefs.length; |
| 340 | } |
| 341 | |
| 342 | private addHave(oid: string): void { |
| 343 | if (this.haveSet.has(oid)) return; |
| 344 | this.haveSet.add(oid); |
| 345 | this.haves.push(oid); |
| 346 | } |
| 347 | |
| 348 | private notHaves(): string { |
| 349 | return this.haves.map((h) => `^${h}\n`).join(""); |
| 350 | } |
| 351 | |
| 352 | /** Bytes on disk of everything reachable from `tip` that the server doesn't have. */ |
| 353 | async sizeOf(tip: string): Promise<number> { |
| 354 | const r = await this.git.run(["rev-list", "--objects", "--disk-usage", "--stdin"], { input: `${tip}\n${this.notHaves()}` }); |
| 355 | return Number(r.stdout.trim()) || 0; |
| 356 | } |
| 357 | |
| 358 | /** Make sure the server has `tip` and everything it needs. */ |
| 359 | async ensurePushed(tip: string, label: string): Promise<void> { |
| 360 | if (this.haveSet.has(tip)) return; |
| 361 | let chain = await this.git.lines(["rev-list", "--first-parent", "--reverse", "--stdin"], `${tip}\n${this.notHaves()}`); |
| 362 | while (chain.length) { |
| 363 | const sizes = new Map<number, number>(); |
| 364 | const best = await furthestWithin( |
| 365 | chain.length, |
| 366 | async (i) => { |
| 367 | const s = await this.sizeOf(chain[i]!); |
| 368 | sizes.set(i, s); |
| 369 | return s; |
| 370 | }, |
| 371 | this.budget, |
| 372 | ); |
| 373 | if (best) { |
| 374 | if (!(await this.pushTemp(chain[best.index]!, best.size, label))) continue; |
| 375 | chain = chain.slice(best.index + 1); |
| 376 | continue; |
| 377 | } |
| 378 | // One first-parent step is over budget: push the side branches it merges first, |
| 379 | // then, if it still is, its new objects in batches. |
| 380 | const commit = chain[0]!; |
| 381 | const parents = (await this.git.lines(["rev-list", "--parents", "-n", "1", commit]))[0]!.split(" ").slice(2); |
| 382 | for (const p of parents) await this.ensurePushed(p, label); |
| 383 | if ((await this.sizeOf(commit)) > this.budget) await this.pushObjectBatches(commit, label); |
| 384 | if (!(await this.pushTemp(commit, await this.sizeOf(commit), label))) continue; |
| 385 | chain = chain.slice(1); |
| 386 | } |
| 387 | } |
| 388 | |
| 389 | /** Upload the blobs `commit` adds through throwaway commits of flat trees, so the commit itself stays small. */ |
| 390 | private async pushObjectBatches(commit: string, label: string): Promise<void> { |
| 391 | const listed = await this.git.lines(["rev-list", "--objects", "--stdin"], `${commit}\n${this.notHaves()}`); |
| 392 | const oids = listed.map((l) => l.split(" ")[0]!); |
| 393 | const check = await this.git.lines(["cat-file", "--batch-check=%(objectname) %(objecttype) %(objectsize:disk)"], `${oids.join("\n")}\n`); |
| 394 | const blobs = check |
| 395 | .map((l) => l.split(" ")) |
| 396 | .filter((p) => p[1] === "blob") |
| 397 | .map((p) => ({ oid: p[0]!, size: Number(p[2]) })); |
| 398 | // Batched at the budget as it stands: when the server refuses a batch the budget shrinks, and |
| 399 | // what is left is batched again at the new one. A lone object is retried as is (it can't be |
| 400 | // split further), until the budget can't shrink any more. |
| 401 | for (let left = blobs, n = 1; left.length; n++) { |
| 402 | const batch = batchBySize(left, this.budget)[0]!; |
| 403 | const tree = (await this.git.run(["mktree"], { input: batch.map((b, j) => `100644 blob ${b.oid}\tobject-${j}\n`).join("") })).stdout.trim(); |
| 404 | const carrier = (await this.git.run(["commit-tree", tree, "-m", `Diffinery import: objects for ${commit.slice(0, 12)} (${n})`])).stdout.trim(); |
| 405 | const size = batch.reduce((s, b) => s + b.size, 0); |
| 406 | if (await this.pushTemp(carrier, size, label)) left = left.slice(batch.length); |
| 407 | } |
| 408 | } |
| 409 | |
| 410 | /** Push one chunk to a temporary ref. Returns false when the budget was lowered and the caller should re-plan. */ |
| 411 | private async pushTemp(oid: string, size: number, label: string): Promise<boolean> { |
| 412 | if (this.haveSet.has(oid)) return true; |
| 413 | const ref = `${TEMP_PREFIX}c${++this.temp}`; |
| 414 | const n = this.result.chunks + 1; |
| 415 | await this.o.progress({ phase: "push", message: `Pushing history of ${label.replace(/^refs\/(heads|tags)\//, "")}: chunk ${n} (${fmtBytes(size)})`, chunks: this.result.chunks, bytes: this.result.bytes }); |
| 416 | const out = await this.push([`${oid}:${ref}`]); |
| 417 | if (out.code === 0 && !parsePorcelain(out.stdout).some((s) => !s.ok)) { |
| 418 | this.tempRefs.push(ref); |
| 419 | this.addHave(oid); |
| 420 | this.result.chunks++; |
| 421 | this.result.bytes += size; |
| 422 | return true; |
| 423 | } |
| 424 | const text = `${out.stdout}\n${out.stderr}`; |
| 425 | if (isSecretRejection(text)) { |
| 426 | for (const f of parseSecretFindings(out.stderr)) this.findings.set(`${f.kind}:${f.blob}`, f); |
| 427 | const found = await this.findingsWithPaths(); |
| 428 | const what = found.map((f) => `${f.kind} in ${f.path ?? `blob ${f.blob}`}`).join("; "); |
| 429 | throw new SecretsFound( |
| 430 | this.spec.secrets === "bypass" |
| 431 | ? `Push protection found possible secrets in the repository's history (${what}), and only a human admin of the target repository can import them with a bypass.` |
| 432 | : `Push protection found possible secrets in the repository's history (${what}). The import was stopped as requested; rotate the secrets, or import again with the bypass option.`, |
| 433 | found, |
| 434 | ); |
| 435 | } |
| 436 | // Past the owner's storage quota nothing more can be stored: say so rather than "push failed". |
| 437 | const full = storageRejection(text); |
| 438 | if (full) throw new ImportError(full); |
| 439 | if (isSizeRejection(text)) { |
| 440 | if (this.budget / 2 >= MIN_BUDGET) { |
| 441 | this.budget = Math.floor(this.budget / 2); |
| 442 | this.o.log(`push of ${fmtBytes(size)} was rejected (${lastLine(out.stderr)}); retrying with chunks of ${fmtBytes(this.budget)}`); |
| 443 | this.temp--; |
| 444 | return false; |
| 445 | } |
| 446 | // Chunking can no longer help. Say what the server answered: the object may be under the |
| 447 | // 100 MB push limit and still be refused (or the server may be failing for another reason). |
| 448 | const said = pushFailure(out); |
| 449 | if (size > this.budget) { |
| 450 | throw new ImportError(`a single Git object of ${fmtBytes(size)} could not be pushed even on its own (${said}). Remove the large file from the repository's history (e.g. with git filter-repo) and import again.`); |
| 451 | } |
| 452 | throw new ImportError(`pushes kept failing even in chunks of ${fmtBytes(this.budget)} (${said})`); |
| 453 | } |
| 454 | throw new ImportError(`push failed: ${pushFailure(out)}`); |
| 455 | } |
| 456 | |
| 457 | /** `git push` to Diffinery in import mode; collects push protection findings from the output. */ |
| 458 | async push(refspecs: string[]): Promise<{ code: number; stdout: string; stderr: string }> { |
| 459 | const out = await this.git.run(["push", "--porcelain", "--no-verify", ...this.pushOpts, this.spec.target.pushUrl, ...refspecs], { allowFail: true }); |
| 460 | if (/Push protection bypassed/i.test(out.stderr)) for (const f of parseSecretFindings(out.stderr)) this.findings.set(`${f.kind}:${f.blob}`, f); |
| 461 | return out; |
| 462 | } |
| 463 | |
| 464 | async deleteTempRefs(): Promise<void> { |
| 465 | for (let i = 0; i < this.tempRefs.length; i += REF_BATCH) { |
| 466 | const batch = this.tempRefs.slice(i, i + REF_BATCH); |
| 467 | const out = await this.push(batch.map((r) => `:${r}`)); |
| 468 | if (out.code !== 0) this.result.warnings.push(`Some temporary import refs could not be removed: ${lastLine(out.stderr)}`); |
| 469 | } |
| 470 | this.tempRefs = []; |
| 471 | } |
| 472 | |
| 473 | /** Findings with a path where each blob appears, so people know what to rotate. */ |
| 474 | async findingsWithPaths(): Promise<ImportSecretFinding[]> { |
| 475 | const out: ImportSecretFinding[] = []; |
| 476 | for (const f of [...this.findings.values()].slice(0, 100)) { |
| 477 | const full = (await this.git.run(["rev-parse", "--verify", "--quiet", `${f.blob}^{blob}`], { allowFail: true })).stdout.trim() || f.blob; |
| 478 | let where: string | undefined = f.path; |
| 479 | if (!where && /^[0-9a-f]{40}$/.test(full)) { |
| 480 | const raw = await this.git.run(["log", "--all", "-n", "1", `--find-object=${full}`, "--raw", "--no-abbrev", "--format="], { allowFail: true }); |
| 481 | const line = raw.stdout.split("\n").find((l) => l.includes(full)); |
| 482 | where = line?.split("\t").slice(-1)[0]; |
| 483 | } |
| 484 | out.push({ kind: f.kind, blob: full, path: where }); |
| 485 | } |
| 486 | return out; |
| 487 | } |
| 488 | } |
| 489 | |
| 490 | function fmtBytes(n: number): string { |
| 491 | if (n < 1024) return `${n} B`; |
| 492 | if (n < 1024 * 1024) return `${(n / 1024).toFixed(1)} KB`; |
| 493 | return `${(n / 1024 / 1024).toFixed(1)} MB`; |
| 494 | } |