Import campaign findings: stale imports, readable reports, an honest large-object error (#103)
The campaign cancels its earlier imports into the same names before starting, and the relay cancels a campaign still running on Diffinery. Report headers are matched after stripping log timestamps. The importer's push refusal now quotes what the server said and only calls it a single large object when it is one. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01AizKXSsengXZRegJxRATkL
Canberk Demirci authored · GitHub committed signed1 parent 14d20cfcommit 227d9a078e75a1b13700a7be72b0ac4d638ba0deBrowse files
4 files changed+33−5
Files (4)
modified.github/workflows/import-campaign.yml+7−0View file
| @@ -75,6 +75,11 @@ | ||
| 75 | 75 | if [ -n "$FOLLOW_RUN" ]; then |
| 76 | 76 | id=$FOLLOW_RUN |
| 77 | 77 | else |
| 78 | + # A new campaign replaces one still running on Diffinery (its imports are cancelled by the new one). | |
| 79 | + for old in $(api GET "/repos/$R/actions/workflows/import-campaign.yml/runs?per_page=20" | jq -r '.workflow_runs[] | select(.status != "completed") | .id'); do | |
| 80 | + echo "cancelling the campaign still running on Diffinery: $old" | |
| 81 | + api POST "/repos/$R/actions/runs/$old/cancel" >/dev/null || true | |
| 82 | + done | |
| 78 | 83 | before=$(api GET "/repos/$R/actions/workflows/import-campaign.yml/runs?per_page=1" | jq -r '.workflow_runs[0].id // 0') |
| 79 | 84 | api POST "/repos/$R/actions/workflows/import-campaign.yml/dispatches" "$(jq -nc --arg r "$REPOS" --arg m "$MINUTES" --arg w "$RUN_WORKFLOWS" '{ref: "main", inputs: {repos: $r, minutes: $m, run_workflows: $w}}')" >/dev/null |
| 80 | 85 | for i in $(seq 1 60); do |
| @@ -91,6 +96,8 @@ | ||
| 91 | 96 | for job in $(jq -r '.jobs[].id' <<<"$jobs"); do |
| 92 | 97 | jq -r --arg id "$job" '.jobs[] | select((.id | tostring) == $id) | "Job \(.name): \(.conclusion // .status)\(if .failure then " (\(.failure.message? // .failure))" else "" end)"' <<<"$jobs" |
| 93 | 98 | log=$(curl -fsS "$DIFFINERY/api/v3/repos/$R/actions/jobs/$job/logs" -H "Authorization: Bearer $TOKEN" || echo "(no log)") |
| 99 | + # Diffinery's job logs start each line with a timestamp. | |
| 100 | + log=$(sed -E 's/^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9:.]+Z //' <<<"$log") | |
| 94 | 101 | if grep -q '^# Import campaign' <<<"$log"; then |
| 95 | 102 | sed -n '/^# Import campaign/,$p' <<<"$log" | tee -a "$GITHUB_STEP_SUMMARY" |
| 96 | 103 | else |
modifiedapps/diffinery/docs/migrate-from-github.md+2−1View file
| @@ -985,7 +985,8 @@ | ||
| 985 | 985 | | Message | Cause and fix | |
| 986 | 986 | | --- | --- | |
| 987 | 987 | | `Push protection found possible secrets in the history (...)` | See [Secrets in old history](#secrets-in-old-history). | |
| 988 | -| `a single Git object is N MB, larger than the 100 MB push limit, so it cannot be imported. ...` | Remove the oversized file from history on GitHub (for example with `git filter-repo`), or move it to Git LFS, and import again. | | |
| 988 | +| `a single Git object of N MB could not be pushed even on its own (...)` | One file in the history is too large for Diffinery to take in one push (the server's answer is in brackets). Remove it from history on GitHub (for example with `git filter-repo`), or move it to Git LFS, and import again. | | |
| 989 | +| `pushes kept failing even in chunks of N KB (...)` | Diffinery kept refusing or dropping the transfer's pushes; the server's answer is in brackets. Retry the import; if it fails the same way, report it. | | |
| 989 | 990 | | `the Git transfer did not finish within 6 hours` | The repository is too large or GitHub was too slow. Retry, or split the repository. | |
| 990 | 991 | | `the import container stopped before finishing (exit code ...)` | The transfer is retried once automatically; after that, retry the import. | |
| 991 | 992 | | `The Git transfer failed and is being retried: ...` (warning) | The first transfer failed and a second one started. Nothing to do. | |
modifiedcontainers/runner/src/importer.ts+7−2View file
| @@ -427,8 +427,13 @@ | ||
| 427 | 427 | this.temp--; |
| 428 | 428 | return false; |
| 429 | 429 | } |
| 430 | - // Chunking can no longer help: a single object is larger than the push limit and cannot be split. | |
| 431 | - throw new ImportError(`a single Git object is ${fmtBytes(size)}, larger than the 100 MB push limit, so it cannot be imported. Remove the oversized file from the repository's history (e.g. with git filter-repo) and import again.`); | |
| 430 | + // Chunking can no longer help. Say what the server answered: the object may be under the | |
| 431 | + // 100 MB push limit and still be refused (or the server may be failing for another reason). | |
| 432 | + const said = lastLine(out.stderr) || lastLine(out.stdout); | |
| 433 | + if (size > this.budget) { | |
| 434 | + throw new ImportError(`a single Git object of ${fmtBytes(size)} could not be pushed even on its own (${said}). Remove the large file from the repository's history (e.g. with git filter-repo) and import again.`); | |
| 435 | + } | |
| 436 | + throw new ImportError(`pushes kept failing even in chunks of ${fmtBytes(this.budget)} (${said})`); | |
| 432 | 437 | } |
| 433 | 438 | throw new ImportError(`push failed: ${lastLine(out.stderr) || lastLine(out.stdout)}`); |
| 434 | 439 | } |
modifiedscripts/import-campaign.sh+17−2View file
| @@ -37,11 +37,26 @@ | ||
| 37 | 37 | queue+=("$src") |
| 38 | 38 | done |
| 39 | 39 | |
| 40 | +# An earlier campaign's import that is still running holds its repository (deleting it gets 423), | |
| 41 | +# so cancel those first. A cancelled import gives its repository back. | |
| 42 | +names=" ${target[*]} " | |
| 43 | +for old in $(api GET /diffinery/imports | jq -r '.[] | select(.status | IN("draft", "queued", "running")) | "\(.id)|\(.target.name // "")"'); do | |
| 44 | + id=${old%%|*} n=${old#*|} | |
| 45 | + [[ "$names" == *" $n "* ]] || continue | |
| 46 | + echo "cancelling an earlier campaign's import into $owner/$n ($id)" | |
| 47 | + code POST "/diffinery/imports/$id/cancel" >/dev/null || true | |
| 48 | +done | |
| 49 | + | |
| 40 | 50 | # Starts one import. Returns 0 when it started or failed for good, 1 when Diffinery is busy. |
| 41 | 51 | start() { |
| 42 | - local src=$1 name=${target[$1]} body c | |
| 52 | + local src=$1 name=${target[$1]} body c d i | |
| 43 | 53 | if [ "$(code GET "/repos/$owner/$name")" = 200 ]; then |
| 44 | - code DELETE "/repos/$owner/$name" >/dev/null | |
| 54 | + for i in 1 2 3 4 5 6; do | |
| 55 | + d=$(code DELETE "/repos/$owner/$name") | |
| 56 | + [ "$d" = 204 ] || [ "$d" = 404 ] && break | |
| 57 | + sleep 10 | |
| 58 | + done | |
| 59 | + [ "$d" = 204 ] || [ "$d" = 404 ] || echo "::warning::$src: deleting the earlier $owner/$name returned $d: $(jq -r '.message // .' /tmp/campaign-out)" | |
| 45 | 60 | fi |
| 46 | 61 | body=$(jq -nc --arg s "$src" --arg o "$owner" --arg n "$name" --arg t "${GITHUB_TOKEN_FOR_IMPORT:-}" \ |
| 47 | 62 | '{source: $s, owner: $o, name: $n, visibility: "private"} + (if $t == "" then {} else {github_token: $t} end)') |