Import campaign can re-run failed jobs; pushes that run out of memory ask to be split (#136)
The Import campaign workflow's new rerun_failed input re-runs a Diffinery run's failed jobs and prints how they end. When unpacking or storing a push runs out of memory, the unpack status says to split the push, which the importer answers by halving its chunks; the importer also reports the server's unpack reason ahead of the per-ref one. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01AizKXSsengXZRegJxRATkL
Canberk Demirci authored · GitHub committed signed1 parent cb1375acommit 2c79ea10db45680d8cba081f6b8e1d8e07d62c8cBrowse files
6 files changed+59−6
Files (6)
modified.github/workflows/import-campaign.yml+22−1View file
| @@ -35,6 +35,10 @@ | ||
| 35 | 35 | description: Instead of starting a campaign, print how the running one is doing (imports, steps, runs) and stop |
| 36 | 36 | type: boolean |
| 37 | 37 | default: false |
| 38 | + rerun_failed: | |
| 39 | + description: Instead of starting a campaign, re-run the failed jobs of this Diffinery run of an imported repository ("<repository> <run id>") and print how they end | |
| 40 | + type: string | |
| 41 | + default: "" | |
| 38 | 42 | |
| 39 | 43 | permissions: |
| 40 | 44 | contents: read |
| @@ -57,7 +61,7 @@ | ||
| 57 | 61 | R: codician-team/diffinery |
| 58 | 62 | steps: |
| 59 | 63 | - name: Copy IMPORT_GITHUB_TOKEN to Diffinery |
| 60 | - if: ${{ inputs.follow_run == '' && !inputs.peek }} | |
| 64 | + if: ${{ inputs.follow_run == '' && !inputs.peek && inputs.rerun_failed == '' }} | |
| 61 | 65 | env: |
| 62 | 66 | IMPORT_GITHUB_TOKEN: ${{ secrets.IMPORT_GITHUB_TOKEN }} |
| 63 | 67 | run: | |
| @@ -75,9 +79,26 @@ | ||
| 75 | 79 | RUN_WORKFLOWS: ${{ inputs.run_workflows }} |
| 76 | 80 | FOLLOW_RUN: ${{ inputs.follow_run }} |
| 77 | 81 | PEEK: ${{ inputs.peek }} |
| 82 | + RERUN_FAILED: ${{ inputs.rerun_failed }} | |
| 78 | 83 | run: | |
| 79 | 84 | set -euo pipefail |
| 80 | 85 | api() { curl -fsS --retry 3 -X "$1" "$DIFFINERY/api/v3$2" -H "Authorization: Bearer $TOKEN" -H "content-type: application/json" ${3:+--data "$3"}; } |
| 86 | + if [ -n "$RERUN_FAILED" ]; then | |
| 87 | + read -r repo run <<<"$RERUN_FAILED" | |
| 88 | + [[ "$repo" =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ && "$run" =~ ^[A-Za-z0-9_-]+$ ]] || { echo "::error::rerun_failed must be \"<owner>/<repository> <run id>\""; exit 1; } | |
| 89 | + api POST "/repos/$repo/actions/runs/$run/rerun-failed-jobs" >/dev/null | |
| 90 | + echo "Re-running the failed jobs of $(api GET "/repos/$repo/actions/runs/$run" | jq -r .html_url)" | |
| 91 | + sleep 30 | |
| 92 | + while [ "$(api GET "/repos/$repo/actions/runs/$run" | jq -r .status)" != completed ]; do sleep 30; done | |
| 93 | + jobs=$(api GET "/repos/$repo/actions/runs/$run/jobs?per_page=100") | |
| 94 | + jq -r '.jobs[] | "\(.name): \(.conclusion // .status)\(if .failure then " (\(.failure.message? // .failure))" else "" end)"' <<<"$jobs" | tee -a "$GITHUB_STEP_SUMMARY" | |
| 95 | + for job in $(jq -r '.jobs[] | select(.conclusion == "failure") | .id' <<<"$jobs"); do | |
| 96 | + echo "::group::$(jq -r --arg id "$job" '.jobs[] | select((.id | tostring) == $id) | .name' <<<"$jobs")" | |
| 97 | + { api GET "/repos/$repo/actions/jobs/$job/logs" || true; } | sed -E 's/^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9:.]+Z //' | tail -80 | |
| 98 | + echo "::endgroup::" | |
| 99 | + done | |
| 100 | + exit 0 | |
| 101 | + fi | |
| 81 | 102 | if [ "$PEEK" = true ]; then |
| 82 | 103 | for id in $(api GET "/repos/$R/actions/workflows/import-campaign.yml/runs?per_page=5" | jq -r '.workflow_runs[] | select(.status != "completed") | .id'); do |
| 83 | 104 | echo "Campaign run $id on Diffinery:" |
modifiedapps/diffinery/src/git/protocol.ts+13−0View file
| @@ -261,6 +261,19 @@ | ||
| 261 | 261 | reason?: string; |
| 262 | 262 | } |
| 263 | 263 | |
| 264 | +/** | |
| 265 | + * The unpack status for a push that couldn't be unpacked or stored. Running out of memory says to | |
| 266 | + * split the push, the wording clients (and the GitHub importer, which then pushes smaller chunks) | |
| 267 | + * already treat as "too big". | |
| 268 | + */ | |
| 269 | +export function unpackFailure(e: unknown): string { | |
| 270 | + const message = e instanceof Error ? e.message : String(e); | |
| 271 | + if (e instanceof RangeError || /allocation failed|out of memory/i.test(message)) { | |
| 272 | + return `the push needs more memory than one push may use; split it into smaller pushes (${message})`; | |
| 273 | + } | |
| 274 | + return message; | |
| 275 | +} | |
| 276 | + | |
| 264 | 277 | export function reportStatus(req: ReceiveRequest, unpackError: string | null, results: RefResult[], messages: string[] = []): Uint8Array { |
| 265 | 278 | const report = new ByteWriter(); |
| 266 | 279 | report.write(pkt(unpackError ? `unpack ${unpackError}\n` : "unpack ok\n")); |
modifiedapps/diffinery/src/repo/RepoDO.ts+3−2View file
| @@ -40,6 +40,7 @@ | ||
| 40 | 40 | handleUploadPackV2, |
| 41 | 41 | parseReceivePack, |
| 42 | 42 | reportStatus, |
| 43 | + unpackFailure, | |
| 43 | 44 | type RefEntry, |
| 44 | 45 | type RefResult, |
| 45 | 46 | } from "../git/protocol.js"; |
| @@ -443,7 +444,7 @@ | ||
| 443 | 444 | try { |
| 444 | 445 | objects = await parsePack(req.pack, (oid) => this.store.read(oid), { largeObjectBytes: LARGE_OBJECT_BYTES }); |
| 445 | 446 | } catch (e) { |
| 446 | - unpackError = (e as Error).message; | |
| 447 | + unpackError = unpackFailure(e); | |
| 447 | 448 | } |
| 448 | 449 | } |
| 449 | 450 | // Commit and tag messages are scanned too, not only files. |
| @@ -494,7 +495,7 @@ | ||
| 494 | 495 | } |
| 495 | 496 | } catch (e) { |
| 496 | 497 | for (const c of req.commands) results.push({ ref: c.ref, ok: false, reason: "unpacker error" }); |
| 497 | - return reportStatus(req, (e as Error).message, results); | |
| 498 | + return reportStatus(req, unpackFailure(e), results); | |
| 498 | 499 | } |
| 499 | 500 | |
| 500 | 501 | // Validate every command before applying any, so `atomic` is honoured. |
addedapps/diffinery/test/unit/unpack-failure.test.ts+13−0View file
| @@ -1,0 +1,13 @@ | ||
| 1 | +import { describe, expect, it } from "vitest"; | |
| 2 | +import { unpackFailure } from "../../src/git/protocol.js"; | |
| 3 | + | |
| 4 | +describe("unpack failures", () => { | |
| 5 | + it("say to split a push that ran out of memory, so clients and the importer push smaller chunks", () => { | |
| 6 | + expect(unpackFailure(new RangeError("Array buffer allocation failed"))).toBe("the push needs more memory than one push may use; split it into smaller pushes (Array buffer allocation failed)"); | |
| 7 | + expect(unpackFailure(new Error("Out of memory"))).toMatch(/split it into smaller pushes/); | |
| 8 | + }); | |
| 9 | + it("pass other errors through as they are", () => { | |
| 10 | + expect(unpackFailure(new Error("pack checksum mismatch"))).toBe("pack checksum mismatch"); | |
| 11 | + expect(unpackFailure("odd")).toBe("odd"); | |
| 12 | + }); | |
| 13 | +}); |
modifiedcontainers/runner/src/importer.ts+4−2View file
| @@ -263,10 +263,12 @@ | ||
| 263 | 263 | ?.slice(0, 300) ?? ""; |
| 264 | 264 | |
| 265 | 265 | /** |
| 266 | - * Why a push failed, in the server's words: the reason on a rejected ref (`--porcelain` puts it on | |
| 267 | - * stdout), else the last line of stderr other than git's generic "failed to push some refs". | |
| 266 | + * Why a push failed, in the server's words: why it couldn't unpack the push, else the reason on a | |
| 267 | + * rejected ref (`--porcelain` puts it on stdout), else the last line of stderr other than git's | |
| 268 | + * generic "failed to push some refs". | |
| 268 | 269 | */ |
| 269 | 270 | export const pushFailure = (out: { stdout: string; stderr: string }) => |
| 271 | + /remote unpack failed: (.+)/.exec(out.stderr)?.[1]?.trim().slice(0, 300) || | |
| 270 | 272 | parsePorcelain(out.stdout).find((s) => !s.ok)?.reason.slice(0, 300) || |
| 271 | 273 | lastLine(out.stderr.split("\n").filter((l) => !/^error: failed to push some refs/.test(l)).join("\n")) || |
| 272 | 274 | lastLine(out.stderr) || |
modifiedcontainers/runner/test/import.test.ts+4−1View file
| @@ -49,6 +49,7 @@ | ||
| 49 | 49 | expect(isSizeRejection("remote: push exceeds 100 MB; split it into smaller pushes")).toBe(true); |
| 50 | 50 | expect(isSizeRejection("error: RPC failed; HTTP 413 curl 22")).toBe(true); |
| 51 | 51 | expect(isSizeRejection("! [rejected] main (non-fast-forward)")).toBe(false); |
| 52 | + expect(isSizeRejection("error: remote unpack failed: the push needs more memory than one push may use; split it into smaller pushes (Array buffer allocation failed)")).toBe(true); | |
| 52 | 53 | expect(parsePorcelain("To x\n*\trefs/heads/a:refs/heads/a\t[new branch]\n!\trefs/heads/b:refs/heads/b\t[remote rejected] (invalid ref name)\nDone\n")).toEqual([ |
| 53 | 54 | { ref: "refs/heads/a", ok: true, reason: "[new branch]" }, |
| 54 | 55 | { ref: "refs/heads/b", ok: false, reason: "invalid ref name" }, |
| @@ -56,7 +57,9 @@ | ||
| 56 | 57 | // A failed push says why in the server's words, not git's generic last line. |
| 57 | 58 | const generic = "error: failed to push some refs to 'https://diffinery.com/o/r.git'"; |
| 58 | 59 | expect(pushFailure({ stdout: "To x\n!\trefs/heads/b:refs/heads/b\t[remote rejected] (missing objects)\nDone\n", stderr: generic })).toBe("missing objects"); |
| 59 | - expect(pushFailure({ stdout: "", stderr: `error: remote unpack failed: unpacker error\n${generic}` })).toBe("error: remote unpack failed: unpacker error"); | |
| 60 | + const unpack = "error: remote unpack failed: pack checksum mismatch"; | |
| 61 | + expect(pushFailure({ stdout: "To x\n!\trefs/heads/b:refs/heads/b\t[remote rejected] (unpacker error)\nDone\n", stderr: `${unpack}\n${generic}` })).toBe("pack checksum mismatch"); | |
| 62 | + expect(pushFailure({ stdout: "", stderr: `remote: Push refused: bad tree\n${generic}` })).toBe("remote: Push refused: bad tree"); | |
| 60 | 63 | expect(pushFailure({ stdout: "", stderr: generic })).toBe(generic); |
| 61 | 64 | }); |
| 62 | 65 | }); |