Import campaign: queue imports past the 3-at-a-time limit, and show why a run failed (#61)
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01AizKXSsengXZRegJxRATkL
Canberk Demirci authored · GitHub committed signed1 parent 811c98dcommit b2caad32af3eb9fc9dda018a14403aee175b68c2Browse files
2 files changed+59−17
modified.github/workflows/import-campaign.yml+31−11View file
| @@ -27,6 +27,10 @@ | ||
| 27 | 27 | description: Also run the imported repositories' push workflows (third-party code) |
| 28 | 28 | type: boolean |
| 29 | 29 | default: false |
| 30 | + follow_run: | |
| 31 | + description: Instead of starting a campaign, print the report and logs of this Diffinery run (its id) | |
| 32 | + type: string | |
| 33 | + default: "" | |
| 30 | 34 | |
| 31 | 35 | permissions: |
| 32 | 36 | contents: read |
| @@ -48,6 +52,7 @@ | ||
| 48 | 52 | R: codician-team/diffinery |
| 49 | 53 | steps: |
| 50 | 54 | - name: Copy IMPORT_GITHUB_TOKEN to Diffinery |
| 55 | + if: ${{ inputs.follow_run == '' }} | |
| 51 | 56 | env: |
| 52 | 57 | IMPORT_GITHUB_TOKEN: ${{ secrets.IMPORT_GITHUB_TOKEN }} |
| 53 | 58 | run: | |
| @@ -63,22 +68,37 @@ | ||
| 63 | 68 | REPOS: ${{ inputs.repos }} |
| 64 | 69 | MINUTES: ${{ inputs.minutes }} |
| 65 | 70 | RUN_WORKFLOWS: ${{ inputs.run_workflows }} |
| 71 | + FOLLOW_RUN: ${{ inputs.follow_run }} | |
| 66 | 72 | run: | |
| 67 | 73 | set -euo pipefail |
| 68 | 74 | api() { curl -fsS --retry 3 -X "$1" "$DIFFINERY/api/v3$2" -H "Authorization: Bearer $TOKEN" -H "content-type: application/json" ${3:+--data "$3"}; } |
| 69 | - before=$(api GET "/repos/$R/actions/workflows/import-campaign.yml/runs?per_page=1" | jq -r '.workflow_runs[0].id // 0') | |
| 70 | - api POST "/repos/$R/actions/workflows/import-campaign.yml/dispatches" "$(jq -nc --arg r "$REPOS" --arg m "$MINUTES" --arg w "$RUN_WORKFLOWS" '{ref: "main", inputs: {repos: $r, minutes: $m, run_workflows: $w}}')" >/dev/null | |
| 71 | - for i in $(seq 1 60); do | |
| 72 | - run=$(api GET "/repos/$R/actions/workflows/import-campaign.yml/runs?per_page=1" | jq -c '.workflow_runs[0] // empty') | |
| 73 | - [ -n "$run" ] && [ "$(jq -r .id <<<"$run")" != "$before" ] && break | |
| 74 | - sleep 5 | |
| 75 | - done | |
| 76 | - id=$(jq -r .id <<<"$run") | |
| 77 | - echo "Diffinery run: $(jq -r .html_url <<<"$run")" | |
| 75 | + if [ -n "$FOLLOW_RUN" ]; then | |
| 76 | + id=$FOLLOW_RUN | |
| 77 | + else | |
| 78 | + before=$(api GET "/repos/$R/actions/workflows/import-campaign.yml/runs?per_page=1" | jq -r '.workflow_runs[0].id // 0') | |
| 79 | + api POST "/repos/$R/actions/workflows/import-campaign.yml/dispatches" "$(jq -nc --arg r "$REPOS" --arg m "$MINUTES" --arg w "$RUN_WORKFLOWS" '{ref: "main", inputs: {repos: $r, minutes: $m, run_workflows: $w}}')" >/dev/null | |
| 80 | + for i in $(seq 1 60); do | |
| 81 | + run=$(api GET "/repos/$R/actions/workflows/import-campaign.yml/runs?per_page=1" | jq -c '.workflow_runs[0] // empty') | |
| 82 | + [ -n "$run" ] && [ "$(jq -r .id <<<"$run")" != "$before" ] && break | |
| 83 | + sleep 5 | |
| 84 | + done | |
| 85 | + id=$(jq -r .id <<<"$run") | |
| 86 | + fi | |
| 87 | + echo "Diffinery run: $(api GET "/repos/$R/actions/runs/$id" | jq -r .html_url)" | |
| 78 | 88 | while [ "$(api GET "/repos/$R/actions/runs/$id" | jq -r .status)" != completed ]; do sleep 30; done |
| 79 | 89 | run=$(api GET "/repos/$R/actions/runs/$id") |
| 80 | - for job in $(api GET "/repos/$R/actions/runs/$id/jobs" | jq -r '.jobs[].id'); do | |
| 81 | - curl -fsS "$DIFFINERY/api/v3/repos/$R/actions/jobs/$job/logs" -H "Authorization: Bearer $TOKEN" | sed -n '/^# Import campaign/,$p' | tee -a "$GITHUB_STEP_SUMMARY" | |
| 90 | + jobs=$(api GET "/repos/$R/actions/runs/$id/jobs") | |
| 91 | + for job in $(jq -r '.jobs[].id' <<<"$jobs"); do | |
| 92 | + jq -r --arg id "$job" '.jobs[] | select((.id | tostring) == $id) | "Job \(.name): \(.conclusion // .status)\(if .failure then " (\(.failure.message? // .failure))" else "" end)"' <<<"$jobs" | |
| 93 | + log=$(curl -fsS "$DIFFINERY/api/v3/repos/$R/actions/jobs/$job/logs" -H "Authorization: Bearer $TOKEN" || echo "(no log)") | |
| 94 | + if grep -q '^# Import campaign' <<<"$log"; then | |
| 95 | + sed -n '/^# Import campaign/,$p' <<<"$log" | tee -a "$GITHUB_STEP_SUMMARY" | |
| 96 | + else | |
| 97 | + # No report: the job stopped before writing one, so show how it ended. | |
| 98 | + echo "::group::Last 150 log lines of job $job" | |
| 99 | + tail -150 <<<"$log" | |
| 100 | + echo "::endgroup::" | |
| 101 | + fi | |
| 82 | 102 | done |
| 83 | 103 | [ "$(jq -r .conclusion <<<"$run")" = success ] || { echo "::error::the campaign run on Diffinery ended $(jq -r .conclusion <<<"$run")"; exit 1; } |
| 84 | 104 |
modifiedscripts/import-campaign.sh+28−6View file
| @@ -14,6 +14,7 @@ | ||
| 14 | 14 | # RUN_WORKFLOWS=true starts the imported repositories' push workflows (off by default: |
| 15 | 15 | # they are third-party code, run with a job token on Diffinery). |
| 16 | 16 | set -euo pipefail |
| 17 | +trap 'echo "::error::import-campaign.sh stopped at line $LINENO: $BASH_COMMAND"' ERR | |
| 17 | 18 | MINUTES=${MINUTES:-180} |
| 18 | 19 | REPORT=${REPORT:-report.md} |
| 19 | 20 | deadline=$(( $(date +%s) + MINUTES * 60 )) |
| @@ -25,11 +26,20 @@ | ||
| 25 | 26 | declare -A import_id target |
| 26 | 27 | work=$(mktemp -d) |
| 27 | 28 | |
| 28 | -# ------------------------------------------------------------ start every import | |
| 29 | +# ------------------------------------------------------------ start the imports and wait for them | |
| 30 | +# Diffinery runs a few repository imports per person at a time (429 beyond that), so the | |
| 31 | +# rest wait their turn and start as earlier ones finish. | |
| 32 | +queue=() | |
| 29 | 33 | for src in "$@"; do |
| 30 | 34 | name="campaign-${src//\//-}" |
| 31 | 35 | name=${name,,} |
| 32 | 36 | target[$src]=$name |
| 37 | + queue+=("$src") | |
| 38 | +done | |
| 39 | + | |
| 40 | +# Starts one import. Returns 0 when it started or failed for good, 1 when Diffinery is busy. | |
| 41 | +start() { | |
| 42 | + local src=$1 name=${target[$1]} body c | |
| 33 | 43 | if [ "$(code GET "/repos/$owner/$name")" = 200 ]; then |
| 34 | 44 | code DELETE "/repos/$owner/$name" >/dev/null |
| 35 | 45 | fi |
| @@ -39,23 +49,35 @@ | ||
| 39 | 49 | if [ "$c" = 201 ]; then |
| 40 | 50 | import_id[$src]=$(jq -r .id /tmp/campaign-out) |
| 41 | 51 | echo "import $src -> $owner/$name (${import_id[$src]})" |
| 52 | + elif [ "$c" = 429 ]; then | |
| 53 | + return 1 | |
| 42 | 54 | else |
| 43 | 55 | echo "::error::$src: starting the import returned $c: $(jq -r '.message // .' /tmp/campaign-out)" |
| 44 | 56 | jq -n --arg s "$src" --arg c "$c" --arg m "$(jq -r '.message // .' /tmp/campaign-out)" '{source: $s, status: "not started", error: "\($c): \($m)"}' > "$work/${name}.import.json" |
| 45 | 57 | fi |
| 46 | -done | |
| 58 | +} | |
| 47 | 59 | |
| 48 | -# ------------------------------------------------------------ wait for the imports | |
| 49 | 60 | while :; do |
| 61 | + while [ "${#queue[@]}" -gt 0 ] && start "${queue[0]}"; do queue=("${queue[@]:1}"); done | |
| 50 | 62 | pending=0 |
| 51 | 63 | for src in "${!import_id[@]}"; do |
| 52 | 64 | f="$work/${target[$src]}.import.json" |
| 53 | 65 | [ -f "$f" ] && jq -e '.status | IN("succeeded", "failed", "cancelled")' "$f" >/dev/null && continue |
| 54 | 66 | api GET "/diffinery/imports/${import_id[$src]}" > "$f" || { pending=$((pending + 1)); continue; } |
| 55 | - jq -e '.status | IN("succeeded", "failed", "cancelled")' "$f" >/dev/null || pending=$((pending + 1)) | |
| 67 | + if jq -e '.status | IN("succeeded", "failed", "cancelled")' "$f" >/dev/null; then | |
| 68 | + echo "$src: import $(jq -r .status "$f")" | |
| 69 | + else | |
| 70 | + pending=$((pending + 1)) | |
| 71 | + fi | |
| 56 | 72 | done |
| 57 | - [ "$pending" = 0 ] && break | |
| 58 | - if [ "$(left)" -le 0 ]; then echo "::warning::$pending imports were still running when the campaign ran out of time"; break; fi | |
| 73 | + [ "$pending" = 0 ] && [ "${#queue[@]}" = 0 ] && break | |
| 74 | + if [ "$(left)" -le 0 ]; then | |
| 75 | + echo "::warning::$pending imports were still running and ${#queue[@]} had not started when the campaign ran out of time" | |
| 76 | + for src in "${queue[@]}"; do | |
| 77 | + jq -n --arg s "$src" '{source: $s, status: "not started", error: "the campaign ran out of time before this import could start"}' > "$work/${target[$src]}.import.json" | |
| 78 | + done | |
| 79 | + break | |
| 80 | + fi | |
| 59 | 81 | sleep 20 |
| 60 | 82 | done |
| 61 | 83 |