| 1 | #!/usr/bin/env bash |
| 2 | # Imports real GitHub repositories into Diffinery as private test repositories, then |
| 3 | # runs their workflows there and reports what came across and what ran. |
| 4 | # |
| 5 | # Each source is imported as <owner>/campaign-<github owner>-<github repo>, replacing |
| 6 | # the copy from an earlier campaign. A commit to the default branch then starts the |
| 7 | # repository's push workflows, which are followed until they finish or MINUTES runs out. |
| 8 | # Jobs for runners Diffinery doesn't host (Windows, macOS) fail at once with a message saying so, |
| 9 | # which the report shows next to the job. |
| 10 | # |
| 11 | # Usage: DIFFINERY=https://diffinery.com TOKEN=... [GITHUB_TOKEN_FOR_IMPORT=...] \ |
| 12 | # import-campaign.sh owner/repo [owner/repo...] |
| 13 | # Env: MINUTES (default 180) bounds the whole campaign; REPORT (default report.md); |
| 14 | # RUN_WORKFLOWS=true starts the imported repositories' push workflows (off by default: |
| 15 | # they are third-party code, run with a job token on Diffinery). |
| 16 | set -euo pipefail |
| 17 | trap 'echo "::error::import-campaign.sh stopped at line $LINENO: $BASH_COMMAND"' ERR |
| 18 | MINUTES=${MINUTES:-180} |
| 19 | REPORT=${REPORT:-report.md} |
| 20 | deadline=$(( $(date +%s) + MINUTES * 60 )) |
| 21 | api() { curl -fsS --retry 3 -X "$1" "$DIFFINERY/api/v3$2" -H "Authorization: Bearer $TOKEN" -H "content-type: application/json" ${3:+--data "$3"}; } |
| 22 | code() { curl -sS -o /tmp/campaign-out -w '%{http_code}' -X "$1" "$DIFFINERY/api/v3$2" -H "Authorization: Bearer $TOKEN" -H "content-type: application/json" ${3:+--data "$3"}; } |
| 23 | left() { echo $(( deadline - $(date +%s) )); } |
| 24 | |
| 25 | owner=$(api GET /user | jq -r .login) |
| 26 | declare -A import_id target |
| 27 | work=$(mktemp -d) |
| 28 | |
| 29 | # ------------------------------------------------------------ start the imports and wait for them |
| 30 | # Diffinery runs a few repository imports per person at a time (429 beyond that), so the |
| 31 | # rest wait their turn and start as earlier ones finish. |
| 32 | queue=() |
| 33 | for src in "$@"; do |
| 34 | name="campaign-${src//\//-}" |
| 35 | name=${name,,} |
| 36 | target[$src]=$name |
| 37 | queue+=("$src") |
| 38 | done |
| 39 | |
| 40 | # An earlier campaign's import that is still running holds its repository (deleting it gets 423), |
| 41 | # so cancel those first. A cancelled import gives its repository back. |
| 42 | names=" ${target[*]} " |
| 43 | for old in $(api GET /diffinery/imports | jq -r '.[] | select(.status | IN("draft", "queued", "running")) | "\(.id)|\(.target.name // "")"'); do |
| 44 | id=${old%%|*} n=${old#*|} |
| 45 | [[ "$names" == *" $n "* ]] || continue |
| 46 | echo "cancelling an earlier campaign's import into $owner/$n ($id)" |
| 47 | code POST "/diffinery/imports/$id/cancel" >/dev/null || true |
| 48 | done |
| 49 | |
| 50 | # Starts one import. Returns 0 when it started or failed for good, 1 when Diffinery is busy. |
| 51 | start() { |
| 52 | local src=$1 name=${target[$1]} body c d i |
| 53 | if [ "$(code GET "/repos/$owner/$name")" = 200 ]; then |
| 54 | for i in 1 2 3 4 5 6; do |
| 55 | d=$(code DELETE "/repos/$owner/$name") |
| 56 | [ "$d" = 204 ] || [ "$d" = 404 ] && break |
| 57 | sleep 10 |
| 58 | done |
| 59 | [ "$d" = 204 ] || [ "$d" = 404 ] || echo "::warning::$src: deleting the earlier $owner/$name returned $d: $(jq -r '.message // .' /tmp/campaign-out)" |
| 60 | fi |
| 61 | body=$(jq -nc --arg s "$src" --arg o "$owner" --arg n "$name" --arg t "${GITHUB_TOKEN_FOR_IMPORT:-}" \ |
| 62 | '{source: $s, owner: $o, name: $n, visibility: "private"} + (if $t == "" then {} else {github_token: $t} end)') |
| 63 | c=$(code POST /diffinery/imports "$body") |
| 64 | if [ "$c" = 201 ]; then |
| 65 | import_id[$src]=$(jq -r .id /tmp/campaign-out) |
| 66 | echo "import $src -> $owner/$name (${import_id[$src]})" |
| 67 | elif [ "$c" = 429 ]; then |
| 68 | return 1 |
| 69 | else |
| 70 | echo "::error::$src: starting the import returned $c: $(jq -r '.message // .' /tmp/campaign-out)" |
| 71 | jq -n --arg s "$src" --arg c "$c" --arg m "$(jq -r '.message // .' /tmp/campaign-out)" '{source: $s, status: "not started", error: "\($c): \($m)"}' > "$work/${name}.import.json" |
| 72 | fi |
| 73 | } |
| 74 | |
| 75 | while :; do |
| 76 | while [ "${#queue[@]}" -gt 0 ] && start "${queue[0]}"; do queue=("${queue[@]:1}"); done |
| 77 | pending=0 |
| 78 | for src in "${!import_id[@]}"; do |
| 79 | f="$work/${target[$src]}.import.json" |
| 80 | [ -f "$f" ] && jq -e '.status | IN("succeeded", "failed", "cancelled")' "$f" >/dev/null && continue |
| 81 | api GET "/diffinery/imports/${import_id[$src]}" > "$f" || { pending=$((pending + 1)); continue; } |
| 82 | if jq -e '.status | IN("succeeded", "failed", "cancelled")' "$f" >/dev/null; then |
| 83 | echo "$src: import $(jq -r .status "$f")" |
| 84 | else |
| 85 | pending=$((pending + 1)) |
| 86 | fi |
| 87 | done |
| 88 | [ "$pending" = 0 ] && [ "${#queue[@]}" = 0 ] && break |
| 89 | if [ "$(left)" -le 0 ]; then |
| 90 | echo "::warning::$pending imports were still running and ${#queue[@]} had not started when the campaign ran out of time" |
| 91 | for src in "${queue[@]}"; do |
| 92 | jq -n --arg s "$src" '{source: $s, status: "not started", error: "the campaign ran out of time before this import could start"}' > "$work/${target[$src]}.import.json" |
| 93 | done |
| 94 | break |
| 95 | fi |
| 96 | sleep 20 |
| 97 | done |
| 98 | |
| 99 | # ------------------------------------------------------------ run each imported repository's push workflows |
| 100 | declare -A sha |
| 101 | for src in "${!import_id[@]}"; do |
| 102 | name=${target[$src]} |
| 103 | f="$work/$name.import.json" |
| 104 | [ "$(jq -r .status "$f")" = succeeded ] || continue |
| 105 | # Test repositories: their runs fail often and shouldn't email whoever runs the campaign. |
| 106 | code PUT "/repos/$owner/$name/subscription" '{"ignored":true}' >/dev/null || true |
| 107 | repo=$(api GET "/repos/$owner/$name") |
| 108 | branch=$(jq -r .default_branch <<<"$repo") |
| 109 | api GET "/repos/$owner/$name/actions/workflows" | jq '[.workflows[] | {name, path, state}]' > "$work/$name.workflows.json" || echo '[]' > "$work/$name.workflows.json" |
| 110 | [ "$(jq length "$work/$name.workflows.json")" -gt 0 ] || continue |
| 111 | [ "${RUN_WORKFLOWS:-false}" = true ] || continue |
| 112 | put=$(jq -nc --arg b "$branch" --arg c "$(printf 'Imported from %s and run on Diffinery\n' "$src" | base64 -w0)" '{message: "Run the workflows on Diffinery", content: $c, branch: $b}') |
| 113 | if out=$(api PUT "/repos/$owner/$name/contents/.diffinery-campaign" "$put"); then |
| 114 | sha[$src]=$(jq -r .commit.sha <<<"$out") |
| 115 | echo "$src: pushed ${sha[$src]} to $branch" |
| 116 | else |
| 117 | echo "::warning::$src: could not push a commit to $branch" |
| 118 | fi |
| 119 | done |
| 120 | |
| 121 | finished() { jq -e '(.workflow_runs | length) > 0 and all(.workflow_runs[]; .status == "completed")' "$1" >/dev/null; } |
| 122 | # Runs can start other runs (workflow_run), so a set counts as finished after three quiet checks. |
| 123 | declare -A quiet |
| 124 | while :; do |
| 125 | pending=0 |
| 126 | for src in "${!sha[@]}"; do |
| 127 | name=${target[$src]} |
| 128 | f="$work/$name.runs.json" |
| 129 | [ "${quiet[$src]:-0}" -ge 3 ] && continue |
| 130 | api GET "/repos/$owner/$name/actions/runs?head_sha=${sha[$src]}&per_page=100" > "$f" || { pending=$((pending + 1)); continue; } |
| 131 | if finished "$f"; then quiet[$src]=$(( ${quiet[$src]:-0} + 1 )); else quiet[$src]=0; fi |
| 132 | [ "${quiet[$src]}" -ge 3 ] || pending=$((pending + 1)) |
| 133 | done |
| 134 | [ "$pending" = 0 ] && break |
| 135 | [ "$(left)" -le 0 ] && break |
| 136 | sleep 20 |
| 137 | done |
| 138 | |
| 139 | # ------------------------------------------------------------ report |
| 140 | { |
| 141 | echo "# Import campaign" |
| 142 | echo |
| 143 | if [ -n "${GITHUB_TOKEN_FOR_IMPORT:-}" ]; then echo "Imported with a GitHub token (IMPORT_GITHUB_TOKEN)."; else echo "Imported without a GitHub token (GitHub's anonymous rate limit)."; fi |
| 144 | echo "Workflows run: ${RUN_WORKFLOWS:-false}." |
| 145 | echo |
| 146 | echo "| Repository | Import | Took | Issues, PRs and comments | Warnings | Workflows | Runs |" |
| 147 | echo "|---|---|---|---|---|---|---|" |
| 148 | } > "$REPORT" |
| 149 | details="" |
| 150 | for src in "$@"; do |
| 151 | name=${target[$src]} |
| 152 | f="$work/$name.import.json" |
| 153 | status=$(jq -r .status "$f" 2>/dev/null || echo "?") |
| 154 | took=$(jq -r 'if .started_at and .finished_at then (((.finished_at | sub("\\.[0-9]+Z$"; "Z") | fromdateiso8601) - (.started_at | sub("\\.[0-9]+Z$"; "Z") | fromdateiso8601)) / 60 | floor | "\(.) min") else "" end' "$f" 2>/dev/null || true) |
| 155 | items=$(jq -r '[.steps[]? | select(.name == "issues" or .name == "comments") | "\(.count // 0) \(.name)"] | join(", ")' "$f" 2>/dev/null || true) |
| 156 | warnings=$(jq -r '.warnings | length' "$f" 2>/dev/null || echo 0) |
| 157 | workflows=$(jq -r 'length' "$work/$name.workflows.json" 2>/dev/null || echo 0) |
| 158 | runs="-" |
| 159 | if [ -f "$work/$name.runs.json" ]; then |
| 160 | runs=$(jq -r '[.workflow_runs | group_by(.conclusion // .status)[] | "\(length) \(.[0].conclusion // .[0].status)"] | join(", ")' "$work/$name.runs.json") |
| 161 | elif [ -n "${sha[$src]:-}" ]; then |
| 162 | runs="none started" |
| 163 | fi |
| 164 | echo "| [$src](https://github.com/$src) → $owner/$name | $status | $took | $items | $warnings | $workflows | $runs |" >> "$REPORT" |
| 165 | |
| 166 | details+=$'\n'"## $src"$'\n\n' |
| 167 | [ "$status" = succeeded ] || details+="Import $status: $(jq -r '.error // "no error recorded"' "$f" 2>/dev/null)"$'\n\n' |
| 168 | details+="$(jq -r '.steps[]? | "- \(.title // .name): \(.status)\(if .message then " (\(.message))" else "" end)"' "$f" 2>/dev/null)"$'\n' |
| 169 | if [ "$warnings" != 0 ]; then details+=$'\n'"Warnings:"$'\n'"$(jq -r '.warnings[] | "- \(.)"' "$f")"$'\n'; fi |
| 170 | [ -f "$work/$name.runs.json" ] || continue |
| 171 | for run in $(jq -r '.workflow_runs[].id' "$work/$name.runs.json"); do |
| 172 | # Diffinery's run and job ids are strings, so they are compared as text. |
| 173 | r=$(jq -c --arg id "$run" '.workflow_runs[] | select((.id | tostring) == $id)' "$work/$name.runs.json") |
| 174 | details+=$'\n'"### $(jq -r '"\(.name) [\(.event)]: \(.conclusion // .status)"' <<<"$r")"$'\n' |
| 175 | jobs=$(api GET "/repos/$owner/$name/actions/runs/$run/jobs?per_page=100" || echo '{"jobs":[]}') |
| 176 | details+="$(jq -r '.jobs[] | "- \(.name): \(.conclusion // .status)\(if .failure then " (\(.failure.message? // .failure))" elif .status != "completed" then " (still \(.status))" else "" end)"' <<<"$jobs")"$'\n' |
| 177 | for job in $(jq -r '.jobs[] | select(.conclusion == "failure") | .id' <<<"$jobs"); do |
| 178 | details+=$'\n'"<details><summary>$(jq -r --arg id "$job" '.jobs[] | select((.id | tostring) == $id) | .name' <<<"$jobs") log (last 40 lines)</summary>"$'\n\n```\n' |
| 179 | details+="$(curl -fsS "$DIFFINERY/api/v3/repos/$owner/$name/actions/jobs/$job/logs" -H "Authorization: Bearer $TOKEN" | tail -40 || echo "(no log)")" |
| 180 | details+=$'\n```\n</details>\n' |
| 181 | done |
| 182 | done |
| 183 | # Runs still waiting (usually for a Windows or macOS runner) are cancelled so they don't sit in the queue. |
| 184 | for run in $(jq -r '.workflow_runs[] | select(.status != "completed") | .id' "$work/$name.runs.json"); do |
| 185 | code POST "/repos/$owner/$name/actions/runs/$run/cancel" >/dev/null || true |
| 186 | done |
| 187 | done |
| 188 | echo "$details" >> "$REPORT" |
| 189 | cat "$REPORT" |