| 1 | #!/usr/bin/env bash |
| 2 | # Imports real GitHub repositories into Diffinery as private test repositories, then |
| 3 | # runs their workflows there and reports what came across and what ran. |
| 4 | # |
| 5 | # Each source is imported as <owner>/campaign-<github owner>-<github repo>, replacing |
| 6 | # the copy from an earlier campaign. A commit to the default branch then starts the |
| 7 | # repository's push workflows, which are followed until they finish or MINUTES runs out. |
| 8 | # Jobs for runners Diffinery doesn't host (Windows, macOS) fail at once with a message saying so, |
| 9 | # which the report shows next to the job. |
| 10 | # |
| 11 | # Usage: DIFFINERY=https://diffinery.com TOKEN=... [GITHUB_TOKEN_FOR_IMPORT=...] \ |
| 12 | # import-campaign.sh owner/repo [owner/repo...] |
| 13 | # Env: MINUTES (default 180) bounds the whole campaign; REPORT (default report.md); |
| 14 | # RUN_WORKFLOWS=true starts the imported repositories' push workflows (off by default: |
| 15 | # they are third-party code, run with a job token on Diffinery). |
| 16 | set -euo pipefail |
| 17 | trap 'echo "::error::import-campaign.sh stopped at line $LINENO: $BASH_COMMAND"' ERR |
| 18 | MINUTES=${MINUTES:-180} |
| 19 | REPORT=${REPORT:-report.md} |
| 20 | deadline=$(( $(date +%s) + MINUTES * 60 )) |
| 21 | api() { curl -fsS --retry 3 -X "$1" "$DIFFINERY/api/v3$2" -H "Authorization: Bearer $TOKEN" -H "content-type: application/json" ${3:+--data "$3"}; } |
| 22 | code() { curl -sS -o /tmp/campaign-out -w '%{http_code}' -X "$1" "$DIFFINERY/api/v3$2" -H "Authorization: Bearer $TOKEN" -H "content-type: application/json" ${3:+--data "$3"}; } |
| 23 | left() { echo $(( deadline - $(date +%s) )); } |
| 24 | |
| 25 | owner=$(api GET /user | jq -r .login) |
| 26 | declare -A import_id target |
| 27 | work=$(mktemp -d) |
| 28 | |
| 29 | # ------------------------------------------------------------ start the imports and wait for them |
| 30 | # Diffinery runs a few repository imports per person at a time (429 beyond that), so the |
| 31 | # rest wait their turn and start as earlier ones finish. |
| 32 | queue=() |
| 33 | for src in "$@"; do |
| 34 | name="campaign-${src//\//-}" |
| 35 | name=${name,,} |
| 36 | target[$src]=$name |
| 37 | queue+=("$src") |
| 38 | done |
| 39 | |
| 40 | # Starts one import. Returns 0 when it started or failed for good, 1 when Diffinery is busy. |
| 41 | start() { |
| 42 | local src=$1 name=${target[$1]} body c |
| 43 | if [ "$(code GET "/repos/$owner/$name")" = 200 ]; then |
| 44 | code DELETE "/repos/$owner/$name" >/dev/null |
| 45 | fi |
| 46 | body=$(jq -nc --arg s "$src" --arg o "$owner" --arg n "$name" --arg t "${GITHUB_TOKEN_FOR_IMPORT:-}" \ |
| 47 | '{source: $s, owner: $o, name: $n, visibility: "private"} + (if $t == "" then {} else {github_token: $t} end)') |
| 48 | c=$(code POST /diffinery/imports "$body") |
| 49 | if [ "$c" = 201 ]; then |
| 50 | import_id[$src]=$(jq -r .id /tmp/campaign-out) |
| 51 | echo "import $src -> $owner/$name (${import_id[$src]})" |
| 52 | elif [ "$c" = 429 ]; then |
| 53 | return 1 |
| 54 | else |
| 55 | echo "::error::$src: starting the import returned $c: $(jq -r '.message // .' /tmp/campaign-out)" |
| 56 | jq -n --arg s "$src" --arg c "$c" --arg m "$(jq -r '.message // .' /tmp/campaign-out)" '{source: $s, status: "not started", error: "\($c): \($m)"}' > "$work/${name}.import.json" |
| 57 | fi |
| 58 | } |
| 59 | |
| 60 | while :; do |
| 61 | while [ "${#queue[@]}" -gt 0 ] && start "${queue[0]}"; do queue=("${queue[@]:1}"); done |
| 62 | pending=0 |
| 63 | for src in "${!import_id[@]}"; do |
| 64 | f="$work/${target[$src]}.import.json" |
| 65 | [ -f "$f" ] && jq -e '.status | IN("succeeded", "failed", "cancelled")' "$f" >/dev/null && continue |
| 66 | api GET "/diffinery/imports/${import_id[$src]}" > "$f" || { pending=$((pending + 1)); continue; } |
| 67 | if jq -e '.status | IN("succeeded", "failed", "cancelled")' "$f" >/dev/null; then |
| 68 | echo "$src: import $(jq -r .status "$f")" |
| 69 | else |
| 70 | pending=$((pending + 1)) |
| 71 | fi |
| 72 | done |
| 73 | [ "$pending" = 0 ] && [ "${#queue[@]}" = 0 ] && break |
| 74 | if [ "$(left)" -le 0 ]; then |
| 75 | echo "::warning::$pending imports were still running and ${#queue[@]} had not started when the campaign ran out of time" |
| 76 | for src in "${queue[@]}"; do |
| 77 | jq -n --arg s "$src" '{source: $s, status: "not started", error: "the campaign ran out of time before this import could start"}' > "$work/${target[$src]}.import.json" |
| 78 | done |
| 79 | break |
| 80 | fi |
| 81 | sleep 20 |
| 82 | done |
| 83 | |
| 84 | # ------------------------------------------------------------ run each imported repository's push workflows |
| 85 | declare -A sha |
| 86 | for src in "${!import_id[@]}"; do |
| 87 | name=${target[$src]} |
| 88 | f="$work/$name.import.json" |
| 89 | [ "$(jq -r .status "$f")" = succeeded ] || continue |
| 90 | repo=$(api GET "/repos/$owner/$name") |
| 91 | branch=$(jq -r .default_branch <<<"$repo") |
| 92 | api GET "/repos/$owner/$name/actions/workflows" | jq '[.workflows[] | {name, path, state}]' > "$work/$name.workflows.json" || echo '[]' > "$work/$name.workflows.json" |
| 93 | [ "$(jq length "$work/$name.workflows.json")" -gt 0 ] || continue |
| 94 | [ "${RUN_WORKFLOWS:-false}" = true ] || continue |
| 95 | put=$(jq -nc --arg b "$branch" --arg c "$(printf 'Imported from %s and run on Diffinery\n' "$src" | base64 -w0)" '{message: "Run the workflows on Diffinery", content: $c, branch: $b}') |
| 96 | if out=$(api PUT "/repos/$owner/$name/contents/.diffinery-campaign" "$put"); then |
| 97 | sha[$src]=$(jq -r .commit.sha <<<"$out") |
| 98 | echo "$src: pushed ${sha[$src]} to $branch" |
| 99 | else |
| 100 | echo "::warning::$src: could not push a commit to $branch" |
| 101 | fi |
| 102 | done |
| 103 | |
| 104 | finished() { jq -e '(.workflow_runs | length) > 0 and all(.workflow_runs[]; .status == "completed")' "$1" >/dev/null; } |
| 105 | # Runs can start other runs (workflow_run), so a set counts as finished after three quiet checks. |
| 106 | declare -A quiet |
| 107 | while :; do |
| 108 | pending=0 |
| 109 | for src in "${!sha[@]}"; do |
| 110 | name=${target[$src]} |
| 111 | f="$work/$name.runs.json" |
| 112 | [ "${quiet[$src]:-0}" -ge 3 ] && continue |
| 113 | api GET "/repos/$owner/$name/actions/runs?head_sha=${sha[$src]}&per_page=100" > "$f" || { pending=$((pending + 1)); continue; } |
| 114 | if finished "$f"; then quiet[$src]=$(( ${quiet[$src]:-0} + 1 )); else quiet[$src]=0; fi |
| 115 | [ "${quiet[$src]}" -ge 3 ] || pending=$((pending + 1)) |
| 116 | done |
| 117 | [ "$pending" = 0 ] && break |
| 118 | [ "$(left)" -le 0 ] && break |
| 119 | sleep 20 |
| 120 | done |
| 121 | |
| 122 | # ------------------------------------------------------------ report |
| 123 | { |
| 124 | echo "# Import campaign" |
| 125 | echo |
| 126 | if [ -n "${GITHUB_TOKEN_FOR_IMPORT:-}" ]; then echo "Imported with a GitHub token (IMPORT_GITHUB_TOKEN)."; else echo "Imported without a GitHub token (GitHub's anonymous rate limit)."; fi |
| 127 | echo "Workflows run: ${RUN_WORKFLOWS:-false}." |
| 128 | echo |
| 129 | echo "| Repository | Import | Took | Issues, PRs and comments | Warnings | Workflows | Runs |" |
| 130 | echo "|---|---|---|---|---|---|---|" |
| 131 | } > "$REPORT" |
| 132 | details="" |
| 133 | for src in "$@"; do |
| 134 | name=${target[$src]} |
| 135 | f="$work/$name.import.json" |
| 136 | status=$(jq -r .status "$f" 2>/dev/null || echo "?") |
| 137 | took=$(jq -r 'if .started_at and .finished_at then (((.finished_at | sub("\\.[0-9]+Z$"; "Z") | fromdateiso8601) - (.started_at | sub("\\.[0-9]+Z$"; "Z") | fromdateiso8601)) / 60 | floor | "\(.) min") else "" end' "$f" 2>/dev/null || true) |
| 138 | items=$(jq -r '[.steps[]? | select(.name == "issues" or .name == "comments") | "\(.count // 0) \(.name)"] | join(", ")' "$f" 2>/dev/null || true) |
| 139 | warnings=$(jq -r '.warnings | length' "$f" 2>/dev/null || echo 0) |
| 140 | workflows=$(jq -r 'length' "$work/$name.workflows.json" 2>/dev/null || echo 0) |
| 141 | runs="-" |
| 142 | if [ -f "$work/$name.runs.json" ]; then |
| 143 | runs=$(jq -r '[.workflow_runs | group_by(.conclusion // .status)[] | "\(length) \(.[0].conclusion // .[0].status)"] | join(", ")' "$work/$name.runs.json") |
| 144 | elif [ -n "${sha[$src]:-}" ]; then |
| 145 | runs="none started" |
| 146 | fi |
| 147 | echo "| [$src](https://github.com/$src) → $owner/$name | $status | $took | $items | $warnings | $workflows | $runs |" >> "$REPORT" |
| 148 | |
| 149 | details+=$'\n'"## $src"$'\n\n' |
| 150 | [ "$status" = succeeded ] || details+="Import $status: $(jq -r '.error // "no error recorded"' "$f" 2>/dev/null)"$'\n\n' |
| 151 | details+="$(jq -r '.steps[]? | "- \(.title // .name): \(.status)\(if .message then " (\(.message))" else "" end)"' "$f" 2>/dev/null)"$'\n' |
| 152 | if [ "$warnings" != 0 ]; then details+=$'\n'"Warnings:"$'\n'"$(jq -r '.warnings[] | "- \(.)"' "$f")"$'\n'; fi |
| 153 | [ -f "$work/$name.runs.json" ] || continue |
| 154 | for run in $(jq -r '.workflow_runs[].id' "$work/$name.runs.json"); do |
| 155 | r=$(jq -c --argjson id "$run" '.workflow_runs[] | select(.id == $id)' "$work/$name.runs.json") |
| 156 | details+=$'\n'"### $(jq -r '"\(.name) [\(.event)]: \(.conclusion // .status)"' <<<"$r")"$'\n' |
| 157 | jobs=$(api GET "/repos/$owner/$name/actions/runs/$run/jobs?per_page=100" || echo '{"jobs":[]}') |
| 158 | details+="$(jq -r '.jobs[] | "- \(.name): \(.conclusion // .status)\(if .failure then " (\(.failure.message? // .failure))" elif .status != "completed" then " (still \(.status))" else "" end)"' <<<"$jobs")"$'\n' |
| 159 | for job in $(jq -r '.jobs[] | select(.conclusion == "failure") | .id' <<<"$jobs"); do |
| 160 | details+=$'\n'"<details><summary>$(jq -r --argjson id "$job" '.jobs[] | select(.id == $id) | .name' <<<"$jobs") log (last 40 lines)</summary>"$'\n\n```\n' |
| 161 | details+="$(curl -fsS "$DIFFINERY/api/v3/repos/$owner/$name/actions/jobs/$job/logs" -H "Authorization: Bearer $TOKEN" | tail -40 || echo "(no log)")" |
| 162 | details+=$'\n```\n</details>\n' |
| 163 | done |
| 164 | done |
| 165 | # Runs still waiting (usually for a Windows or macOS runner) are cancelled so they don't sit in the queue. |
| 166 | for run in $(jq -r '.workflow_runs[] | select(.status != "completed") | .id' "$work/$name.runs.json"); do |
| 167 | code POST "/repos/$owner/$name/actions/runs/$run/cancel" >/dev/null || true |
| 168 | done |
| 169 | done |
| 170 | echo "$details" >> "$REPORT" |
| 171 | cat "$REPORT" |