| 1 | #!/usr/bin/env bash |
| 2 | # Imports real GitHub repositories into Diffinery as private test repositories, then |
| 3 | # runs their workflows there and reports what came across and what ran. |
| 4 | # |
| 5 | # Each source is imported as <owner>/campaign-<github owner>-<github repo>, replacing |
| 6 | # the copy from an earlier campaign. A commit to the default branch then starts the |
| 7 | # repository's push workflows, which are followed until they finish or MINUTES runs out. |
| 8 | # Jobs for runners Diffinery doesn't host (Windows, macOS) fail at once with a message saying so, |
| 9 | # which the report shows next to the job. |
| 10 | # |
| 11 | # Usage: DIFFINERY=https://diffinery.com TOKEN=... [GITHUB_TOKEN_FOR_IMPORT=...] \ |
| 12 | # import-campaign.sh owner/repo [owner/repo...] |
| 13 | # Env: MINUTES (default 180) bounds the whole campaign; REPORT (default report.md). |
| 14 | set -euo pipefail |
| 15 | MINUTES=${MINUTES:-180} |
| 16 | REPORT=${REPORT:-report.md} |
| 17 | deadline=$(( $(date +%s) + MINUTES * 60 )) |
| 18 | api() { curl -fsS --retry 3 -X "$1" "$DIFFINERY/api/v3$2" -H "Authorization: Bearer $TOKEN" -H "content-type: application/json" ${3:+--data "$3"}; } |
| 19 | code() { curl -sS -o /tmp/campaign-out -w '%{http_code}' -X "$1" "$DIFFINERY/api/v3$2" -H "Authorization: Bearer $TOKEN" -H "content-type: application/json" ${3:+--data "$3"}; } |
| 20 | left() { echo $(( deadline - $(date +%s) )); } |
| 21 | |
| 22 | owner=$(api GET /user | jq -r .login) |
| 23 | declare -A import_id target |
| 24 | work=$(mktemp -d) |
| 25 | |
| 26 | # ------------------------------------------------------------ start every import |
| 27 | for src in "$@"; do |
| 28 | name="campaign-${src//\//-}" |
| 29 | name=${name,,} |
| 30 | target[$src]=$name |
| 31 | if [ "$(code GET "/repos/$owner/$name")" = 200 ]; then |
| 32 | code DELETE "/repos/$owner/$name" >/dev/null |
| 33 | fi |
| 34 | body=$(jq -nc --arg s "$src" --arg o "$owner" --arg n "$name" --arg t "${GITHUB_TOKEN_FOR_IMPORT:-}" \ |
| 35 | '{source: $s, owner: $o, name: $n, visibility: "private"} + (if $t == "" then {} else {github_token: $t} end)') |
| 36 | c=$(code POST /diffinery/imports "$body") |
| 37 | if [ "$c" = 201 ]; then |
| 38 | import_id[$src]=$(jq -r .id /tmp/campaign-out) |
| 39 | echo "import $src -> $owner/$name (${import_id[$src]})" |
| 40 | else |
| 41 | echo "::error::$src: starting the import returned $c: $(jq -r '.message // .' /tmp/campaign-out)" |
| 42 | jq -n --arg s "$src" --arg c "$c" --arg m "$(jq -r '.message // .' /tmp/campaign-out)" '{source: $s, status: "not started", error: "\($c): \($m)"}' > "$work/${name}.import.json" |
| 43 | fi |
| 44 | done |
| 45 | |
| 46 | # ------------------------------------------------------------ wait for the imports |
| 47 | while :; do |
| 48 | pending=0 |
| 49 | for src in "${!import_id[@]}"; do |
| 50 | f="$work/${target[$src]}.import.json" |
| 51 | [ -f "$f" ] && jq -e '.status | IN("succeeded", "failed", "cancelled")' "$f" >/dev/null && continue |
| 52 | api GET "/diffinery/imports/${import_id[$src]}" > "$f" || { pending=$((pending + 1)); continue; } |
| 53 | jq -e '.status | IN("succeeded", "failed", "cancelled")' "$f" >/dev/null || pending=$((pending + 1)) |
| 54 | done |
| 55 | [ "$pending" = 0 ] && break |
| 56 | if [ "$(left)" -le 0 ]; then echo "::warning::$pending imports were still running when the campaign ran out of time"; break; fi |
| 57 | sleep 20 |
| 58 | done |
| 59 | |
| 60 | # ------------------------------------------------------------ run each imported repository's push workflows |
| 61 | declare -A sha |
| 62 | for src in "${!import_id[@]}"; do |
| 63 | name=${target[$src]} |
| 64 | f="$work/$name.import.json" |
| 65 | [ "$(jq -r .status "$f")" = succeeded ] || continue |
| 66 | repo=$(api GET "/repos/$owner/$name") |
| 67 | branch=$(jq -r .default_branch <<<"$repo") |
| 68 | api GET "/repos/$owner/$name/actions/workflows" | jq '[.workflows[] | {name, path, state}]' > "$work/$name.workflows.json" || echo '[]' > "$work/$name.workflows.json" |
| 69 | [ "$(jq length "$work/$name.workflows.json")" -gt 0 ] || continue |
| 70 | put=$(jq -nc --arg b "$branch" --arg c "$(printf 'Imported from %s and run on Diffinery\n' "$src" | base64 -w0)" '{message: "Run the workflows on Diffinery", content: $c, branch: $b}') |
| 71 | if out=$(api PUT "/repos/$owner/$name/contents/.diffinery-campaign" "$put"); then |
| 72 | sha[$src]=$(jq -r .commit.sha <<<"$out") |
| 73 | echo "$src: pushed ${sha[$src]} to $branch" |
| 74 | else |
| 75 | echo "::warning::$src: could not push a commit to $branch" |
| 76 | fi |
| 77 | done |
| 78 | |
| 79 | finished() { jq -e '(.workflow_runs | length) > 0 and all(.workflow_runs[]; .status == "completed")' "$1" >/dev/null; } |
| 80 | # Runs can start other runs (workflow_run), so a set counts as finished after three quiet checks. |
| 81 | declare -A quiet |
| 82 | while :; do |
| 83 | pending=0 |
| 84 | for src in "${!sha[@]}"; do |
| 85 | name=${target[$src]} |
| 86 | f="$work/$name.runs.json" |
| 87 | [ "${quiet[$src]:-0}" -ge 3 ] && continue |
| 88 | api GET "/repos/$owner/$name/actions/runs?head_sha=${sha[$src]}&per_page=100" > "$f" || { pending=$((pending + 1)); continue; } |
| 89 | if finished "$f"; then quiet[$src]=$(( ${quiet[$src]:-0} + 1 )); else quiet[$src]=0; fi |
| 90 | [ "${quiet[$src]}" -ge 3 ] || pending=$((pending + 1)) |
| 91 | done |
| 92 | [ "$pending" = 0 ] && break |
| 93 | [ "$(left)" -le 0 ] && break |
| 94 | sleep 20 |
| 95 | done |
| 96 | |
| 97 | # ------------------------------------------------------------ report |
| 98 | { |
| 99 | echo "# Import campaign" |
| 100 | echo |
| 101 | echo "| Repository | Import | Took | Issues, PRs and comments | Warnings | Workflows | Runs |" |
| 102 | echo "|---|---|---|---|---|---|---|" |
| 103 | } > "$REPORT" |
| 104 | details="" |
| 105 | for src in "$@"; do |
| 106 | name=${target[$src]} |
| 107 | f="$work/$name.import.json" |
| 108 | status=$(jq -r .status "$f" 2>/dev/null || echo "?") |
| 109 | took=$(jq -r 'if .started_at and .finished_at then (((.finished_at | sub("\\.[0-9]+Z$"; "Z") | fromdateiso8601) - (.started_at | sub("\\.[0-9]+Z$"; "Z") | fromdateiso8601)) / 60 | floor | "\(.) min") else "" end' "$f" 2>/dev/null || true) |
| 110 | items=$(jq -r '[.steps[]? | select(.name == "issues" or .name == "comments") | "\(.count // 0) \(.name)"] | join(", ")' "$f" 2>/dev/null || true) |
| 111 | warnings=$(jq -r '.warnings | length' "$f" 2>/dev/null || echo 0) |
| 112 | workflows=$(jq -r 'length' "$work/$name.workflows.json" 2>/dev/null || echo 0) |
| 113 | runs="-" |
| 114 | if [ -f "$work/$name.runs.json" ]; then |
| 115 | runs=$(jq -r '[.workflow_runs | group_by(.conclusion // .status)[] | "\(length) \(.[0].conclusion // .[0].status)"] | join(", ")' "$work/$name.runs.json") |
| 116 | elif [ -n "${sha[$src]:-}" ]; then |
| 117 | runs="none started" |
| 118 | fi |
| 119 | echo "| [$src](https://github.com/$src) → $owner/$name | $status | $took | $items | $warnings | $workflows | $runs |" >> "$REPORT" |
| 120 | |
| 121 | details+=$'\n'"## $src"$'\n\n' |
| 122 | [ "$status" = succeeded ] || details+="Import $status: $(jq -r '.error // "no error recorded"' "$f" 2>/dev/null)"$'\n\n' |
| 123 | details+="$(jq -r '.steps[]? | "- \(.title // .name): \(.status)\(if .message then " (\(.message))" else "" end)"' "$f" 2>/dev/null)"$'\n' |
| 124 | if [ "$warnings" != 0 ]; then details+=$'\n'"Warnings:"$'\n'"$(jq -r '.warnings[] | "- \(.)"' "$f")"$'\n'; fi |
| 125 | [ -f "$work/$name.runs.json" ] || continue |
| 126 | for run in $(jq -r '.workflow_runs[].id' "$work/$name.runs.json"); do |
| 127 | r=$(jq -c --argjson id "$run" '.workflow_runs[] | select(.id == $id)' "$work/$name.runs.json") |
| 128 | details+=$'\n'"### $(jq -r '"\(.name) [\(.event)]: \(.conclusion // .status)"' <<<"$r")"$'\n' |
| 129 | jobs=$(api GET "/repos/$owner/$name/actions/runs/$run/jobs?per_page=100" || echo '{"jobs":[]}') |
| 130 | details+="$(jq -r '.jobs[] | "- \(.name): \(.conclusion // .status)\(if .failure then " (\(.failure.message? // .failure))" elif .status != "completed" then " (still \(.status))" else "" end)"' <<<"$jobs")"$'\n' |
| 131 | for job in $(jq -r '.jobs[] | select(.conclusion == "failure") | .id' <<<"$jobs"); do |
| 132 | details+=$'\n'"<details><summary>$(jq -r --argjson id "$job" '.jobs[] | select(.id == $id) | .name' <<<"$jobs") log (last 40 lines)</summary>"$'\n\n```\n' |
| 133 | details+="$(curl -fsS "$DIFFINERY/api/v3/repos/$owner/$name/actions/jobs/$job/logs" -H "Authorization: Bearer $TOKEN" | tail -40 || echo "(no log)")" |
| 134 | details+=$'\n```\n</details>\n' |
| 135 | done |
| 136 | done |
| 137 | # Runs still waiting (usually for a Windows or macOS runner) are cancelled so they don't sit in the queue. |
| 138 | for run in $(jq -r '.workflow_runs[] | select(.status != "completed") | .id' "$work/$name.runs.json"); do |
| 139 | code POST "/repos/$owner/$name/actions/runs/$run/cancel" >/dev/null || true |
| 140 | done |
| 141 | done |
| 142 | echo "$details" >> "$REPORT" |
| 143 | cat "$REPORT" |