| 1 | name: Import campaign |
| 2 | |
| 3 | # Imports a varied set of public GitHub repositories into Diffinery as private test |
| 4 | # repositories (<owner>/campaign-*), optionally runs their push workflows there, and reports what |
| 5 | # came across and what ran (scripts/import-campaign.sh). Started by hand. |
| 6 | # |
| 7 | # Dispatched on GitHub, it starts the same workflow on Diffinery and prints its report; |
| 8 | # on Diffinery it does the work. Imports use the IMPORT_GITHUB_TOKEN secret when it is |
| 9 | # set (a read-only token for public repositories), otherwise GitHub's anonymous rate |
| 10 | # limit applies and large repositories take hours. Set on GitHub, it is copied to |
| 11 | # Diffinery's repository secrets before the run starts (the main sync doesn't copy it). |
| 12 | on: |
| 13 | workflow_dispatch: |
| 14 | inputs: |
| 15 | repos: |
| 16 | description: GitHub repositories to import, separated by spaces |
| 17 | type: string |
| 18 | default: >- |
| 19 | octocat/git-consortium sindresorhus/slugify expressjs/cors spf13/pflag |
| 20 | pallets/markupsafe BurntSushi/walkdir actions/hello-world-javascript-action |
| 21 | actions/hello-world-docker-action jekyll/minima |
| 22 | minutes: |
| 23 | description: How long to wait for imports and workflow runs, in minutes |
| 24 | type: string |
| 25 | default: "300" |
| 26 | run_workflows: |
| 27 | description: Also run the imported repositories' push workflows (third-party code) |
| 28 | type: boolean |
| 29 | default: false |
| 30 | follow_run: |
| 31 | description: Instead of starting a campaign, print the report and logs of this Diffinery run (its id) |
| 32 | type: string |
| 33 | default: "" |
| 34 | peek: |
| 35 | description: Instead of starting a campaign, print how the running one is doing (imports, steps, runs) and stop |
| 36 | type: boolean |
| 37 | default: false |
| 38 | rerun_failed: |
| 39 | description: Instead of starting a campaign, re-run the failed jobs of this Diffinery run of an imported repository ("<repository> <run id>") and print how they end |
| 40 | type: string |
| 41 | default: "" |
| 42 | inspect_run: |
| 43 | description: Instead of starting a campaign, print the jobs of this Diffinery run of an imported repository ("<repository> [<run id>]", the newest run without an id), with each failed job's runner, steps and log |
| 44 | type: string |
| 45 | default: "" |
| 46 | |
| 47 | permissions: |
| 48 | contents: read |
| 49 | |
| 50 | # A peek or an inspection runs next to the campaign it looks at; campaigns queue behind each other. |
| 51 | concurrency: |
| 52 | group: ${{ (inputs.peek || inputs.inspect_run != '') && 'import-campaign-peek' || 'import-campaign' }} |
| 53 | cancel-in-progress: false |
| 54 | |
| 55 | jobs: |
| 56 | relay: |
| 57 | name: Start the campaign on Diffinery and print its report |
| 58 | if: ${{ github.server_url == 'https://github.com' }} |
| 59 | runs-on: ubuntu-latest |
| 60 | timeout-minutes: 360 |
| 61 | environment: production |
| 62 | env: |
| 63 | DIFFINERY: https://diffinery.com |
| 64 | TOKEN: ${{ secrets.DIFFINERY_LIVE_TOKEN }} |
| 65 | R: codician-team/diffinery |
| 66 | steps: |
| 67 | - name: Copy IMPORT_GITHUB_TOKEN to Diffinery |
| 68 | if: ${{ inputs.follow_run == '' && !inputs.peek && inputs.rerun_failed == '' && inputs.inspect_run == '' }} |
| 69 | env: |
| 70 | IMPORT_GITHUB_TOKEN: ${{ secrets.IMPORT_GITHUB_TOKEN }} |
| 71 | run: | |
| 72 | set -euo pipefail |
| 73 | if [ -n "$IMPORT_GITHUB_TOKEN" ]; then |
| 74 | jq -n --arg v "$IMPORT_GITHUB_TOKEN" '{value: $v}' | curl -fsS --retry 4 --retry-delay 30 -o /dev/null -X PUT "$DIFFINERY/api/v3/repos/$R/actions/secrets/IMPORT_GITHUB_TOKEN" -H "Authorization: Bearer $TOKEN" -H "content-type: application/json" --data-binary @- |
| 75 | echo "IMPORT_GITHUB_TOKEN is set on GitHub and was copied to Diffinery" |
| 76 | else |
| 77 | echo "IMPORT_GITHUB_TOKEN is not set on GitHub; Diffinery's own copy is used if it has one" |
| 78 | fi |
| 79 | - name: Dispatch and follow |
| 80 | env: |
| 81 | REPOS: ${{ inputs.repos }} |
| 82 | MINUTES: ${{ inputs.minutes }} |
| 83 | RUN_WORKFLOWS: ${{ inputs.run_workflows }} |
| 84 | FOLLOW_RUN: ${{ inputs.follow_run }} |
| 85 | PEEK: ${{ inputs.peek }} |
| 86 | RERUN_FAILED: ${{ inputs.rerun_failed }} |
| 87 | INSPECT_RUN: ${{ inputs.inspect_run }} |
| 88 | run: | |
| 89 | set -euo pipefail |
| 90 | api() { curl -fsS --retry 3 -X "$1" "$DIFFINERY/api/v3$2" -H "Authorization: Bearer $TOKEN" -H "content-type: application/json" ${3:+--data "$3"}; } |
| 91 | if [ -n "$RERUN_FAILED$INSPECT_RUN" ]; then |
| 92 | read -r repo run <<<"${RERUN_FAILED:-$INSPECT_RUN}" |
| 93 | if [ -n "$INSPECT_RUN" ] && [ -z "$run" ]; then |
| 94 | # No run id: list the repository's latest runs and inspect the newest. |
| 95 | runs=$(api GET "/repos/$repo/actions/runs?per_page=10") |
| 96 | jq -r '.workflow_runs[] | "\(.id) \(.name) \(.created_at) \(.status) \(.conclusion // "")"' <<<"$runs" |
| 97 | run=$(jq -r '.workflow_runs[0].id // ""' <<<"$runs") |
| 98 | fi |
| 99 | [[ "$repo" =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ && "$run" =~ ^[A-Za-z0-9_-]+$ ]] || { echo "::error::rerun_failed takes \"<owner>/<repository> <run id>\", inspect_run \"<owner>/<repository> [<run id>]\""; exit 1; } |
| 100 | if [ -n "$RERUN_FAILED" ]; then |
| 101 | # Diffinery answers with the run that re-runs it ({run_id}); GitHub re-runs in place. |
| 102 | run=$(api POST "/repos/$repo/actions/runs/$run/rerun-failed-jobs" | jq -r --arg run "$run" '.run_id // $run') |
| 103 | echo "Re-running the failed jobs as $(api GET "/repos/$repo/actions/runs/$run" | jq -r .html_url)" |
| 104 | sleep 30 |
| 105 | while [ "$(api GET "/repos/$repo/actions/runs/$run" | jq -r .status)" != completed ]; do sleep 30; done |
| 106 | else |
| 107 | echo "Jobs of $(api GET "/repos/$repo/actions/runs/$run" | jq -r '"\(.html_url) (\(.status) \(.conclusion // ""), attempt \(.run_attempt // 1))"')" |
| 108 | fi |
| 109 | jobs=$(api GET "/repos/$repo/actions/runs/$run/jobs?per_page=100") |
| 110 | jq -r '.jobs[] | "\(.name): \(.conclusion // .status)\(if .failure then " (\(.failure.message? // .failure))" else "" end)"' <<<"$jobs" | tee -a "$GITHUB_STEP_SUMMARY" |
| 111 | for job in $(jq -r '.jobs[] | select(.conclusion == "failure") | .id' <<<"$jobs"); do |
| 112 | echo "::group::$(jq -r --arg id "$job" '.jobs[] | select((.id | tostring) == $id) | .name' <<<"$jobs")" |
| 113 | # Where it ran and how far it got, for a job that stopped without a log. |
| 114 | jq -r --arg id "$job" '.jobs[] | select((.id | tostring) == $id) | "runner: \(.runner // .runner_name // "none")", "started \(.started_at // "-"), completed \(.completed_at // "-")", (.steps[]? | " step \(.number) \(.name): \(.status) \(.conclusion // "")")' <<<"$jobs" |
| 115 | { api GET "/repos/$repo/actions/jobs/$job/logs" || echo "(no log: the request failed)"; } | sed -E 's/^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9:.]+Z //' | tail -80 |
| 116 | echo "::endgroup::" |
| 117 | done |
| 118 | exit 0 |
| 119 | fi |
| 120 | if [ "$PEEK" = true ]; then |
| 121 | for id in $(api GET "/repos/$R/actions/workflows/import-campaign.yml/runs?per_page=5" | jq -r '.workflow_runs[] | select(.status != "completed") | .id'); do |
| 122 | echo "Campaign run $id on Diffinery:" |
| 123 | api GET "/repos/$R/actions/runs/$id/jobs" | jq -r '.jobs[] | " job \(.name): \(.status) \(.conclusion // "")"' |
| 124 | done |
| 125 | owner=$(api GET /user | jq -r .login) |
| 126 | since=$(date -u -d '-12 hours' +%Y-%m-%dT%H:%M:%S) |
| 127 | imports=$(api GET /diffinery/imports | jq -c --arg since "$since" '[.[] | select((.target.name // "") | startswith("campaign-")) | select(.created_at >= $since)] | sort_by(.created_at)') |
| 128 | jq -r '.[] | "\(.target.name): \(.status) (attempt \(.attempt))\(if .error then ": \(.error)" else "" end)", |
| 129 | ([.steps[] | select(.status == "running" or .status == "failed")][] | " step \(.title): \(.status) \(.message // "") \(.count // "")"), |
| 130 | (if .git then " git \(.git.phase // "") \(.git.message // "") (\(.git.bytes) bytes, \(.git.chunks) chunks, \(.git.tries) tries)" else empty end), |
| 131 | (.warnings[]? | " warning: \(.)")' <<<"$imports" |
| 132 | # Runs on the imported repositories, with each failed job's reason and the end of its log. |
| 133 | for name in $(jq -r '.[] | select(.status == "succeeded") | .target.name' <<<"$imports"); do |
| 134 | runs=$(api GET "/repos/$owner/$name/actions/runs?per_page=30") || continue |
| 135 | jq -r --arg n "$name" '.workflow_runs[] | "\($n) run \(.name): \(.status) \(.conclusion // "")"' <<<"$runs" |
| 136 | for run in $(jq -r '.workflow_runs[] | select(.conclusion == "failure") | .id' <<<"$runs"); do |
| 137 | jobs=$(api GET "/repos/$owner/$name/actions/runs/$run/jobs?per_page=100") || continue |
| 138 | for job in $(jq -r '.jobs[] | select(.conclusion == "failure") | .id' <<<"$jobs"); do |
| 139 | echo "::group::$name: $(jq -r --arg id "$job" '.jobs[] | select((.id | tostring) == $id) | "\(.name)\(if .failure then " (\(.failure.message? // .failure))" else "" end)"' <<<"$jobs")" |
| 140 | { api GET "/repos/$owner/$name/actions/jobs/$job/logs" || true; } | sed -E 's/^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9:.]+Z //' | tail -60 |
| 141 | echo "::endgroup::" |
| 142 | done |
| 143 | done |
| 144 | done |
| 145 | exit 0 |
| 146 | fi |
| 147 | if [ -n "$FOLLOW_RUN" ]; then |
| 148 | id=$FOLLOW_RUN |
| 149 | else |
| 150 | # A new campaign replaces one still running on Diffinery (its imports are cancelled by the new one). |
| 151 | for old in $(api GET "/repos/$R/actions/workflows/import-campaign.yml/runs?per_page=20" | jq -r '.workflow_runs[] | select(.status != "completed") | .id'); do |
| 152 | echo "cancelling the campaign still running on Diffinery: $old" |
| 153 | api POST "/repos/$R/actions/runs/$old/cancel" >/dev/null || true |
| 154 | done |
| 155 | before=$(api GET "/repos/$R/actions/workflows/import-campaign.yml/runs?per_page=1" | jq -r '.workflow_runs[0].id // 0') |
| 156 | api POST "/repos/$R/actions/workflows/import-campaign.yml/dispatches" "$(jq -nc --arg r "$REPOS" --arg m "$MINUTES" --arg w "$RUN_WORKFLOWS" '{ref: "main", inputs: {repos: $r, minutes: $m, run_workflows: $w}}')" >/dev/null |
| 157 | for i in $(seq 1 60); do |
| 158 | run=$(api GET "/repos/$R/actions/workflows/import-campaign.yml/runs?per_page=1" | jq -c '.workflow_runs[0] // empty') |
| 159 | [ -n "$run" ] && [ "$(jq -r .id <<<"$run")" != "$before" ] && break |
| 160 | sleep 5 |
| 161 | done |
| 162 | id=$(jq -r .id <<<"$run") |
| 163 | fi |
| 164 | echo "Diffinery run: $(api GET "/repos/$R/actions/runs/$id" | jq -r .html_url)" |
| 165 | while [ "$(api GET "/repos/$R/actions/runs/$id" | jq -r .status)" != completed ]; do sleep 30; done |
| 166 | run=$(api GET "/repos/$R/actions/runs/$id") |
| 167 | jobs=$(api GET "/repos/$R/actions/runs/$id/jobs") |
| 168 | for job in $(jq -r '.jobs[].id' <<<"$jobs"); do |
| 169 | jq -r --arg id "$job" '.jobs[] | select((.id | tostring) == $id) | "Job \(.name): \(.conclusion // .status)\(if .failure then " (\(.failure.message? // .failure))" else "" end)"' <<<"$jobs" |
| 170 | log=$(curl -fsS "$DIFFINERY/api/v3/repos/$R/actions/jobs/$job/logs" -H "Authorization: Bearer $TOKEN" || echo "(no log)") |
| 171 | # Diffinery's job logs start each line with a timestamp. |
| 172 | log=$(sed -E 's/^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9:.]+Z //' <<<"$log") |
| 173 | if grep -q '^# Import campaign' <<<"$log"; then |
| 174 | sed -n '/^# Import campaign/,$p' <<<"$log" | tee -a "$GITHUB_STEP_SUMMARY" |
| 175 | else |
| 176 | # No report: the job stopped before writing one, so show how it ended. |
| 177 | echo "::group::Last 150 log lines of job $job" |
| 178 | tail -150 <<<"$log" |
| 179 | echo "::endgroup::" |
| 180 | fi |
| 181 | done |
| 182 | [ "$(jq -r .conclusion <<<"$run")" = success ] || { echo "::error::the campaign run on Diffinery ended $(jq -r .conclusion <<<"$run")"; exit 1; } |
| 183 | |
| 184 | campaign: |
| 185 | name: Import, run and report |
| 186 | if: ${{ github.server_url != 'https://github.com' }} |
| 187 | runs-on: ubuntu-latest |
| 188 | timeout-minutes: 360 |
| 189 | env: |
| 190 | DIFFINERY: ${{ github.server_url }} |
| 191 | TOKEN: ${{ secrets.DIFFINERY_LIVE_TOKEN }} |
| 192 | GITHUB_TOKEN_FOR_IMPORT: ${{ secrets.IMPORT_GITHUB_TOKEN }} |
| 193 | steps: |
| 194 | - uses: actions/checkout@v4 |
| 195 | - name: Import and run |
| 196 | env: |
| 197 | REPOS: ${{ inputs.repos }} |
| 198 | MINUTES: ${{ inputs.minutes }} |
| 199 | RUN_WORKFLOWS: ${{ inputs.run_workflows }} |
| 200 | run: | |
| 201 | if [ -n "$GITHUB_TOKEN_FOR_IMPORT" ]; then |
| 202 | echo "::notice::Mode: imports use the IMPORT_GITHUB_TOKEN secret" |
| 203 | else |
| 204 | echo "::warning::Mode: token-free. IMPORT_GITHUB_TOKEN is not set, so imports use GitHub's anonymous rate limit" |
| 205 | fi |
| 206 | # shellcheck disable=SC2086 |
| 207 | REPORT=report.md scripts/import-campaign.sh $REPOS |
| 208 | cat report.md >> "$GITHUB_STEP_SUMMARY" |
| 209 | - uses: actions/upload-artifact@v4 |
| 210 | if: always() |
| 211 | with: |
| 212 | name: import-campaign-report |
| 213 | path: report.md |