| 1 | name: Import campaign |
| 2 | |
| 3 | # Imports a varied set of public GitHub repositories into Diffinery as private test |
| 4 | # repositories (<owner>/campaign-*), optionally runs their push workflows there, and reports what |
| 5 | # came across and what ran (scripts/import-campaign.sh). Started by hand. |
| 6 | # |
| 7 | # Dispatched on GitHub, it starts the same workflow on Diffinery and prints its report; |
| 8 | # on Diffinery it does the work. Imports use the IMPORT_GITHUB_TOKEN secret when it is |
| 9 | # set (a read-only token for public repositories), otherwise GitHub's anonymous rate |
| 10 | # limit applies and large repositories take hours. Set on GitHub, it is copied to |
| 11 | # Diffinery's repository secrets before the run starts (the main sync doesn't copy it). |
| 12 | on: |
| 13 | workflow_dispatch: |
| 14 | inputs: |
| 15 | repos: |
| 16 | description: GitHub repositories to import, separated by spaces |
| 17 | type: string |
| 18 | default: >- |
| 19 | octocat/git-consortium sindresorhus/slugify expressjs/cors spf13/pflag |
| 20 | pallets/markupsafe BurntSushi/walkdir actions/hello-world-javascript-action |
| 21 | actions/hello-world-docker-action jekyll/minima |
| 22 | minutes: |
| 23 | description: How long to wait for imports and workflow runs, in minutes |
| 24 | type: string |
| 25 | default: "300" |
| 26 | run_workflows: |
| 27 | description: Also run the imported repositories' push workflows (third-party code) |
| 28 | type: boolean |
| 29 | default: false |
| 30 | follow_run: |
| 31 | description: Instead of starting a campaign, print the report and logs of this Diffinery run (its id) |
| 32 | type: string |
| 33 | default: "" |
| 34 | peek: |
| 35 | description: Instead of starting a campaign, print how the running one is doing (imports, steps, runs) and stop |
| 36 | type: boolean |
| 37 | default: false |
| 38 | rerun_failed: |
| 39 | description: Instead of starting a campaign, re-run the failed jobs of this Diffinery run of an imported repository ("<repository> <run id>"), or only the job whose name contains the rest ("<repository> <run id> <job name part>"), and print how they end |
| 40 | type: string |
| 41 | default: "" |
| 42 | push_test: |
| 43 | description: Instead of starting a campaign, push this ref of a GitHub repository ("<repository> <ref>") to a new private Diffinery repository in one push, and clone it back |
| 44 | type: string |
| 45 | default: "" |
| 46 | inspect_run: |
| 47 | description: Instead of starting a campaign, print the jobs of this Diffinery run of an imported repository ("<repository> [<run id>]", the newest run without an id), with each failed job's runner, steps and log |
| 48 | type: string |
| 49 | default: "" |
| 50 | |
| 51 | permissions: |
| 52 | contents: read |
| 53 | |
| 54 | # A peek or an inspection runs next to the campaign it looks at; campaigns queue behind each other. |
| 55 | concurrency: |
| 56 | group: ${{ (inputs.peek || inputs.inspect_run != '' || inputs.push_test != '') && 'import-campaign-peek' || 'import-campaign' }} |
| 57 | cancel-in-progress: false |
| 58 | |
| 59 | jobs: |
| 60 | relay: |
| 61 | name: Start the campaign on Diffinery and print its report |
| 62 | if: ${{ github.server_url == 'https://github.com' }} |
| 63 | runs-on: ubuntu-latest |
| 64 | timeout-minutes: 360 |
| 65 | environment: production |
| 66 | env: |
| 67 | DIFFINERY: https://diffinery.com |
| 68 | TOKEN: ${{ secrets.DIFFINERY_LIVE_TOKEN }} |
| 69 | R: codician-team/diffinery |
| 70 | steps: |
| 71 | - name: Copy IMPORT_GITHUB_TOKEN to Diffinery |
| 72 | if: ${{ inputs.follow_run == '' && !inputs.peek && inputs.rerun_failed == '' && inputs.inspect_run == '' && inputs.push_test == '' }} |
| 73 | env: |
| 74 | IMPORT_GITHUB_TOKEN: ${{ secrets.IMPORT_GITHUB_TOKEN }} |
| 75 | run: | |
| 76 | set -euo pipefail |
| 77 | if [ -n "$IMPORT_GITHUB_TOKEN" ]; then |
| 78 | jq -n --arg v "$IMPORT_GITHUB_TOKEN" '{value: $v}' | curl -fsS --retry 4 --retry-delay 30 -o /dev/null -X PUT "$DIFFINERY/api/v3/repos/$R/actions/secrets/IMPORT_GITHUB_TOKEN" -H "Authorization: Bearer $TOKEN" -H "content-type: application/json" --data-binary @- |
| 79 | echo "IMPORT_GITHUB_TOKEN is set on GitHub and was copied to Diffinery" |
| 80 | else |
| 81 | echo "IMPORT_GITHUB_TOKEN is not set on GitHub; Diffinery's own copy is used if it has one" |
| 82 | fi |
| 83 | - name: Dispatch and follow |
| 84 | env: |
| 85 | REPOS: ${{ inputs.repos }} |
| 86 | MINUTES: ${{ inputs.minutes }} |
| 87 | RUN_WORKFLOWS: ${{ inputs.run_workflows }} |
| 88 | FOLLOW_RUN: ${{ inputs.follow_run }} |
| 89 | PEEK: ${{ inputs.peek }} |
| 90 | RERUN_FAILED: ${{ inputs.rerun_failed }} |
| 91 | INSPECT_RUN: ${{ inputs.inspect_run }} |
| 92 | PUSH_TEST: ${{ inputs.push_test }} |
| 93 | run: | |
| 94 | set -euo pipefail |
| 95 | api() { curl -fsS --retry 3 -X "$1" "$DIFFINERY/api/v3$2" -H "Authorization: Bearer $TOKEN" -H "content-type: application/json" ${3:+--data "$3"}; } |
| 96 | if [ -n "$PUSH_TEST" ]; then |
| 97 | # One large push, as `git push` sends it: no import, no chunks. |
| 98 | read -r src ref <<<"$PUSH_TEST" |
| 99 | [[ "$src" =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ && "$ref" =~ ^refs/[A-Za-z0-9_./-]+$ ]] || { echo "::error::push_test takes \"<owner>/<repository> <ref>\""; exit 1; } |
| 100 | owner=$(api GET /user | jq -r .login) |
| 101 | name="campaign-push-$GITHUB_RUN_ID" |
| 102 | api POST /user/repos "$(jq -nc --arg n "$name" '{name: $n, private: true}')" >/dev/null |
| 103 | git init -q --bare one && cd one |
| 104 | git fetch -q "https://github.com/$src" "+$ref:refs/heads/one-push" |
| 105 | echo "Pack for $src $ref: $(git pack-objects --revs --stdout -q <<<refs/heads/one-push | wc -c) bytes, $(git rev-list --objects refs/heads/one-push | wc -l) objects" |
| 106 | auth="Authorization: Basic $(printf 'x-token:%s' "$TOKEN" | base64 -w0)" |
| 107 | started=$(date +%s) |
| 108 | if git -c "http.extraHeader=$auth" -c http.postBuffer=1048576 push "$DIFFINERY/$owner/$name.git" refs/heads/one-push:refs/heads/main 2>&1 | tail -20; then :; fi |
| 109 | echo "Push took $(( $(date +%s) - started )) s" |
| 110 | want=$(git rev-parse refs/heads/one-push) |
| 111 | got=$(git -c "http.extraHeader=$auth" ls-remote "$DIFFINERY/$owner/$name.git" refs/heads/main | cut -f1) |
| 112 | echo "Diffinery has main at ${got:-nothing}; pushed $want" | tee -a "$GITHUB_STEP_SUMMARY" |
| 113 | [ "$got" = "$want" ] || { echo "::error::the push did not land"; exit 1; } |
| 114 | cd .. && git -c "http.extraHeader=$auth" clone -q --bare "$DIFFINERY/$owner/$name.git" back |
| 115 | git -C back fsck --full --strict && echo "Cloned back and checked: $DIFFINERY/$owner/$name" | tee -a "$GITHUB_STEP_SUMMARY" |
| 116 | exit 0 |
| 117 | fi |
| 118 | if [ -n "$RERUN_FAILED$INSPECT_RUN" ]; then |
| 119 | read -r repo run only <<<"${RERUN_FAILED:-$INSPECT_RUN}" |
| 120 | if [ -n "$INSPECT_RUN" ] && [ -z "$run" ]; then |
| 121 | # No run id: list the repository's latest runs and inspect the newest. |
| 122 | runs=$(api GET "/repos/$repo/actions/runs?per_page=10") |
| 123 | jq -r '.workflow_runs[] | "\(.id) \(.name) \(.created_at) \(.status) \(.conclusion // "")"' <<<"$runs" |
| 124 | run=$(jq -r '.workflow_runs[0].id // ""' <<<"$runs") |
| 125 | fi |
| 126 | [[ "$repo" =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ && "$run" =~ ^[A-Za-z0-9_-]+$ ]] || { echo "::error::rerun_failed takes \"<owner>/<repository> <run id>\", inspect_run \"<owner>/<repository> [<run id>]\""; exit 1; } |
| 127 | if [ -n "$RERUN_FAILED" ] && [ -n "$only" ]; then |
| 128 | # One job: the first whose name contains the rest of the input (e.g. "Node.js 16.x"). |
| 129 | job=$(api GET "/repos/$repo/actions/runs/$run/jobs?per_page=100" | jq -r --arg n "$only" '[.jobs[] | select(.name | contains($n))][0].id // ""') |
| 130 | [ -n "$job" ] || { echo "::error::no job of run $run has \"$only\" in its name"; exit 1; } |
| 131 | run=$(api POST "/repos/$repo/actions/jobs/$job/rerun" | jq -r --arg run "$run" '.run_id // $run') |
| 132 | echo "Re-running job $job as $(api GET "/repos/$repo/actions/runs/$run" | jq -r .html_url)" |
| 133 | sleep 30 |
| 134 | while [ "$(api GET "/repos/$repo/actions/runs/$run" | jq -r .status)" != completed ]; do sleep 30; done |
| 135 | elif [ -n "$RERUN_FAILED" ]; then |
| 136 | # Diffinery answers with the run that re-runs it ({run_id}); GitHub re-runs in place. |
| 137 | run=$(api POST "/repos/$repo/actions/runs/$run/rerun-failed-jobs" | jq -r --arg run "$run" '.run_id // $run') |
| 138 | echo "Re-running the failed jobs as $(api GET "/repos/$repo/actions/runs/$run" | jq -r .html_url)" |
| 139 | sleep 30 |
| 140 | while [ "$(api GET "/repos/$repo/actions/runs/$run" | jq -r .status)" != completed ]; do sleep 30; done |
| 141 | else |
| 142 | echo "Jobs of $(api GET "/repos/$repo/actions/runs/$run" | jq -r '"\(.html_url) (\(.status) \(.conclusion // ""), attempt \(.run_attempt // 1))"')" |
| 143 | fi |
| 144 | jobs=$(api GET "/repos/$repo/actions/runs/$run/jobs?per_page=100") |
| 145 | jq -r '.jobs[] | "\(.name): \(.conclusion // .status)\(if .failure then " (\(.failure.message? // .failure))" else "" end)"' <<<"$jobs" | tee -a "$GITHUB_STEP_SUMMARY" |
| 146 | for job in $(jq -r '.jobs[] | select(.conclusion == "failure") | .id' <<<"$jobs"); do |
| 147 | echo "::group::$(jq -r --arg id "$job" '.jobs[] | select((.id | tostring) == $id) | .name' <<<"$jobs")" |
| 148 | # Where it ran and how far it got, for a job that stopped without a log. |
| 149 | jq -r --arg id "$job" '.jobs[] | select((.id | tostring) == $id) | "runner: \(.runner // .runner_name // "none")", "started \(.started_at // "-"), completed \(.completed_at // "-")", (.steps[]? | " step \(.number) \(.name): \(.status) \(.conclusion // "")")' <<<"$jobs" |
| 150 | { api GET "/repos/$repo/actions/jobs/$job/logs" || echo "(no log: the request failed)"; } | sed -E 's/^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9:.]+Z //' | tail -80 |
| 151 | echo "::endgroup::" |
| 152 | done |
| 153 | exit 0 |
| 154 | fi |
| 155 | if [ "$PEEK" = true ]; then |
| 156 | for id in $(api GET "/repos/$R/actions/workflows/import-campaign.yml/runs?per_page=5" | jq -r '.workflow_runs[] | select(.status != "completed") | .id'); do |
| 157 | echo "Campaign run $id on Diffinery:" |
| 158 | api GET "/repos/$R/actions/runs/$id/jobs" | jq -r '.jobs[] | " job \(.name): \(.status) \(.conclusion // "")"' |
| 159 | done |
| 160 | owner=$(api GET /user | jq -r .login) |
| 161 | since=$(date -u -d '-12 hours' +%Y-%m-%dT%H:%M:%S) |
| 162 | imports=$(api GET /diffinery/imports | jq -c --arg since "$since" '[.[] | select((.target.name // "") | startswith("campaign-")) | select(.created_at >= $since)] | sort_by(.created_at)') |
| 163 | jq -r '.[] | "\(.target.name): \(.status) (attempt \(.attempt))\(if .error then ": \(.error)" else "" end)", |
| 164 | ([.steps[] | select(.status == "running" or .status == "failed")][] | " step \(.title): \(.status) \(.message // "") \(.count // "")"), |
| 165 | (if .git then " git \(.git.phase // "") \(.git.message // "") (\(.git.bytes) bytes, \(.git.chunks) chunks, \(.git.tries) tries)" else empty end), |
| 166 | (.warnings[]? | " warning: \(.)")' <<<"$imports" |
| 167 | # Runs on the imported repositories, with each failed job's reason and the end of its log. |
| 168 | for name in $(jq -r '.[] | select(.status == "succeeded") | .target.name' <<<"$imports"); do |
| 169 | runs=$(api GET "/repos/$owner/$name/actions/runs?per_page=30") || continue |
| 170 | jq -r --arg n "$name" '.workflow_runs[] | "\($n) run \(.name): \(.status) \(.conclusion // "")"' <<<"$runs" |
| 171 | for run in $(jq -r '.workflow_runs[] | select(.conclusion == "failure") | .id' <<<"$runs"); do |
| 172 | jobs=$(api GET "/repos/$owner/$name/actions/runs/$run/jobs?per_page=100") || continue |
| 173 | for job in $(jq -r '.jobs[] | select(.conclusion == "failure") | .id' <<<"$jobs"); do |
| 174 | echo "::group::$name: $(jq -r --arg id "$job" '.jobs[] | select((.id | tostring) == $id) | "\(.name)\(if .failure then " (\(.failure.message? // .failure))" else "" end)"' <<<"$jobs")" |
| 175 | { api GET "/repos/$owner/$name/actions/jobs/$job/logs" || true; } | sed -E 's/^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9:.]+Z //' | tail -60 |
| 176 | echo "::endgroup::" |
| 177 | done |
| 178 | done |
| 179 | done |
| 180 | exit 0 |
| 181 | fi |
| 182 | if [ -n "$FOLLOW_RUN" ]; then |
| 183 | id=$FOLLOW_RUN |
| 184 | else |
| 185 | # A new campaign replaces one still running on Diffinery (its imports are cancelled by the new one). |
| 186 | for old in $(api GET "/repos/$R/actions/workflows/import-campaign.yml/runs?per_page=20" | jq -r '.workflow_runs[] | select(.status != "completed") | .id'); do |
| 187 | echo "cancelling the campaign still running on Diffinery: $old" |
| 188 | api POST "/repos/$R/actions/runs/$old/cancel" >/dev/null || true |
| 189 | done |
| 190 | before=$(api GET "/repos/$R/actions/workflows/import-campaign.yml/runs?per_page=1" | jq -r '.workflow_runs[0].id // 0') |
| 191 | api POST "/repos/$R/actions/workflows/import-campaign.yml/dispatches" "$(jq -nc --arg r "$REPOS" --arg m "$MINUTES" --arg w "$RUN_WORKFLOWS" '{ref: "main", inputs: {repos: $r, minutes: $m, run_workflows: $w}}')" >/dev/null |
| 192 | for i in $(seq 1 60); do |
| 193 | run=$(api GET "/repos/$R/actions/workflows/import-campaign.yml/runs?per_page=1" | jq -c '.workflow_runs[0] // empty') |
| 194 | [ -n "$run" ] && [ "$(jq -r .id <<<"$run")" != "$before" ] && break |
| 195 | sleep 5 |
| 196 | done |
| 197 | id=$(jq -r .id <<<"$run") |
| 198 | fi |
| 199 | echo "Diffinery run: $(api GET "/repos/$R/actions/runs/$id" | jq -r .html_url)" |
| 200 | while [ "$(api GET "/repos/$R/actions/runs/$id" | jq -r .status)" != completed ]; do sleep 30; done |
| 201 | run=$(api GET "/repos/$R/actions/runs/$id") |
| 202 | jobs=$(api GET "/repos/$R/actions/runs/$id/jobs") |
| 203 | for job in $(jq -r '.jobs[].id' <<<"$jobs"); do |
| 204 | jq -r --arg id "$job" '.jobs[] | select((.id | tostring) == $id) | "Job \(.name): \(.conclusion // .status)\(if .failure then " (\(.failure.message? // .failure))" else "" end)"' <<<"$jobs" |
| 205 | log=$(curl -fsS "$DIFFINERY/api/v3/repos/$R/actions/jobs/$job/logs" -H "Authorization: Bearer $TOKEN" || echo "(no log)") |
| 206 | # Diffinery's job logs start each line with a timestamp. |
| 207 | log=$(sed -E 's/^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9:.]+Z //' <<<"$log") |
| 208 | if grep -q '^# Import campaign' <<<"$log"; then |
| 209 | sed -n '/^# Import campaign/,$p' <<<"$log" | tee -a "$GITHUB_STEP_SUMMARY" |
| 210 | else |
| 211 | # No report: the job stopped before writing one, so show how it ended. |
| 212 | echo "::group::Last 150 log lines of job $job" |
| 213 | tail -150 <<<"$log" |
| 214 | echo "::endgroup::" |
| 215 | fi |
| 216 | done |
| 217 | [ "$(jq -r .conclusion <<<"$run")" = success ] || { echo "::error::the campaign run on Diffinery ended $(jq -r .conclusion <<<"$run")"; exit 1; } |
| 218 | |
| 219 | campaign: |
| 220 | name: Import, run and report |
| 221 | if: ${{ github.server_url != 'https://github.com' }} |
| 222 | runs-on: ubuntu-latest |
| 223 | timeout-minutes: 360 |
| 224 | env: |
| 225 | DIFFINERY: ${{ github.server_url }} |
| 226 | TOKEN: ${{ secrets.DIFFINERY_LIVE_TOKEN }} |
| 227 | GITHUB_TOKEN_FOR_IMPORT: ${{ secrets.IMPORT_GITHUB_TOKEN }} |
| 228 | steps: |
| 229 | - uses: actions/checkout@v4 |
| 230 | - name: Import and run |
| 231 | env: |
| 232 | REPOS: ${{ inputs.repos }} |
| 233 | MINUTES: ${{ inputs.minutes }} |
| 234 | RUN_WORKFLOWS: ${{ inputs.run_workflows }} |
| 235 | run: | |
| 236 | if [ -n "$GITHUB_TOKEN_FOR_IMPORT" ]; then |
| 237 | echo "::notice::Mode: imports use the IMPORT_GITHUB_TOKEN secret" |
| 238 | else |
| 239 | echo "::warning::Mode: token-free. IMPORT_GITHUB_TOKEN is not set, so imports use GitHub's anonymous rate limit" |
| 240 | fi |
| 241 | # shellcheck disable=SC2086 |
| 242 | REPORT=report.md scripts/import-campaign.sh $REPOS |
| 243 | cat report.md >> "$GITHUB_STEP_SUMMARY" |
| 244 | - uses: actions/upload-artifact@v4 |
| 245 | if: always() |
| 246 | with: |
| 247 | name: import-campaign-report |
| 248 | path: report.md |