| 1 | name: Import campaign |
| 2 | |
| 3 | # Imports a varied set of public GitHub repositories into Diffinery as private test |
| 4 | # repositories (<owner>/campaign-*), optionally runs their push workflows there, and reports what |
| 5 | # came across and what ran (scripts/import-campaign.sh). Started by hand. |
| 6 | # |
| 7 | # Dispatched on GitHub, it starts the same workflow on Diffinery and prints its report; |
| 8 | # on Diffinery it does the work. Imports use the IMPORT_GITHUB_TOKEN secret when it is |
| 9 | # set (a read-only token for public repositories), otherwise GitHub's anonymous rate |
| 10 | # limit applies and large repositories take hours. Set on GitHub, it is copied to |
| 11 | # Diffinery's repository secrets before the run starts (the main sync doesn't copy it). |
| 12 | on: |
| 13 | workflow_dispatch: |
| 14 | inputs: |
| 15 | repos: |
| 16 | description: GitHub repositories to import, separated by spaces |
| 17 | type: string |
| 18 | default: >- |
| 19 | octocat/git-consortium sindresorhus/slugify expressjs/cors spf13/pflag |
| 20 | pallets/markupsafe BurntSushi/walkdir actions/hello-world-javascript-action |
| 21 | actions/hello-world-docker-action jekyll/minima |
| 22 | minutes: |
| 23 | description: How long to wait for imports and workflow runs, in minutes |
| 24 | type: string |
| 25 | default: "300" |
| 26 | run_workflows: |
| 27 | description: Also run the imported repositories' push workflows (third-party code) |
| 28 | type: boolean |
| 29 | default: false |
| 30 | follow_run: |
| 31 | description: Instead of starting a campaign, print the report and logs of this Diffinery run (its id) |
| 32 | type: string |
| 33 | default: "" |
| 34 | |
| 35 | permissions: |
| 36 | contents: read |
| 37 | |
| 38 | concurrency: |
| 39 | group: import-campaign |
| 40 | cancel-in-progress: false |
| 41 | |
| 42 | jobs: |
| 43 | relay: |
| 44 | name: Start the campaign on Diffinery and print its report |
| 45 | if: ${{ github.server_url == 'https://github.com' }} |
| 46 | runs-on: ubuntu-latest |
| 47 | timeout-minutes: 360 |
| 48 | environment: production |
| 49 | env: |
| 50 | DIFFINERY: https://diffinery.com |
| 51 | TOKEN: ${{ secrets.DIFFINERY_LIVE_TOKEN }} |
| 52 | R: codician-team/diffinery |
| 53 | steps: |
| 54 | - name: Copy IMPORT_GITHUB_TOKEN to Diffinery |
| 55 | if: ${{ inputs.follow_run == '' }} |
| 56 | env: |
| 57 | IMPORT_GITHUB_TOKEN: ${{ secrets.IMPORT_GITHUB_TOKEN }} |
| 58 | run: | |
| 59 | set -euo pipefail |
| 60 | if [ -n "$IMPORT_GITHUB_TOKEN" ]; then |
| 61 | jq -n --arg v "$IMPORT_GITHUB_TOKEN" '{value: $v}' | curl -fsS -o /dev/null -X PUT "$DIFFINERY/api/v3/repos/$R/actions/secrets/IMPORT_GITHUB_TOKEN" -H "Authorization: Bearer $TOKEN" -H "content-type: application/json" --data-binary @- |
| 62 | echo "IMPORT_GITHUB_TOKEN is set on GitHub and was copied to Diffinery" |
| 63 | else |
| 64 | echo "IMPORT_GITHUB_TOKEN is not set on GitHub; Diffinery's own copy is used if it has one" |
| 65 | fi |
| 66 | - name: Dispatch and follow |
| 67 | env: |
| 68 | REPOS: ${{ inputs.repos }} |
| 69 | MINUTES: ${{ inputs.minutes }} |
| 70 | RUN_WORKFLOWS: ${{ inputs.run_workflows }} |
| 71 | FOLLOW_RUN: ${{ inputs.follow_run }} |
| 72 | run: | |
| 73 | set -euo pipefail |
| 74 | api() { curl -fsS --retry 3 -X "$1" "$DIFFINERY/api/v3$2" -H "Authorization: Bearer $TOKEN" -H "content-type: application/json" ${3:+--data "$3"}; } |
| 75 | if [ -n "$FOLLOW_RUN" ]; then |
| 76 | id=$FOLLOW_RUN |
| 77 | else |
| 78 | # A new campaign replaces one still running on Diffinery (its imports are cancelled by the new one). |
| 79 | for old in $(api GET "/repos/$R/actions/workflows/import-campaign.yml/runs?per_page=20" | jq -r '.workflow_runs[] | select(.status != "completed") | .id'); do |
| 80 | echo "cancelling the campaign still running on Diffinery: $old" |
| 81 | api POST "/repos/$R/actions/runs/$old/cancel" >/dev/null || true |
| 82 | done |
| 83 | before=$(api GET "/repos/$R/actions/workflows/import-campaign.yml/runs?per_page=1" | jq -r '.workflow_runs[0].id // 0') |
| 84 | api POST "/repos/$R/actions/workflows/import-campaign.yml/dispatches" "$(jq -nc --arg r "$REPOS" --arg m "$MINUTES" --arg w "$RUN_WORKFLOWS" '{ref: "main", inputs: {repos: $r, minutes: $m, run_workflows: $w}}')" >/dev/null |
| 85 | for i in $(seq 1 60); do |
| 86 | run=$(api GET "/repos/$R/actions/workflows/import-campaign.yml/runs?per_page=1" | jq -c '.workflow_runs[0] // empty') |
| 87 | [ -n "$run" ] && [ "$(jq -r .id <<<"$run")" != "$before" ] && break |
| 88 | sleep 5 |
| 89 | done |
| 90 | id=$(jq -r .id <<<"$run") |
| 91 | fi |
| 92 | echo "Diffinery run: $(api GET "/repos/$R/actions/runs/$id" | jq -r .html_url)" |
| 93 | while [ "$(api GET "/repos/$R/actions/runs/$id" | jq -r .status)" != completed ]; do sleep 30; done |
| 94 | run=$(api GET "/repos/$R/actions/runs/$id") |
| 95 | jobs=$(api GET "/repos/$R/actions/runs/$id/jobs") |
| 96 | for job in $(jq -r '.jobs[].id' <<<"$jobs"); do |
| 97 | jq -r --arg id "$job" '.jobs[] | select((.id | tostring) == $id) | "Job \(.name): \(.conclusion // .status)\(if .failure then " (\(.failure.message? // .failure))" else "" end)"' <<<"$jobs" |
| 98 | log=$(curl -fsS "$DIFFINERY/api/v3/repos/$R/actions/jobs/$job/logs" -H "Authorization: Bearer $TOKEN" || echo "(no log)") |
| 99 | # Diffinery's job logs start each line with a timestamp. |
| 100 | log=$(sed -E 's/^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9:.]+Z //' <<<"$log") |
| 101 | if grep -q '^# Import campaign' <<<"$log"; then |
| 102 | sed -n '/^# Import campaign/,$p' <<<"$log" | tee -a "$GITHUB_STEP_SUMMARY" |
| 103 | else |
| 104 | # No report: the job stopped before writing one, so show how it ended. |
| 105 | echo "::group::Last 150 log lines of job $job" |
| 106 | tail -150 <<<"$log" |
| 107 | echo "::endgroup::" |
| 108 | fi |
| 109 | done |
| 110 | [ "$(jq -r .conclusion <<<"$run")" = success ] || { echo "::error::the campaign run on Diffinery ended $(jq -r .conclusion <<<"$run")"; exit 1; } |
| 111 | |
| 112 | campaign: |
| 113 | name: Import, run and report |
| 114 | if: ${{ github.server_url != 'https://github.com' }} |
| 115 | runs-on: ubuntu-latest |
| 116 | timeout-minutes: 360 |
| 117 | env: |
| 118 | DIFFINERY: ${{ github.server_url }} |
| 119 | TOKEN: ${{ secrets.DIFFINERY_LIVE_TOKEN }} |
| 120 | GITHUB_TOKEN_FOR_IMPORT: ${{ secrets.IMPORT_GITHUB_TOKEN }} |
| 121 | steps: |
| 122 | - uses: actions/checkout@v4 |
| 123 | - name: Import and run |
| 124 | env: |
| 125 | REPOS: ${{ inputs.repos }} |
| 126 | MINUTES: ${{ inputs.minutes }} |
| 127 | RUN_WORKFLOWS: ${{ inputs.run_workflows }} |
| 128 | run: | |
| 129 | if [ -n "$GITHUB_TOKEN_FOR_IMPORT" ]; then |
| 130 | echo "::notice::Mode: imports use the IMPORT_GITHUB_TOKEN secret" |
| 131 | else |
| 132 | echo "::warning::Mode: token-free. IMPORT_GITHUB_TOKEN is not set, so imports use GitHub's anonymous rate limit" |
| 133 | fi |
| 134 | # shellcheck disable=SC2086 |
| 135 | REPORT=report.md scripts/import-campaign.sh $REPOS |
| 136 | cat report.md >> "$GITHUB_STEP_SUMMARY" |
| 137 | - uses: actions/upload-artifact@v4 |
| 138 | if: always() |
| 139 | with: |
| 140 | name: import-campaign-report |
| 141 | path: report.md |