| 1 | #!/usr/bin/env bash |
| 2 | # Waits for every Diffinery Actions run that pushing a commit started (push and |
| 3 | # pull_request runs, and the workflow_run followers they start) and prints each run and |
| 4 | # job. Prints the logs of failed jobs and exits non-zero if any run did not succeed. |
| 5 | # Runs that someone started by hand or on a schedule while that commit was the branch |
| 6 | # head (workflow_dispatch, schedule, repository_dispatch) share its sha but not the push, |
| 7 | # so they are listed as skipped and not waited for. |
| 8 | # |
| 9 | # Usage: DIFFINERY=https://diffinery.com TOKEN=... diffinery-wait-runs.sh owner/repo sha [minutes] |
| 10 | set -euo pipefail |
| 11 | repo=$1 sha=$2 minutes=${3:-60} |
| 12 | # --retry-all-errors also retries a reset connection, which plain --retry gives up on. |
| 13 | api() { curl -fsS --retry 4 --retry-all-errors --retry-delay 3 "$DIFFINERY/api/v3/repos/$repo$1" -H "Authorization: Bearer $TOKEN"; } |
| 14 | unrelated='["workflow_dispatch", "schedule", "repository_dispatch"]' |
| 15 | |
| 16 | deadline=$(( $(date +%s) + minutes * 60 )) |
| 17 | settled=0 |
| 18 | while :; do |
| 19 | # One failed poll (the network, a deploy in progress) is not a result: poll again. |
| 20 | if ! all=$(api "/actions/runs?head_sha=$sha&per_page=100"); then |
| 21 | echo "::warning::could not list the runs at $(date -u +%T); polling again" |
| 22 | [ "$(date +%s)" -lt "$deadline" ] || { echo "::error::Diffinery was unreachable at the $minutes minute limit"; exit 1; } |
| 23 | sleep 15 |
| 24 | continue |
| 25 | fi |
| 26 | runs=$(jq --argjson u "$unrelated" '.workflow_runs |= map(select(.event as $e | $u | index($e) | not))' <<<"$all") |
| 27 | total=$(jq '.workflow_runs | length' <<<"$runs") |
| 28 | pending=$(jq '[.workflow_runs[] | select(.status != "completed")] | length' <<<"$runs") |
| 29 | if [ "$total" -gt 0 ] && [ "$pending" = 0 ]; then |
| 30 | # Finished runs can start followers (workflow_run), so make sure nothing new appears. |
| 31 | settled=$((settled + 1)) |
| 32 | [ "$settled" -ge 3 ] && break |
| 33 | else |
| 34 | settled=0 |
| 35 | fi |
| 36 | if [ "$(date +%s)" -ge "$deadline" ]; then |
| 37 | echo "::error::Diffinery runs for $sha were not finished after $minutes minutes" |
| 38 | jq -r '.workflow_runs[] | " \(.name) [\(.event)]: \(.status)"' <<<"$runs" |
| 39 | exit 1 |
| 40 | fi |
| 41 | sleep 15 |
| 42 | done |
| 43 | |
| 44 | jq -r --argjson u "$unrelated" '.workflow_runs[] | select(.event as $e | $u | index($e)) | "skipped \(.name) [\(.event)]: \(.status) \(.html_url)"' <<<"$all" |
| 45 | failed=0 |
| 46 | for id in $(jq -r '.workflow_runs[].id' <<<"$runs"); do |
| 47 | run=$(api "/actions/runs/$id") |
| 48 | jq -r '"\(.name) [\(.event)]: \(.conclusion)\(try " in \((.updated_at | fromdateiso8601) - (.created_at | fromdateiso8601))s" catch "") \(.html_url)"' <<<"$run" |
| 49 | jobs=$(api "/actions/runs/$id/jobs") |
| 50 | # Each job's time and runner image, and its steps that took over 20 seconds, to see where a sync |
| 51 | # spends it (and whether a job a rollout stopped had started on the previous image). |
| 52 | jq -r 'def secs: try ((.completed_at | fromdateiso8601) - (.started_at | fromdateiso8601)) catch null; |
| 53 | .jobs[] | " \(.name): \(.conclusion)\(if .failure then " - " + (.failure.message // "") else "" end)\(if secs then " (\(secs)s)" else "" end)\(if .runner.image then " on \(.runner.image)" else "" end)", |
| 54 | (.steps[]? | select((secs // 0) > 20) | " \(secs)s \(.name)")' <<<"$jobs" |
| 55 | # A deploy that went on past a problem says so in warnings (the token missing a permission the |
| 56 | # bootstrap needs, say): repeat them here, where they can be read without access to Diffinery. |
| 57 | if [ "$(jq -r .name <<<"$run")" = Deploy ]; then |
| 58 | for job in $(jq -r '.jobs[] | select(.conclusion != "skipped") | .id' <<<"$jobs"); do |
| 59 | log=$(api "/actions/jobs/$job/logs" 2>/dev/null) || true |
| 60 | sed -nE 's/^.*(##\[warning\]|::warning[^:]*::)//p' <<<"$log" | head -n 40 | while IFS= read -r w; do |
| 61 | echo "::warning::Diffinery Deploy: $w" |
| 62 | done || true |
| 63 | # What the deploy did about Turnstile (site keys are public; the secret key is never printed). |
| 64 | grep -E 'Turnstile widget|TURNSTILE_SECRET|site key:|/security/report' <<<"$log" | grep -v 'warning' | head -n 10 | sed 's/^/ deploy: /' || true |
| 65 | done |
| 66 | fi |
| 67 | # A skipped run (every job's `if` was false) is fine; a skipped Deploy after a failed CI |
| 68 | # still fails the sync through CI. |
| 69 | if ! grep -qxE 'success|skipped' <<<"$(jq -r .conclusion <<<"$run")"; then |
| 70 | failed=1 |
| 71 | for job in $(jq -r '.jobs[] | select(.conclusion == "failure" or .conclusion == "timed_out") | .id' <<<"$jobs"); do |
| 72 | echo "::group::Last log lines of job $job" |
| 73 | api "/actions/jobs/$job/logs" | tail -n 120 |
| 74 | echo "::endgroup::" |
| 75 | done |
| 76 | fi |
| 77 | done |
| 78 | if [ "$failed" != 0 ]; then |
| 79 | img=$(curl -fsS --retry 2 "$DIFFINERY/api/v3/meta" | jq -r '.runner_image // "unknown"') || img=unknown |
| 80 | echo "the last deploy's runner image is $img" |
| 81 | fi |
| 82 | [ "$failed" = 0 ] && echo "every Diffinery run on $sha succeeded" |
| 83 | exit "$failed" |