| 1 | #!/usr/bin/env bash |
| 2 | # Idempotent production setup for Diffinery on a Cloudflare account. |
| 3 | # |
| 4 | # Creates (only when missing): the D1 database, R2 bucket, event and mail queues, the |
| 5 | # subscription that sends Email Sending's bounces and complaints to the diffinery-mail-events |
| 6 | # queue, the Email Sending setup for the domain (scripts/email-sending.sh), the Turnstile |
| 7 | # widget for the security report form (scripts/turnstile.sh) and the |
| 8 | # Cloudflare Access application for /auth/access, the email-based account |
| 9 | # recovery path. Everyday sign-in uses passkeys; git, the API, MCP and runners |
| 10 | # use Diffinery tokens. |
| 11 | # Then renders apps/diffinery/wrangler.deploy.jsonc and applies D1 migrations. |
| 12 | # |
| 13 | # Requires: CLOUDFLARE_API_TOKEN, CLOUDFLARE_ACCOUNT_ID, node, jq, curl. |
| 14 | # Optional: DIFFINERY_DOMAIN (default diffinery.com; the Worker's route, PUBLIC_URL |
| 15 | # and sender follow it), DIFFINERY_ZONE (the Cloudflare zone holding that host, |
| 16 | # default its last two labels), MAIL_DOMAIN (default DIFFINERY_DOMAIN), |
| 17 | # ACCESS_TEAM_DOMAIN (e.g. example.cloudflareaccess.com) when the token cannot read it, |
| 18 | # BOOTSTRAP_ADMINS (comma-separated emails that become site administrators on first sign-in; |
| 19 | # when unset, the deployed Worker keeps the list it has: see scripts/bootstrap-admins.sh). |
| 20 | set -euo pipefail |
| 21 | |
| 22 | : "${CLOUDFLARE_API_TOKEN:?set CLOUDFLARE_API_TOKEN}" |
| 23 | : "${CLOUDFLARE_ACCOUNT_ID:?set CLOUDFLARE_ACCOUNT_ID}" |
| 24 | DOMAIN="${DIFFINERY_DOMAIN:-diffinery.com}" |
| 25 | ROOT="$(cd "$(dirname "$0")/.." && pwd)" |
| 26 | APP="$ROOT/apps/diffinery" |
| 27 | WRANGLER=(pnpm --dir "$APP" exec wrangler) |
| 28 | API="https://api.cloudflare.com/client/v4/accounts/$CLOUDFLARE_ACCOUNT_ID" |
| 29 | |
| 30 | cf() { # cf METHOD PATH [JSON] |
| 31 | local method="$1" path="$2" body="${3:-}" |
| 32 | if [[ -n "$body" ]]; then |
| 33 | curl -sS -X "$method" "$API$path" -H "Authorization: Bearer $CLOUDFLARE_API_TOKEN" -H "Content-Type: application/json" --data "$body" |
| 34 | else |
| 35 | curl -sS -X "$method" "$API$path" -H "Authorization: Bearer $CLOUDFLARE_API_TOKEN" |
| 36 | fi |
| 37 | } |
| 38 | log() { echo "::group::$*"; } |
| 39 | endlog() { echo "::endgroup::"; } |
| 40 | warn() { echo "::warning::$*" >&2; } |
| 41 | |
| 42 | # ------------------------------------------------------------------ D1 |
| 43 | log "D1 database" |
| 44 | D1_ID="$(cf GET "/d1/database?name=diffinery&per_page=100" | jq -r '.result[]? | select(.name=="diffinery") | .uuid' | head -n1)" |
| 45 | if [[ -z "$D1_ID" ]]; then |
| 46 | D1_ID="$(cf POST /d1/database '{"name":"diffinery"}' | jq -r '.result.uuid // empty')" |
| 47 | [[ -n "$D1_ID" ]] || { echo "could not create D1 database (does the token have D1: Edit?)" >&2; exit 1; } |
| 48 | echo "created D1 database $D1_ID" |
| 49 | else |
| 50 | echo "D1 database exists: $D1_ID" |
| 51 | fi |
| 52 | echo "read replication: $(cf GET "/d1/database/$D1_ID" | jq -r '.result.read_replication.mode // "unknown"') (turn it on with the 'D1 read replication' workflow)" |
| 53 | endlog |
| 54 | |
| 55 | # ------------------------------------------------------------------ R2 |
| 56 | log "R2 bucket" |
| 57 | if cf GET "/r2/buckets/diffinery-data" | jq -e '.success' >/dev/null; then |
| 58 | echo "R2 bucket exists" |
| 59 | else |
| 60 | cf POST /r2/buckets '{"name":"diffinery-data"}' | jq -e '.success' >/dev/null || { echo "could not create R2 bucket (R2 enabled? token has R2: Edit?)" >&2; exit 1; } |
| 61 | echo "created R2 bucket diffinery-data" |
| 62 | fi |
| 63 | # Lifecycle rules (layout: apps/diffinery/src/lib/storage-keys.ts). Only transient prefixes |
| 64 | # age out; repos/ (git objects, LFS, release assets), registry/ and actions-cache/ (evicted |
| 65 | # by Diffinery) have no age rule. Multipart uploads left unfinished are aborted after a day. |
| 66 | LIFECYCLE='{"rules":[ |
| 67 | {"id":"abort-multipart-1d","enabled":true,"conditions":{"prefix":""},"abortMultipartUploadsTransition":{"condition":{"type":"Age","maxAge":86400}}}, |
| 68 | {"id":"artifacts-90d","enabled":true,"conditions":{"prefix":"artifacts/"},"deleteObjectsTransition":{"condition":{"type":"Age","maxAge":7776000}}}, |
| 69 | {"id":"logs-90d","enabled":true,"conditions":{"prefix":"logs/"},"deleteObjectsTransition":{"condition":{"type":"Age","maxAge":7776000}}}, |
| 70 | {"id":"actions-staging-1d","enabled":true,"conditions":{"prefix":"actions-staging/"},"deleteObjectsTransition":{"condition":{"type":"Age","maxAge":86400}}}, |
| 71 | {"id":"archives-30d","enabled":true,"conditions":{"prefix":"archives/"},"deleteObjectsTransition":{"condition":{"type":"Age","maxAge":2592000}}}, |
| 72 | {"id":"exports-8d","enabled":true,"conditions":{"prefix":"exports/"},"deleteObjectsTransition":{"condition":{"type":"Age","maxAge":691200}}}, |
| 73 | {"id":"events-400d","enabled":true,"conditions":{"prefix":"events/"},"deleteObjectsTransition":{"condition":{"type":"Age","maxAge":34560000}}}, |
| 74 | {"id":"action-mirror-30d","enabled":true,"conditions":{"prefix":"action-mirror/"},"deleteObjectsTransition":{"condition":{"type":"Age","maxAge":2592000}}} |
| 75 | ]}' |
| 76 | # Applied on every deploy; a failure stops it, since a wrong rule can delete data. |
| 77 | RESULT="$(cf PUT "/r2/buckets/diffinery-data/lifecycle" "$LIFECYCLE")" |
| 78 | if ! jq -e '.success' >/dev/null <<<"$RESULT"; then |
| 79 | echo "could not set the R2 lifecycle rules (token has R2: Edit?): $(jq -c '.errors // .' <<<"$RESULT" 2>/dev/null || echo "$RESULT")" >&2 |
| 80 | exit 1 |
| 81 | fi |
| 82 | CURRENT="$(cf GET "/r2/buckets/diffinery-data/lifecycle")" |
| 83 | if jq -e '[.result.rules[]? | select(.deleteObjectsTransition != null and ((.conditions.prefix // "") | startswith("repos/") or . == ""))] | length > 0' >/dev/null <<<"$CURRENT"; then |
| 84 | echo "an R2 lifecycle rule still deletes objects under repos/ (git objects, LFS); refusing to continue" >&2 |
| 85 | exit 1 |
| 86 | fi |
| 87 | echo "R2 lifecycle rules set" |
| 88 | endlog |
| 89 | |
| 90 | # ------------------------------------------------------------------ Queues |
| 91 | log "Queues" |
| 92 | EXISTING_QUEUES="$(cf GET "/queues?per_page=100" | jq -r '.result[]?.queue_name')" |
| 93 | for q in diffinery-events diffinery-events-dlq diffinery-mail diffinery-mail-dlq diffinery-mail-events; do |
| 94 | if grep -qx "$q" <<<"$EXISTING_QUEUES"; then |
| 95 | echo "queue $q exists" |
| 96 | else |
| 97 | cf POST /queues "{\"queue_name\":\"$q\"}" | jq -e '.success' >/dev/null || { echo "could not create queue $q (token has Queues: Edit?)" >&2; exit 1; } |
| 98 | echo "created queue $q" |
| 99 | fi |
| 100 | done |
| 101 | endlog |
| 102 | |
| 103 | # ------------------------------------------------------------------ Email Sending events |
| 104 | # Bounces and complaints reach the Worker through a Queues event subscription: source email.sending |
| 105 | # (one per sending domain), events message.bounced and message.complained, destination the |
| 106 | # diffinery-mail-events queue. Created only when missing; one that lacks an event gets it added. |
| 107 | # Create: POST /accounts/{account_id}/event_subscriptions/subscriptions |
| 108 | # https://developers.cloudflare.com/api/resources/queues/subresources/subscriptions/methods/create/ |
| 109 | # List: GET /accounts/{account_id}/event_subscriptions/subscriptions?queue_id=... |
| 110 | # Update: PATCH /accounts/{account_id}/event_subscriptions/subscriptions/{id} |
| 111 | # Source and event names: https://developers.cloudflare.com/queues/event-subscriptions/events-schemas/ |
| 112 | # and https://developers.cloudflare.com/email-service/platform/event-subscriptions/. The email.sending |
| 113 | # source takes zone_id and domain, as `wrangler queues subscription create --source email.sending |
| 114 | # --zone-id ... --domain ...` sends them (https://developers.cloudflare.com/workers/wrangler/commands/queues/). |
| 115 | # The token needs Account → Queues → Edit (the API reference's "Queues Write") for this, and |
| 116 | # Zone → Zone → Read on the sending domain to find its zone. Without them this warns and the deploy |
| 117 | # goes on: bounces and complaints then don't reach the suppression list until someone subscribes the |
| 118 | # queue in the dashboard (docs/admin/site-administration.md). |
| 119 | log "Email Sending event subscription" |
| 120 | SEND_DOMAIN="${MAIL_DOMAIN:-$DOMAIN}" |
| 121 | SUB_EVENTS='["message.bounced","message.complained"]' |
| 122 | SUB_PERMISSION="Account → Queues → Edit (\"Queues Write\" in the API reference)" |
| 123 | api_errors() { jq -r '[.errors[]? | "\(.code): \(.message)"] | join("; ")' <<<"$1" 2>/dev/null || echo "$1"; } |
| 124 | subscribe_mail_events() { |
| 125 | local qid zone="" name="$SEND_DOMAIN" list existing id body res |
| 126 | qid="$(cf GET "/queues?per_page=100" | jq -r '.result[]? | select(.queue_name=="diffinery-mail-events") | .queue_id' | head -n1 || true)" |
| 127 | if [[ -z "$qid" ]]; then warn "Could not find the diffinery-mail-events queue, so it isn't subscribed to Email Sending's bounce and complaint events."; return; fi |
| 128 | # The zone that holds the sending domain (the domain itself or a parent), as scripts/email-sending.sh finds it. |
| 129 | while [[ "$name" == *.* ]]; do |
| 130 | zone="$(curl -sS "https://api.cloudflare.com/client/v4/zones?name=$name" -H "Authorization: Bearer $CLOUDFLARE_API_TOKEN" | jq -r '.result[0].id // empty' || true)" |
| 131 | [[ -n "$zone" ]] && break |
| 132 | name="${name#*.}" |
| 133 | done |
| 134 | if [[ -z "$zone" ]]; then warn "Could not find the Cloudflare zone for $SEND_DOMAIN (the token needs Zone → Zone → Read on it), so diffinery-mail-events isn't subscribed to Email Sending's bounce and complaint events. Subscribe it in the dashboard: see Site administration → Email in the docs."; return; fi |
| 135 | list="$(cf GET "/event_subscriptions/subscriptions?queue_id=$qid&per_page=100" || true)" |
| 136 | if ! jq -e '.success' >/dev/null 2>&1 <<<"$list"; then |
| 137 | warn "Could not read the queue's event subscriptions, so diffinery-mail-events may not get bounces and complaints. The token needs $SUB_PERMISSION. Cloudflare said: $(api_errors "$list")" |
| 138 | return |
| 139 | fi |
| 140 | existing="$(jq -c --arg d "$SEND_DOMAIN" --arg q "$qid" '[.result[]? | select(.source.type == "email.sending" and .source.domain == $d and (.destination.queue_id // $q) == $q)][0] // empty' <<<"$list")" |
| 141 | if [[ -n "$existing" ]]; then |
| 142 | id="$(jq -r '.id' <<<"$existing")" |
| 143 | if jq -e --argjson want "$SUB_EVENTS" '.enabled != false and ((.events // []) as $have | all($want[]; . as $e | $have | index($e)))' >/dev/null <<<"$existing"; then |
| 144 | echo "event subscription $id sends $SEND_DOMAIN's bounces and complaints to diffinery-mail-events" |
| 145 | return |
| 146 | fi |
| 147 | body="$(jq -c --argjson want "$SUB_EVENTS" '{events: ((.events // []) + $want | unique), enabled: true}' <<<"$existing")" |
| 148 | res="$(cf PATCH "/event_subscriptions/subscriptions/$id" "$body" || true)" |
| 149 | if jq -e '.success' >/dev/null 2>&1 <<<"$res"; then echo "event subscription $id now includes bounced and complained events"; else warn "Could not update event subscription $id. The token needs $SUB_PERMISSION. Cloudflare said: $(api_errors "$res")"; fi |
| 150 | return |
| 151 | fi |
| 152 | body="$(jq -nc --arg d "$SEND_DOMAIN" --arg z "$zone" --arg q "$qid" --argjson events "$SUB_EVENTS" '{ |
| 153 | name: ("diffinery-mail-events " + $d), enabled: true, |
| 154 | source: {type: "email.sending", zone_id: $z, domain: $d}, |
| 155 | destination: {type: "queues.queue", queue_id: $q}, |
| 156 | events: $events |
| 157 | }')" |
| 158 | res="$(cf POST /event_subscriptions/subscriptions "$body" || true)" |
| 159 | if jq -e '.success' >/dev/null 2>&1 <<<"$res"; then |
| 160 | echo "created event subscription $(jq -r '.result.id' <<<"$res"): $SEND_DOMAIN's bounces and complaints go to diffinery-mail-events" |
| 161 | else |
| 162 | warn "Could not subscribe diffinery-mail-events to Email Sending's bounce and complaint events for $SEND_DOMAIN. The token needs $SUB_PERMISSION (and $SEND_DOMAIN onboarded to Email Sending). Cloudflare said: $(api_errors "$res")" |
| 163 | fi |
| 164 | } |
| 165 | subscribe_mail_events |
| 166 | endlog |
| 167 | |
| 168 | # ------------------------------------------------------------------ Email Sending |
| 169 | # notifications@<MAIL_DOMAIN or DIFFINERY_DOMAIN>; warns (and email is skipped) until the domain is onboarded. |
| 170 | DIFFINERY_DOMAIN="$DOMAIN" "$ROOT/scripts/email-sending.sh" || warn "Email Sending check failed; email may not be delivered." |
| 171 | |
| 172 | # ------------------------------------------------------------------ Turnstile |
| 173 | # The widget "diffinery <DIFFINERY_DOMAIN>", created when missing, so people who aren't signed in can |
| 174 | # send security reports (security.txt points researchers at the form). Its site key is deployed as the |
| 175 | # TURNSTILE_SITE_KEY var (render-config below); ensure-secrets.sh sets TURNSTILE_SECRET when it's missing, |
| 176 | # and turnstile.sh replaces it here, before the deploy, when it goes with another widget. Needs |
| 177 | # Account → Turnstile → Edit; without it turnstile.sh warns, naming it, and the deploy goes on with |
| 178 | # anonymous reports refused. Sign-up stays as SIGNUP says: keys alone don't open it. |
| 179 | log "Turnstile" |
| 180 | TURNSTILE_SITE_KEY="$(DIFFINERY_DOMAIN="$DOMAIN" "$ROOT/scripts/turnstile.sh")" || { warn "scripts/turnstile.sh failed; anonymous security reports stay refused."; TURNSTILE_SITE_KEY=""; } |
| 181 | echo "site key: ${TURNSTILE_SITE_KEY:-none}" |
| 182 | endlog |
| 183 | |
| 184 | # ------------------------------------------------------------------ Access |
| 185 | log "Cloudflare Access" |
| 186 | TEAM="${ACCESS_TEAM_DOMAIN:-}" |
| 187 | if [[ -z "$TEAM" ]]; then |
| 188 | TEAM="$(cf GET /access/organizations | jq -r '.result.auth_domain // empty' || true)" |
| 189 | fi |
| 190 | APPS="$(cf GET "/access/apps?per_page=100" || echo '{}')" |
| 191 | AUD="" |
| 192 | if ! jq -e '.success' >/dev/null <<<"$APPS"; then |
| 193 | warn "Could not read Access applications (token needs Access: Apps and Policies Edit). Passkeys work; email recovery stays off." |
| 194 | else |
| 195 | # People sign in with passkeys. Access guards only /auth/access, the email-based |
| 196 | # recovery path, so everything else (web, git, API, MCP, runners) reaches the Worker directly. |
| 197 | WEB_BODY="$(jq -nc --arg d "$DOMAIN" '{ |
| 198 | name: "Diffinery account recovery", type: "self_hosted", domain: ($d + "/auth/access"), |
| 199 | destinations: [{type: "public", uri: ($d + "/auth/access")}], |
| 200 | session_duration: "15m", app_launcher_visible: false, http_only_cookie_attribute: true, same_site_cookie_attribute: "lax" |
| 201 | }')" |
| 202 | POLICY='[{"name": "Anyone who proves their email (Diffinery decides access)", "decision": "allow", "include": [{"everyone": {}}], "precedence": 1}]' |
| 203 | WEB_ID="$(jq -r '.result[] | select(.name=="Diffinery account recovery" or .name=="Diffinery") | .id' <<<"$APPS" | head -n1)" |
| 204 | if [[ -z "$WEB_ID" ]]; then |
| 205 | WEB_APP="$(cf POST /access/apps "$(jq -c --argjson p "$POLICY" '. + {policies: $p}' <<<"$WEB_BODY")" | jq -c '.result // empty')" |
| 206 | [[ -n "$WEB_APP" ]] && echo "created Access app for $DOMAIN/auth/access" || warn "Could not create the Access application; email recovery stays off." |
| 207 | else |
| 208 | WEB_APP="$(cf PUT "/access/apps/$WEB_ID" "$WEB_BODY" | jq -c '.result // empty')" |
| 209 | [[ -n "$WEB_APP" ]] && echo "Access app covers $DOMAIN/auth/access" || warn "Could not update the Access application." |
| 210 | fi |
| 211 | AUD="$(jq -r '.aud // empty' <<<"${WEB_APP:-null}")" |
| 212 | |
| 213 | # Earlier versions put Access in front of the whole site and needed a bypass app; remove it. |
| 214 | BYPASS_ID="$(jq -r '.result[] | select(.name=="Diffinery machine endpoints") | .id' <<<"$APPS" | head -n1)" |
| 215 | if [[ -n "$BYPASS_ID" ]]; then |
| 216 | cf DELETE "/access/apps/$BYPASS_ID" | jq -e '.success' >/dev/null && echo "removed the old Access bypass app" || warn "Could not remove the old Access bypass application." |
| 217 | fi |
| 218 | fi |
| 219 | if [[ -z "$TEAM" ]]; then |
| 220 | # Fall back to the sign-in redirect, which names the team domain. |
| 221 | TEAM="$(curl -sS -o /dev/null -w '%{redirect_url}' "https://$DOMAIN/auth/access" | sed -nE 's#^https://([^/]+\.cloudflareaccess\.com)/.*#\1#p' || true)" |
| 222 | fi |
| 223 | [[ -n "$TEAM" ]] || warn "Access team domain unknown; set the ACCESS_TEAM_DOMAIN repository variable (Zero Trust → Settings → Custom pages shows it)." |
| 224 | echo "team=${TEAM:-?} aud=${AUD:+set}" |
| 225 | endlog |
| 226 | |
| 227 | # ------------------------------------------------------------------ Site administrators |
| 228 | # BOOTSTRAP_ADMINS from the environment, else wrangler.jsonc, else what the deployed Worker has; it |
| 229 | # stops the deploy if the Worker's list can't be read, rather than deploy an empty one in its place. |
| 230 | log "Bootstrap administrators" |
| 231 | ADMINS="$("$ROOT/scripts/bootstrap-admins.sh")" |
| 232 | endlog |
| 233 | |
| 234 | # ------------------------------------------------------------------ config + migrations |
| 235 | log "Render config and migrate D1" |
| 236 | TURNSTILE_SITE_KEY="$TURNSTILE_SITE_KEY" node "$ROOT/scripts/render-config.mjs" --d1 "$D1_ID" ${TEAM:+--team "$TEAM"} ${AUD:+--aud "$AUD"} --public-url "https://$DOMAIN" ${DIFFINERY_ZONE:+--zone "$DIFFINERY_ZONE"} --mail-from "notifications@${MAIL_DOMAIN:-$DOMAIN}" --admins "$ADMINS" |
| 237 | "${WRANGLER[@]}" d1 migrations apply diffinery --remote -c wrangler.deploy.jsonc |
| 238 | endlog |
| 239 | |
| 240 | if [[ -n "${GITHUB_OUTPUT:-}" ]]; then |
| 241 | { echo "d1_id=$D1_ID"; echo "access_team=$TEAM"; echo "access_aud_set=${AUD:+true}"; } >>"$GITHUB_OUTPUT" |
| 242 | fi |