Diffinery

codician/diffinerypublic

Open-source, agent-first GitHub and GitHub Actions alternative that runs on your own Cloudflare account: git hosting, pull requests, code review and drop-in GitHub Actions YAML on Workers, Durable Objects, R2 and Containers.

Read D1 through the Sessions API, ready for read replication (#9)

Each request now runs its D1 queries in one session. Browsers keep the
latest bookmark in a cookie so a page loaded after a form post sees that
post's writes; token clients (git, CLI, MCP, runners) read from the primary
unless they send back the x-d1-bookmark header every response carries.
A new manual workflow checks the Sessions API on a throwaway database with
read replication on, then turns replication on for production.


Claude-Session: https://claude.ai/code/session_01AizKXSsengXZRegJxRATkL

Co-authored-by: Claude <noreply@anthropic.com>
Canberk Demirci authored · GitHub committed signed1 parent bc0428bcommit 55e6c724bb688e00c50dd0af7e1ec59839cf96b0Browse files
9 files changed+256−1
Files (9)
added.github/workflows/d1-replication.yml+108−0View file
@@ -1,0 +1,108 @@
1+name: D1 read replication
2+
3+# Checks the D1 Sessions API with read replication on a throwaway database, then
4+# (when asked) turns read replication on for the production database.
5+# Uses the repository secrets CLOUDFLARE_API_TOKEN (D1 and Workers Scripts: Edit)
6+# and CLOUDFLARE_ACCOUNT_ID.
7+on:
8+ workflow_dispatch:
9+ inputs:
10+ enable_production:
11+ description: Turn on read replication for the production database after the check passes
12+ type: boolean
13+ default: false
14+
15+permissions:
16+ contents: read
17+
18+jobs:
19+ check:
20+ name: Check sessions on a test database
21+ runs-on: ubuntu-latest
22+ timeout-minutes: 20
23+ env:
24+ CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
25+ CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
26+ NAME: diffinery-d1-probe-${{ github.run_id }}
27+ steps:
28+ - uses: actions/checkout@v4
29+ - uses: pnpm/action-setup@v4
30+ - uses: actions/setup-node@v4
31+ with:
32+ node-version: 22
33+ cache: pnpm
34+ - run: pnpm install --frozen-lockfile
35+ - name: Create a test database with read replication
36+ run: |
37+ set -euo pipefail
38+ API="https://api.cloudflare.com/client/v4/accounts/$CLOUDFLARE_ACCOUNT_ID"
39+ auth=(-H "Authorization: Bearer $CLOUDFLARE_API_TOKEN" -H "Content-Type: application/json")
40+ id=$(curl -sS -X POST "$API/d1/database" "${auth[@]}" --data "{\"name\":\"$NAME\"}" | jq -r '.result.uuid // empty')
41+ [ -n "$id" ] || { echo "::error::could not create the test database"; exit 1; }
42+ echo "D1_ID=$id" >> "$GITHUB_ENV"
43+ curl -sS -X PUT "$API/d1/database/$id" "${auth[@]}" --data '{"read_replication":{"mode":"auto"}}' | jq -e '.success' >/dev/null
44+ mode=$(curl -sS "$API/d1/database/$id" "${auth[@]}" | jq -r '.result.read_replication.mode')
45+ echo "read replication on the test database: $mode"
46+ [ "$mode" = auto ] || { echo "::error::read replication did not turn on"; exit 1; }
47+ - name: Deploy the probe Worker
48+ working-directory: scripts/d1-probe
49+ run: |
50+ set -euo pipefail
51+ cat > wrangler.json <<JSON
52+ {"name":"$NAME","main":"worker.js","compatibility_date":"2026-10-01","workers_dev":true,
53+ "d1_databases":[{"binding":"DB","database_name":"$NAME","database_id":"$D1_ID"}]}
54+ JSON
55+ W=(pnpm --dir "$GITHUB_WORKSPACE/apps/diffinery" exec wrangler)
56+ "${W[@]}" d1 execute "$NAME" --remote -c "$PWD/wrangler.json" --command "CREATE TABLE probe (k TEXT PRIMARY KEY, v TEXT NOT NULL)"
57+ "${W[@]}" deploy -c "$PWD/wrangler.json"
58+ sub=$(curl -sS "https://api.cloudflare.com/client/v4/accounts/$CLOUDFLARE_ACCOUNT_ID/workers/subdomain" -H "Authorization: Bearer $CLOUDFLARE_API_TOKEN" | jq -r '.result.subdomain // empty')
59+ [ -n "$sub" ] || { echo "::error::the account has no workers.dev subdomain"; exit 1; }
60+ echo "PROBE_URL=https://$NAME.$sub.workers.dev" >> "$GITHUB_ENV"
61+ - name: Read your own writes through bookmarks
62+ run: |
63+ set -uo pipefail
64+ for i in $(seq 1 30); do
65+ body=$(curl -sS "$PROBE_URL/check?rounds=25") && jq -e . >/dev/null <<<"$body" && break
66+ sleep 5
67+ done
68+ echo "$body" | jq .
69+ jq -e '.ok' >/dev/null <<<"$body" || { echo "::error::a read after a bookmark missed the write, or a query failed"; exit 1; }
70+ echo "every read from a bookmark saw its write ($(jq -r '.replicaReads' <<<"$body") served by replicas)"
71+ - name: Remove the probe and the test database
72+ if: always()
73+ run: |
74+ cd scripts/d1-probe
75+ [ -f wrangler.json ] && pnpm --dir "$GITHUB_WORKSPACE/apps/diffinery" exec wrangler delete -c "$PWD/wrangler.json" --force || true
76+ [ -n "${D1_ID:-}" ] && curl -sS -X DELETE "https://api.cloudflare.com/client/v4/accounts/$CLOUDFLARE_ACCOUNT_ID/d1/database/$D1_ID" -H "Authorization: Bearer $CLOUDFLARE_API_TOKEN" | jq -c '{success}' || true
77+
78+ enable:
79+ name: Turn on read replication in production
80+ needs: check
81+ if: ${{ inputs.enable_production }}
82+ runs-on: ubuntu-latest
83+ environment:
84+ name: production
85+ url: https://diffinery.com
86+ env:
87+ CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
88+ CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
89+ steps:
90+ - name: Enable and verify
91+ run: |
92+ set -euo pipefail
93+ API="https://api.cloudflare.com/client/v4/accounts/$CLOUDFLARE_ACCOUNT_ID"
94+ auth=(-H "Authorization: Bearer $CLOUDFLARE_API_TOKEN" -H "Content-Type: application/json")
95+ id=$(curl -sS "$API/d1/database?name=diffinery&per_page=100" "${auth[@]}" | jq -r '.result[]? | select(.name=="diffinery") | .uuid' | head -n1)
96+ [ -n "$id" ] || { echo "::error::production database not found"; exit 1; }
97+ curl -sS -X PUT "$API/d1/database/$id" "${auth[@]}" --data '{"read_replication":{"mode":"auto"}}' | jq -e '.success' >/dev/null
98+ mode=$(curl -sS "$API/d1/database/$id" "${auth[@]}" | jq -r '.result.read_replication.mode')
99+ echo "read replication on diffinery: $mode"
100+ [ "$mode" = auto ] || { echo "::error::read replication did not turn on"; exit 1; }
101+ - name: Check the site still reads and writes
102+ run: |
103+ set -euo pipefail
104+ for p in /healthz /login /sitemap.xml; do
105+ code=$(curl -sS -o /dev/null -w '%{http_code}' "https://diffinery.com$p"); echo "$p -> $code"; [ "$code" = 200 ]
106+ done
107+ bm=$(curl -sS -D - -o /dev/null https://diffinery.com/sitemap.xml | tr -d '\r' | sed -n 's/^x-d1-bookmark: //Ip')
108+ echo "x-d1-bookmark: $bm"; [ -n "$bm" ]
modified.github/workflows/deploy.yml+3−0View file
@@ -95,3 +95,6 @@
9595 grep -qF "$tag" <<<"$home" || { echo "::error::the home page is missing $tag"; exit 1; }
9696 done
9797 echo "the home page carries the favicon, canonical, Open Graph and structured data tags"
98+ bm=$(curl -sS -D - -o /dev/null https://diffinery.com/sitemap.xml | tr -d '\r' | sed -n 's/^x-d1-bookmark: //Ip')
99+ [ -n "$bm" ] || { echo "::error::D1 queries don't run in a session (no x-d1-bookmark header)"; exit 1; }
100+ echo "D1 sessions are on (bookmark $bm)"
modifiedREADME.md+1−1View file
@@ -25,7 +25,7 @@
2525 | HTTP, git smart HTTP, REST API, MCP, web UI | Workers (Hono) |
2626 | One repository: refs, objects, PRs, issues, reviews, checks, runs | Durable Object with SQLite (`RepoDO`), one per repository |
2727 | Large objects, Git LFS, logs, caches, artifacts, event archive | R2 |
28-| Accounts, orgs, tokens, invites, notifications, audit log | D1 |
28+| Accounts, orgs, tokens, invites, notifications, audit log | D1, read from nearby replicas through the Sessions API (each browser keeps a bookmark so it reads its own writes) |
2929 | Webhooks, notifications, audit fan-out | Queues (with a dead-letter queue) |
3030 | Workflow runs (durable, survive restarts, 6 h job timeout) | Workflows |
3131 | Job runners and agent sandboxes | Containers (Ubuntu 24.04 image in `containers/runner`) |
modifiedapps/diffinery/src/app.ts+15−0View file
@@ -1,9 +1,12 @@
11 import { Hono } from "hono";
2+import { getCookie } from "hono/cookie";
23 import type { DiffineryEvent } from "@diffinery/shared";
34 import { cronMatches } from "@diffinery/workflow-yaml";
45 import type { Env } from "./env.js";
56 import { credentialFrom, resolvePrincipal, type AppVars } from "./auth/principal.js";
67 import { Directory } from "./lib/directory.js";
8+import { BOOKMARK_COOKIE, BOOKMARK_HEADER, d1Session, sessionConstraint } from "./lib/d1.js";
9+import { SESSION_COOKIE } from "./auth/sessions.js";
710 import { hmacSha256Hex, newId } from "./lib/util.js";
811 import { errorResponse, loadRepo, repoStub } from "./http/common.js";
912 import { git } from "./http/git.js";
@@ -27,12 +30,24 @@
2730 return c.json({ message: "cross-site request refused" }, 403);
2831 }
2932 }
33+ // One D1 session per request: replica reads that still see this client's earlier writes (see lib/d1.ts).
34+ const token = !!credentialFrom(c.req.raw);
35+ const cookie = getCookie(c, BOOKMARK_COOKIE);
36+ const d1 = d1Session(c.env, sessionConstraint({ header: c.req.header(BOOKMARK_HEADER), cookie, token }));
37+ c.env = d1.env;
3038 const dir = new Directory(c.env);
3139 c.set("dir", dir);
3240 c.set("principal", await resolvePrincipal(c.req.raw, c.env, dir));
3341 await next();
3442 if (c.res.status === 101) return;
3543 const h = c.res.headers;
44+ const bookmark = d1.bookmark();
45+ if (bookmark) {
46+ h.set(BOOKMARK_HEADER, bookmark);
47+ // Browsers: remember it once they have written something or are signed in.
48+ const browser = !token && (!["GET", "HEAD"].includes(c.req.method) || cookie !== undefined || getCookie(c, SESSION_COOKIE) !== undefined);
49+ if (browser && bookmark !== cookie) h.append("set-cookie", `${BOOKMARK_COOKIE}=${bookmark}; Max-Age=86400; Path=/; HttpOnly; Secure; SameSite=Lax`);
50+ }
3651 h.set("strict-transport-security", "max-age=63072000; includeSubDomains; preload");
3752 h.set("x-content-type-options", "nosniff");
3853 h.set("referrer-policy", "strict-origin-when-cross-origin");
addedapps/diffinery/src/lib/d1.ts+39−0View file
@@ -1,0 +1,39 @@
1+import type { Env } from "../env.js";
2+
3+/**
4+ * D1 read replication through the Sessions API.
5+ *
6+ * Every request runs its D1 queries in one session, so reads can come from a nearby
7+ * replica while still seeing everything the same client wrote before:
8+ *
9+ * - Browsers carry the latest bookmark in a cookie. A request continues from it, so a
10+ * page loaded after a form post sees that post's writes (sign-in, new repositories…).
11+ * - Token clients (git, CLI, MCP, runners) don't keep cookies, so they read from the
12+ * primary unless they send back the `x-d1-bookmark` header from an earlier response.
13+ * - Anonymous first visits take the nearest copy ("first-unconstrained").
14+ *
15+ * Durable Objects, queue consumers and cron use the binding directly, which always
16+ * goes to the primary.
17+ */
18+export const BOOKMARK_COOKIE = "__Host-dfy_d1";
19+export const BOOKMARK_HEADER = "x-d1-bookmark";
20+
21+/** Bookmarks look like `00000085-0000024c-00004c6d-8e61117bf38d7adb71b934ebbf891683`. */
22+export const isBookmark = (s: string | null | undefined): s is string => !!s && s.length <= 128 && /^[0-9a-f]+(-[0-9a-f]+){1,7}$/i.test(s);
23+
24+export type D1Session = { env: Env; bookmark(): string | null };
25+
26+/** An env whose DB runs every query in one session that starts at `constraint`. */
27+export function d1Session(env: Env, constraint: string): D1Session {
28+ // Test doubles and old runtimes without sessions keep using the binding as is.
29+ if (typeof env.DB.withSession !== "function") return { env, bookmark: () => null };
30+ const session = env.DB.withSession(constraint);
31+ return { env: { ...env, DB: session as unknown as D1Database }, bookmark: () => session.getBookmark() };
32+}
33+
34+/** Where a request's session starts. */
35+export function sessionConstraint(o: { header?: string | null; cookie?: string | null; token: boolean }): string {
36+ if (isBookmark(o.header)) return o.header;
37+ if (!o.token && isBookmark(o.cookie)) return o.cookie;
38+ return o.token ? "first-primary" : "first-unconstrained";
39+}
modifiedapps/diffinery/test/e2e/e2e.test.mjs+17−0View file
@@ -511,3 +511,20 @@
511511 assert.match(JSON.stringify(hosted), /hosted Diffinery runners don't have/);
512512 assert.equal(jobs.find((j) => j.name === "boxed").conclusion, "success");
513513 });
514+
515+test("D1 sessions: bookmarks for token clients and browsers", async () => {
516+ // Token clients get the bookmark in a header and can send it back to read their own writes.
517+ const r = await fetch(`${base}/api/v3/user`, { headers: { authorization: `Bearer ${token}` } });
518+ assert.equal(r.status, 200);
519+ const bookmark = r.headers.get("x-d1-bookmark");
520+ assert.match(bookmark ?? "", /^[0-9a-f]+(-[0-9a-f]+)+$/i);
521+ assert.equal(r.headers.get("set-cookie"), null);
522+ const again = await fetch(`${base}/api/v3/user`, { headers: { authorization: `Bearer ${token}`, "x-d1-bookmark": bookmark } });
523+ assert.equal(again.status, 200);
524+ // A junk bookmark is ignored rather than breaking the request.
525+ assert.equal((await fetch(`${base}/api/v3/user`, { headers: { authorization: `Bearer ${token}`, "x-d1-bookmark": "nope" } })).status, 200);
526+ // Anonymous page views don't get a cookie; a browser that writes (any non-GET) does.
527+ assert.equal((await fetch(`${base}/`)).headers.get("set-cookie")?.includes("__Host-dfy_d1") ?? false, false);
528+ const post = await fetch(`${base}/auth/logout`, { method: "POST", redirect: "manual", headers: { "sec-fetch-site": "same-origin" } });
529+ assert.match(post.headers.get("set-cookie") ?? "", /__Host-dfy_d1=[0-9a-f-]+; Max-Age=86400; Path=\/; HttpOnly; Secure; SameSite=Lax/);
530+});
addedapps/diffinery/test/unit/d1.test.ts+36−0View file
@@ -1,0 +1,36 @@
1+import { describe, expect, it } from "vitest";
2+import { d1Session, isBookmark, sessionConstraint } from "../../src/lib/d1.js";
3+import type { Env } from "../../src/env.js";
4+
5+const BM = "00000085-0000024c-00004c6d-8e61117bf38d7adb71b934ebbf891683";
6+
7+describe("D1 sessions", () => {
8+ it("recognises bookmarks and nothing else", () => {
9+ expect(isBookmark(BM)).toBe(true);
10+ expect(isBookmark("first-primary")).toBe(false);
11+ expect(isBookmark("x; Path=/")).toBe(false);
12+ expect(isBookmark(`${BM}${"0".repeat(200)}`)).toBe(false);
13+ expect(isBookmark(undefined)).toBe(false);
14+ });
15+
16+ it("starts browsers from their cookie and token clients from the primary", () => {
17+ expect(sessionConstraint({ token: false })).toBe("first-unconstrained");
18+ expect(sessionConstraint({ token: false, cookie: BM })).toBe(BM);
19+ expect(sessionConstraint({ token: false, cookie: "junk" })).toBe("first-unconstrained");
20+ expect(sessionConstraint({ token: true, cookie: BM })).toBe("first-primary");
21+ expect(sessionConstraint({ token: true, header: BM })).toBe(BM);
22+ expect(sessionConstraint({ token: false, header: BM, cookie: "00000001-00000001" })).toBe(BM);
23+ });
24+
25+ it("wraps the binding in a session and reports its bookmark", () => {
26+ const seen: string[] = [];
27+ const DB = { withSession: (c: string) => (seen.push(c), { prepare: () => null, batch: async () => [], getBookmark: () => BM }) };
28+ const s = d1Session({ DB } as unknown as Env, "first-primary");
29+ expect(seen).toEqual(["first-primary"]);
30+ expect(s.bookmark()).toBe(BM);
31+ expect(typeof s.env.DB.prepare).toBe("function");
32+ // Without withSession (test doubles), the binding is used as is.
33+ const plain = { prepare: () => null } as unknown as D1Database;
34+ expect(d1Session({ DB: plain } as unknown as Env, "first-primary").env.DB).toBe(plain);
35+ });
36+});
modifiedscripts/bootstrap.sh+1−0View file
@@ -42,6 +42,7 @@
4242 else
4343 echo "D1 database exists: $D1_ID"
4444 fi
45+echo "read replication: $(cf GET "/d1/database/$D1_ID" | jq -r '.result.read_replication.mode // "unknown"') (turn it on with the 'D1 read replication' workflow)"
4546 endlog
4647
4748 # ------------------------------------------------------------------ R2
addedscripts/d1-probe/worker.js+36−0View file
@@ -1,0 +1,36 @@
1+// Probe for .github/workflows/d1-replication.yml: checks the D1 Sessions API on a
2+// throwaway database with read replication on, the way apps/diffinery uses it
3+// (src/lib/d1.ts): write in one session, then read in a new session started from
4+// the first one's bookmark, and expect to see the write every time.
5+export default {
6+ async fetch(req, env) {
7+ const url = new URL(req.url);
8+ if (url.pathname !== "/check") return new Response("not found", { status: 404 });
9+ const rounds = Number(url.searchParams.get("rounds") ?? 20);
10+ const out = { rounds, stale: 0, replicaReads: 0, primaryReads: 0, regions: {}, errors: [] };
11+ for (let i = 1; i <= rounds; i++) {
12+ try {
13+ const w = env.DB.withSession("first-primary");
14+ const value = `${Date.now()}-${i}-${crypto.randomUUID()}`;
15+ await w.prepare("INSERT INTO probe (k, v) VALUES ('latest', ?) ON CONFLICT (k) DO UPDATE SET v = excluded.v").bind(value).run();
16+ const bookmark = w.getBookmark();
17+ // A later request: new session from the bookmark (as a browser cookie would carry it).
18+ const r = env.DB.withSession(bookmark);
19+ const row = await r.prepare("SELECT v FROM probe WHERE k = 'latest'").first();
20+ const res = await r.prepare("SELECT v FROM probe WHERE k = 'latest'").run();
21+ const meta = res.meta ?? {};
22+ if (row?.v !== value) out.stale++;
23+ if (meta.served_by_primary === false) out.replicaReads++;
24+ else out.primaryReads++;
25+ const region = meta.served_by_region ?? "unknown";
26+ out.regions[region] = (out.regions[region] ?? 0) + 1;
27+ // An unconstrained session may read an older value, but must never fail.
28+ await env.DB.withSession("first-unconstrained").prepare("SELECT v FROM probe WHERE k = 'latest'").first();
29+ } catch (e) {
30+ out.errors.push(String(e));
31+ }
32+ }
33+ out.ok = out.stale === 0 && out.errors.length === 0;
34+ return Response.json(out, { status: out.ok ? 200 : 500 });
35+ },
36+};