Faster main syncs: fix the registry relay's slow uploads, gate Deploy on CI (#46)
The registry relay re-copied the pending chunk for every received piece (a 150 MB layer took 332s locally, now about 1s). Deploy follows CI with workflow_run, deploys only commits CI passed, and no longer repeats typecheck and tests; the e2e suite moves to e2e.yml. The 50,000-block test pages one snapshot and has a 20s timeout. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Ruyx3Qr4jYrZEpPi466deb
Canberk Demirci authored · GitHub committed signed1 parent 59f5247commit 6d95237237e322aef1b6254551e9fc5bc25b73dbBrowse files
6 files changed+68−31
Files (6)
modified.github/workflows/ci.yml+3−16View file
| @@ -1,5 +1,8 @@ | ||
| 1 | 1 | name: CI |
| 2 | 2 | |
| 3 | +# Deploy waits for this workflow to pass on main. The end-to-end suite is its own | |
| 4 | +# workflow (e2e.yml), so deploys don't wait for it too. | |
| 5 | + | |
| 3 | 6 | on: |
| 4 | 7 | push: |
| 5 | 8 | branches: [main] |
| @@ -34,19 +37,3 @@ | ||
| 34 | 37 | - name: Bundle the Worker (dry run) |
| 35 | 38 | working-directory: apps/diffinery |
| 36 | 39 | run: pnpm exec wrangler deploy --dry-run --env dev --outdir dist |
| 37 | - | |
| 38 | - e2e: | |
| 39 | - name: End-to-end (git, PRs, Actions, passkeys) | |
| 40 | - runs-on: ubuntu-latest | |
| 41 | - timeout-minutes: 20 | |
| 42 | - steps: | |
| 43 | - - uses: actions/checkout@v4 | |
| 44 | - - uses: pnpm/action-setup@v4 | |
| 45 | - - uses: actions/setup-node@v4 | |
| 46 | - with: | |
| 47 | - node-version: 22 | |
| 48 | - cache: pnpm | |
| 49 | - - run: pnpm install --frozen-lockfile | |
| 50 | - - run: pnpm -r --if-present build | |
| 51 | - - run: pnpm --filter @diffinery/app exec playwright install --with-deps chromium | |
| 52 | - - run: pnpm --filter @diffinery/app e2e |
modified.github/workflows/deploy.yml+10−4View file
| @@ -7,8 +7,13 @@ | ||
| 7 | 7 | # DIFFINERY_ZONE the Cloudflare zone for the route (default: the domain's last two labels) |
| 8 | 8 | # MAIL_DOMAIN sends from notifications@<MAIL_DOMAIN> (default DIFFINERY_DOMAIN) |
| 9 | 9 | # ACCESS_TEAM_DOMAIN e.g. codician.cloudflareaccess.com |
| 10 | +# | |
| 11 | +# It deploys a main commit only once CI (typecheck, unit tests and the Worker bundle) has | |
| 12 | +# passed on it, so it doesn't run those itself. End-to-end tests run alongside the deploy. | |
| 10 | 13 | on: |
| 11 | - push: | |
| 14 | + workflow_run: | |
| 15 | + workflows: [CI] | |
| 16 | + types: [completed] | |
| 12 | 17 | branches: [main] |
| 13 | 18 | workflow_dispatch: |
| 14 | 19 | |
| @@ -25,7 +30,7 @@ | ||
| 25 | 30 | # Production deploys from Diffinery itself, on its own hosted runners (any |
| 26 | 31 | # Diffinery, so self-hosted copies deploy themselves too). Running this |
| 27 | 32 | # workflow by hand on GitHub still deploys, as a fallback. |
| 28 | - if: ${{ github.server_url != 'https://github.com' || github.event_name == 'workflow_dispatch' }} | |
| 33 | + if: ${{ github.event_name == 'workflow_dispatch' || (github.server_url != 'https://github.com' && github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.event == 'push') }} | |
| 29 | 34 | runs-on: ubuntu-latest |
| 30 | 35 | timeout-minutes: 45 |
| 31 | 36 | environment: |
| @@ -40,6 +45,9 @@ | ||
| 40 | 45 | ACCESS_TEAM_DOMAIN: ${{ vars.ACCESS_TEAM_DOMAIN }} |
| 41 | 46 | steps: |
| 42 | 47 | - uses: actions/checkout@v4 |
| 48 | + with: | |
| 49 | + # The commit CI passed on, not whatever main is by now. | |
| 50 | + ref: ${{ github.event.workflow_run.head_sha || github.sha }} | |
| 43 | 51 | - uses: pnpm/action-setup@v4 |
| 44 | 52 | - uses: actions/setup-node@v4 |
| 45 | 53 | with: |
| @@ -48,8 +56,6 @@ | ||
| 48 | 56 | - run: pnpm install --frozen-lockfile |
| 49 | 57 | - name: Build runner agent and web islands |
| 50 | 58 | run: pnpm -r --if-present build |
| 51 | - - name: Typecheck and test | |
| 52 | - run: pnpm -r --if-present typecheck && pnpm -r --if-present test | |
| 53 | 59 | |
| 54 | 60 | - name: Create or update Cloudflare resources |
| 55 | 61 | id: bootstrap |
added.github/workflows/e2e.yml+30−0View file
| @@ -1,0 +1,30 @@ | ||
| 1 | +name: End-to-end | |
| 2 | + | |
| 3 | +on: | |
| 4 | + push: | |
| 5 | + branches: [main] | |
| 6 | + pull_request: | |
| 7 | + | |
| 8 | +permissions: | |
| 9 | + contents: read | |
| 10 | + | |
| 11 | +concurrency: | |
| 12 | + group: e2e-${{ github.ref }} | |
| 13 | + cancel-in-progress: true | |
| 14 | + | |
| 15 | +jobs: | |
| 16 | + e2e: | |
| 17 | + name: End-to-end (git, PRs, Actions, passkeys) | |
| 18 | + runs-on: ubuntu-latest | |
| 19 | + timeout-minutes: 20 | |
| 20 | + steps: | |
| 21 | + - uses: actions/checkout@v4 | |
| 22 | + - uses: pnpm/action-setup@v4 | |
| 23 | + - uses: actions/setup-node@v4 | |
| 24 | + with: | |
| 25 | + node-version: 22 | |
| 26 | + cache: pnpm | |
| 27 | + - run: pnpm install --frozen-lockfile | |
| 28 | + - run: pnpm -r --if-present build | |
| 29 | + - run: pnpm --filter @diffinery/app exec playwright install --with-deps chromium | |
| 30 | + - run: pnpm --filter @diffinery/app e2e |
modifiedREADME.md+4−2View file
| @@ -60,8 +60,10 @@ | ||
| 60 | 60 | |
| 61 | 61 | Diffinery deploys itself: this repository lives at |
| 62 | 62 | `diffinery.com/codician-team/diffinery`, and every push to `main` there runs |
| 63 | -`.github/workflows/deploy.yml` on Diffinery's hosted runners, followed by the | |
| 64 | -live test (`.github/workflows/live-test.yml`). GitHub stays the upstream for | |
| 63 | +CI (`.github/workflows/ci.yml`: typecheck, unit tests and the Worker bundle) on | |
| 64 | +Diffinery's hosted runners. When it passes, `.github/workflows/deploy.yml` | |
| 65 | +deploys that commit, followed by the live test (`.github/workflows/live-test.yml`); | |
| 66 | +the end-to-end suite (`.github/workflows/e2e.yml`) runs alongside. GitHub stays the upstream for | |
| 65 | 67 | now: `.github/workflows/sync-to-diffinery.yml` copies every branch and tag, and |
| 66 | 68 | the secrets and variables below, to Diffinery on each push, then waits for the |
| 67 | 69 | Actions runs there and fails if any of them fail. Running Deploy by hand on |
modifiedapps/diffinery/test/unit/actions-runtime.test.ts+6−2View file
| @@ -101,6 +101,7 @@ | ||
| 101 | 101 | const uploads = new Map<string, { key: string; parts: Map<number, Uint8Array> }>(); |
| 102 | 102 | const stats = { multipart: 0, parts: [] as number[], ops: 0 }; |
| 103 | 103 | let n = 0; |
| 104 | + let listing: { prefix: string; all: Array<{ key: string; size: number }> } | undefined; | |
| 104 | 105 | const bytes = async (v: ReadableStream | ArrayBuffer | ArrayBufferView) => |
| 105 | 106 | v instanceof ReadableStream ? new Uint8Array(await new Response(v).arrayBuffer()) : v instanceof ArrayBuffer ? new Uint8Array(v) : new Uint8Array(v.buffer, v.byteOffset, v.byteLength).slice(); |
| 106 | 107 | const store = (key: string, data: Uint8Array) => { |
| @@ -158,8 +159,10 @@ | ||
| 158 | 159 | }, |
| 159 | 160 | async list({ prefix, cursor, limit }) { |
| 160 | 161 | stats.ops++; |
| 161 | - const all = [...objects].filter(([k]) => k.startsWith(prefix)).map(([key, v]) => ({ key, size: v.data.length })); | |
| 162 | + // A listing pages through one snapshot (the cursor is an index into it), so 50k objects aren't filtered again for every page. | |
| 162 | 163 | const start = Number(cursor ?? 0); |
| 164 | + if (!start || listing?.prefix !== prefix) listing = { prefix, all: [...objects].filter(([k]) => k.startsWith(prefix)).map(([key, v]) => ({ key, size: v.data.length })) }; | |
| 165 | + const all = listing.all; | |
| 163 | 166 | const page = all.slice(start, start + (limit ?? 1000)); |
| 164 | 167 | const truncated = start + page.length < all.length; |
| 165 | 168 | return { objects: page, truncated, cursor: truncated ? String(start + page.length) : undefined }; |
| @@ -279,7 +282,8 @@ | ||
| 279 | 282 | expect(plan.parts.length).toBeLessThanOrEqual(10_000); |
| 280 | 283 | expect(plan.parts.slice(0, -1).every((p) => p.reduce((n, s) => n + s.length, 0) === plan.partSize)).toBe(true); |
| 281 | 284 | } |
| 282 | - }); | |
| 285 | + // Writes and composes 50k objects: about 1.5s alone, and several times that on a busy CI runner. | |
| 286 | + }, 20_000); | |
| 283 | 287 | }); |
| 284 | 288 | |
| 285 | 289 | describe("deleting a repository's transient objects", () => { |
modifiedcontainers/runner/src/relay.ts+15−7View file
| @@ -98,7 +98,17 @@ | ||
| 98 | 98 | const declaredStart = /^(?:bytes[ =])?(\d+)-/.exec(String(req.headers["content-range"] ?? "")); |
| 99 | 99 | if (declaredStart && Number(declaredStart[1]) !== offset) return reply(res, status, 416); |
| 100 | 100 | |
| 101 | - let buf: Buffer = Buffer.alloc(0); | |
| 101 | + // Pieces received since the last chunk went out. Joining them only when a chunk is full | |
| 102 | + // copies each byte once; growing one buffer per piece copied whole chunks thousands of times. | |
| 103 | + let pieces: Buffer[] = []; | |
| 104 | + let held = 0; | |
| 105 | + const take = (n: number): Buffer => { | |
| 106 | + const all = pieces.length === 1 ? pieces[0]! : Buffer.concat(pieces, held); | |
| 107 | + const rest = all.subarray(n); | |
| 108 | + pieces = rest.length ? [rest] : []; | |
| 109 | + held = rest.length; | |
| 110 | + return all.subarray(0, n); | |
| 111 | + }; | |
| 102 | 112 | let last: Awaited<ReturnType<typeof send>> | null = null; |
| 103 | 113 | const flush = async (data: Buffer) => { |
| 104 | 114 | for (let attempt = 1; ; attempt++) { |
| @@ -128,13 +138,11 @@ | ||
| 128 | 138 | } |
| 129 | 139 | }; |
| 130 | 140 | for await (const piece of req as AsyncIterable<Buffer>) { |
| 131 | - buf = buf.length ? Buffer.concat([buf, piece]) : piece; | |
| 132 | - while (buf.length >= chunkSize) { | |
| 133 | - const chunk = buf.subarray(0, chunkSize); | |
| 134 | - buf = Buffer.from(buf.subarray(chunkSize)); | |
| 135 | - if (!(await flush(chunk))) return reply(res, last!); | |
| 136 | - } | |
| 141 | + pieces.push(piece); | |
| 142 | + held += piece.length; | |
| 143 | + while (held >= chunkSize) if (!(await flush(take(chunkSize)))) return reply(res, last!); | |
| 137 | 144 | } |
| 145 | + const buf = take(held); | |
| 138 | 146 | if (req.method === "PUT") { |
| 139 | 147 | const r = await send("PUT", `${location}${url.search}`, { ...headers, ...(buf.length ? { "content-type": "application/octet-stream", "content-range": `${offset}-${offset + buf.length - 1}` } : {}) }, buf); |
| 140 | 148 | return reply(res, r); |