Diffinery

codician/diffinerypublic

Open-source, agent-first GitHub and GitHub Actions alternative that runs on your own Cloudflare account: git hosting, pull requests, code review and drop-in GitHub Actions YAML on Workers, Durable Objects, R2 and Containers.

Scale: D1 indexes and retention, affiliated repository lists, batched notification reads, size-cut mail batches (#190)

* WIP (paused): D1 indexes and retention, affiliated repo lists, batched notification reads, size-cut mail batches
* Keep roleFor's comment on roleFor
* Exports still cover public repositories; run emails and oversized jobs fit the queue's message limit
test-bot authored 1 parent 1998051commit be909c86151b78b94d76ad5cb2332b14ffd64f29Browse files
17 files changed+397−57
Files (17)
modifiedapps/diffinery/docs/reference/limits.md+2−2View file
@@ -56,7 +56,7 @@
5656 | Commits searched for a path | The latest 2,000 commits | `GET /repos/{owner}/{repo}/commits?path=...` and the history of a file in the web UI only look at the 2,000 most recent commits. |
5757 | Package versions per page | 30 by default, 100 at most | `GET .../packages/container/{name}/versions`. |
5858 | Repository lists | 500 repositories | `GET /user/repos`, `/users/{login}/repos`, `/orgs/{org}/repos` page over at most 500 repositories (most recently pushed first). |
59-| Notifications | 100 | `GET /api/v3/notifications` returns your 100 most recent (unread unless `all=true`). |
59+| Notifications | 100 | `GET /api/v3/notifications` returns your 100 most recent (unread unless `all=true`). Notifications older than 180 days are deleted. |
6060 | Email log | 50 by default, 500 at most | `GET /api/v3/diffinery/emails?per_page=`. |
6161 | Job log lines per call while a job runs | 5,000 | `GET /repos/{owner}/{repo}/actions/jobs/{id}/logs`. Pass `after=<seq>` with `format=json` to read the next lines. A finished job's log comes back whole. |
6262 | Recursive tree listing | 100,000 entries | `GET .../git/trees/{sha}?recursive=1` stops there and returns `truncated: true`. |
@@ -356,7 +356,7 @@
356356 | Limit | Value | Applies to |
357357 | --- | --- | --- |
358358 | Email log retention | 90 days | **Settings → Email** and `GET /api/v3/diffinery/emails` show emails from the last 90 days. Older entries are deleted once a day. |
359-| Audit log retention | 2 years | Entries older than 730 days are deleted once a day (see [Audit log](/docs/security/audit-log#how-long-entries-are-kept)). |
359+| Audit log retention | 2 years | Entries older than 730 days are deleted every hour (see [Audit log](/docs/security/audit-log#how-long-entries-are-kept)). |
360360 | Sending retries | 6 attempts | A temporary sending failure (rate limit, internal error or delivery failure reported by Cloudflare) is retried with backoff (starting at 30 seconds and doubling, at most 10 minutes). Other failures are logged as failed at once. After 6 attempts the email is logged as failed. |
361361 | Message body | 20,000 characters | Longer issue, pull request or comment text in an email is cut and ends with `…`. |
362362 | Subject | 900 characters | |
modifiedapps/diffinery/docs/security/audit-log.md+1−1View file
@@ -27,7 +27,7 @@
2727
2828 ## How long entries are kept
2929
30-Entries are kept for 2 years (730 days). Older ones are deleted once a day, at about 03:00 UTC. An account deletion doesn't remove the deleted account's entries; they go when they reach 2 years like the rest.
30+Entries are kept for 2 years (730 days). Older ones are deleted every hour. An account deletion doesn't remove the deleted account's entries; they go when they reach 2 years like the rest.
3131
3232 ## Entry fields
3333
addedapps/diffinery/migrations/0035_scale_indexes.sql+16−0View file
@@ -1,0 +1,16 @@
1+-- Indexes for D1 queries that scanned whole tables as they grew (scalability evaluation, 2026-10-14).
2+-- Suspended people, read on every issue and pull request page (lib/moderation.ts suspendedIds).
3+CREATE INDEX IF NOT EXISTS accounts_suspended ON accounts(suspended_at) WHERE suspended_at IS NOT NULL;
4+-- Pending invitations by address, read at every sign-up and sign-in (lib/directory.ts).
5+CREATE INDEX IF NOT EXISTS invites_email ON invites(email);
6+-- An owner's imports in progress (lib/directory.ts), and the cron that expires drafts.
7+CREATE INDEX IF NOT EXISTS imports_target_owner ON imports(target_owner_id, status);
8+CREATE INDEX IF NOT EXISTS imports_status ON imports(status, created_at);
9+-- Token expiry reminders (mail/notify.ts mailCron, every five minutes).
10+CREATE INDEX IF NOT EXISTS tokens_expires ON tokens(expires_at);
11+-- Container registry cleanup (registry/store.ts, every five minutes).
12+CREATE INDEX IF NOT EXISTS registry_blobs_created ON registry_blobs(created_at);
13+CREATE INDEX IF NOT EXISTS package_blobs_created ON package_blobs(created_at);
14+-- Retention (lib/retention.ts): expired sessions and old notifications are deleted.
15+CREATE INDEX IF NOT EXISTS sessions_expires ON sessions(expires_at);
16+CREATE INDEX IF NOT EXISTS notifications_created ON notifications(created_at);
modifiedapps/diffinery/src/accounts/export.ts+7−2View file
@@ -109,6 +109,9 @@
109109 }
110110 }
111111
112+/** Public repositories of other people an export looks through for what the account wrote. */
113+const EXPORT_PUBLIC_REPOS = 1000;
114+
112115 /** First step: mark the export running and list the repositories it covers. */
113116 export async function startExport(env: Env, id: string): Promise<{ owned: string[]; visible: string[] }> {
114117 const r = await row(env, id);
@@ -116,8 +119,10 @@
116119 await env.DB.prepare("UPDATE account_exports SET status = 'running' WHERE id = ?").bind(id).run();
117120 const dir = new Directory(env);
118121 const owned = (await env.DB.prepare("SELECT id FROM repos WHERE owner_id = ? ORDER BY name").bind(account.id).all<{ id: string }>()).results.map((x) => x.id);
119- const visible = (await dir.reposVisibleTo(account)).map((x) => x.id);
120- return { owned, visible: [...new Set([...owned, ...visible])] };
122+ // Issues and pull requests the account wrote: in the repositories it works with, and in public
123+ // ones (reposVisibleTo leaves those out), the most recently pushed first.
124+ const [visible, open] = await Promise.all([dir.reposVisibleTo(account), dir.publicRepos(EXPORT_PUBLIC_REPOS)]);
125+ return { owned, visible: [...new Set([...owned, ...visible.map((x) => x.id), ...open.map((x) => x.id)])] };
121126 }
122127
123128 /** account.json: the profile and settings. Tokens and passkeys by name only: nothing in the archive signs anyone in. */
modifiedapps/diffinery/src/agentic/today.ts+10−6View file
@@ -186,12 +186,16 @@
186186 // The morning digest passes its own start: the last digest, which can be days ago after a weekend.
187187 const since = opts.since ?? new Date(now.getTime() - SINCE_HOURS * 3600_000).toISOString();
188188 const cutoff = new Date(now.getTime() - ACTIVE_DAYS * 86400_000).toISOString();
189- const [visible, activityRows, orgs] = await Promise.all([
190- dir.reposVisibleTo(me),
191- env.DB.prepare("SELECT repo_id, at FROM repo_activity WHERE at >= ?").bind(cutoff).all<{ repo_id: string; at: string }>(),
192- me.kind === "user" ? dir.orgsFor(me.id) : Promise.resolve([]),
193- ]);
194- const activity = new Map(activityRows.results.map((r) => [r.repo_id, r.at]));
189+ const [visible, orgs] = await Promise.all([dir.reposVisibleTo(me), me.kind === "user" ? dir.orgsFor(me.id) : Promise.resolve([])]);
190+ // Activity of these repositories only, not of every repository active on the instance.
191+ const activity = new Map<string, string>();
192+ for (let i = 0; i < visible.length; i += 90) {
193+ const ids = visible.slice(i, i + 90).map((r) => r.id);
194+ const rows = await env.DB.prepare(`SELECT repo_id, at FROM repo_activity WHERE repo_id IN (${ids.map(() => "?").join(", ")}) AND at >= ?`)
195+ .bind(...ids, cutoff)
196+ .all<{ repo_id: string; at: string }>();
197+ for (const r of rows.results) activity.set(r.repo_id, r.at);
198+ }
195199 const allowed = opts.repoIds ? visible.filter((r) => opts.repoIds!.includes(r.id)) : visible;
196200 const { repos, skipped } = pickRepos(allowed, activity, now);
197201 const owners = me.kind === "agent" ? [] : [me.id, ...orgs.map((o) => o.id)];
modifiedapps/diffinery/src/app.ts+5−2View file
@@ -4,6 +4,7 @@
44 import type { Env } from "./env.js";
55 import { credentialFrom, resolvePrincipal, type AppVars } from "./auth/principal.js";
66 import { Directory, pruneAuditLog } from "./lib/directory.js";
7+import { pruneExpired } from "./lib/retention.js";
78 import { getFirst } from "./lib/storage-keys.js";
89 import { BOOKMARK_COOKIE, BOOKMARK_HEADER, d1Session, sessionConstraint } from "./lib/d1.js";
910 import { SESSION_COOKIE } from "./auth/sessions.js";
@@ -251,8 +252,10 @@
251252 schedulerStub(env).ensure().catch((e) => console.error("scheduler alarm check failed", e)),
252253 // Token expiry reminders and email log pruning.
253254 mailCron(env, new Directory(env), at).catch((e) => console.error("mail cron failed", e)),
254- // Audit log entries past their 2 years, once a day.
255- (at.getUTCHours() === 3 && at.getUTCMinutes() < 5 ? pruneAuditLog(env.DB, at) : Promise.resolve(0)).catch((e) => console.error("audit log pruning failed", e)),
255+ // Audit log entries past their 2 years, and expired sessions, old notifications and compute
256+ // records (lib/retention.ts): a bounded batch each hour, which keeps up with a busy instance.
257+ (at.getUTCMinutes() < 5 ? pruneAuditLog(env.DB, at) : Promise.resolve(0)).catch((e) => console.error("audit log pruning failed", e)),
258+ (at.getUTCMinutes() < 5 ? pruneExpired(env.DB, at) : Promise.resolve(null)).catch((e) => console.error("retention pruning failed", e)),
256259 // CLI sign-ins a day past their expiry (auth/cli-login.ts).
257260 new CliLogins(env).prune().catch((e) => console.error("cli sign-in pruning failed", e)),
258261 // Account exports past their 7 days (accounts/export.ts).
modifiedapps/diffinery/src/http/search.ts+11−5View file
@@ -26,18 +26,23 @@
2626 * candidate is checked again with callerFor, which applies roles, token
2727 * scopes and repository restrictions.
2828 */
29-async function candidates(c: AppContext): Promise<Repo[]> {
29+async function candidates(c: AppContext, q?: { text: string[]; owners: string[] }): Promise<Repo[]> {
3030 const p = c.get("principal");
3131 const dir = c.get("dir");
32+ // Public repositories are looked up by the query's words and owners in D1, not taken from a fixed
33+ // number of the most recent ones, so any of them can be found however many there are.
34+ const publicOnes = () => dir.publicRepos(500, { words: q?.text.filter((w) => !w.startsWith("-")), owners: q?.owners });
3235 if (p.type === "account") {
3336 if (p.account.status !== "active") return [];
34- return dir.reposVisibleTo(p.account);
37+ const [mine, open] = await Promise.all([dir.reposVisibleTo(p.account), publicOnes()]);
38+ const seen = new Set(mine.map((r) => r.id));
39+ return [...mine, ...open.filter((r) => !seen.has(r.id))];
3540 }
3641 if (p.type === "repo-token") {
3742 const r = await dir.repoById(p.repoId);
3843 return r ? [r] : [];
3944 }
40- return dir.publicRepos(500);
45+ return publicOnes();
4146 }
4247
4348 /** The first `limit` repositories (in order) the caller can actually read, and whether more were left unchecked. */
@@ -63,7 +68,8 @@
6368 const q = parseIssueQuery(queryOf(c), { sort: c.req.query("sort"), order: c.req.query("order") });
6469 const p = pageOf(c);
6570 if (p.offset >= MAX_RESULTS) throw new HttpError(422, "Only the first 1000 search results are available");
66- let repos = await candidates(c);
71+ const owners = [...q.owners, ...q.repos.map((r) => r.split("/")[0]!)];
72+ let repos = await candidates(c, { text: [], owners });
6773 if (q.repos.length) {
6874 const want = new Set(q.repos.map((r) => r.toLowerCase()));
6975 repos = repos.filter((r) => want.has(fullName(r)));
@@ -112,7 +118,7 @@
112118 const fields = q.in.length ? q.in : ["name", "description", "topics"];
113119 const owners = new Set(q.owners.map((o) => o.toLowerCase()));
114120 const names = new Set(q.repos.map((r) => r.toLowerCase()));
115- const filtered = (await candidates(c)).filter((r) => {
121+ const filtered = (await candidates(c, q)).filter((r) => {
116122 if (owners.size && !owners.has(r.owner_login.toLowerCase())) return false;
117123 if (names.size && !names.has(fullName(r))) return false;
118124 if (q.visibility && (q.visibility === "public") !== (r.visibility === "public")) return false;
modifiedapps/diffinery/src/lib/directory.ts+93−16View file
@@ -712,7 +712,13 @@
712712 return this.db.prepare("SELECT r.*, a.login AS owner_login, a.kind AS owner_kind, a.suspended_at AS owner_suspended_at FROM repos r JOIN accounts a ON a.id = r.owner_id WHERE r.id = ?").bind(id).first<Repo>();
713713 }
714714
715- /** Repositories an account can see. */
715+ /**
716+ * Repositories an account works with, most recently pushed first (at most 500): its own, its
717+ * organizations' (as its role there allows) and those shared with it, like GitHub's /user/repos.
718+ * With `ownerId`, that owner's repositories the account can see, public ones included. Without it,
719+ * other people's public repositories aren't listed: there may be any number of them (see
720+ * publicRepos), and listing them pushed the account's own out of the 500.
721+ */
716722 async reposVisibleTo(account: Account, ownerId?: string): Promise<Repo[]> {
717723 if (account.status !== "active") return [];
718724 if (account.kind === "agent") {
@@ -722,28 +728,46 @@
722728 const params: unknown[] = [];
723729 let q = "SELECT r.*, a.login AS owner_login, a.kind AS owner_kind, a.suspended_at AS owner_suspended_at FROM repos r JOIN accounts a ON a.id = r.owner_id";
724730 const subjectId = account.kind === "agent" ? account.owner_id! : account.id;
725- if (!account.site_admin) {
726- q += ` WHERE (r.visibility = 'public' OR r.owner_id = ? OR r.owner_id IN (SELECT m.org_id FROM org_members m JOIN accounts o ON o.id = m.org_id WHERE m.user_id = ? AND (m.role != 'member' OR COALESCE(o.base_role, 'read') != 'none')) OR r.id IN (SELECT repo_id FROM repo_access WHERE account_id IN (?, ?)))`;
727- params.push(subjectId, subjectId, account.id, subjectId);
728- // A suspended person's repositories are hidden from everyone but site administrators.
729- q += " AND a.suspended_at IS NULL";
730- } else q += " WHERE 1 = 1";
731731 if (ownerId) {
732- q += " AND r.owner_id = ?";
732+ q += " WHERE r.owner_id = ?";
733733 params.push(ownerId);
734+ if (!account.site_admin) {
735+ q += ` AND (r.visibility = 'public' OR r.owner_id = ? OR r.owner_id IN (SELECT m.org_id FROM org_members m JOIN accounts o ON o.id = m.org_id WHERE m.user_id = ? AND (m.role != 'member' OR COALESCE(o.base_role, 'read') != 'none')) OR r.id IN (SELECT repo_id FROM repo_access WHERE account_id IN (?, ?)))`;
736+ params.push(subjectId, subjectId, account.id, subjectId);
737+ }
738+ } else {
739+ // Each part is an indexed lookup (repos by owner, org_members by user, repo_access by account).
740+ q += ` WHERE r.id IN (SELECT id FROM repos WHERE owner_id = ?
741+ UNION SELECT o.id FROM org_members m JOIN accounts oa ON oa.id = m.org_id JOIN repos o ON o.owner_id = m.org_id WHERE m.user_id = ? AND (m.role != 'member' OR COALESCE(oa.base_role, 'read') != 'none' OR o.visibility = 'public')
742+ UNION SELECT repo_id FROM repo_access WHERE account_id IN (?, ?))`;
743+ params.push(subjectId, subjectId, account.id, subjectId);
734744 }
745+ // A suspended person's repositories are hidden from everyone but site administrators.
746+ if (!account.site_admin) q += " AND a.suspended_at IS NULL";
735747 q += " ORDER BY COALESCE(r.pushed_at, r.updated_at) DESC LIMIT 500";
736748 return (await this.db.prepare(q).bind(...params).all<Repo>()).results;
737749 }
738750
739- /** Public repositories, most recently pushed first (for the sitemap). */
740- async publicRepos(limit: number): Promise<Repo[]> {
741- return (
742- await this.db
743- .prepare("SELECT r.*, a.login AS owner_login, a.kind AS owner_kind, a.suspended_at AS owner_suspended_at FROM repos r JOIN accounts a ON a.id = r.owner_id WHERE r.visibility = 'public' AND a.suspended_at IS NULL ORDER BY COALESCE(r.pushed_at, r.updated_at) DESC LIMIT ?")
744- .bind(limit)
745- .all<Repo>()
746- ).results;
751+ /**
752+ * Public repositories, most recently pushed first (for the sitemap and search). `filter.words` keeps
753+ * those whose owner/name, description or topics contain every word; `filter.owners` those of these
754+ * owners (logins).
755+ */
756+ async publicRepos(limit: number, filter: { words?: string[]; owners?: string[] } = {}): Promise<Repo[]> {
757+ let q = "SELECT r.*, a.login AS owner_login, a.kind AS owner_kind, a.suspended_at AS owner_suspended_at FROM repos r JOIN accounts a ON a.id = r.owner_id WHERE r.visibility = 'public' AND a.suspended_at IS NULL";
758+ const params: unknown[] = [];
759+ const owners = (filter.owners ?? []).slice(0, 20);
760+ if (owners.length) {
761+ q += ` AND a.login IN (${owners.map(() => "?").join(", ")})`;
762+ params.push(...owners);
763+ }
764+ for (const w of (filter.words ?? []).slice(0, 10)) {
765+ q += " AND (a.login || '/' || r.name || ' ' || COALESCE(r.description, '') || ' ' || r.topics) LIKE ? ESCAPE '\\'";
766+ params.push(`%${w.replace(/[\\%_]/g, (ch) => `\\${ch}`)}%`);
767+ }
768+ q += " ORDER BY COALESCE(r.pushed_at, r.updated_at) DESC LIMIT ?";
769+ params.push(limit);
770+ return (await this.db.prepare(q).bind(...params).all<Repo>()).results;
747771 }
748772
749773 /** `o.authorized`: the caller already checked that `creator` may create repositories for `owner` (GitHub imports). */
@@ -1022,6 +1046,59 @@
10221046 if (repo) await enqueueMail(this.env, [{ kind: "repo_access", data: { repo: `${repo.owner_login}/${repo.name}`, role, previous }, to: { accountId }, reason: "account", at: now() }]);
10231047 }
10241048
1049+ /** Accounts by id, in chunks that stay under D1's 100 bound parameters (missing ids are left out). */
1050+ async accountsByIds(ids: string[]): Promise<Account[]> {
1051+ const out: Account[] = [];
1052+ const unique = [...new Set(ids)];
1053+ for (let i = 0; i < unique.length; i += 90) {
1054+ const part = unique.slice(i, i + 90);
1055+ out.push(...(await this.db.prepare(`SELECT * FROM accounts WHERE id IN (${part.map(() => "?").join(", ")})`).bind(...part).all<Account>()).results);
1056+ }
1057+ return out;
1058+ }
1059+
1060+ /** Accounts by login, case-insensitively (unknown logins are left out). */
1061+ async accountsByLogins(logins: string[]): Promise<Account[]> {
1062+ const out: Account[] = [];
1063+ const unique = [...new Set(logins.map((l) => l.toLowerCase()))];
1064+ for (let i = 0; i < unique.length; i += 90) {
1065+ const part = unique.slice(i, i + 90);
1066+ out.push(...(await this.db.prepare(`SELECT * FROM accounts WHERE login IN (${part.map(() => "?").join(", ")})`).bind(...part).all<Account>()).results);
1067+ }
1068+ return out;
1069+ }
1070+
1071+ /**
1072+ * The ids of `accounts` that can read `repo` (roleFor isn't null), with a few queries for all of
1073+ * them rather than two each: a notification about a busy thread checks up to 200 people.
1074+ */
1075+ async readersOf(repo: Repo, accounts: Account[]): Promise<Set<string>> {
1076+ const out = new Set<string>();
1077+ const users: Account[] = [];
1078+ for (const a of accounts) {
1079+ if (a.status !== "active") continue;
1080+ if (a.kind === "agent") {
1081+ if (await this.roleFor(a, repo)) out.add(a.id);
1082+ } else if (a.kind === "user") users.push(a);
1083+ }
1084+ for (const a of users) if (a.site_admin || repo.visibility !== "private" || repo.owner_id === a.id) out.add(a.id);
1085+ const rest = users.filter((a) => !out.has(a.id));
1086+ for (let i = 0; i < rest.length; i += 90) {
1087+ const ids = rest.slice(i, i + 90).map((a) => a.id);
1088+ const marks = ids.map(() => "?").join(", ");
1089+ const shared = await this.db.prepare(`SELECT account_id AS id FROM repo_access WHERE repo_id = ? AND account_id IN (${marks})`).bind(repo.id, ...ids).all<{ id: string }>();
1090+ for (const r of shared.results) out.add(r.id);
1091+ if (repo.owner_kind === "org") {
1092+ const members = await this.db
1093+ .prepare(`SELECT m.user_id AS id, m.role, o.base_role FROM org_members m JOIN accounts o ON o.id = m.org_id WHERE m.org_id = ? AND m.user_id IN (${marks})`)
1094+ .bind(repo.owner_id, ...ids)
1095+ .all<{ id: string; role: string; base_role: string | null }>();
1096+ for (const m of members.results) if (m.role === "owner" || m.role === "admin" || (m.role === "member" && baseRole(m) !== "none")) out.add(m.id);
1097+ }
1098+ }
1099+ return out;
1100+ }
1101+
10251102 /** Effective role of an account on a repository (null = no access). */
10261103 async roleFor(account: Account, repo: Repo): Promise<Role | null> {
10271104 if (account.status !== "active") return null;
modifiedapps/diffinery/src/lib/quotas.ts+9−3View file
@@ -76,6 +76,12 @@
7676
7777 /** What an account uses now. */
7878 export async function usageOf(env: Env, accountId: string, at = new Date()): Promise<Usage> {
79+ const [compute, repos, storage] = await Promise.all([computeUsage(env, accountId, at), env.DB.prepare("SELECT count(*) AS n FROM repos WHERE owner_id = ?").bind(accountId).first<{ n: number }>(), storedBytes(env, accountId)]);
80+ return { ...compute, repositories: repos?.n ?? 0, storage_bytes: storage };
81+}
82+
83+/** The hosted minutes, jobs and sandboxes an account uses now: all a job or sandbox start needs to know. */
84+export async function computeUsage(env: Env, accountId: string, at = new Date()): Promise<Pick<Usage, "hosted_minutes" | "hosted_jobs" | "sandboxes">> {
7985 const t = at.getTime();
8086 const month = monthStart(at);
8187 // Anything that started earlier ended before the month began (or is counted as ended by then).
@@ -96,8 +102,7 @@
96102 else sandboxes++;
97103 }
98104 }
99- const repos = await env.DB.prepare("SELECT count(*) AS n FROM repos WHERE owner_id = ?").bind(accountId).first<{ n: number }>();
100- return { hosted_minutes: minutes, hosted_jobs: jobs, sandboxes, repositories: repos?.n ?? 0, storage_bytes: await storedBytes(env, accountId) };
105+ return { hosted_minutes: minutes, hosted_jobs: jobs, sandboxes };
101106 }
102107
103108 /** Repositories not measured yet (made before storage was counted) measured by one check. */
@@ -151,7 +156,8 @@
151156 export async function mayStart(env: Env, accountId: string, kind: "job" | "sandbox"): Promise<{ ok: true } | { ok: false; wait: boolean; message: string }> {
152157 const q = await quotasFor(env, accountId);
153158 if (q.exempt) return { ok: true };
154- const u = await usageOf(env, accountId);
159+ // Not usageOf: a start doesn't need the repository count or the storage total (up to 25 repositories asked).
160+ const u = await computeUsage(env, accountId);
155161 if (!within(q.hosted_minutes, u.hosted_minutes))
156162 return { ok: false, wait: false, message: `this account has used its ${q.hosted_minutes} hosted minutes for this month (Actions jobs on hosted runners and agent sandboxes); they renew on the 1st (UTC). Self-hosted runners still work` };
157163 if (kind === "job" && !within(q.hosted_jobs, u.hosted_jobs)) return { ok: false, wait: true, message: `waiting for one of this account's ${q.hosted_jobs} hosted jobs to finish` };
addedapps/diffinery/src/lib/retention.ts+24−0View file
@@ -1,0 +1,24 @@
1+/**
2+ * Rows D1 doesn't need to keep, deleted a bounded batch per table each hour so the tables stop
3+ * growing with every request, notification and job (scalability evaluation, 2026-10-14):
4+ *
5+ * - sessions a day past their expiry (a session is refused once expired; the day keeps a just-expired
6+ * one visible on the sessions page meanwhile);
7+ * - notifications older than NOTIFICATION_RETENTION_DAYS, as GitHub drops them after five months;
8+ * - hosted compute records (lib/quotas.ts) ended more than COMPUTE_RETENTION_DAYS ago, past any month
9+ * the usage pages show.
10+ */
11+export const NOTIFICATION_RETENTION_DAYS = 180;
12+export const COMPUTE_RETENTION_DAYS = 400;
13+/** Rows deleted per table per run: an hourly run keeps up with 120,000 new rows a day per table. */
14+export const RETENTION_BATCH = 5000;
15+
16+export async function pruneExpired(db: D1Database, at = new Date(), batch = RETENTION_BATCH): Promise<Record<"sessions" | "notifications" | "compute_usage", number>> {
17+ const ago = (days: number) => new Date(at.getTime() - days * 86400_000).toISOString();
18+ const [sessions, notifications, compute] = await db.batch([
19+ db.prepare("DELETE FROM sessions WHERE id IN (SELECT id FROM sessions WHERE expires_at < ? LIMIT ?)").bind(ago(1), batch),
20+ db.prepare("DELETE FROM notifications WHERE id IN (SELECT id FROM notifications WHERE created_at < ? LIMIT ?)").bind(ago(NOTIFICATION_RETENTION_DAYS), batch),
21+ db.prepare("DELETE FROM compute_usage WHERE id IN (SELECT id FROM compute_usage WHERE ended_at < ? LIMIT ?)").bind(ago(COMPUTE_RETENTION_DAYS), batch),
22+ ]);
23+ return { sessions: sessions!.meta.changes ?? 0, notifications: notifications!.meta.changes ?? 0, compute_usage: compute!.meta.changes ?? 0 };
24+}
modifiedapps/diffinery/src/mail/notify.ts+9−14View file
@@ -56,23 +56,16 @@
5656 }
5757
5858 async function accountsByLogin(dir: Directory, logins: string[]): Promise<Account[]> {
59- const out: Account[] = [];
60- for (const l of [...new Set(logins.map((x) => x.toLowerCase()))].slice(0, 60)) {
61- const a = await dir.accountByLogin(l);
62- if (a) out.push(a);
63- }
64- return out;
59+ return dir.accountsByLogins([...new Set(logins.map((x) => x.toLowerCase()))].slice(0, 60));
6560 }
6661
6762 /** Recipients who may still read the repository, with their accounts. */
6863 async function readable(dir: Directory, repo: Repo, ids: string[]): Promise<Account[]> {
69- const out: Account[] = [];
70- for (const id of ids.slice(0, 200)) {
71- const a = await dir.account(id);
72- if (!a || a.kind === "org" || a.status !== "active") continue;
73- if (await dir.roleFor(a, repo)) out.push(a);
74- }
75- return out;
64+ const first = ids.slice(0, 200);
65+ const byId = new Map((await dir.accountsByIds(first)).map((a) => [a.id, a]));
66+ const accounts = first.map((id) => byId.get(id)).filter((a): a is Account => !!a && a.kind !== "org" && a.status === "active");
67+ const readers = await dir.readersOf(repo, accounts);
68+ return accounts.filter((a) => readers.has(a.id));
7669 }
7770
7871 async function threadEvent(env: Env, dir: Directory, ev: ThreadEvent): Promise<void> {
@@ -209,7 +202,9 @@
209202 sha: run.headSha || run.sha,
210203 actor: run.actor.login,
211204 url,
212- jobs: run.jobs.map((j) => ({ name: j.name, conclusion: j.conclusion, url: `${url}/job/${j.id}`, failure: j.failure?.message ?? null })),
205+ // What the email shows (mail/templates.ts): the first 20 jobs, each failure's first 160
206+ // characters. A large matrix with long failures would pass the queue's 128 KB per message.
207+ jobs: run.jobs.slice(0, 20).map((j) => ({ name: j.name, conclusion: j.conclusion, url: `${url}/job/${j.id}`, failure: j.failure?.message?.slice(0, 160) ?? null })),
213208 };
214209 jobs = mailTo.map((a) => ({ kind: "run" as const, data, to: { accountId: a.id }, reason: "ci_activity" as const, at: ev.at }));
215210 }
modifiedapps/diffinery/src/mail/send.ts+49−4View file
@@ -15,6 +15,48 @@
1515 * (suppression.ts), and an address Email Sending refuses for good goes on it.
1616 */
1717
18+/**
19+ * Queues limit a message to 128 KB and a sendBatch to 100 messages and 256 KB. A long comment sent to
20+ * many watchers passed 256 KB in one batch of 50, and the whole batch was refused and lost. Batches are
21+ * cut by size now, and a body longer than MAX_QUEUED_BODY is shortened (the email links to the thread).
22+ */
23+const MAX_BATCH_BYTES = 200_000;
24+const MAX_BATCH_JOBS = 50;
25+/** Under Queues' 128 KB per message, with room for its own envelope. */
26+const MAX_MESSAGE_BYTES = 120_000;
27+export const MAX_QUEUED_BODY = 32_000;
28+
29+/** A job as it goes on the queue: the thread body cut to MAX_QUEUED_BODY characters. */
30+export function queuedJob(job: MailJob): MailJob {
31+ const data = (job as { data?: { body?: unknown } }).data;
32+ if (!data || typeof data.body !== "string" || data.body.length <= MAX_QUEUED_BODY) return job;
33+ return { ...job, data: { ...data, body: `${data.body.slice(0, MAX_QUEUED_BODY)}\n\n…` } } as MailJob;
34+}
35+
36+/** Jobs in sendBatch-sized groups: at most MAX_BATCH_JOBS and about MAX_BATCH_BYTES of JSON each. */
37+export function mailBatches(jobs: MailJob[]): MailJob[][] {
38+ const out: MailJob[][] = [];
39+ let cur: MailJob[] = [];
40+ let bytes = 0;
41+ for (const job of jobs.map(queuedJob)) {
42+ const size = new TextEncoder().encode(JSON.stringify(job)).length + 100;
43+ // Queues would refuse it, and the batch it is in with it: it is left out on its own.
44+ if (size > MAX_MESSAGE_BYTES) {
45+ console.error("email too large to queue; dropped", job.kind, size);
46+ continue;
47+ }
48+ if (cur.length && (cur.length >= MAX_BATCH_JOBS || bytes + size > MAX_BATCH_BYTES)) {
49+ out.push(cur);
50+ cur = [];
51+ bytes = 0;
52+ }
53+ cur.push(job);
54+ bytes += size;
55+ }
56+ if (cur.length) out.push(cur);
57+ return out;
58+}
59+
1860 /** Queue emails. Never throws: email must not break the action that caused it. */
1961 export async function enqueueMail(env: Env, jobs: MailJob[]): Promise<void> {
2062 if (!jobs.length) return;
@@ -22,10 +64,13 @@
2264 console.warn("mail queue not bound; dropping", jobs.length, "email(s)");
2365 return;
2466 }
25- try {
26- for (let i = 0; i < jobs.length; i += 50) await env.MAIL.sendBatch(jobs.slice(i, i + 50).map((body) => ({ body })));
27- } catch (e) {
28- console.error("could not queue email", e);
67+ // One refused batch doesn't stop the others.
68+ for (const batch of mailBatches(jobs)) {
69+ try {
70+ await env.MAIL.sendBatch(batch.map((body) => ({ body })));
71+ } catch (e) {
72+ console.error("could not queue email", batch.length, e);
73+ }
2974 }
3075 }
3176
modifiedapps/diffinery/test/unit/audit-retention.test.ts+1−1View file
@@ -2,7 +2,7 @@
22 import { AUDIT_RETENTION_DAYS, pruneAuditLog } from "../../src/lib/directory.js";
33 import { testD1 } from "./stubs/d1.js";
44
5-// The privacy policy says the audit log is kept 2 years: entries older than that are deleted once a day.
5+// The privacy policy says the audit log is kept 2 years: entries older than that are deleted every hour.
66 describe("audit log retention", () => {
77 it("deletes entries older than two years and keeps the rest", async () => {
88 const DB = testD1();
addedapps/diffinery/test/unit/mail-batches.test.ts+38−0View file
@@ -1,0 +1,38 @@
1+import { describe, expect, it } from "vitest";
2+import { MAX_QUEUED_BODY, enqueueMail, mailBatches } from "../../src/mail/send.js";
3+import type { MailJob } from "../../src/mail/types.js";
4+import type { Env } from "../../src/env.js";
5+
6+const thread = (i: number, body: string): MailJob =>
7+ ({ kind: "thread", data: { repo: "acme/app", repoId: "r", number: 1, kind: "issue", title: "t", event: "comment", actor: "ada", url: "u", body }, to: { accountId: `u${i}` }, reason: "subscribed", at: "2026-10-14T00:00:00Z" }) as MailJob;
8+
9+describe("queueing email in batches the queue accepts", () => {
10+ it("cuts batches by size, under the 256 KB sendBatch limit", () => {
11+ const jobs = Array.from({ length: 200 }, (_, i) => thread(i, "x".repeat(20_000)));
12+ const batches = mailBatches(jobs);
13+ expect(batches.flat()).toHaveLength(200);
14+ for (const b of batches) expect(new TextEncoder().encode(JSON.stringify(b.map((body) => ({ body })))).length).toBeLessThan(256_000);
15+ // Small jobs still go 50 at a time.
16+ expect(mailBatches(Array.from({ length: 120 }, (_, i) => thread(i, "hi"))).map((b) => b.length)).toEqual([50, 50, 20]);
17+ });
18+
19+ it("shortens a body too long for one queue message", () => {
20+ const [[job]] = mailBatches([thread(0, "y".repeat(500_000))]) as [[MailJob]];
21+ const body = (job as unknown as { data: { body: string } }).data.body;
22+ expect(body.length).toBeLessThan(MAX_QUEUED_BODY + 10);
23+ expect(body.endsWith("…")).toBe(true);
24+ });
25+
26+ it("leaves out a job too large for one queue message, not the batch it would be in", () => {
27+ const huge = { ...thread(1, "hi"), data: { ...(thread(1, "hi") as unknown as { data: object }).data, title: "z".repeat(200_000) } } as unknown as MailJob;
28+ expect(mailBatches([thread(0, "hi"), huge, thread(2, "hi")]).flat().map((j) => j.to)).toEqual([{ accountId: "u0" }, { accountId: "u2" }]);
29+ });
30+
31+ it("queues every batch even when one is refused", async () => {
32+ const sent: number[] = [];
33+ let calls = 0;
34+ const env = { MAIL: { sendBatch: async (b: unknown[]) => { if (calls++ === 0) throw new Error("refused"); sent.push(b.length); } } } as unknown as Env;
35+ await enqueueMail(env, Array.from({ length: 120 }, (_, i) => thread(i, "hi")));
36+ expect(sent).toEqual([50, 20]);
37+ });
38+});
addedapps/diffinery/test/unit/retention.test.ts+38−0View file
@@ -1,0 +1,38 @@
1+import { describe, expect, it } from "vitest";
2+import { COMPUTE_RETENTION_DAYS, NOTIFICATION_RETENTION_DAYS, pruneExpired } from "../../src/lib/retention.js";
3+import { testD1 } from "./stubs/d1.js";
4+
5+describe("retention", () => {
6+ it("deletes expired sessions, old notifications and old compute records, and keeps the rest", async () => {
7+ const DB = testD1();
8+ const at = new Date("2026-10-14T10:00:00Z");
9+ const ago = (d: number) => new Date(at.getTime() - d * 86400_000).toISOString();
10+ DB.sqlite.prepare("INSERT INTO accounts (id, login, kind, status, created_at) VALUES ('u', 'u', 'user', 'active', ?)").run(ago(1000));
11+ const session = DB.sqlite.prepare("INSERT INTO sessions (id, account_id, method, created_at, last_seen_at, expires_at) VALUES (?, 'u', 'passkey', ?, ?, ?)");
12+ session.run("expired", ago(40), ago(10), ago(2));
13+ session.run("just-expired", ago(40), ago(1), ago(0.5));
14+ session.run("live", ago(1), ago(0), ago(-29));
15+ const note = DB.sqlite.prepare("INSERT INTO notifications (id, account_id, kind, title, url, created_at) VALUES (?, 'u', 'issues', 't', 'u', ?)");
16+ note.run("old", ago(NOTIFICATION_RETENTION_DAYS + 1));
17+ note.run("recent", ago(NOTIFICATION_RETENTION_DAYS - 1));
18+ const usage = DB.sqlite.prepare("INSERT INTO compute_usage (id, account_id, repo_id, kind, started_at, ended_at) VALUES (?, 'u', 'r', 'job', ?, ?)");
19+ usage.run("old-job", ago(COMPUTE_RETENTION_DAYS + 2), ago(COMPUTE_RETENTION_DAYS + 1));
20+ usage.run("recent-job", ago(30), ago(30));
21+ usage.run("running", ago(0), null);
22+ expect(await pruneExpired(DB, at)).toEqual({ sessions: 1, notifications: 1, compute_usage: 1 });
23+ const ids = (t: string) => (DB.sqlite.prepare(`SELECT id FROM ${t} ORDER BY id`).all() as Array<{ id: string }>).map((r) => r.id);
24+ expect(ids("sessions")).toEqual(["just-expired", "live"]);
25+ expect(ids("notifications")).toEqual(["recent"]);
26+ expect(ids("compute_usage")).toEqual(["recent-job", "running"]);
27+ });
28+
29+ it("deletes a bounded batch at a time", async () => {
30+ const DB = testD1();
31+ const at = new Date("2026-10-14T10:00:00Z");
32+ DB.sqlite.prepare("INSERT INTO accounts (id, login, kind, status, created_at) VALUES ('u', 'u', 'user', 'active', '2020-01-01T00:00:00Z')").run();
33+ const note = DB.sqlite.prepare("INSERT INTO notifications (id, account_id, kind, title, url, created_at) VALUES (?, 'u', 'issues', 't', 'u', '2020-01-01T00:00:00Z')");
34+ for (let i = 0; i < 5; i++) note.run(`n${i}`);
35+ expect((await pruneExpired(DB, at, 3)).notifications).toBe(3);
36+ expect((await pruneExpired(DB, at, 3)).notifications).toBe(2);
37+ });
38+});
addedapps/diffinery/test/unit/visibility.test.ts+81−0View file
@@ -1,0 +1,81 @@
1+import { describe, expect, it } from "vitest";
2+import { Directory, type Account, type Repo } from "../../src/lib/directory.js";
3+import { testD1 } from "./stubs/d1.js";
4+import type { Env } from "../../src/env.js";
5+
6+/**
7+ * Who can see what, by the batched lookups (reposVisibleTo, publicRepos, readersOf) against the
8+ * one-at-a-time roleFor they stand in for.
9+ */
10+function world() {
11+ const DB = testD1();
12+ const t = "2026-10-01T00:00:00.000Z";
13+ const acct = DB.sqlite.prepare("INSERT INTO accounts (id, login, kind, status, site_admin, owner_id, created_at) VALUES (?, ?, ?, ?, ?, ?, ?)");
14+ acct.run("ada", "ada", "user", "active", 0, null, t);
15+ acct.run("bob", "bob", "user", "active", 0, null, t);
16+ acct.run("cy", "cy", "user", "active", 0, null, t);
17+ acct.run("dee", "dee", "user", "disabled", 0, null, t);
18+ acct.run("root", "root", "user", "active", 1, null, t);
19+ acct.run("bot", "ada-bot", "agent", "active", 0, "ada", t);
20+ acct.run("acme", "acme", "org", "active", 0, null, t);
21+ acct.run("shut", "shut", "org", "active", 0, null, t);
22+ DB.sqlite.prepare("UPDATE accounts SET base_role = 'none' WHERE id = 'shut'").run();
23+ const member = DB.sqlite.prepare("INSERT INTO org_members (org_id, user_id, role, created_at) VALUES (?, ?, ?, '2026-10-01')");
24+ member.run("acme", "bob", "member");
25+ member.run("shut", "bob", "member");
26+ member.run("shut", "cy", "admin");
27+ const repo = DB.sqlite.prepare("INSERT INTO repos (id, owner_id, name, visibility, created_at, updated_at, pushed_at) VALUES (?, ?, ?, ?, ?, ?, ?)");
28+ repo.run("r-ada", "ada", "notes", "private", t, t, "2026-10-02T00:00:00.000Z");
29+ repo.run("r-acme", "acme", "app", "private", t, t, "2026-10-03T00:00:00.000Z");
30+ repo.run("r-shut", "shut", "vault", "private", t, t, "2026-10-04T00:00:00.000Z");
31+ repo.run("r-shut-pub", "shut", "site", "public", t, t, "2026-10-05T00:00:00.000Z");
32+ repo.run("r-pub", "cy", "tool", "public", t, t, "2026-10-06T00:00:00.000Z");
33+ DB.sqlite.prepare("UPDATE repos SET description = 'A fast 100% thing', topics = '[\"cli\"]' WHERE id = 'r-pub'").run();
34+ DB.sqlite.prepare("INSERT INTO repo_access (repo_id, account_id, role) VALUES ('r-ada', 'cy', 'read')").run();
35+ const dir = new Directory({ DB } as unknown as Env);
36+ return { dir };
37+}
38+
39+const ids = (rs: Repo[]) => rs.map((r) => r.id);
40+
41+describe("repository visibility at scale", () => {
42+ it("lists the repositories an account works with, not every public one", async () => {
43+ const { dir } = world();
44+ const get = async (id: string) => (await dir.account(id))!;
45+ expect(ids(await dir.reposVisibleTo(await get("ada")))).toEqual(["r-ada"]);
46+ // bob: acme's repository as a member (base read), shut's public one but not its private one (base none).
47+ expect(ids(await dir.reposVisibleTo(await get("bob")))).toEqual(["r-shut-pub", "r-acme"]);
48+ // cy: own public repo, shut's as an admin, ada's shared with them.
49+ expect(ids(await dir.reposVisibleTo(await get("cy")))).toEqual(["r-pub", "r-shut-pub", "r-shut", "r-ada"]);
50+ // An agent sees what its owner works with.
51+ expect(ids(await dir.reposVisibleTo(await get("bot")))).toEqual(["r-ada"]);
52+ expect(await dir.reposVisibleTo(await get("dee"))).toEqual([]);
53+ // One owner's repositories still include its public ones for anyone.
54+ expect(ids(await dir.reposVisibleTo(await get("ada"), "cy"))).toEqual(["r-pub"]);
55+ expect(ids(await dir.reposVisibleTo(await get("ada"), "shut"))).toEqual(["r-shut-pub"]);
56+ });
57+
58+ it("finds public repositories by words and owners in D1", async () => {
59+ const { dir } = world();
60+ expect(ids(await dir.publicRepos(10))).toEqual(["r-pub", "r-shut-pub"]);
61+ expect(ids(await dir.publicRepos(10, { words: ["TOOL"] }))).toEqual(["r-pub"]);
62+ expect(ids(await dir.publicRepos(10, { words: ["100%", "cli"] }))).toEqual(["r-pub"]);
63+ // % and _ are matched literally.
64+ expect(ids(await dir.publicRepos(10, { words: ["1_0"] }))).toEqual([]);
65+ expect(ids(await dir.publicRepos(10, { words: ["shut/"] }))).toEqual(["r-shut-pub"]);
66+ expect(ids(await dir.publicRepos(10, { owners: ["SHUT"] }))).toEqual(["r-shut-pub"]);
67+ });
68+
69+ it("finds a repository's readers in a few queries, as roleFor would one by one", async () => {
70+ const { dir } = world();
71+ const everyone = (await dir.accountsByIds(["ada", "bob", "cy", "dee", "root", "bot", "acme", "nobody"])) as Account[];
72+ expect(everyone.map((a) => a.id).sort()).toEqual(["acme", "ada", "bob", "bot", "cy", "dee", "root"]);
73+ for (const id of ["r-ada", "r-acme", "r-shut", "r-shut-pub", "r-pub"]) {
74+ const repo = (await dir.repoById(id))!;
75+ const want = new Set<string>();
76+ for (const a of everyone) if (a.kind !== "org" && (await dir.roleFor(a, repo))) want.add(a.id);
77+ expect([...(await dir.readersOf(repo, everyone))].sort(), id).toEqual([...want].sort());
78+ }
79+ expect((await dir.accountsByLogins(["ADA", "ada-bot", "ghost"])).map((a) => a.id).sort()).toEqual(["ada", "bot"]);
80+ });
81+});
modifiedapps/diffinery/wrangler.jsonc+3−1View file
@@ -33,7 +33,9 @@
3333 { "binding": "MAIL", "queue": "diffinery-mail" }
3434 ],
3535 "consumers": [
36- { "queue": "diffinery-events", "max_batch_size": 50, "max_batch_timeout": 2, "max_retries": 8, "dead_letter_queue": "diffinery-events-dlq" },
36+ // 20 events per batch: each makes D1 queries (webhooks, notifications, audit), and one consumer
37+ // invocation may make at most 1,000.
38+ { "queue": "diffinery-events", "max_batch_size": 20, "max_batch_timeout": 2, "max_retries": 8, "dead_letter_queue": "diffinery-events-dlq" },
3739 { "queue": "diffinery-mail", "max_batch_size": 20, "max_batch_timeout": 2, "max_retries": 8, "dead_letter_queue": "diffinery-mail-dlq" },
3840 // Bounces and complaints: subscribe this queue to Email Sending's events (docs/admin/site-administration.md).
3941 { "queue": "diffinery-mail-events", "max_batch_size": 50, "max_batch_timeout": 5, "max_retries": 8 }