Diffinery

codician/diffinerypublic

Open-source, agent-first GitHub and GitHub Actions alternative that runs on your own Cloudflare account: git hosting, pull requests, code review and drop-in GitHub Actions YAML on Workers, Durable Objects, R2 and Containers.

245 lines · 17.9 KBRaw History
1/** SQLite schema of a repository Durable Object. Applied idempotently on construction. */
2export const REPO_SCHEMA = `
3CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT NOT NULL);
4CREATE TABLE IF NOT EXISTS refs (name TEXT PRIMARY KEY, oid TEXT NOT NULL);
5CREATE TABLE IF NOT EXISTS objects (
6 oid TEXT PRIMARY KEY, type INTEGER NOT NULL, size INTEGER NOT NULL, data BLOB, r2 INTEGER NOT NULL DEFAULT 0
7);
8CREATE TABLE IF NOT EXISTS counters (name TEXT PRIMARY KEY, value INTEGER NOT NULL);
9-- Git LFS objects stored in R2 (repos/<id>/lfs/...), for the owner's storage quota (lib/quotas.ts).
10CREATE TABLE IF NOT EXISTS lfs_objects (oid TEXT PRIMARY KEY, size INTEGER NOT NULL);
11
12CREATE TABLE IF NOT EXISTS issues (
13 number INTEGER PRIMARY KEY, kind TEXT NOT NULL, title TEXT NOT NULL, body TEXT NOT NULL DEFAULT '',
14 state TEXT NOT NULL DEFAULT 'open', state_reason TEXT, author_id TEXT NOT NULL, author_login TEXT NOT NULL,
15 author_kind TEXT NOT NULL DEFAULT 'user', assignees TEXT NOT NULL DEFAULT '[]', labels TEXT NOT NULL DEFAULT '[]',
16 created_at TEXT NOT NULL, updated_at TEXT NOT NULL, closed_at TEXT, locked INTEGER NOT NULL DEFAULT 0
17);
18CREATE TABLE IF NOT EXISTS pulls (
19 number INTEGER PRIMARY KEY, head_ref TEXT NOT NULL, head_sha TEXT NOT NULL, base_ref TEXT NOT NULL,
20 draft INTEGER NOT NULL DEFAULT 0, merged INTEGER NOT NULL DEFAULT 0, merged_at TEXT, merged_by TEXT,
21 merge_commit TEXT, requested_reviewers TEXT NOT NULL DEFAULT '[]', intent TEXT
22);
23CREATE INDEX IF NOT EXISTS pulls_head ON pulls(head_ref);
24CREATE TABLE IF NOT EXISTS comments (
25 id TEXT PRIMARY KEY, number INTEGER NOT NULL, author_id TEXT NOT NULL, author_login TEXT NOT NULL,
26 author_kind TEXT NOT NULL DEFAULT 'user', body TEXT NOT NULL, created_at TEXT NOT NULL, updated_at TEXT NOT NULL,
27 review_id TEXT, path TEXT, line INTEGER, side TEXT, commit_id TEXT, in_reply_to TEXT, resolved INTEGER NOT NULL DEFAULT 0
28);
29CREATE INDEX IF NOT EXISTS comments_number ON comments(number);
30CREATE TABLE IF NOT EXISTS reviews (
31 id TEXT PRIMARY KEY, number INTEGER NOT NULL, author_id TEXT NOT NULL, author_login TEXT NOT NULL,
32 author_kind TEXT NOT NULL DEFAULT 'user', state TEXT NOT NULL, body TEXT NOT NULL DEFAULT '', commit_id TEXT NOT NULL,
33 submitted_at TEXT NOT NULL
34);
35CREATE INDEX IF NOT EXISTS reviews_number ON reviews(number);
36CREATE TABLE IF NOT EXISTS labels (name TEXT PRIMARY KEY, color TEXT NOT NULL, description TEXT NOT NULL DEFAULT '');
37CREATE TABLE IF NOT EXISTS protections (pattern TEXT PRIMARY KEY, rules TEXT NOT NULL);
38CREATE TABLE IF NOT EXISTS secrets (name TEXT PRIMARY KEY, ciphertext TEXT NOT NULL, updated_at TEXT NOT NULL);
39CREATE TABLE IF NOT EXISTS variables (name TEXT PRIMARY KEY, value TEXT NOT NULL, updated_at TEXT NOT NULL);
40CREATE TABLE IF NOT EXISTS webhooks (
41 id TEXT PRIMARY KEY, url TEXT NOT NULL, secret TEXT NOT NULL, events TEXT NOT NULL, active INTEGER NOT NULL DEFAULT 1,
42 created_at TEXT NOT NULL, last_status INTEGER, last_delivery_at TEXT
43);
44CREATE TABLE IF NOT EXISTS checks (
45 id TEXT PRIMARY KEY, sha TEXT NOT NULL, name TEXT NOT NULL, status TEXT NOT NULL, conclusion TEXT,
46 run_id TEXT, job_id TEXT, details_url TEXT, output TEXT, app TEXT NOT NULL DEFAULT 'diffinery-actions',
47 started_at TEXT, completed_at TEXT
48);
49CREATE INDEX IF NOT EXISTS checks_sha ON checks(sha);
50CREATE TABLE IF NOT EXISTS runs (
51 id TEXT PRIMARY KEY, run_number INTEGER NOT NULL, workflow_path TEXT NOT NULL, workflow_name TEXT NOT NULL,
52 event TEXT NOT NULL, ref TEXT NOT NULL, sha TEXT NOT NULL, actor TEXT NOT NULL, status TEXT NOT NULL,
53 conclusion TEXT, created_at TEXT NOT NULL, updated_at TEXT NOT NULL, pr_number INTEGER, attempt INTEGER NOT NULL DEFAULT 1,
54 event_payload TEXT NOT NULL, inputs TEXT NOT NULL DEFAULT '{}', concurrency_group TEXT, display_title TEXT NOT NULL DEFAULT ''
55);
56CREATE INDEX IF NOT EXISTS runs_created ON runs(created_at);
57CREATE TABLE IF NOT EXISTS jobs (
58 id TEXT PRIMARY KEY, run_id TEXT NOT NULL, job_key TEXT NOT NULL, name TEXT NOT NULL, status TEXT NOT NULL,
59 conclusion TEXT, matrix TEXT NOT NULL DEFAULT '{}', strategy TEXT NOT NULL DEFAULT '{}', started_at TEXT, completed_at TEXT,
60 outputs TEXT NOT NULL DEFAULT '{}', steps TEXT NOT NULL DEFAULT '[]', failure TEXT, token_hash TEXT, summary TEXT,
61 container TEXT, timeout_minutes INTEGER NOT NULL DEFAULT 360, labels TEXT
62);
63CREATE INDEX IF NOT EXISTS jobs_run ON jobs(run_id);
64CREATE TABLE IF NOT EXISTS logs (job_id TEXT NOT NULL, seq INTEGER NOT NULL, t TEXT NOT NULL, step INTEGER NOT NULL, text TEXT NOT NULL, PRIMARY KEY (job_id, seq));
65CREATE TABLE IF NOT EXISTS caches (key TEXT PRIMARY KEY, size INTEGER NOT NULL, created_at TEXT NOT NULL, last_used TEXT NOT NULL);
66-- Where older runners' artifacts were recorded before rt_artifacts; migrateRepoSchema moves them over.
67CREATE TABLE IF NOT EXISTS artifacts (id TEXT PRIMARY KEY, run_id TEXT NOT NULL, name TEXT NOT NULL, size INTEGER NOT NULL, created_at TEXT NOT NULL, UNIQUE(run_id, name));
68CREATE TABLE IF NOT EXISTS merge_queue (
69 number INTEGER PRIMARY KEY, base_ref TEXT NOT NULL, seq INTEGER NOT NULL, head_sha TEXT NOT NULL, method TEXT NOT NULL,
70 delete_branch INTEGER NOT NULL DEFAULT 0, enqueued_by TEXT NOT NULL, enqueued_at TEXT NOT NULL,
71 parent_sha TEXT, group_sha TEXT, group_ref TEXT, built_at TEXT
72);
73CREATE TABLE IF NOT EXISTS sandboxes (id TEXT PRIMARY KEY, secret TEXT NOT NULL, created_by TEXT NOT NULL, ref TEXT NOT NULL, created_at TEXT NOT NULL, status TEXT NOT NULL);
74
75-- Actions runtime services (repo/runtime.ts): the cache and artifact services the
76-- official toolkit talks to, with numeric ids like GitHub's. Content lives in R2.
77CREATE TABLE IF NOT EXISTS rt_caches (
78 id INTEGER PRIMARY KEY AUTOINCREMENT, key TEXT NOT NULL, version TEXT NOT NULL, scope TEXT NOT NULL,
79 size INTEGER NOT NULL DEFAULT 0, status TEXT NOT NULL, job_id TEXT, created_at TEXT NOT NULL, last_used TEXT NOT NULL,
80 UNIQUE(key, version, scope)
81);
82CREATE INDEX IF NOT EXISTS rt_caches_used ON rt_caches(last_used);
83CREATE TABLE IF NOT EXISTS rt_artifacts (
84 id INTEGER PRIMARY KEY AUTOINCREMENT, run_id TEXT NOT NULL, job_id TEXT NOT NULL, name TEXT NOT NULL,
85 size INTEGER NOT NULL DEFAULT 0, digest TEXT, mime_type TEXT NOT NULL DEFAULT 'application/zip', status TEXT NOT NULL,
86 created_at TEXT NOT NULL, updated_at TEXT NOT NULL, expires_at TEXT NOT NULL
87);
88CREATE INDEX IF NOT EXISTS rt_artifacts_run ON rt_artifacts(run_id, name);
89-- What a job's GITHUB_TOKEN may do, from the workflow's permissions (repo/runtime.ts).
90CREATE TABLE IF NOT EXISTS job_grants (job_id TEXT PRIMARY KEY, role TEXT NOT NULL, permissions TEXT NOT NULL);
91-- GitHub-compatible extras (see ./releases.ts, ./statuses.ts, ./deployments.ts, ./social.ts).
92CREATE TABLE IF NOT EXISTS releases (
93 id INTEGER PRIMARY KEY AUTOINCREMENT, tag_name TEXT NOT NULL, target_commitish TEXT NOT NULL, name TEXT NOT NULL DEFAULT '',
94 body TEXT NOT NULL DEFAULT '', draft INTEGER NOT NULL DEFAULT 0, prerelease INTEGER NOT NULL DEFAULT 0,
95 author_id TEXT NOT NULL, author_login TEXT NOT NULL, author_kind TEXT NOT NULL DEFAULT 'user',
96 created_at TEXT NOT NULL, published_at TEXT, updated_at TEXT NOT NULL
97);
98CREATE INDEX IF NOT EXISTS releases_tag ON releases(tag_name);
99CREATE TABLE IF NOT EXISTS release_assets (
100 id INTEGER PRIMARY KEY AUTOINCREMENT, release_id INTEGER NOT NULL, name TEXT NOT NULL, label TEXT NOT NULL DEFAULT '',
101 content_type TEXT NOT NULL, size INTEGER NOT NULL DEFAULT 0, state TEXT NOT NULL DEFAULT 'starter', download_count INTEGER NOT NULL DEFAULT 0,
102 uploader_id TEXT, uploader_login TEXT, uploader_kind TEXT, r2_key TEXT NOT NULL, created_at TEXT NOT NULL, updated_at TEXT NOT NULL,
103 UNIQUE (release_id, name)
104);
105CREATE TABLE IF NOT EXISTS statuses (
106 id INTEGER PRIMARY KEY AUTOINCREMENT, sha TEXT NOT NULL, state TEXT NOT NULL, context TEXT NOT NULL, description TEXT, target_url TEXT,
107 creator_id TEXT NOT NULL, creator_login TEXT NOT NULL, creator_kind TEXT NOT NULL, created_at TEXT NOT NULL, updated_at TEXT NOT NULL
108);
109CREATE INDEX IF NOT EXISTS statuses_sha ON statuses(sha, context);
110CREATE TABLE IF NOT EXISTS deployments (
111 id INTEGER PRIMARY KEY AUTOINCREMENT, sha TEXT NOT NULL, ref TEXT NOT NULL, task TEXT NOT NULL DEFAULT 'deploy', environment TEXT NOT NULL,
112 original_environment TEXT NOT NULL, description TEXT, payload TEXT NOT NULL DEFAULT '{}', transient INTEGER NOT NULL DEFAULT 0,
113 production INTEGER NOT NULL DEFAULT 0, creator_id TEXT NOT NULL, creator_login TEXT NOT NULL, creator_kind TEXT NOT NULL,
114 job_id TEXT, created_at TEXT NOT NULL, updated_at TEXT NOT NULL
115);
116CREATE INDEX IF NOT EXISTS deployments_env ON deployments(environment);
117CREATE TABLE IF NOT EXISTS deployment_statuses (
118 id INTEGER PRIMARY KEY AUTOINCREMENT, deployment_id INTEGER NOT NULL, state TEXT NOT NULL, description TEXT NOT NULL DEFAULT '',
119 environment TEXT NOT NULL, environment_url TEXT NOT NULL DEFAULT '', log_url TEXT NOT NULL DEFAULT '',
120 creator_id TEXT NOT NULL, creator_login TEXT NOT NULL, creator_kind TEXT NOT NULL, created_at TEXT NOT NULL
121);
122CREATE INDEX IF NOT EXISTS deployment_statuses_dep ON deployment_statuses(deployment_id);
123CREATE TABLE IF NOT EXISTS environments (
124 id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT NOT NULL UNIQUE COLLATE NOCASE, branch_policy TEXT, created_at TEXT NOT NULL, updated_at TEXT NOT NULL
125);
126-- Required reviewers' decisions on a run's environments (repo/deployments.ts): one approval per run and environment is enough.
127CREATE TABLE IF NOT EXISTS deployment_reviews (
128 id INTEGER PRIMARY KEY AUTOINCREMENT, run_id TEXT NOT NULL, environment TEXT NOT NULL COLLATE NOCASE, state TEXT NOT NULL, comment TEXT NOT NULL DEFAULT '',
129 user_id TEXT NOT NULL, user_login TEXT NOT NULL, created_at TEXT NOT NULL
130);
131CREATE INDEX IF NOT EXISTS deployment_reviews_run ON deployment_reviews(run_id);
132CREATE TABLE IF NOT EXISTS environment_secrets (environment TEXT NOT NULL COLLATE NOCASE, name TEXT NOT NULL, ciphertext TEXT NOT NULL, updated_at TEXT NOT NULL, PRIMARY KEY (environment, name));
133CREATE TABLE IF NOT EXISTS environment_variables (environment TEXT NOT NULL COLLATE NOCASE, name TEXT NOT NULL, value TEXT NOT NULL, created_at TEXT NOT NULL, updated_at TEXT NOT NULL, PRIMARY KEY (environment, name));
134CREATE TABLE IF NOT EXISTS reactions (
135 id INTEGER PRIMARY KEY AUTOINCREMENT, subject TEXT NOT NULL, content TEXT NOT NULL, user_id TEXT NOT NULL, user_login TEXT NOT NULL,
136 user_kind TEXT NOT NULL, created_at TEXT NOT NULL, UNIQUE (subject, content, user_id)
137);
138CREATE TABLE IF NOT EXISTS issue_events (
139 id INTEGER PRIMARY KEY AUTOINCREMENT, number INTEGER NOT NULL, event TEXT NOT NULL, actor_id TEXT NOT NULL, actor_login TEXT NOT NULL,
140 actor_kind TEXT NOT NULL, detail TEXT NOT NULL DEFAULT '{}', created_at TEXT NOT NULL
141);
142CREATE INDEX IF NOT EXISTS issue_events_number ON issue_events(number);
143-- Who last moved each ref: tags made by agents never match environment tag patterns
144-- (./deployments.ts), and runs of code an agent pushed act as the agent (./token-policy.ts).
145CREATE TABLE IF NOT EXISTS ref_actors (ref TEXT PRIMARY KEY, actor_id TEXT NOT NULL, actor_kind TEXT NOT NULL, updated_at TEXT NOT NULL);
146-- GitHub imports and milestones (see ./imports.ts).
147CREATE TABLE IF NOT EXISTS milestones (
148 number INTEGER PRIMARY KEY, title TEXT NOT NULL, description TEXT NOT NULL DEFAULT '', state TEXT NOT NULL DEFAULT 'open',
149 due_on TEXT, created_at TEXT NOT NULL, updated_at TEXT NOT NULL, closed_at TEXT, creator TEXT
150);
151CREATE TABLE IF NOT EXISTS issue_milestones (number INTEGER PRIMARY KEY, milestone INTEGER NOT NULL);
152CREATE INDEX IF NOT EXISTS issue_milestones_milestone ON issue_milestones(milestone);
153CREATE TABLE IF NOT EXISTS import_findings (blob TEXT NOT NULL, kind TEXT NOT NULL, at TEXT NOT NULL, PRIMARY KEY (blob, kind));
154-- Refs a running import wrote, with the oid it set, so a retry only moves refs it owns (never a ref people adopted).
155CREATE TABLE IF NOT EXISTS import_refs (ref TEXT PRIMARY KEY, oid TEXT NOT NULL, import_id TEXT NOT NULL);
156-- A copy of the owning organization's Actions variables (D1 org_variables), kept in sync by the Directory.
157CREATE TABLE IF NOT EXISTS org_variables (name TEXT PRIMARY KEY, value TEXT NOT NULL);
158-- Self-hosted runners registered with a registration token (repo/actions.ts registerRunner). A runner
159-- claims jobs with its own token, never a person's; an ephemeral one's token ends with its first job.
160CREATE TABLE IF NOT EXISTS runners (id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT NOT NULL, labels TEXT NOT NULL, token_hash TEXT, ephemeral INTEGER NOT NULL DEFAULT 0, created_by TEXT NOT NULL, created_at TEXT NOT NULL, last_seen_at TEXT);
161CREATE TABLE IF NOT EXISTS runner_registrations (token_hash TEXT PRIMARY KEY, created_by TEXT NOT NULL, expires_at TEXT NOT NULL);
162`;
163
164/** Columns added after a table was first created; added with ALTER TABLE when missing. */
165const ADDED_COLUMNS: Array<[table: string, column: string, definition: string]> = [
166 // The commit a run reports its checks on: the PR head for pull_request runs, whose
167 // GITHUB_SHA is the test merge commit. Null means the run's own sha.
168 ["runs", "head_sha", "TEXT"],
169 // 1 when the code a run executes came from an agent (repo/actions.ts agentOrigin): its jobs act as an agent.
170 ["runs", "agent_origin", "INTEGER NOT NULL DEFAULT 0"],
171 // The repository a pull request's head branch lives in, when it is not this one (a fork).
172 ["pulls", "head_repo_id", "TEXT"],
173 // Bytes an upload in progress has staged (counted against the budgets), and the
174 // committed object's etag (a later overwrite of the object is refused).
175 ["rt_caches", "staged", "INTEGER NOT NULL DEFAULT 0"],
176 ["rt_caches", "etag", "TEXT"],
177 ["rt_artifacts", "staged", "INTEGER NOT NULL DEFAULT 0"],
178 ["rt_artifacts", "etag", "TEXT"],
179 // 1 for an artifact an older runner uploaded through the runner API: a tarball at its legacy key (repo/runtime.ts).
180 ["rt_artifacts", "legacy", "INTEGER NOT NULL DEFAULT 0"],
181 // Who created a check run over the REST API (null for Diffinery Actions), and GitHub's numeric id.
182 ["checks", "creator_id", "TEXT"],
183 ["checks", "creator_login", "TEXT"],
184 ["checks", "creator_kind", "TEXT"],
185 ["checks", "seq", "INTEGER"],
186 // Per matrix leg: job-level continue-on-error, and the job-level concurrency group with its cancel-in-progress.
187 ["jobs", "continue_on_error", "INTEGER NOT NULL DEFAULT 0"],
188 ["jobs", "concurrency_group", "TEXT"],
189 ["jobs", "concurrency_cancel", "INTEGER NOT NULL DEFAULT 0"],
190 // Characters of log text stored so far (repo/actions.ts MAX_JOB_LOG).
191 ["jobs", "log_chars", "INTEGER NOT NULL DEFAULT 0"],
192 // 1 while a queued job waits for one of the repository's hosted runner slots (repo/actions.ts MAX_HOSTED_JOBS).
193 ["jobs", "fleet_held", "INTEGER NOT NULL DEFAULT 0"],
194 // The registered self-hosted runner that claimed the job (null for hosted jobs and personal-token claims).
195 ["jobs", "runner_id", "INTEGER"],
196 // Times a hosted job's container couldn't get a machine and the job went back in line (repo/actions.ts MAX_PLACEMENT_TRIES).
197 ["jobs", "placement_tries", "INTEGER NOT NULL DEFAULT 0"],
198 // What the runner reported when it fetched the job (JSON RunnerInfo: image, location, instance), for investigations.
199 ["jobs", "runner_info", "TEXT"],
200 // The GitHub import that wrote this row (null for everything else); a retry may only overwrite its own rows.
201 ["issues", "imported", "TEXT"],
202 ["pulls", "imported", "TEXT"],
203 ["comments", "imported", "TEXT"],
204 ["reviews", "imported", "TEXT"],
205 ["labels", "imported", "TEXT"],
206 ["milestones", "imported", "TEXT"],
207 ["protections", "imported", "TEXT"],
208 // Auto-merge while it is on (repo/auto-merge.ts AutoMergeState as JSON); null when off.
209 ["pulls", "auto_merge", "TEXT"],
210 ["releases", "imported", "TEXT"],
211 // "task" for an agent task's sandbox (agentic/): no public sandbox call reaches it, and its git
212 // credential also needs the person who started the task (task_person) to still be allowed.
213 ["sandboxes", "kind", "TEXT NOT NULL DEFAULT 'api'"],
214 ["sandboxes", "task_person", "TEXT"],
215 // Environment protection rules (repo/deployments.ts): required reviewers as JSON [{type: "User", id, login}],
216 // a wait timer in minutes, and whether whoever started a run may approve it.
217 ["environments", "reviewers", "TEXT"],
218 ["environments", "wait_timer", "INTEGER NOT NULL DEFAULT 0"],
219 ["environments", "prevent_self_review", "INTEGER NOT NULL DEFAULT 0"],
220 // The job's `environment:` name, evaluated when it is launched; runnerSpec uses this, never a second evaluation.
221 ["jobs", "environment", "TEXT"],
222 // While a job waits on its environment's protection rules: when it started waiting, and "released" once they passed.
223 ["jobs", "gate_since", "TEXT"],
224 ["jobs", "gate_state", "TEXT"],
225 // Logins that started earlier attempts of a re-run (JSON list): prevent_self_review covers them too.
226 ["runs", "prior_actors", "TEXT"],
227 // Bytes an object takes as stored (compressed, in SQLite or R2); null in objects stored before it was kept.
228 ["objects", "stored", "INTEGER"],
229];
230
231export function migrateRepoSchema(sql: SqlStorage): void {
232 for (const [table, column, definition] of ADDED_COLUMNS) {
233 const has = sql.exec<{ n: number }>(`SELECT count(*) AS n FROM pragma_table_info('${table}') WHERE name = ?`, column).one().n > 0;
234 if (!has) sql.exec(`ALTER TABLE ${table} ADD COLUMN ${column} ${definition}`);
235 }
236 // Older runners' artifacts were first recorded in `artifacts`, which the REST API didn't list: they
237 // move to rt_artifacts, kept for 90 days from their upload. One shadowed by an artifact service
238 // upload of the same name was never shown and is dropped (its object goes with the bucket's age rule).
239 sql.exec(
240 `INSERT INTO rt_artifacts (run_id, job_id, name, size, mime_type, status, created_at, updated_at, expires_at, legacy)
241 SELECT a.run_id, '', a.name, a.size, 'application/gzip', 'ready', a.created_at, a.created_at, strftime('%Y-%m-%dT%H:%M:%fZ', a.created_at, '+90 days'), 1 FROM artifacts a
242 WHERE NOT EXISTS (SELECT 1 FROM rt_artifacts r WHERE r.run_id = a.run_id AND r.name = a.name)`,
243 );
244 sql.exec("DELETE FROM artifacts");
245}