| 1 | import type { ImportedComment, ImportedIssue, ImportedMilestone, ImportedReview } from "../imports/mapping.js"; |
| 2 | import { HttpError, now } from "../lib/util.js"; |
| 3 | |
| 4 | /** Refs only a GitHub import writes (pull request heads and chunks in flight); see containers/runner/src/importer.ts. */ |
| 5 | export const IMPORT_REF_PREFIX = "refs/diffinery-import/"; |
| 6 | |
| 7 | export type Milestone = { |
| 8 | number: number; |
| 9 | title: string; |
| 10 | description: string; |
| 11 | state: "open" | "closed"; |
| 12 | dueOn: string | null; |
| 13 | createdAt: string; |
| 14 | updatedAt: string; |
| 15 | closedAt: string | null; |
| 16 | creator: string | null; |
| 17 | openIssues: number; |
| 18 | closedIssues: number; |
| 19 | }; |
| 20 | |
| 21 | type MilestoneRow = { |
| 22 | number: number; |
| 23 | title: string; |
| 24 | description: string; |
| 25 | state: "open" | "closed"; |
| 26 | due_on: string | null; |
| 27 | created_at: string; |
| 28 | updated_at: string; |
| 29 | closed_at: string | null; |
| 30 | creator: string | null; |
| 31 | }; |
| 32 | |
| 33 | /** |
| 34 | * Repository side of GitHub imports, plus milestones. Imported rows keep their |
| 35 | * GitHub numbers, timestamps and people, and every write is an upsert keyed by |
| 36 | * the GitHub id, so a retried import step never duplicates anything. Nothing |
| 37 | * here emits events: an import fires no webhooks, workflows or notifications. |
| 38 | */ |
| 39 | export class RepoImports { |
| 40 | constructor(private readonly sql: SqlStorage) {} |
| 41 | |
| 42 | // ---------------------------------------------------------------- labels and milestones |
| 43 | |
| 44 | labels(list: Array<{ name: string; color: string; description: string }>, importId: string): number { |
| 45 | for (const l of list) |
| 46 | this.sql.exec( |
| 47 | `INSERT INTO labels (name, color, description, imported) VALUES (?, ?, ?, ?) |
| 48 | ON CONFLICT(name) DO UPDATE SET color = excluded.color, description = excluded.description, imported = excluded.imported |
| 49 | WHERE labels.imported = excluded.imported`, |
| 50 | l.name, |
| 51 | (l.color || "888888").replace(/^#/, ""), |
| 52 | l.description ?? "", |
| 53 | importId, |
| 54 | ); |
| 55 | return list.length; |
| 56 | } |
| 57 | |
| 58 | milestones(list: ImportedMilestone[], importId: string): number { |
| 59 | for (const m of list) { |
| 60 | this.sql.exec( |
| 61 | `INSERT INTO milestones (number, title, description, state, due_on, created_at, updated_at, closed_at, creator, imported) |
| 62 | VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) |
| 63 | ON CONFLICT(number) DO UPDATE SET title = excluded.title, description = excluded.description, state = excluded.state, |
| 64 | due_on = excluded.due_on, created_at = excluded.created_at, updated_at = excluded.updated_at, closed_at = excluded.closed_at, |
| 65 | creator = excluded.creator, imported = excluded.imported |
| 66 | WHERE milestones.imported = excluded.imported`, |
| 67 | m.number, |
| 68 | m.title, |
| 69 | m.description, |
| 70 | m.state, |
| 71 | m.dueOn, |
| 72 | m.createdAt, |
| 73 | m.updatedAt, |
| 74 | m.closedAt, |
| 75 | m.creator, |
| 76 | importId, |
| 77 | ); |
| 78 | } |
| 79 | this.raiseCounter("milestone", Math.max(0, ...list.map((m) => m.number))); |
| 80 | return list.length; |
| 81 | } |
| 82 | |
| 83 | // ---------------------------------------------------------------- issues and pull requests |
| 84 | |
| 85 | /** Insert issues and pull requests with their original numbers; only overwrites rows this import owns. */ |
| 86 | issues(list: ImportedIssue[], importId: string): number { |
| 87 | for (const i of list) { |
| 88 | this.sql.exec( |
| 89 | `INSERT INTO issues (number, kind, title, body, state, state_reason, author_id, author_login, author_kind, assignees, labels, created_at, updated_at, closed_at, locked, imported) |
| 90 | VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) |
| 91 | ON CONFLICT(number) DO UPDATE SET kind = excluded.kind, title = excluded.title, body = excluded.body, state = excluded.state, |
| 92 | state_reason = excluded.state_reason, author_id = excluded.author_id, author_login = excluded.author_login, author_kind = excluded.author_kind, |
| 93 | assignees = excluded.assignees, labels = excluded.labels, created_at = excluded.created_at, updated_at = excluded.updated_at, |
| 94 | closed_at = excluded.closed_at, locked = excluded.locked, imported = excluded.imported |
| 95 | WHERE issues.imported = excluded.imported`, |
| 96 | i.number, |
| 97 | i.kind, |
| 98 | i.title, |
| 99 | i.body, |
| 100 | i.state, |
| 101 | i.stateReason, |
| 102 | i.author.id, |
| 103 | i.author.login, |
| 104 | i.author.kind, |
| 105 | JSON.stringify(i.assignees), |
| 106 | JSON.stringify(i.labels), |
| 107 | i.createdAt, |
| 108 | i.updatedAt, |
| 109 | i.closedAt, |
| 110 | i.locked ? 1 : 0, |
| 111 | importId, |
| 112 | ); |
| 113 | if (i.pull) { |
| 114 | const p = i.pull; |
| 115 | this.sql.exec( |
| 116 | `INSERT INTO pulls (number, head_ref, head_repo_id, head_sha, base_ref, draft, merged, merged_at, merged_by, merge_commit, requested_reviewers, imported) |
| 117 | VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) |
| 118 | ON CONFLICT(number) DO UPDATE SET head_ref = excluded.head_ref, head_repo_id = excluded.head_repo_id, head_sha = excluded.head_sha, base_ref = excluded.base_ref, draft = excluded.draft, |
| 119 | merged = excluded.merged, merged_at = excluded.merged_at, merged_by = COALESCE(excluded.merged_by, pulls.merged_by), |
| 120 | merge_commit = excluded.merge_commit, requested_reviewers = excluded.requested_reviewers, imported = excluded.imported |
| 121 | WHERE pulls.imported = excluded.imported`, |
| 122 | i.number, |
| 123 | p.headRef, |
| 124 | // A fork on GitHub: marks the pull request as one whose runs get no write and no secrets. |
| 125 | p.headRepo ? `github:${p.headRepo}` : null, |
| 126 | p.headSha, |
| 127 | p.baseRef, |
| 128 | p.draft ? 1 : 0, |
| 129 | p.merged ? 1 : 0, |
| 130 | p.mergedAt, |
| 131 | p.mergedBy, |
| 132 | p.mergeCommit, |
| 133 | JSON.stringify(p.requestedReviewers), |
| 134 | importId, |
| 135 | ); |
| 136 | } |
| 137 | if (i.milestone) this.sql.exec("INSERT OR REPLACE INTO issue_milestones (number, milestone) VALUES (?, ?)", i.number, i.milestone); |
| 138 | else this.sql.exec("DELETE FROM issue_milestones WHERE number = ?", i.number); |
| 139 | } |
| 140 | this.raiseCounter("issue", Math.max(0, ...list.map((i) => i.number))); |
| 141 | return list.length; |
| 142 | } |
| 143 | |
| 144 | /** GitHub reports who merged a pull request only on the single pull request. */ |
| 145 | setMergedBy(number: number, login: string): void { |
| 146 | this.sql.exec("UPDATE pulls SET merged_by = ? WHERE number = ? AND merged = 1", login, number); |
| 147 | } |
| 148 | |
| 149 | /** Pull request numbers after `after`, for the reviews pass. */ |
| 150 | pullNumbers(after: number, limit: number): Array<{ number: number; merged: boolean }> { |
| 151 | return this.sql |
| 152 | .exec<{ number: number; merged: number }>("SELECT number, merged FROM pulls WHERE number > ? ORDER BY number LIMIT ?", after, limit) |
| 153 | .toArray() |
| 154 | .map((r) => ({ number: r.number, merged: !!r.merged })); |
| 155 | } |
| 156 | |
| 157 | comments(list: ImportedComment[], importId: string): number { |
| 158 | let n = 0; |
| 159 | for (const c of list) { |
| 160 | if (!this.sql.exec<{ n: number }>("SELECT count(*) AS n FROM issues WHERE number = ?", c.number).one().n) continue; |
| 161 | // Not INSERT OR REPLACE: that reassigns the rowid, and comment "seq" (the public id) is the rowid. |
| 162 | this.sql.exec( |
| 163 | `INSERT INTO comments (id, number, author_id, author_login, author_kind, body, created_at, updated_at, review_id, path, line, side, commit_id, in_reply_to, imported) |
| 164 | VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) |
| 165 | ON CONFLICT(id) DO UPDATE SET number = excluded.number, author_id = excluded.author_id, author_login = excluded.author_login, |
| 166 | author_kind = excluded.author_kind, body = excluded.body, created_at = excluded.created_at, updated_at = excluded.updated_at, |
| 167 | review_id = excluded.review_id, path = excluded.path, line = excluded.line, side = excluded.side, commit_id = excluded.commit_id, |
| 168 | in_reply_to = excluded.in_reply_to, imported = excluded.imported |
| 169 | WHERE comments.imported = excluded.imported`, |
| 170 | c.id, |
| 171 | c.number, |
| 172 | c.author.id, |
| 173 | c.author.login, |
| 174 | c.author.kind, |
| 175 | c.body, |
| 176 | c.createdAt, |
| 177 | c.updatedAt, |
| 178 | c.reviewId ?? null, |
| 179 | c.path ?? null, |
| 180 | c.line ?? null, |
| 181 | c.side ?? null, |
| 182 | c.commitId ?? null, |
| 183 | c.inReplyTo ?? null, |
| 184 | importId, |
| 185 | ); |
| 186 | n++; |
| 187 | } |
| 188 | return n; |
| 189 | } |
| 190 | |
| 191 | reviews(list: ImportedReview[], importId: string): number { |
| 192 | for (const r of list) { |
| 193 | this.sql.exec( |
| 194 | `INSERT INTO reviews (id, number, author_id, author_login, author_kind, state, body, commit_id, submitted_at, imported) |
| 195 | VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) |
| 196 | ON CONFLICT(id) DO UPDATE SET number = excluded.number, author_id = excluded.author_id, author_login = excluded.author_login, |
| 197 | author_kind = excluded.author_kind, state = excluded.state, body = excluded.body, commit_id = excluded.commit_id, |
| 198 | submitted_at = excluded.submitted_at, imported = excluded.imported |
| 199 | WHERE reviews.imported = excluded.imported`, |
| 200 | r.id, |
| 201 | r.number, |
| 202 | r.author.id, |
| 203 | r.author.login, |
| 204 | r.author.kind, |
| 205 | r.state, |
| 206 | r.body, |
| 207 | r.commitId, |
| 208 | r.submittedAt, |
| 209 | importId, |
| 210 | ); |
| 211 | } |
| 212 | return list.length; |
| 213 | } |
| 214 | |
| 215 | /** New issues continue after the highest imported number. */ |
| 216 | private raiseCounter(name: string, value: number): void { |
| 217 | if (value > 0) this.sql.exec("INSERT INTO counters (name, value) VALUES (?, ?) ON CONFLICT(name) DO UPDATE SET value = max(value, excluded.value)", name, value); |
| 218 | } |
| 219 | |
| 220 | counts(): { issues: number; pulls: number; comments: number; reviews: number; labels: number; milestones: number; maxNumber: number } { |
| 221 | const q = (s: string) => this.sql.exec<{ n: number }>(s).one().n; |
| 222 | return { |
| 223 | issues: q("SELECT count(*) AS n FROM issues WHERE kind = 'issue'"), |
| 224 | pulls: q("SELECT count(*) AS n FROM issues WHERE kind = 'pull'"), |
| 225 | comments: q("SELECT count(*) AS n FROM comments"), |
| 226 | reviews: q("SELECT count(*) AS n FROM reviews"), |
| 227 | labels: q("SELECT count(*) AS n FROM labels"), |
| 228 | milestones: q("SELECT count(*) AS n FROM milestones"), |
| 229 | maxNumber: q("SELECT COALESCE(max(number), 0) AS n FROM issues"), |
| 230 | }; |
| 231 | } |
| 232 | |
| 233 | // ---------------------------------------------------------------- push protection findings |
| 234 | |
| 235 | /** Secrets an import pushed with an admin bypass, so the summary can list what to rotate. */ |
| 236 | recordFindings(leaks: Array<{ oid: string; kind: string }>): void { |
| 237 | for (const l of leaks.slice(0, 500)) this.sql.exec("INSERT OR IGNORE INTO import_findings (blob, kind, at) VALUES (?, ?, ?)", l.oid, l.kind, now()); |
| 238 | } |
| 239 | |
| 240 | findings(): Array<{ blob: string; kind: string; at: string }> { |
| 241 | return this.sql.exec<{ blob: string; kind: string; at: string }>("SELECT blob, kind, at FROM import_findings ORDER BY at, blob").toArray(); |
| 242 | } |
| 243 | |
| 244 | /** Imported pull requests and the head commit each one pins. */ |
| 245 | pullHeads(): Array<{ number: number; headSha: string }> { |
| 246 | return this.sql.exec<{ number: number; head_sha: string }>("SELECT number, head_sha FROM pulls ORDER BY number").toArray().map((r) => ({ number: r.number, headSha: r.head_sha })); |
| 247 | } |
| 248 | |
| 249 | // ---------------------------------------------------------------- import lock and provenance (H2) |
| 250 | |
| 251 | /** |
| 252 | * True when the repository already holds any issue, pull request, comment, review, label, |
| 253 | * milestone, protection or release that this import did not write. A retry refuses to reuse a repo |
| 254 | * that people have since adopted, so an import never clobbers real content. |
| 255 | */ |
| 256 | hasForeignContent(importId: string): boolean { |
| 257 | const tables = ["issues", "pulls", "comments", "reviews", "labels", "milestones", "protections", "releases"]; |
| 258 | for (const t of tables) { |
| 259 | const n = this.sql.exec<{ n: number }>(`SELECT count(*) AS n FROM ${t} WHERE imported IS NULL OR imported <> ?`, importId).one().n; |
| 260 | if (n > 0) return true; |
| 261 | } |
| 262 | // A milestone assignment or issue-milestone link is content too; covered by issues/milestones above. |
| 263 | return false; |
| 264 | } |
| 265 | |
| 266 | /** Record that this import set `ref` to `oid` (so a later update may move only refs it owns). */ |
| 267 | recordImportRef(ref: string, oid: string, importId: string): void { |
| 268 | this.sql.exec("INSERT INTO import_refs (ref, oid, import_id) VALUES (?, ?, ?) ON CONFLICT(ref) DO UPDATE SET oid = excluded.oid, import_id = excluded.import_id", ref, oid, importId); |
| 269 | } |
| 270 | |
| 271 | /** The oid this import last wrote to `ref`, or null if the import never wrote it. */ |
| 272 | importRefOid(ref: string, importId: string): string | null { |
| 273 | const r = this.sql.exec<{ oid: string }>("SELECT oid FROM import_refs WHERE ref = ? AND import_id = ?", ref, importId).toArray()[0]; |
| 274 | return r ? r.oid : null; |
| 275 | } |
| 276 | |
| 277 | dropImportRefs(importId: string): void { |
| 278 | this.sql.exec("DELETE FROM import_refs WHERE import_id = ?", importId); |
| 279 | } |
| 280 | |
| 281 | /** Write imported branch protection, only overwriting a rule this import owns. */ |
| 282 | protection(pattern: string, rules: string, importId: string): void { |
| 283 | this.sql.exec( |
| 284 | `INSERT INTO protections (pattern, rules, imported) VALUES (?, ?, ?) |
| 285 | ON CONFLICT(pattern) DO UPDATE SET rules = excluded.rules, imported = excluded.imported |
| 286 | WHERE protections.imported = excluded.imported`, |
| 287 | pattern, |
| 288 | rules, |
| 289 | importId, |
| 290 | ); |
| 291 | } |
| 292 | |
| 293 | // ---------------------------------------------------------------- organization variables |
| 294 | |
| 295 | /** Replace the copy of the owning organization's Actions variables. */ |
| 296 | setOrgVariables(vars: Record<string, string>): void { |
| 297 | this.sql.exec("DELETE FROM org_variables"); |
| 298 | for (const [name, value] of Object.entries(vars)) this.sql.exec("INSERT INTO org_variables (name, value) VALUES (?, ?)", name, value); |
| 299 | } |
| 300 | |
| 301 | // ---------------------------------------------------------------- milestones |
| 302 | |
| 303 | private milestoneRow(m: MilestoneRow): Milestone { |
| 304 | const counts = this.sql |
| 305 | .exec<{ state: string; n: number }>("SELECT i.state, count(*) AS n FROM issue_milestones x JOIN issues i ON i.number = x.number WHERE x.milestone = ? GROUP BY i.state", m.number) |
| 306 | .toArray(); |
| 307 | return { |
| 308 | number: m.number, |
| 309 | title: m.title, |
| 310 | description: m.description, |
| 311 | state: m.state, |
| 312 | dueOn: m.due_on, |
| 313 | createdAt: m.created_at, |
| 314 | updatedAt: m.updated_at, |
| 315 | closedAt: m.closed_at, |
| 316 | creator: m.creator, |
| 317 | openIssues: counts.find((c) => c.state === "open")?.n ?? 0, |
| 318 | closedIssues: counts.find((c) => c.state === "closed")?.n ?? 0, |
| 319 | }; |
| 320 | } |
| 321 | |
| 322 | listMilestones(state: "open" | "closed" | "all" = "open"): Milestone[] { |
| 323 | const rows = this.sql |
| 324 | .exec<MilestoneRow>(`SELECT * FROM milestones ${state === "all" ? "" : "WHERE state = ?"} ORDER BY COALESCE(due_on, '9999'), number`, ...(state === "all" ? [] : [state])) |
| 325 | .toArray(); |
| 326 | return rows.map((r) => this.milestoneRow(r)); |
| 327 | } |
| 328 | |
| 329 | milestone(number: number): Milestone { |
| 330 | const r = this.sql.exec<MilestoneRow>("SELECT * FROM milestones WHERE number = ?", number).toArray()[0]; |
| 331 | if (!r) throw new HttpError(404, "milestone not found"); |
| 332 | return this.milestoneRow(r); |
| 333 | } |
| 334 | |
| 335 | saveMilestone(number: number | null, input: { title?: string; description?: string; state?: "open" | "closed"; dueOn?: string | null }, creator: string): Milestone { |
| 336 | const t = now(); |
| 337 | if (number === null) { |
| 338 | if (!input.title?.trim()) throw new HttpError(422, "title is required"); |
| 339 | this.sql.exec("INSERT INTO counters (name, value) VALUES ('milestone', 1) ON CONFLICT(name) DO UPDATE SET value = value + 1"); |
| 340 | const n = this.sql.exec<{ value: number }>("SELECT value FROM counters WHERE name = 'milestone'").one().value; |
| 341 | const state = input.state ?? "open"; |
| 342 | this.sql.exec( |
| 343 | "INSERT INTO milestones (number, title, description, state, due_on, created_at, updated_at, creator, closed_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", |
| 344 | n, |
| 345 | input.title.trim(), |
| 346 | input.description ?? "", |
| 347 | state, |
| 348 | input.dueOn ?? null, |
| 349 | t, |
| 350 | t, |
| 351 | creator, |
| 352 | state === "closed" ? t : null, |
| 353 | ); |
| 354 | return this.milestone(n); |
| 355 | } |
| 356 | const m = this.milestone(number); |
| 357 | const state = input.state ?? m.state; |
| 358 | this.sql.exec( |
| 359 | "UPDATE milestones SET title = ?, description = ?, state = ?, due_on = ?, updated_at = ?, closed_at = ? WHERE number = ?", |
| 360 | input.title?.trim() || m.title, |
| 361 | input.description ?? m.description, |
| 362 | state, |
| 363 | input.dueOn === undefined ? m.dueOn : input.dueOn, |
| 364 | t, |
| 365 | state === "closed" ? (m.closedAt ?? t) : null, |
| 366 | number, |
| 367 | ); |
| 368 | return this.milestone(number); |
| 369 | } |
| 370 | |
| 371 | setIssueMilestone(number: number, milestone: number | null): void { |
| 372 | if (!this.sql.exec<{ n: number }>("SELECT count(*) AS n FROM issues WHERE number = ?", number).one().n) throw new HttpError(404, `#${number} not found`); |
| 373 | if (milestone === null) { |
| 374 | this.sql.exec("DELETE FROM issue_milestones WHERE number = ?", number); |
| 375 | return; |
| 376 | } |
| 377 | this.milestone(milestone); |
| 378 | this.sql.exec("INSERT OR REPLACE INTO issue_milestones (number, milestone) VALUES (?, ?)", number, milestone); |
| 379 | } |
| 380 | |
| 381 | milestoneOf(number: number): { number: number; title: string; state: "open" | "closed"; dueOn: string | null } | null { |
| 382 | const r = this.sql |
| 383 | .exec<{ number: number; title: string; state: "open" | "closed"; due_on: string | null }>("SELECT m.number, m.title, m.state, m.due_on FROM issue_milestones x JOIN milestones m ON m.number = x.milestone WHERE x.number = ?", number) |
| 384 | .toArray()[0]; |
| 385 | return r ? { number: r.number, title: r.title, state: r.state, dueOn: r.due_on } : null; |
| 386 | } |
| 387 | } |