| 1 | import { Hono } from "hono"; |
| 2 | import { Apps } from "../apps/store.js"; |
| 3 | import type { Env } from "../env.js"; |
| 4 | import { callerFor, type AppContext, type AppVars, type Principal } from "../auth/principal.js"; |
| 5 | import type { Repo } from "../lib/directory.js"; |
| 6 | import { matchesText, parseIssueQuery, parseRepoQuery } from "../lib/search-query.js"; |
| 7 | import { HttpError } from "../lib/util.js"; |
| 8 | import type { Issue } from "../repo/collab.js"; |
| 9 | import { hiddenAuthors, repoStub } from "./common.js"; |
| 10 | import { baseOf, ghIssue, ghRepo, pageOf, setLinks, topicsOf } from "./gh.js"; |
| 11 | import { issueCtx } from "./ghctx.js"; |
| 12 | |
| 13 | /** GET /search/issues and /search/repositories over the repositories the caller can read. */ |
| 14 | export const search = new Hono<{ Bindings: Env; Variables: AppVars }>(); |
| 15 | |
| 16 | /** Searches fan out to at most this many repositories; more sets `incomplete_results`. */ |
| 17 | const MAX_REPOS = 50; |
| 18 | /** Anonymous searches fan out to fewer: each repository searched is a Durable Object request, and anyone can send them. */ |
| 19 | const MAX_ANONYMOUS_REPOS = 10; |
| 20 | |
| 21 | /** How many repositories one issue search may fan out to for this caller. */ |
| 22 | export const searchFanOut = (p: Principal): number => (p.type === "anonymous" ? MAX_ANONYMOUS_REPOS : MAX_REPOS); |
| 23 | const MAX_RESULTS = 1000; |
| 24 | /** GitHub's limit on the length of a search query. */ |
| 25 | const MAX_QUERY = 256; |
| 26 | /** Repository search checks access on at most this many candidates. */ |
| 27 | const MAX_REPO_CANDIDATES = 300; |
| 28 | |
| 29 | /** |
| 30 | * Repositories the caller might read: only active accounts (and agents whose |
| 31 | * owner is active) get anything beyond public repositories, and every |
| 32 | * candidate is checked again with callerFor, which applies roles, token |
| 33 | * scopes and repository restrictions. |
| 34 | */ |
| 35 | async function candidates(c: AppContext, q?: { text: string[]; owners: string[] }): Promise<Repo[]> { |
| 36 | const p = c.get("principal"); |
| 37 | const dir = c.get("dir"); |
| 38 | // Public repositories are looked up by the query's words and owners in D1, not taken from a fixed |
| 39 | // number of the most recent ones, so any of them can be found however many there are. |
| 40 | const publicOnes = () => dir.publicRepos(500, { words: q?.text.filter((w) => !w.startsWith("-")), owners: q?.owners }); |
| 41 | if (p.type === "account") { |
| 42 | if (p.account.status !== "active") return []; |
| 43 | const [mine, open] = await Promise.all([dir.reposVisibleTo(p.account), publicOnes()]); |
| 44 | const seen = new Set(mine.map((r) => r.id)); |
| 45 | return [...mine, ...open.filter((r) => !seen.has(r.id))]; |
| 46 | } |
| 47 | if (p.type === "repo-token") { |
| 48 | const r = await dir.repoById(p.repoId); |
| 49 | return r ? [r] : []; |
| 50 | } |
| 51 | if (p.type === "installation") { |
| 52 | const only = p.auth.repoIds; |
| 53 | const [mine, open] = await Promise.all([new Apps(c.env, dir).reposOf(p.auth.installation), publicOnes()]); |
| 54 | const own = only ? mine.filter((r) => only.includes(r.id)) : mine; |
| 55 | const seen = new Set(own.map((r) => r.id)); |
| 56 | return [...own, ...open.filter((r) => !seen.has(r.id))]; |
| 57 | } |
| 58 | return publicOnes(); |
| 59 | } |
| 60 | |
| 61 | /** The first `limit` repositories (in order) the caller can actually read, and whether more were left unchecked. */ |
| 62 | async function authorized(c: AppContext, repos: Repo[], limit: number): Promise<{ repos: Repo[]; incomplete: boolean }> { |
| 63 | const out: Repo[] = []; |
| 64 | for (let i = 0; i < repos.length; i++) { |
| 65 | if (out.length >= limit) return { repos: out, incomplete: true }; |
| 66 | if (await callerFor(c, repos[i]!)) out.push(repos[i]!); |
| 67 | } |
| 68 | return { repos: out, incomplete: false }; |
| 69 | } |
| 70 | |
| 71 | const fullName = (r: Repo) => `${r.owner_login}/${r.name}`.toLowerCase(); |
| 72 | |
| 73 | function queryOf(c: AppContext): string { |
| 74 | const raw = c.req.query("q") ?? ""; |
| 75 | if (!raw.trim()) throw new HttpError(422, "Validation Failed", { errors: [{ resource: "Search", field: "q", code: "missing" }] }); |
| 76 | if (raw.length > MAX_QUERY) throw new HttpError(422, `The search is longer than ${MAX_QUERY} characters.`, { errors: [{ resource: "Search", field: "q", code: "invalid" }] }); |
| 77 | return raw; |
| 78 | } |
| 79 | |
| 80 | search.get("/search/issues", async (c) => { |
| 81 | const q = parseIssueQuery(queryOf(c), { sort: c.req.query("sort"), order: c.req.query("order") }); |
| 82 | const p = pageOf(c); |
| 83 | if (p.offset >= MAX_RESULTS) throw new HttpError(422, "Only the first 1000 search results are available"); |
| 84 | const owners = [...q.owners, ...q.repos.map((r) => r.split("/")[0]!)]; |
| 85 | let repos = await candidates(c, { text: [], owners }); |
| 86 | if (q.repos.length) { |
| 87 | const want = new Set(q.repos.map((r) => r.toLowerCase())); |
| 88 | repos = repos.filter((r) => want.has(fullName(r))); |
| 89 | } |
| 90 | if (q.owners.length) { |
| 91 | const owners = new Set(q.owners.map((o) => o.toLowerCase())); |
| 92 | repos = repos.filter((r) => owners.has(r.owner_login.toLowerCase())); |
| 93 | } |
| 94 | const allowed = await authorized(c, repos, searchFanOut(c.get("principal"))); |
| 95 | // Each repository returns at most `want` hits as (number, sort key); only the page is loaded afterwards. |
| 96 | const want = Math.min(p.offset + p.perPage, MAX_RESULTS); |
| 97 | const results = await Promise.all(allowed.repos.map(async (r) => ({ repo: r, ...(await repoStub(c.env, r.id).searchIssues(q, want)) }))); |
| 98 | const total = results.reduce((n, r) => n + r.total, 0); |
| 99 | const sort = q.sort ?? { field: "created", dir: "desc" }; |
| 100 | const page = results |
| 101 | .flatMap((r) => r.hits.map((h) => ({ repo: r.repo, number: h.number, key: h.key }))) |
| 102 | .sort((a, b) => { |
| 103 | const d = a.key < b.key ? -1 : a.key > b.key ? 1 : 0; |
| 104 | return (sort.dir === "asc" ? d : -d) || (a.repo.id < b.repo.id ? -1 : a.repo.id > b.repo.id ? 1 : b.number - a.number); |
| 105 | }) |
| 106 | .slice(p.offset, p.offset + p.perPage); |
| 107 | const byRepo = new Map<string, { repo: Repo; numbers: number[] }>(); |
| 108 | for (const h of page) { |
| 109 | const entry = byRepo.get(h.repo.id) ?? { repo: h.repo, numbers: [] }; |
| 110 | entry.numbers.push(h.number); |
| 111 | byRepo.set(h.repo.id, entry); |
| 112 | } |
| 113 | const shaped = new Map<string, ReturnType<typeof ghIssue>>(); |
| 114 | for (const { repo, numbers } of byRepo.values()) { |
| 115 | const stub = repoStub(c.env, repo.id); |
| 116 | const issues = (await stub.issuesByNumber(numbers, await hiddenAuthors(c))) as unknown as Issue[]; |
| 117 | const x = await issueCtx(c, repo, stub, issues); |
| 118 | for (const i of issues) shaped.set(`${repo.id}#${i.number}`, ghIssue(x, i)); |
| 119 | } |
| 120 | setLinks(c, p, Math.min(total, MAX_RESULTS)); |
| 121 | const items = page.flatMap((h) => { |
| 122 | const item = shaped.get(`${h.repo.id}#${h.number}`); |
| 123 | return item ? [{ ...item, score: 1 }] : []; |
| 124 | }); |
| 125 | return c.json({ total_count: total, incomplete_results: allowed.incomplete, items }); |
| 126 | }); |
| 127 | |
| 128 | search.get("/search/repositories", async (c) => { |
| 129 | const q = parseRepoQuery(queryOf(c)); |
| 130 | const p = pageOf(c); |
| 131 | const fields = q.in.length ? q.in : ["name", "description", "topics"]; |
| 132 | const owners = new Set(q.owners.map((o) => o.toLowerCase())); |
| 133 | const names = new Set(q.repos.map((r) => r.toLowerCase())); |
| 134 | const filtered = (await candidates(c, q)).filter((r) => { |
| 135 | if (owners.size && !owners.has(r.owner_login.toLowerCase())) return false; |
| 136 | if (names.size && !names.has(fullName(r))) return false; |
| 137 | if (q.visibility && (q.visibility === "public") !== (r.visibility === "public")) return false; |
| 138 | if (q.archived !== undefined && !!r.archived !== q.archived) return false; |
| 139 | const topics = topicsOf(r); |
| 140 | if (q.topics.some((t) => !topics.includes(t))) return false; |
| 141 | const hay: string[] = []; |
| 142 | if (fields.includes("name")) hay.push(r.name, fullName(r)); |
| 143 | if (fields.includes("description")) hay.push(r.description ?? ""); |
| 144 | if (fields.includes("topics")) hay.push(topics.join(" ")); |
| 145 | return matchesText(q.text, hay); |
| 146 | }); |
| 147 | const allowed = await authorized(c, filtered, MAX_REPO_CANDIDATES); |
| 148 | const hits = allowed.repos; |
| 149 | const order = c.req.query("order") === "asc" ? 1 : -1; |
| 150 | if (c.req.query("sort") === "updated") hits.sort((a, b) => ((a.pushed_at ?? a.updated_at) < (b.pushed_at ?? b.updated_at) ? -order : order)); |
| 151 | setLinks(c, p, hits.length); |
| 152 | return c.json({ total_count: hits.length, incomplete_results: allowed.incomplete, items: hits.slice(p.offset, p.offset + p.perPage).map((r) => ({ ...ghRepo(baseOf(c), r), score: 1 })) }); |
| 153 | }); |