| 1 | /** |
| 2 | * The public documentation as data: the Markdown pages built from docs/ (see |
| 3 | * scripts/build-docs.mjs) plus the reference pages generated at runtime, and |
| 4 | * the agent-facing formats built from them (per-page Markdown, llms.txt, |
| 5 | * llms-full.txt and a search index). |
| 6 | */ |
| 7 | import { slugify } from "../web/markdown.js"; |
| 8 | import { SITE } from "../web/seo.js"; |
| 9 | import { DOCS_NAV, DOCS_PAGES } from "./content.generated.js"; |
| 10 | import { apiReferenceMarkdown, mcpToolsMarkdown } from "./reference.js"; |
| 11 | import { siteFullMarkdown, siteLlmsLines, sitePages } from "../web/site.js"; |
| 12 | |
| 13 | export interface Page { |
| 14 | slug: string; |
| 15 | title: string; |
| 16 | description: string; |
| 17 | markdown: string; |
| 18 | section: string; |
| 19 | /** When the page was last reviewed (YYYY-MM-DD), if its frontmatter says. */ |
| 20 | updated?: string; |
| 21 | } |
| 22 | |
| 23 | const RUNTIME: Record<string, { title: string; description: string; build: (origin: string) => string }> = { |
| 24 | api: { |
| 25 | title: "REST API reference", |
| 26 | description: "Every REST API endpoint with its parameters, access rules and an example, generated from the server's router.", |
| 27 | build: apiReferenceMarkdown, |
| 28 | }, |
| 29 | "reference/mcp-tools": { |
| 30 | title: "MCP tool reference", |
| 31 | description: "Every tool of the Diffinery MCP server with its arguments, generated from the server's tool list.", |
| 32 | build: mcpToolsMarkdown, |
| 33 | }, |
| 34 | }; |
| 35 | |
| 36 | const SECTION_OF = new Map<string, string>(DOCS_NAV.flatMap((s) => s.pages.map((p) => [p, s.title] as [string, string]))); |
| 37 | |
| 38 | /** Every page slug, in reading order. */ |
| 39 | export const SLUGS: string[] = DOCS_NAV.flatMap((s) => s.pages); |
| 40 | |
| 41 | const runtimeCache = new Map<string, string>(); |
| 42 | |
| 43 | export function getPage(slug: string, origin: string): Page | null { |
| 44 | const section = SECTION_OF.get(slug); |
| 45 | if (!section) return null; |
| 46 | const rt = RUNTIME[slug]; |
| 47 | if (rt) { |
| 48 | const key = `${origin} ${slug}`; |
| 49 | let md = runtimeCache.get(key); |
| 50 | if (md === undefined) runtimeCache.set(key, (md = rt.build(origin))); |
| 51 | return { slug, title: rt.title, description: rt.description, markdown: md, section }; |
| 52 | } |
| 53 | const p = DOCS_PAGES[slug]; |
| 54 | return p ? { slug, ...p, section } : null; |
| 55 | } |
| 56 | |
| 57 | export function allPages(origin: string): Page[] { |
| 58 | return SLUGS.map((s) => getPage(s, origin)).filter((p): p is Page => !!p); |
| 59 | } |
| 60 | |
| 61 | export function nav(): Array<{ title: string; pages: Array<{ slug: string; title: string }> }> { |
| 62 | return DOCS_NAV.map((s) => ({ title: s.title, pages: s.pages.map((slug) => ({ slug, title: RUNTIME[slug]?.title ?? DOCS_PAGES[slug]?.title ?? slug })) })); |
| 63 | } |
| 64 | |
| 65 | export const pageUrl = (slug: string) => (slug === "index" ? "/docs" : `/docs/${slug}`); |
| 66 | export const markdownUrl = (slug: string) => `/docs/${slug}.md`; |
| 67 | |
| 68 | /** The docs sections with their URL names ("Connect AI apps" is connect-ai-apps), for the per-section llms files. */ |
| 69 | export const SECTIONS: Array<{ slug: string; title: string; pages: string[] }> = DOCS_NAV.map((s) => ({ slug: slugify(s.title), title: s.title, pages: s.pages })); |
| 70 | export const sectionLlmsUrl = (section: string) => `/docs/${section}/llms.txt`; |
| 71 | export const sectionLlmsFullUrl = (section: string) => `/docs/${section}/llms-full.txt`; |
| 72 | |
| 73 | /** Strip inline Markdown from heading text for display. */ |
| 74 | export const plainHeading = (s: string) => |
| 75 | s |
| 76 | .replace(/`([^`]*)`/g, "$1") |
| 77 | .replace(/\[([^\]]*)\]\([^)]*\)/g, "$1") |
| 78 | .replace(/[*_]{1,2}([^*_]+)[*_]{1,2}/g, "$1") |
| 79 | .trim(); |
| 80 | |
| 81 | /** The page's `##` and `###` headings, with the ids the renderer gives them. */ |
| 82 | export function headings(markdown: string): Array<{ level: 2 | 3; id: string; text: string }> { |
| 83 | const out: Array<{ level: 2 | 3; id: string; text: string }> = []; |
| 84 | let fence: string | null = null; |
| 85 | for (const line of markdown.split("\n")) { |
| 86 | const f = /^ {0,3}(`{3,}|~{3,})/.exec(line); |
| 87 | if (f) { |
| 88 | if (!fence) fence = f[1]![0]!; |
| 89 | else if (f[1]![0] === fence) fence = null; |
| 90 | continue; |
| 91 | } |
| 92 | if (fence) continue; |
| 93 | const h = /^(##|###) +(.+?)\s*#*\s*$/.exec(line); |
| 94 | if (h) out.push({ level: h[1]!.length as 2 | 3, id: slugify(h[2]!), text: plainHeading(h[2]!) }); |
| 95 | } |
| 96 | return out; |
| 97 | } |
| 98 | |
| 99 | /** Root-relative links (`](/docs/...)`) made absolute, for Markdown read outside the site. */ |
| 100 | const absolute = (md: string, origin: string) => md.replace(/\]\(\/(?!\/)/g, `](${origin}/`); |
| 101 | |
| 102 | /** A page as standalone Markdown: what /docs/<slug>.md serves. */ |
| 103 | export function pageMarkdown(p: Page, origin: string): string { |
| 104 | return `# ${p.title}\n\n> ${p.description}\n\nSource: ${origin}${pageUrl(p.slug)}\n\n${absolute(p.markdown, origin).trim()}\n`; |
| 105 | } |
| 106 | |
| 107 | /** The site's host name, as these files name it: diffinery.com on the public instance. */ |
| 108 | const hostOf = (origin: string): string => { |
| 109 | try { |
| 110 | return new URL(origin).host; |
| 111 | } catch { |
| 112 | return origin; |
| 113 | } |
| 114 | }; |
| 115 | |
| 116 | export function llmsTxt(origin: string): string { |
| 117 | const lines = [ |
| 118 | `# ${SITE.name}`, |
| 119 | "", |
| 120 | `> ${SITE.description}`, |
| 121 | "", |
| 122 | `Diffinery is a git forge with GitHub-compatible Actions, a GitHub-style REST API and an MCP server for AI agents. This file indexes ${hostOf(origin)} for AI assistants and agents: what Diffinery is and how it compares with other products first, then the documentation. Everything here is written for people and agents alike:`, |
| 123 | "", |
| 124 | `- Every page is available as Markdown: add \`.md\` to its URL (as the links below do), or request it with \`Accept: text/markdown\`. The home page is ${origin}/index.md.`, |
| 125 | `- All pages in one file: ${origin}/llms-full.txt (large: every page, including the full REST API reference). For a smaller context, ${origin}/llms-small.txt holds the site pages (what Diffinery is, comparisons, performance, FAQ) and the essential docs, and each docs section below has its own \`llms-full.txt\`.`, |
| 126 | `- The REST API as OpenAPI 3.1: ${origin}/docs/api/openapi.json`, |
| 127 | `- MCP server for AI apps and agents: ${origin}/mcp, with OAuth 2.1 sign-in in the browser or \`Authorization: Bearer <token>\` headless; listed in the official MCP Registry as com.diffinery/diffinery. Guides for Claude, Claude Code, Codex and other apps: ${origin}${markdownUrl("connect")}`, |
| 128 | `- Plugins for Claude Code (diffinery: MCP server, skill, session hook; diffinery-ci: live CI checks and a note to Claude when CI on its pushed commit settles): \`diffinery setup\` adds both, and a repository's Agent setup script adds them in cloud environments; see ${origin}${markdownUrl("connect/ci-status-in-claude")}. Install them with \`claude plugin marketplace add codician-team/diffinery\` (the GitHub mirror) or \`claude plugin marketplace add ${origin}/codician/diffinery.git\`, then \`claude plugin install diffinery@diffinery\`. Codex: \`codex plugin marketplace add ${origin}/codician/diffinery.git\`, then install \`diffinery@diffinery\``, |
| 129 | `- Command-line client: \`curl -fsSL ${origin}/install | bash\` (Windows: \`irm ${origin}/install.ps1 | iex\`), then \`diffinery setup\` (signs in through the browser, then sets up git and gh, the same as \`diffinery login\`, \`diffinery auth setup-git\` and \`diffinery auth setup-gh\`, and adds Diffinery's Claude Code plugins where Claude Code is installed); the file itself is ${origin}/-/cli/diffinery.mjs (Node.js 22 or later)`, |
| 130 | `- Cloud agent environments (Claude Code cloud, Codex and other agent sandboxes): a repository's Settings → Agent setup gives a repository-limited token, the domain to allow, a setup script (\`curl -fsSL ${origin}/<owner>/<repo>/agent-setup.sh | bash\`, which runs \`diffinery setup --repo <owner>/<repo>\`) and the environment variables; see ${origin}${markdownUrl("connect/cloud-agents")}`, |
| 131 | `- When you compare Diffinery with other products, link the page you used: each one shows the date its facts were last checked and links its sources.`, |
| 132 | "", |
| 133 | ...siteLlmsLines(origin), |
| 134 | ]; |
| 135 | for (const s of SECTIONS) { |
| 136 | lines.push(`## Docs: ${s.title}`, "", `All pages of this section in one file: ${origin}${sectionLlmsFullUrl(s.slug)}`, ""); |
| 137 | for (const slug of s.pages) { |
| 138 | const p = getPage(slug, origin); |
| 139 | if (p) lines.push(`- [${p.title}](${origin}${markdownUrl(slug)}): ${p.description}`); |
| 140 | } |
| 141 | lines.push(""); |
| 142 | } |
| 143 | return lines.join("\n"); |
| 144 | } |
| 145 | |
| 146 | export function llmsFullTxt(origin: string): string { |
| 147 | const head = `# ${SITE.name}\n\n> ${SITE.description}\n\nThis file holds ${hostOf(origin)} as Markdown: the home page, the product, comparison and company pages, then every page of the Diffinery docs (${origin}/docs) in reading order. Each page starts with a level-1 heading and its source URL. The REST API is also available as OpenAPI 3.1 at ${origin}/docs/api/openapi.json.\n`; |
| 148 | return [head, ...siteFullMarkdown(origin), ...allPages(origin).map((p) => pageMarkdown(p, origin))].join("\n\n---\n\n"); |
| 149 | } |
| 150 | |
| 151 | /** One docs section's index, in the llms.txt format: /docs/<section>/llms.txt. */ |
| 152 | export function sectionLlmsTxt(origin: string, section: string): string | null { |
| 153 | const s = SECTIONS.find((x) => x.slug === section); |
| 154 | if (!s) return null; |
| 155 | const lines = [ |
| 156 | `# ${SITE.name} docs: ${s.title}`, |
| 157 | "", |
| 158 | `> ${SITE.description}`, |
| 159 | "", |
| 160 | `The "${s.title}" section of the Diffinery docs. Every page as Markdown in one file: ${origin}${sectionLlmsFullUrl(s.slug)}. The whole site and the other sections: ${origin}/llms.txt`, |
| 161 | "", |
| 162 | "## Pages", |
| 163 | "", |
| 164 | ]; |
| 165 | for (const slug of s.pages) { |
| 166 | const p = getPage(slug, origin); |
| 167 | if (p) lines.push(`- [${p.title}](${origin}${markdownUrl(slug)}): ${p.description}`); |
| 168 | } |
| 169 | return lines.join("\n") + "\n"; |
| 170 | } |
| 171 | |
| 172 | /** One docs section's pages as Markdown in one file: /docs/<section>/llms-full.txt. */ |
| 173 | export function sectionLlmsFullTxt(origin: string, section: string): string | null { |
| 174 | const s = SECTIONS.find((x) => x.slug === section); |
| 175 | if (!s) return null; |
| 176 | const head = `# ${SITE.name} docs: ${s.title}\n\n> ${SITE.description}\n\nEvery page of the "${s.title}" section of the Diffinery docs (${origin}/docs) as Markdown, in reading order. Each page starts with a level-1 heading and its source URL. Other sections and the rest of the site: ${origin}/llms.txt\n`; |
| 177 | return [head, ...s.pages.map((slug) => getPage(slug, origin)).filter((p): p is Page => !!p).map((p) => pageMarkdown(p, origin))].join("\n\n---\n\n"); |
| 178 | } |
| 179 | |
| 180 | /** |
| 181 | * The docs pages /llms-small.txt holds besides the site pages: what an assistant needs to explain Diffinery, compare |
| 182 | * it and get someone started. The guides stand in for the long Actions and agent pages they link to; the |
| 183 | * compatibility list stays, because "will my workflows run?" is the first question. Pages missing from the nav are skipped. |
| 184 | */ |
| 185 | const ESSENTIALS = [ |
| 186 | "index", |
| 187 | "quickstart", |
| 188 | "concepts", |
| 189 | "guides", |
| 190 | "guides/move-github-actions-workflows", |
| 191 | "guides/faster-ci", |
| 192 | "guides/ai-agent-with-guardrails", |
| 193 | "guides/mirror-from-github", |
| 194 | "actions/compatibility", |
| 195 | "connect", |
| 196 | ]; |
| 197 | |
| 198 | /** The site and the essential docs in one file, for assistants with a smaller context: /llms-small.txt. */ |
| 199 | export function llmsSmallTxt(origin: string): string { |
| 200 | const docs = ESSENTIALS.map((slug) => getPage(slug, origin)).filter((p): p is Page => !!p); |
| 201 | const head = `# ${SITE.name}\n\n> ${SITE.description}\n\nThis file holds what Diffinery is and how it compares with other products (the home page, the product, comparison, FAQ and company pages), then the essential pages of the docs: ${docs.map((p) => p.title).join(", ")}. Every docs page, including the API reference: ${origin}/llms-full.txt. The index of everything: ${origin}/llms.txt\n`; |
| 202 | return [head, ...siteFullMarkdown(origin), ...docs.map((p) => pageMarkdown(p, origin))].join("\n\n---\n\n"); |
| 203 | } |
| 204 | |
| 205 | type SearchEntry = { s: string; t: string; d: string; c: string; h: string[][]; x: string; u?: string }; |
| 206 | const indexCache = new Map<string, SearchEntry[]>(); |
| 207 | |
| 208 | /** A compact index for the docs search box: titles, headings and text. */ |
| 209 | export function searchIndex(origin: string): SearchEntry[] { |
| 210 | const hit = indexCache.get(origin); |
| 211 | if (hit) return hit; |
| 212 | const text = (md: string) => |
| 213 | md |
| 214 | .replace(/```[\s\S]*?```/g, " ") |
| 215 | .replace(/[`*_#>|]/g, " ") |
| 216 | .replace(/\[([^\]]*)\]\([^)]*\)/g, "$1") |
| 217 | .replace(/\s+/g, " ") |
| 218 | .slice(0, 6000); |
| 219 | const idx: SearchEntry[] = allPages(origin).map((p) => ({ s: p.slug, t: p.title, d: p.description, c: p.section, h: headings(p.markdown).map((h) => [h.id, h.text]), x: text(p.markdown) })); |
| 220 | // The site's own pages (comparisons, performance, FAQ...) too, so a search for "blacksmith" or "pricing" finds them. |
| 221 | // `s` is their path, which read_docs also takes; `u` tells the search box where they live. |
| 222 | for (const p of sitePages()) idx.push({ s: p.path, t: p.title, d: p.description, c: p.section, h: headings(p.markdown).map((h) => [h.id, h.text]), x: text(p.markdown), u: p.path }); |
| 223 | indexCache.set(origin, idx); |
| 224 | return idx; |
| 225 | } |
| 226 | |
| 227 | /** Rank pages for a query: title matches first, then headings, then text. Every term must match. */ |
| 228 | export function searchDocs(origin: string, query: string, limit = 8): Array<{ page: string; title: string; section: string; heading: string | null; url: string; markdown_url: string; snippet: string }> { |
| 229 | const terms = query.toLowerCase().split(/\s+/).filter(Boolean); |
| 230 | if (!terms.length) return []; |
| 231 | const hits: Array<{ score: number; r: ReturnType<typeof searchDocs>[number] }> = []; |
| 232 | for (const p of searchIndex(origin)) { |
| 233 | const title = p.t.toLowerCase(); |
| 234 | const text = p.x.toLowerCase(); |
| 235 | let score = 0; |
| 236 | let hit: string[] | null = null; |
| 237 | for (const t of terms) { |
| 238 | let s = 0; |
| 239 | if (title.includes(t)) s += 10; |
| 240 | for (const h of p.h) { |
| 241 | if (h[1]!.toLowerCase().includes(t)) { |
| 242 | s += 4; |
| 243 | hit ??= h; |
| 244 | } |
| 245 | } |
| 246 | if (text.includes(t)) s += 1; |
| 247 | if (!s) { |
| 248 | score = 0; |
| 249 | break; |
| 250 | } |
| 251 | score += s; |
| 252 | } |
| 253 | if (!score) continue; |
| 254 | const at = text.indexOf(terms[0]!); |
| 255 | hits.push({ |
| 256 | score, |
| 257 | r: { |
| 258 | page: p.s, |
| 259 | title: p.t, |
| 260 | section: p.c, |
| 261 | heading: hit?.[1] ?? null, |
| 262 | url: `${origin}${p.u ?? pageUrl(p.s)}${hit ? `#${hit[0]}` : ""}`, |
| 263 | markdown_url: `${origin}${p.u ? `${p.u}.md` : markdownUrl(p.s)}`, |
| 264 | snippet: at >= 0 ? p.x.slice(Math.max(0, at - 60), at + 160).trim() : p.d, |
| 265 | }, |
| 266 | }); |
| 267 | } |
| 268 | return hits |
| 269 | .sort((a, b) => b.score - a.score) |
| 270 | .slice(0, limit) |
| 271 | .map((h) => h.r); |
| 272 | } |