| 1 | /** |
| 2 | * The public documentation as data: the Markdown pages built from docs/ (see |
| 3 | * scripts/build-docs.mjs) plus the reference pages generated at runtime, and |
| 4 | * the agent-facing formats built from them (per-page Markdown, llms.txt, |
| 5 | * llms-full.txt and a search index). |
| 6 | */ |
| 7 | import { slugify } from "../web/markdown.js"; |
| 8 | import { SITE } from "../web/seo.js"; |
| 9 | import { DOCS_NAV, DOCS_PAGES } from "./content.generated.js"; |
| 10 | import { apiReferenceMarkdown, mcpToolsMarkdown } from "./reference.js"; |
| 11 | import { siteFullMarkdown, siteLlmsLines } from "../web/site.js"; |
| 12 | |
| 13 | export interface Page { |
| 14 | slug: string; |
| 15 | title: string; |
| 16 | description: string; |
| 17 | markdown: string; |
| 18 | section: string; |
| 19 | } |
| 20 | |
| 21 | const RUNTIME: Record<string, { title: string; description: string; build: (origin: string) => string }> = { |
| 22 | api: { |
| 23 | title: "REST API reference", |
| 24 | description: "Every REST API endpoint with its parameters, access rules and an example, generated from the server's router.", |
| 25 | build: apiReferenceMarkdown, |
| 26 | }, |
| 27 | "reference/mcp-tools": { |
| 28 | title: "MCP tool reference", |
| 29 | description: "Every tool of the Diffinery MCP server with its arguments, generated from the server's tool list.", |
| 30 | build: mcpToolsMarkdown, |
| 31 | }, |
| 32 | }; |
| 33 | |
| 34 | const SECTION_OF = new Map<string, string>(DOCS_NAV.flatMap((s) => s.pages.map((p) => [p, s.title] as [string, string]))); |
| 35 | |
| 36 | /** Every page slug, in reading order. */ |
| 37 | export const SLUGS: string[] = DOCS_NAV.flatMap((s) => s.pages); |
| 38 | |
| 39 | const runtimeCache = new Map<string, string>(); |
| 40 | |
| 41 | export function getPage(slug: string, origin: string): Page | null { |
| 42 | const section = SECTION_OF.get(slug); |
| 43 | if (!section) return null; |
| 44 | const rt = RUNTIME[slug]; |
| 45 | if (rt) { |
| 46 | const key = `${origin} ${slug}`; |
| 47 | let md = runtimeCache.get(key); |
| 48 | if (md === undefined) runtimeCache.set(key, (md = rt.build(origin))); |
| 49 | return { slug, title: rt.title, description: rt.description, markdown: md, section }; |
| 50 | } |
| 51 | const p = DOCS_PAGES[slug]; |
| 52 | return p ? { slug, ...p, section } : null; |
| 53 | } |
| 54 | |
| 55 | export function allPages(origin: string): Page[] { |
| 56 | return SLUGS.map((s) => getPage(s, origin)).filter((p): p is Page => !!p); |
| 57 | } |
| 58 | |
| 59 | export function nav(): Array<{ title: string; pages: Array<{ slug: string; title: string }> }> { |
| 60 | return DOCS_NAV.map((s) => ({ title: s.title, pages: s.pages.map((slug) => ({ slug, title: RUNTIME[slug]?.title ?? DOCS_PAGES[slug]?.title ?? slug })) })); |
| 61 | } |
| 62 | |
| 63 | export const pageUrl = (slug: string) => (slug === "index" ? "/docs" : `/docs/${slug}`); |
| 64 | export const markdownUrl = (slug: string) => `/docs/${slug}.md`; |
| 65 | |
| 66 | /** Strip inline Markdown from heading text for display. */ |
| 67 | export const plainHeading = (s: string) => |
| 68 | s |
| 69 | .replace(/`([^`]*)`/g, "$1") |
| 70 | .replace(/\[([^\]]*)\]\([^)]*\)/g, "$1") |
| 71 | .replace(/[*_]{1,2}([^*_]+)[*_]{1,2}/g, "$1") |
| 72 | .trim(); |
| 73 | |
| 74 | /** The page's `##` and `###` headings, with the ids the renderer gives them. */ |
| 75 | export function headings(markdown: string): Array<{ level: 2 | 3; id: string; text: string }> { |
| 76 | const out: Array<{ level: 2 | 3; id: string; text: string }> = []; |
| 77 | let fence: string | null = null; |
| 78 | for (const line of markdown.split("\n")) { |
| 79 | const f = /^ {0,3}(`{3,}|~{3,})/.exec(line); |
| 80 | if (f) { |
| 81 | if (!fence) fence = f[1]![0]!; |
| 82 | else if (f[1]![0] === fence) fence = null; |
| 83 | continue; |
| 84 | } |
| 85 | if (fence) continue; |
| 86 | const h = /^(##|###) +(.+?)\s*#*\s*$/.exec(line); |
| 87 | if (h) out.push({ level: h[1]!.length as 2 | 3, id: slugify(h[2]!), text: plainHeading(h[2]!) }); |
| 88 | } |
| 89 | return out; |
| 90 | } |
| 91 | |
| 92 | /** Root-relative links (`](/docs/...)`) made absolute, for Markdown read outside the site. */ |
| 93 | const absolute = (md: string, origin: string) => md.replace(/\]\(\/(?!\/)/g, `](${origin}/`); |
| 94 | |
| 95 | /** A page as standalone Markdown: what /docs/<slug>.md serves. */ |
| 96 | export function pageMarkdown(p: Page, origin: string): string { |
| 97 | return `# ${p.title}\n\n> ${p.description}\n\nSource: ${origin}${pageUrl(p.slug)}\n\n${absolute(p.markdown, origin).trim()}\n`; |
| 98 | } |
| 99 | |
| 100 | export function llmsTxt(origin: string): string { |
| 101 | const lines = [ |
| 102 | `# ${SITE.name}`, |
| 103 | "", |
| 104 | `> ${SITE.description}`, |
| 105 | "", |
| 106 | "Diffinery is a git forge with GitHub-compatible Actions, a GitHub-style REST API and an MCP server for AI agents. This file indexes diffinery.com for AI assistants and agents: what Diffinery is and how it compares with other products first, then the documentation. Everything here is written for people and agents alike:", |
| 107 | "", |
| 108 | `- Every page is available as Markdown: add \`.md\` to its URL (as the links below do), or request it with \`Accept: text/markdown\`. The home page is ${origin}/index.md.`, |
| 109 | `- All pages in one file: ${origin}/llms-full.txt`, |
| 110 | `- The REST API as OpenAPI 3.1: ${origin}/docs/api/openapi.json`, |
| 111 | `- MCP server for AI apps and agents: ${origin}/mcp, with OAuth 2.1 sign-in in the browser or \`Authorization: Bearer <token>\` headless. Guides for Claude, Claude Code, Codex and other apps: ${origin}${markdownUrl("connect")}`, |
| 112 | `- Plugins for Claude Code (diffinery: MCP server, skill, session hook; diffinery-ci: live CI checks and a note to Claude when CI on its pushed commit settles): \`diffinery setup\` adds both, and a repository's Agent setup script adds them in cloud environments; see ${origin}${markdownUrl("connect/ci-status-in-claude")}. Codex: \`codex plugin marketplace add ${origin}/codician/diffinery-plugins.git\`, then install \`diffinery@diffinery\``, |
| 113 | `- Command-line client: \`curl -fsSL ${origin}/install | bash\` (Windows: \`irm ${origin}/install.ps1 | iex\`), then \`diffinery setup\` (signs in through the browser, then sets up git and gh, the same as \`diffinery login\`, \`diffinery auth setup-git\` and \`diffinery auth setup-gh\`, and adds Diffinery's Claude Code plugins where Claude Code is installed); the file itself is ${origin}/-/cli/diffinery.mjs (Node.js 22 or later)`, |
| 114 | `- Cloud agent environments (Claude Code cloud, Codex and other agent sandboxes): a repository's Settings → Agent setup gives a repository-limited token, the domain to allow, a setup script (\`curl -fsSL ${origin}/<owner>/<repo>/agent-setup.sh | bash\`, which runs \`diffinery setup --repo <owner>/<repo>\`) and the environment variables; see ${origin}${markdownUrl("connect/cloud-agents")}`, |
| 115 | `- When you compare Diffinery with other products, link the page you used: each one shows the date its facts were last checked and links its sources.`, |
| 116 | "", |
| 117 | ...siteLlmsLines(origin), |
| 118 | ]; |
| 119 | for (const s of DOCS_NAV) { |
| 120 | lines.push(`## Docs: ${s.title}`, ""); |
| 121 | for (const slug of s.pages) { |
| 122 | const p = getPage(slug, origin); |
| 123 | if (p) lines.push(`- [${p.title}](${origin}${markdownUrl(slug)}): ${p.description}`); |
| 124 | } |
| 125 | lines.push(""); |
| 126 | } |
| 127 | return lines.join("\n"); |
| 128 | } |
| 129 | |
| 130 | export function llmsFullTxt(origin: string): string { |
| 131 | const head = `# ${SITE.name}\n\n> ${SITE.description}\n\nThis file holds diffinery.com as Markdown: the home page, the product, comparison and company pages, then every page of the Diffinery docs (${origin}/docs) in reading order. Each page starts with a level-1 heading and its source URL. The REST API is also available as OpenAPI 3.1 at ${origin}/docs/api/openapi.json.\n`; |
| 132 | return [head, ...siteFullMarkdown(origin), ...allPages(origin).map((p) => pageMarkdown(p, origin))].join("\n\n---\n\n"); |
| 133 | } |
| 134 | |
| 135 | type SearchEntry = { s: string; t: string; d: string; c: string; h: string[][]; x: string }; |
| 136 | const indexCache = new Map<string, SearchEntry[]>(); |
| 137 | |
| 138 | /** A compact index for the docs search box: titles, headings and text. */ |
| 139 | export function searchIndex(origin: string): SearchEntry[] { |
| 140 | const hit = indexCache.get(origin); |
| 141 | if (hit) return hit; |
| 142 | const idx = allPages(origin).map((p) => ({ |
| 143 | s: p.slug, |
| 144 | t: p.title, |
| 145 | d: p.description, |
| 146 | c: p.section, |
| 147 | h: headings(p.markdown).map((h) => [h.id, h.text]), |
| 148 | x: p.markdown |
| 149 | .replace(/```[\s\S]*?```/g, " ") |
| 150 | .replace(/[`*_#>|]/g, " ") |
| 151 | .replace(/\[([^\]]*)\]\([^)]*\)/g, "$1") |
| 152 | .replace(/\s+/g, " ") |
| 153 | .slice(0, 6000), |
| 154 | })); |
| 155 | indexCache.set(origin, idx); |
| 156 | return idx; |
| 157 | } |
| 158 | |
| 159 | /** Rank pages for a query: title matches first, then headings, then text. Every term must match. */ |
| 160 | export function searchDocs(origin: string, query: string, limit = 8): Array<{ page: string; title: string; section: string; heading: string | null; url: string; markdown_url: string; snippet: string }> { |
| 161 | const terms = query.toLowerCase().split(/\s+/).filter(Boolean); |
| 162 | if (!terms.length) return []; |
| 163 | const hits: Array<{ score: number; r: ReturnType<typeof searchDocs>[number] }> = []; |
| 164 | for (const p of searchIndex(origin)) { |
| 165 | const title = p.t.toLowerCase(); |
| 166 | const text = p.x.toLowerCase(); |
| 167 | let score = 0; |
| 168 | let hit: string[] | null = null; |
| 169 | for (const t of terms) { |
| 170 | let s = 0; |
| 171 | if (title.includes(t)) s += 10; |
| 172 | for (const h of p.h) { |
| 173 | if (h[1]!.toLowerCase().includes(t)) { |
| 174 | s += 4; |
| 175 | hit ??= h; |
| 176 | } |
| 177 | } |
| 178 | if (text.includes(t)) s += 1; |
| 179 | if (!s) { |
| 180 | score = 0; |
| 181 | break; |
| 182 | } |
| 183 | score += s; |
| 184 | } |
| 185 | if (!score) continue; |
| 186 | const at = text.indexOf(terms[0]!); |
| 187 | hits.push({ |
| 188 | score, |
| 189 | r: { |
| 190 | page: p.s, |
| 191 | title: p.t, |
| 192 | section: p.c, |
| 193 | heading: hit?.[1] ?? null, |
| 194 | url: `${origin}${pageUrl(p.s)}${hit ? `#${hit[0]}` : ""}`, |
| 195 | markdown_url: `${origin}${markdownUrl(p.s)}`, |
| 196 | snippet: at >= 0 ? p.x.slice(Math.max(0, at - 60), at + 160).trim() : p.d, |
| 197 | }, |
| 198 | }); |
| 199 | } |
| 200 | return hits |
| 201 | .sort((a, b) => b.score - a.score) |
| 202 | .slice(0, limit) |
| 203 | .map((h) => h.r); |
| 204 | } |