| 1 | /** |
| 2 | * The public documentation as data: the Markdown pages built from docs/ (see |
| 3 | * scripts/build-docs.mjs) plus the reference pages generated at runtime, and |
| 4 | * the agent-facing formats built from them (per-page Markdown, llms.txt, |
| 5 | * llms-full.txt and a search index). |
| 6 | */ |
| 7 | import { slugify } from "../web/markdown.js"; |
| 8 | import { SITE } from "../web/seo.js"; |
| 9 | import { DOCS_NAV, DOCS_PAGES } from "./content.generated.js"; |
| 10 | import { apiReferenceMarkdown, mcpToolsMarkdown } from "./reference.js"; |
| 11 | |
| 12 | export interface Page { |
| 13 | slug: string; |
| 14 | title: string; |
| 15 | description: string; |
| 16 | markdown: string; |
| 17 | section: string; |
| 18 | } |
| 19 | |
| 20 | const RUNTIME: Record<string, { title: string; description: string; build: (origin: string) => string }> = { |
| 21 | api: { |
| 22 | title: "REST API reference", |
| 23 | description: "Every REST API endpoint with its parameters, access rules and an example, generated from the server's router.", |
| 24 | build: apiReferenceMarkdown, |
| 25 | }, |
| 26 | "reference/mcp-tools": { |
| 27 | title: "MCP tool reference", |
| 28 | description: "Every tool of the Diffinery MCP server with its arguments, generated from the server's tool list.", |
| 29 | build: mcpToolsMarkdown, |
| 30 | }, |
| 31 | }; |
| 32 | |
| 33 | const SECTION_OF = new Map<string, string>(DOCS_NAV.flatMap((s) => s.pages.map((p) => [p, s.title] as [string, string]))); |
| 34 | |
| 35 | /** Every page slug, in reading order. */ |
| 36 | export const SLUGS: string[] = DOCS_NAV.flatMap((s) => s.pages); |
| 37 | |
| 38 | const runtimeCache = new Map<string, string>(); |
| 39 | |
| 40 | export function getPage(slug: string, origin: string): Page | null { |
| 41 | const section = SECTION_OF.get(slug); |
| 42 | if (!section) return null; |
| 43 | const rt = RUNTIME[slug]; |
| 44 | if (rt) { |
| 45 | const key = `${origin} ${slug}`; |
| 46 | let md = runtimeCache.get(key); |
| 47 | if (md === undefined) runtimeCache.set(key, (md = rt.build(origin))); |
| 48 | return { slug, title: rt.title, description: rt.description, markdown: md, section }; |
| 49 | } |
| 50 | const p = DOCS_PAGES[slug]; |
| 51 | return p ? { slug, ...p, section } : null; |
| 52 | } |
| 53 | |
| 54 | export function allPages(origin: string): Page[] { |
| 55 | return SLUGS.map((s) => getPage(s, origin)).filter((p): p is Page => !!p); |
| 56 | } |
| 57 | |
| 58 | export function nav(): Array<{ title: string; pages: Array<{ slug: string; title: string }> }> { |
| 59 | return DOCS_NAV.map((s) => ({ title: s.title, pages: s.pages.map((slug) => ({ slug, title: RUNTIME[slug]?.title ?? DOCS_PAGES[slug]?.title ?? slug })) })); |
| 60 | } |
| 61 | |
| 62 | export const pageUrl = (slug: string) => (slug === "index" ? "/docs" : `/docs/${slug}`); |
| 63 | export const markdownUrl = (slug: string) => `/docs/${slug}.md`; |
| 64 | |
| 65 | /** Strip inline Markdown from heading text for display. */ |
| 66 | export const plainHeading = (s: string) => |
| 67 | s |
| 68 | .replace(/`([^`]*)`/g, "$1") |
| 69 | .replace(/\[([^\]]*)\]\([^)]*\)/g, "$1") |
| 70 | .replace(/[*_]{1,2}([^*_]+)[*_]{1,2}/g, "$1") |
| 71 | .trim(); |
| 72 | |
| 73 | /** The page's `##` and `###` headings, with the ids the renderer gives them. */ |
| 74 | export function headings(markdown: string): Array<{ level: 2 | 3; id: string; text: string }> { |
| 75 | const out: Array<{ level: 2 | 3; id: string; text: string }> = []; |
| 76 | let fence: string | null = null; |
| 77 | for (const line of markdown.split("\n")) { |
| 78 | const f = /^ {0,3}(`{3,}|~{3,})/.exec(line); |
| 79 | if (f) { |
| 80 | if (!fence) fence = f[1]![0]!; |
| 81 | else if (f[1]![0] === fence) fence = null; |
| 82 | continue; |
| 83 | } |
| 84 | if (fence) continue; |
| 85 | const h = /^(##|###) +(.+?)\s*#*\s*$/.exec(line); |
| 86 | if (h) out.push({ level: h[1]!.length as 2 | 3, id: slugify(h[2]!), text: plainHeading(h[2]!) }); |
| 87 | } |
| 88 | return out; |
| 89 | } |
| 90 | |
| 91 | /** Root-relative links (`](/docs/...)`) made absolute, for Markdown read outside the site. */ |
| 92 | const absolute = (md: string, origin: string) => md.replace(/\]\(\/(?!\/)/g, `](${origin}/`); |
| 93 | |
| 94 | /** A page as standalone Markdown: what /docs/<slug>.md serves. */ |
| 95 | export function pageMarkdown(p: Page, origin: string): string { |
| 96 | return `# ${p.title}\n\n> ${p.description}\n\nSource: ${origin}${pageUrl(p.slug)}\n\n${absolute(p.markdown, origin).trim()}\n`; |
| 97 | } |
| 98 | |
| 99 | export function llmsTxt(origin: string): string { |
| 100 | const lines = [ |
| 101 | `# ${SITE.name}`, |
| 102 | "", |
| 103 | `> ${SITE.description}`, |
| 104 | "", |
| 105 | "Diffinery is a git forge with GitHub-compatible Actions, a GitHub-style REST API and an MCP server for AI agents. These docs are written for people and agents alike:", |
| 106 | "", |
| 107 | `- Every page is available as Markdown: add \`.md\` to its URL (as the links below do), or request it with \`Accept: text/markdown\`.`, |
| 108 | `- All pages in one file: ${origin}/llms-full.txt`, |
| 109 | `- The REST API as OpenAPI 3.1: ${origin}/docs/api/openapi.json`, |
| 110 | `- MCP server for agents: ${origin}/mcp (send \`Authorization: Bearer <token>\`)`, |
| 111 | `- Command-line client: ${origin}/-/cli/diffinery.mjs (Node.js 22 or later)`, |
| 112 | "", |
| 113 | ]; |
| 114 | for (const s of DOCS_NAV) { |
| 115 | lines.push(`## ${s.title}`, ""); |
| 116 | for (const slug of s.pages) { |
| 117 | const p = getPage(slug, origin); |
| 118 | if (p) lines.push(`- [${p.title}](${origin}${markdownUrl(slug)}): ${p.description}`); |
| 119 | } |
| 120 | lines.push(""); |
| 121 | } |
| 122 | return lines.join("\n"); |
| 123 | } |
| 124 | |
| 125 | export function llmsFullTxt(origin: string): string { |
| 126 | const head = `# ${SITE.name} documentation\n\n> ${SITE.description}\n\nThis file holds every page of the Diffinery docs (${origin}/docs) as Markdown, in reading order. Each page starts with a level-1 heading and its source URL. The REST API is also available as OpenAPI 3.1 at ${origin}/docs/api/openapi.json.\n`; |
| 127 | return [head, ...allPages(origin).map((p) => pageMarkdown(p, origin))].join("\n\n---\n\n"); |
| 128 | } |
| 129 | |
| 130 | type SearchEntry = { s: string; t: string; d: string; c: string; h: string[][]; x: string }; |
| 131 | const indexCache = new Map<string, SearchEntry[]>(); |
| 132 | |
| 133 | /** A compact index for the docs search box: titles, headings and text. */ |
| 134 | export function searchIndex(origin: string): SearchEntry[] { |
| 135 | const hit = indexCache.get(origin); |
| 136 | if (hit) return hit; |
| 137 | const idx = allPages(origin).map((p) => ({ |
| 138 | s: p.slug, |
| 139 | t: p.title, |
| 140 | d: p.description, |
| 141 | c: p.section, |
| 142 | h: headings(p.markdown).map((h) => [h.id, h.text]), |
| 143 | x: p.markdown |
| 144 | .replace(/```[\s\S]*?```/g, " ") |
| 145 | .replace(/[`*_#>|]/g, " ") |
| 146 | .replace(/\[([^\]]*)\]\([^)]*\)/g, "$1") |
| 147 | .replace(/\s+/g, " ") |
| 148 | .slice(0, 6000), |
| 149 | })); |
| 150 | indexCache.set(origin, idx); |
| 151 | return idx; |
| 152 | } |
| 153 | |
| 154 | /** Rank pages for a query: title matches first, then headings, then text. Every term must match. */ |
| 155 | export function searchDocs(origin: string, query: string, limit = 8): Array<{ page: string; title: string; section: string; heading: string | null; url: string; markdown_url: string; snippet: string }> { |
| 156 | const terms = query.toLowerCase().split(/\s+/).filter(Boolean); |
| 157 | if (!terms.length) return []; |
| 158 | const hits: Array<{ score: number; r: ReturnType<typeof searchDocs>[number] }> = []; |
| 159 | for (const p of searchIndex(origin)) { |
| 160 | const title = p.t.toLowerCase(); |
| 161 | const text = p.x.toLowerCase(); |
| 162 | let score = 0; |
| 163 | let hit: string[] | null = null; |
| 164 | for (const t of terms) { |
| 165 | let s = 0; |
| 166 | if (title.includes(t)) s += 10; |
| 167 | for (const h of p.h) { |
| 168 | if (h[1]!.toLowerCase().includes(t)) { |
| 169 | s += 4; |
| 170 | hit ??= h; |
| 171 | } |
| 172 | } |
| 173 | if (text.includes(t)) s += 1; |
| 174 | if (!s) { |
| 175 | score = 0; |
| 176 | break; |
| 177 | } |
| 178 | score += s; |
| 179 | } |
| 180 | if (!score) continue; |
| 181 | const at = text.indexOf(terms[0]!); |
| 182 | hits.push({ |
| 183 | score, |
| 184 | r: { |
| 185 | page: p.s, |
| 186 | title: p.t, |
| 187 | section: p.c, |
| 188 | heading: hit?.[1] ?? null, |
| 189 | url: `${origin}${pageUrl(p.s)}${hit ? `#${hit[0]}` : ""}`, |
| 190 | markdown_url: `${origin}${markdownUrl(p.s)}`, |
| 191 | snippet: at >= 0 ? p.x.slice(Math.max(0, at - 60), at + 160).trim() : p.d, |
| 192 | }, |
| 193 | }); |
| 194 | } |
| 195 | return hits |
| 196 | .sort((a, b) => b.score - a.score) |
| 197 | .slice(0, limit) |
| 198 | .map((h) => h.r); |
| 199 | } |