Diffinery

codician/diffinerypublic

Open-source, agent-first GitHub and GitHub Actions alternative that runs on your own Cloudflare account: git hosting, pull requests, code review and drop-in GitHub Actions YAML on Workers, Durable Objects, R2 and Containers.

c101f89

Branches

claude/early-access-cta claude/github-discoverability claude/readme-reorganize claude/readme-screenshots-up claude/showcase-agents-plugins lists-archived main marketplace-builds mcp-registry-auth mcp-registry-docs plugins-any-instance privacy-page

Tags

v0.1.0
diffinery/apps/diffinery/scripts/build-docs.mjs
342 lines · 19.0 KBRaw History
1// Builds the public documentation into src/docs/content.generated.ts:
2// - every Markdown page under docs/ (with its frontmatter), in the order of docs/nav.json
3// - the public site's pages under site/ (the product, comparison and company pages around the
4// home page), in the order of site/nav.json
5// - the CLI reference, generated from the CLI's own command registry
6// - the CLI itself as one file, served at /-/cli/diffinery.mjs, and its install scripts
7// (apps/cli/install), served at /install and /install.ps1
8// - the editor extension for VS Code and Cursor (apps/vscode), packed as a .vsix, served at
9// /-/vscode/diffinery.vsix, and its install scripts (apps/vscode/install), served at /install-editor
10// and /install-editor.ps1
11// - the plugin for JetBrains IDEs (apps/jetbrains), built and checked in at apps/jetbrains/dist (the build here has no
12// JDK; scripts/plugin.mjs there refuses a zip built from older sources), served at /-/jetbrains/
13// - the Claude Code plugins packed by codician/diffinery-plugins, checked in at claude-plugins/, served at /-/claude/
14// The REST API and MCP tool references are generated at runtime from the live
15// router and tool list (src/docs/reference.ts), so none of the references can drift.
16//
17// Run by `pnpm build` and `pnpm dev`. Fails on a page missing from the nav (or the
18// other way round) and on missing or overlong frontmatter.
19import { build } from "esbuild";
20import { createHash } from "node:crypto";
21import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from "node:fs/promises";
22import { tmpdir } from "node:os";
23import path from "node:path";
24import { fileURLToPath, pathToFileURL } from "node:url";
25
26const here = path.dirname(fileURLToPath(import.meta.url));
27const app = path.resolve(here, "..");
28const docsDir = path.join(app, "docs");
29const cliSrc = path.resolve(app, "../cli/src");
30const cliInstall = path.resolve(app, "../cli/install");
31const editorPackager = path.resolve(app, "../vscode/scripts/package.mjs");
32const editorInstall = path.resolve(app, "../vscode/install");
33const jetbrainsReader = path.resolve(app, "../jetbrains/scripts/plugin.mjs");
34const siteDir = path.join(app, "site");
35const claudePluginsDir = path.join(app, "claude-plugins");
36/** The Claude Code plugins Diffinery serves for upload to claude.ai, by name. */
37const CLAUDE_PLUGIN_NAMES = new Set(["diffinery", "diffinery-ci"]);
38const out = path.join(app, "src/docs/content.generated.ts");
39
40/** Pages produced in code rather than from a Markdown file. */
41const RUNTIME_PAGES = new Set(["api", "reference/mcp-tools"]);
42const BUILD_PAGES = new Set(["reference/cli"]);
43
44const problems = [];
45
46async function walk(dir) {
47 const files = [];
48 for (const e of await readdir(dir, { withFileTypes: true })) {
49 const p = path.join(dir, e.name);
50 if (e.isDirectory()) files.push(...(await walk(p)));
51 else if (e.name.endsWith(".md")) files.push(p);
52 }
53 return files;
54}
55
56function parsePage(slug, text) {
57 const m = /^---\n([\s\S]*?)\n---\n?/.exec(text.replace(/\r\n?/g, "\n"));
58 if (!m) {
59 problems.push(`${slug}.md: missing frontmatter (--- title / description ---)`);
60 return { title: slug, description: "", markdown: text };
61 }
62 const meta = {};
63 for (const line of m[1].split("\n")) {
64 const kv = /^(\w+):\s*(.*)$/.exec(line);
65 if (kv) meta[kv[1]] = kv[2].trim().replace(/^(["'])(.*)\1$/, "$2");
66 }
67 if (!meta.title) problems.push(`${slug}.md: frontmatter has no title`);
68 if (!meta.description) problems.push(`${slug}.md: frontmatter has no description`);
69 else if (meta.description.length > 200) problems.push(`${slug}.md: description is ${meta.description.length} characters (keep it under 200)`);
70 const markdown = text.replace(/\r\n?/g, "\n").slice(m[0].length).trim() + "\n";
71 if (/^# /m.test(markdown.replace(/```[\s\S]*?```/g, ""))) problems.push(`${slug}.md: don't use an H1 (the title is the H1)`);
72 return { title: meta.title ?? slug, description: meta.description ?? "", markdown };
73}
74
75// ---------------------------------------------------------------- the CLI
76
77// Lets bundled CommonJS dependencies call require() (same banner as apps/cli's own build).
78const REQUIRE_SHIM = "import { createRequire as __diffineryCreateRequire } from 'node:module'; const require = __diffineryCreateRequire(import.meta.url);";
79
80async function bundleCli(tmp) {
81 // The registry, for the reference page.
82 const regFile = path.join(tmp, "registry.mjs");
83 await build({
84 stdin: {
85 contents: `export { registry } from "./main.ts"; export { optionsFor } from "./cli.ts"; export { VERSION } from "./version.ts";`,
86 resolveDir: cliSrc,
87 loader: "ts",
88 },
89 bundle: true,
90 platform: "node",
91 target: "node22",
92 format: "esm",
93 outfile: regFile,
94 banner: { js: REQUIRE_SHIM },
95 logLevel: "error",
96 });
97 // The CLI itself, minified, for /-/cli/diffinery.mjs.
98 const binFile = path.join(tmp, "diffinery.mjs");
99 await build({
100 entryPoints: [path.join(cliSrc, "bin.ts")],
101 bundle: true,
102 minify: true,
103 platform: "node",
104 target: "node22",
105 format: "esm",
106 outfile: binFile,
107 banner: { js: `#!/usr/bin/env node\n${REQUIRE_SHIM}` },
108 logLevel: "error",
109 });
110 const mod = await import(pathToFileURL(regFile).href);
111 return { ...mod, bin: await readFile(binFile, "utf8") };
112}
113
114function cliReference({ registry, optionsFor, VERSION }) {
115 const all = [...registry.groups.flatMap((g) => g.commands), ...registry.commands];
116 const groups = [...registry.groups].sort((a, b) => a.name.localeCompare(b.name));
117 const singles = [...registry.commands].sort((a, b) => a.name.localeCompare(b.name));
118 const lines = [
119 `This page lists every command of the \`diffinery\` command-line client (version ${VERSION}) with all of its options. It is generated from the CLI's own command definitions, so it always matches \`diffinery <command> --help\`.`,
120 "",
121 "## Install",
122 "",
123 "```sh",
124 "curl -fsSL https://diffinery.com/install | bash # Linux and macOS",
125 "diffinery setup # sign in through your browser, set up git and gh",
126 "```",
127 "",
128 "On Windows, in PowerShell: `powershell -c \"irm https://diffinery.com/install.ps1 | iex\"`. The CLI runs on Node.js 22 or later; the script installs one for it when yours is older or missing. `diffinery upgrade` updates it. See [Install and sign in to the CLI](/docs/accounts/cli-sign-in).",
129 "",
130 "## Configuration",
131 "",
132 "| Setting | Where | Notes |",
133 "| --- | --- | --- |",
134 "| Server URL | `DIFFINERY_URL`, else `url` in the config file, else `https://diffinery.com` | Set this for your own Diffinery instance. |",
135 "| Token | `DIFFINERY_TOKEN`, else `token` in the config file | `diffinery login` signs in through the browser and stores the CLI's token in the config file. |",
136 "| Config file | `~/.config/diffinery/config.json` (`$XDG_CONFIG_HOME/diffinery/config.json` when set) | Written by `diffinery login`. |",
137 "",
138 "Commands that work on a repository take `-R OWNER/NAME`. Without it, the CLI uses the `origin` remote of the git repository you are in, when that remote points at the configured server.",
139 "",
140 "Most read commands take `--json` to print the raw API response, for scripts and agents.",
141 "",
142 "## Exit codes",
143 "",
144 "| Code | Meaning |",
145 "| --- | --- |",
146 "| `0` | Success. |",
147 "| `1` | The command failed (an API error, a failed check, a failed run when waiting). |",
148 "| `2` | Usage error: unknown command, bad option or missing argument. Help is printed to stderr. |",
149 "",
150 "## Commands",
151 "",
152 "| Command | Summary |",
153 "| --- | --- |",
154 ...[...groups.map((g) => [g.name, g.summary]), ...singles.map((c) => [c.name, c.summary])].sort((a, b) => a[0].localeCompare(b[0])).map(([n, s]) => `| [\`${n}\`](#${slug(`diffinery ${n}`)}) | ${s} |`),
155 "",
156 ];
157 const cmdSection = (cmd, level) => {
158 const opts = optionsFor(cmd);
159 const usage = `diffinery ${cmd.name}${cmd.args ? ` ${cmd.args}` : ""}${Object.keys(opts).length > 1 ? " [options]" : ""}`;
160 lines.push(`${"#".repeat(level)} diffinery ${cmd.name}`, "", cmd.summary.replace(/\.?$/, "."), "", "```text", usage, "```", "");
161 if (cmd.description) lines.push(cmd.description.trim(), "");
162 const rows = Object.entries(opts).filter(([k]) => k !== "help");
163 if (rows.length) {
164 lines.push("| Option | Description |", "| --- | --- |");
165 for (const [k, o] of rows) {
166 const flag = `${o.short ? `-${o.short}, ` : ""}--${k}${o.type === "string" ? ` ${o.arg ?? "VALUE"}` : ""}`;
167 lines.push(`| \`${flag}\` | ${o.desc.replace(/\|/g, "\\|")}${o.multiple ? " Repeatable." : ""} |`);
168 }
169 lines.push("");
170 }
171 };
172 for (const g of groups) {
173 lines.push(`### diffinery ${g.name}`, "", g.summary.replace(/\.?$/, "."), "");
174 for (const c of [...g.commands].sort((a, b) => a.name.localeCompare(b.name))) cmdSection(c, 4);
175 }
176 for (const c of singles) cmdSection(c, 3);
177 if (all.length < 10) problems.push("the CLI registry looks empty");
178 return lines.join("\n");
179}
180
181// Same rules as slugify() in src/web/markdown.ts.
182function slug(text) {
183 return text
184 .toLowerCase()
185 .replace(/<[^>]*>/g, "")
186 .replace(/[^\p{L}\p{N}\s_-]/gu, "")
187 .trim()
188 .replace(/\s+/g, "-")
189 .slice(0, 80);
190}
191
192// ---------------------------------------------------------------- main
193
194const nav = JSON.parse(await readFile(path.join(docsDir, "nav.json"), "utf8"));
195const navSlugs = nav.flatMap((s) => s.pages);
196const dupes = navSlugs.filter((s, i) => navSlugs.indexOf(s) !== i);
197if (dupes.length) problems.push(`docs/nav.json lists ${dupes.join(", ")} more than once`);
198
199const files = await walk(docsDir);
200const pages = {};
201for (const f of files) {
202 const s = path.relative(docsDir, f).replace(/\\/g, "/").replace(/\.md$/, "");
203 pages[s] = parsePage(s, await readFile(f, "utf8"));
204}
205// DOCS_DRAFT=1 tolerates pages that aren't written yet (local work in progress only).
206for (const s of navSlugs) if (!pages[s] && !RUNTIME_PAGES.has(s) && !BUILD_PAGES.has(s) && !process.env.DOCS_DRAFT) problems.push(`docs/nav.json lists "${s}" but docs/${s}.md doesn't exist`);
207for (const s of Object.keys(pages)) if (!navSlugs.includes(s)) problems.push(`docs/${s}.md isn't listed in docs/nav.json`);
208
209// Screenshots and other pictures in docs/images/, served at /docs/images/<file>. Every one must be used by
210// a page, and every /docs/images/ reference must exist. Raster formats only (an SVG is a document that can
211// run script on this origin).
212const IMAGE_TYPES = { ".png": "image/png", ".webp": "image/webp", ".jpg": "image/jpeg" };
213const MAX_IMAGE = 400 * 1024;
214const images = {};
215const imagesDir = path.join(docsDir, "images");
216for (const e of await readdir(imagesDir, { withFileTypes: true }).catch(() => [])) {
217 if (!e.isFile()) continue;
218 const type = IMAGE_TYPES[path.extname(e.name).toLowerCase()];
219 if (!type || !/^[a-z0-9][a-z0-9.-]*$/.test(e.name)) {
220 problems.push(`docs/images/${e.name}: use lowercase names ending in .png, .webp or .jpg`);
221 continue;
222 }
223 const data = await readFile(path.join(imagesDir, e.name));
224 if (data.length > MAX_IMAGE) problems.push(`docs/images/${e.name} is ${Math.round(data.length / 1024)} KiB (keep it under ${MAX_IMAGE / 1024} KiB)`);
225 images[e.name] = { type, data: data.toString("base64") };
226}
227const usedImages = new Set();
228for (const [s, p] of Object.entries(pages)) {
229 for (const m of p.markdown.matchAll(/!\[([^\]]*)\]\(\/docs\/images\/([^)\s]+)/g)) {
230 usedImages.add(m[2]);
231 if (!images[m[2]]) problems.push(`docs/${s}.md shows /docs/images/${m[2]}, which doesn't exist`);
232 if (!m[1].trim()) problems.push(`docs/${s}.md shows /docs/images/${m[2]} without alt text (agents and screen readers read it)`);
233 }
234}
235for (const name of Object.keys(images)) if (!usedImages.has(name)) problems.push(`docs/images/${name} isn't used by any page`);
236
237const tmp = await mkdtemp(path.join(tmpdir(), "diffinery-docs-"));
238let cli;
239try {
240 cli = await bundleCli(tmp);
241} finally {
242 await rm(tmp, { recursive: true, force: true });
243}
244const editor = await (await import(pathToFileURL(editorPackager).href)).packageExtension();
245let jetbrains;
246try {
247 jetbrains = await (await import(pathToFileURL(jetbrainsReader).href)).readPlugin();
248} catch (e) {
249 problems.push(e.message);
250}
251// claude-plugins/<name>-<version>.zip: one per plugin, each a zip with the plugin's manifest at its root.
252const claudePlugins = {};
253for (const e of await readdir(claudePluginsDir, { withFileTypes: true }).catch(() => [])) {
254 if (!e.isFile() || e.name === "README.md") continue;
255 const m = /^([a-z][a-z0-9-]*?)-(\d+\.\d+\.\d+)\.zip$/.exec(e.name);
256 if (!m || !CLAUDE_PLUGIN_NAMES.has(m[1])) {
257 problems.push(`claude-plugins/${e.name}: expected <name>-<version>.zip for one of ${[...CLAUDE_PLUGIN_NAMES].join(", ")}`);
258 continue;
259 }
260 if (claudePlugins[m[1]]) {
261 problems.push(`claude-plugins/ holds two versions of ${m[1]}: keep only the newest`);
262 continue;
263 }
264 const zip = await readFile(path.join(claudePluginsDir, e.name));
265 // Zip entry names are stored as written: the manifest's path shows in the bytes of a zip that has one.
266 if (zip.readUInt32LE(0) !== 0x04034b50 || !zip.includes(Buffer.from(".claude-plugin/plugin.json"))) {
267 problems.push(`claude-plugins/${e.name} isn't a zip with .claude-plugin/plugin.json at its root`);
268 continue;
269 }
270 claudePlugins[m[1]] = { version: m[2], sha256: createHash("sha256").update(zip).digest("hex"), zip: zip.toString("base64") };
271}
272for (const name of CLAUDE_PLUGIN_NAMES) if (!claudePlugins[name]) problems.push(`claude-plugins/ has no ${name}-<version>.zip`);
273
274pages["reference/cli"] = {
275 title: "CLI reference",
276 description: "Every command and option of the diffinery command-line client, generated from its source.",
277 markdown: cliReference(cli) + "\n",
278};
279
280// ---------------------------------------------------------------- the public site (site/)
281
282// site/<path>.md is served at /<path> (site/compare/index.md at /compare; the home page is web/site.tsx), with the same frontmatter as a
283// docs page plus `updated` (the date its facts were last checked, shown on the page) and optional
284// `seo_title` (the <title>, when the H1 is too long or too plain for a search result).
285const siteNav = JSON.parse(await readFile(path.join(siteDir, "nav.json"), "utf8"));
286const sitePaths = siteNav.flatMap((s) => s.pages);
287const site = {};
288for (const f of await walk(siteDir)) {
289 const rel = path.relative(siteDir, f).replace(/\\/g, "/").replace(/\.md$/, "");
290 const url = `/${rel.replace(/(^|\/)index$/, "")}`.replace(/(.)\/$/, "$1");
291 const text = (await readFile(f, "utf8")).replace(/\r\n?/g, "\n");
292 const page = parsePage(`site/${rel}`, text);
293 const meta = Object.fromEntries([...(/^---\n([\s\S]*?)\n---/.exec(text)?.[1] ?? "").matchAll(/^(\w+):\s*(.*)$/gm)].map((m) => [m[1], m[2].trim().replace(/^(["'])(.*)\1$/, "$2")]));
294 if (!/^\d{4}-\d{2}-\d{2}$/.test(meta.updated ?? "")) problems.push(`site/${rel}.md: frontmatter needs updated: YYYY-MM-DD`);
295 if (meta.seo_title && meta.seo_title.length > 70) problems.push(`site/${rel}.md: seo_title is ${meta.seo_title.length} characters (keep it to 70)`);
296 if (page.description.length > 160) problems.push(`site/${rel}.md: description is ${page.description.length} characters (search results show about 160)`);
297 if (!sitePaths.includes(url)) problems.push(`site/${rel}.md isn't listed in site/nav.json`);
298 site[url] = { ...page, updated: meta.updated ?? "", seoTitle: meta.seo_title ?? null };
299}
300for (const u of sitePaths) if (!site[u]) problems.push(`site/nav.json lists "${u}" but there's no page for it in site/`);
301
302if (problems.length) {
303 console.error(`docs: ${problems.length} problem(s):\n${problems.map((p) => ` - ${p}`).join("\n")}`);
304 process.exit(1);
305}
306
307const ordered = Object.fromEntries(navSlugs.filter((s) => pages[s]).map((s) => [s, pages[s]]));
308const src = `// Generated by scripts/build-docs.mjs from docs/, site/, claude-plugins/, apps/cli, apps/vscode and apps/jetbrains/dist. Do not edit.
309/* eslint-disable */
310export interface DocPage { title: string; description: string; markdown: string }
311export interface DocSection { title: string; pages: string[] }
312export const DOCS_NAV: DocSection[] = ${JSON.stringify(nav)};
313export const DOCS_PAGES: Record<string, DocPage> = ${JSON.stringify(ordered)};
314/** docs/images/: content type and base64 bytes by file name. */
315export interface SitePage { title: string; description: string; markdown: string; updated: string; seoTitle: string | null }
316export interface SiteSection { title: string; pages: string[] }
317/** The public site's pages (site/), by URL path, and their sections in reading order. The home page is code (web/site.tsx). */
318export const SITE_NAV: SiteSection[] = ${JSON.stringify(siteNav)};
319export const SITE_PAGES: Record<string, SitePage> = ${JSON.stringify(Object.fromEntries(sitePaths.filter((u) => site[u]).map((u) => [u, site[u]])))};
320export const DOCS_IMAGES: Record<string, { type: string; data: string }> = ${JSON.stringify(images)};
321export const CLI_VERSION = ${JSON.stringify(cli.VERSION)};
322export const CLI_BUNDLE = ${JSON.stringify(cli.bin)};
323export const EDITOR_EXTENSION_VERSION = ${JSON.stringify(editor.version)};
324export const EDITOR_EXTENSION_SHA256 = ${JSON.stringify(createHash("sha256").update(editor.vsix).digest("hex"))};
325export const EDITOR_EXTENSION_VSIX_B64 = ${JSON.stringify(editor.vsix.toString("base64"))};
326export const EDITOR_INSTALL_SH = ${JSON.stringify(await readFile(path.join(editorInstall, "install.sh"), "utf8"))};
327export const EDITOR_INSTALL_PS1 = ${JSON.stringify(await readFile(path.join(editorInstall, "install.ps1"), "utf8"))};
328export const JETBRAINS_PLUGIN_VERSION = ${JSON.stringify(jetbrains.version)};
329export const JETBRAINS_PLUGIN_SINCE_BUILD = ${JSON.stringify(jetbrains.sinceBuild)};
330export const JETBRAINS_PLUGIN_DESCRIPTION = ${JSON.stringify(jetbrains.description)};
331export const JETBRAINS_PLUGIN_SHA256 = ${JSON.stringify(createHash("sha256").update(jetbrains.zip).digest("hex"))};
332export const JETBRAINS_PLUGIN_ZIP_B64 = ${JSON.stringify(jetbrains.zip.toString("base64"))};
333/** claude-plugins/: the Claude Code plugins served at /-/claude/<name>.zip, base64. */
334export const CLAUDE_PLUGINS: Record<string, { version: string; sha256: string; zip: string }> = ${JSON.stringify(claudePlugins)};
335export const CLI_INSTALL_SH = ${JSON.stringify(await readFile(path.join(cliInstall, "install.sh"), "utf8"))};
336export const CLI_INSTALL_PS1 = ${JSON.stringify(await readFile(path.join(cliInstall, "install.ps1"), "utf8"))};
337export const CLI_SHA256 = ${JSON.stringify(createHash("sha256").update(cli.bin).digest("hex"))};
338export const AGENT_SETUP_SH = ${JSON.stringify(await readFile(path.join(cliInstall, "agent-setup.sh"), "utf8"))};
339`;
340await mkdir(path.dirname(out), { recursive: true });
341await writeFile(out, src);
342console.log(`docs: ${Object.keys(ordered).length} pages (+${RUNTIME_PAGES.size} generated at runtime), ${Object.keys(site).length} site pages, ${Object.keys(images).length} images, CLI ${cli.VERSION} (${Math.round(cli.bin.length / 1024)} KiB), editor extension ${editor.version} (${Math.round(editor.vsix.length / 1024)} KiB), JetBrains plugin ${jetbrains.version} (${Math.round(jetbrains.zip.length / 1024)} KiB), Claude Code plugins ${Object.entries(claudePlugins).map(([n, p]) => `${n} ${p.version}`).join(", ")}`);