| 1 | /** |
| 2 | * The public site: the home page and the product, comparison and company pages in site/. Every page |
| 3 | * renders for visitors with its search and social tags and structured data, is served as Markdown, |
| 4 | * links only to pages that exist, and is listed in the sitemap, llms.txt and llms-full.txt. |
| 5 | * Needs src/docs/content.generated.ts (`pnpm build` or `pnpm build:docs`). |
| 6 | */ |
| 7 | import { describe, expect, it } from "vitest"; |
| 8 | import { app } from "../../src/app.js"; |
| 9 | import { web } from "../../src/web/routes.js"; |
| 10 | import { slugify } from "../../src/web/markdown.js"; |
| 11 | import type { Env } from "../../src/env.js"; |
| 12 | import { DOCS_IMAGES } from "../../src/docs/content.generated.js"; |
| 13 | import { getPage, llmsFullTxt, llmsTxt } from "../../src/docs/pages.js"; |
| 14 | import { CLOUDFLARE_CLAIM, faqs, homeMarkdown, sitePage, sitePages } from "../../src/web/site.js"; |
| 15 | import { RESERVED_LOGINS } from "../../src/lib/util.js"; |
| 16 | |
| 17 | app.route("/", web); |
| 18 | |
| 19 | const ORIGIN = "https://diffinery.example"; |
| 20 | const stmt = { bind: () => stmt, all: async () => ({ results: [] }), first: async () => null, run: async () => ({ success: true, meta: {} }) }; |
| 21 | const env = { PUBLIC_URL: ORIGIN, DB: { prepare: () => stmt } } as unknown as Env; |
| 22 | const ctx = { waitUntil() {}, passThroughOnException() {}, props: {} } as unknown as ExecutionContext; |
| 23 | const get = (path: string, headers: Record<string, string> = {}) => app.fetch(new Request(`${ORIGIN}${path}`, { headers }), env, ctx); |
| 24 | |
| 25 | const pages = sitePages(); |
| 26 | |
| 27 | /** The JSON-LD graph of an HTML page. */ |
| 28 | function graph(html: string): Array<Record<string, unknown>> { |
| 29 | const m = /<script type="application\/ld\+json"[^>]*>([\s\S]*?)<\/script>/.exec(html); |
| 30 | expect(m, "structured data").toBeTruthy(); |
| 31 | const ld = JSON.parse(m![1]!) as { "@context": string; "@graph": Array<Record<string, unknown>> }; |
| 32 | expect(ld["@context"]).toBe("https://schema.org"); |
| 33 | return ld["@graph"]; |
| 34 | } |
| 35 | |
| 36 | const types = (g: Array<Record<string, unknown>>) => g.map((x) => x["@type"]); |
| 37 | |
| 38 | /** Markdown without code, so examples aren't checked as links. */ |
| 39 | const prose = (md: string) => md.replace(/^```[\s\S]*?^```/gm, "").replace(/`[^`\n]*`/g, ""); |
| 40 | |
| 41 | /** Paths outside /docs and the site pages that the site may link to. */ |
| 42 | const OTHER_PATHS = [/^\/$/, /^\/llms(-full)?\.txt$/, /^\/early-access$/, /^\/status$/, /^\/security\/report$/, /^\/\.well-known\/security\.txt$/, /^\/docs\/api\/openapi\.json$/]; |
| 43 | |
| 44 | function linkProblems(md: string): string[] { |
| 45 | const problems: string[] = []; |
| 46 | for (const m of prose(md).matchAll(/\]\(([^)\s]+)\)/g)) { |
| 47 | const href = m[1]!; |
| 48 | if (/^https?:\/\//.test(href)) { |
| 49 | if (!href.startsWith("https://")) problems.push(`insecure link: ${href}`); |
| 50 | continue; |
| 51 | } |
| 52 | if (!href.startsWith("/")) { |
| 53 | problems.push(`relative link: ${href}`); |
| 54 | continue; |
| 55 | } |
| 56 | const [path, frag] = href.split("#") as [string, string | undefined]; |
| 57 | // Screenshots, served from docs/images (scripts/build-docs.mjs checks them). |
| 58 | if (path.startsWith("/docs/images/")) { |
| 59 | if (!(path.slice("/docs/images/".length) in DOCS_IMAGES)) problems.push(`no image ${path}`); |
| 60 | continue; |
| 61 | } |
| 62 | if (path === "/docs" || path.startsWith("/docs/")) { |
| 63 | if (OTHER_PATHS.some((re) => re.test(path))) continue; |
| 64 | const doc = getPage(path === "/docs" ? "index" : path.slice(6), ORIGIN); |
| 65 | if (!doc) problems.push(`no docs page ${path}`); |
| 66 | else if (frag && !doc.markdown.split("\n").some((l) => /^#{2,6} /.test(l) && slugify(l.replace(/^#+ /, "")) === frag)) problems.push(`no heading ${href}`); |
| 67 | continue; |
| 68 | } |
| 69 | const site = sitePage(path); |
| 70 | if (site) { |
| 71 | if (frag && !site.markdown.split("\n").some((l) => /^#{2,6} /.test(l) && slugify(l.replace(/^#+ /, "")) === frag)) problems.push(`no heading ${href}`); |
| 72 | continue; |
| 73 | } |
| 74 | if (!OTHER_PATHS.some((re) => re.test(path))) problems.push(`link to ${path} isn't a page`); |
| 75 | } |
| 76 | return problems; |
| 77 | } |
| 78 | |
| 79 | describe("site pages", () => { |
| 80 | it("has the product, comparison and company pages", () => { |
| 81 | expect(pages.map((p) => p.path)).toEqual(expect.arrayContaining(["/features", "/agents", "/performance", "/security", "/built-on-cloudflare", "/about", "/faq", "/blog", "/compare", "/compare/github", "/compare/gitlab", "/compare/blacksmith", "/compare/depot"])); |
| 82 | for (const p of pages) { |
| 83 | expect(p.description.length, p.path).toBeLessThanOrEqual(160); |
| 84 | expect(p.updated, p.path).toMatch(/^\d{4}-\d{2}-\d{2}$/); |
| 85 | // The blog's index only lists its posts. |
| 86 | if (p.path !== "/blog") expect(p.markdown.length, `${p.path} is too short`).toBeGreaterThan(1500); |
| 87 | } |
| 88 | }); |
| 89 | |
| 90 | it("keeps the pages' top-level paths away from account names", () => { |
| 91 | for (const p of pages) expect(RESERVED_LOGINS.has(p.path.split("/")[1]!), p.path).toBe(true); |
| 92 | }); |
| 93 | |
| 94 | it.each(pages.map((p) => [p.path, p] as const))("%s: links resolve", (_path, p) => { |
| 95 | expect(linkProblems(p.markdown)).toEqual([]); |
| 96 | }); |
| 97 | |
| 98 | it("home page links resolve", () => { |
| 99 | expect(linkProblems(homeMarkdown(""))).toEqual([]); |
| 100 | }); |
| 101 | |
| 102 | it("every comparison names its sources and the date they were checked", () => { |
| 103 | for (const p of pages.filter((x) => x.path.startsWith("/compare/"))) { |
| 104 | expect(p.markdown, p.path).toMatch(/\[source\]\(https:\/\//); |
| 105 | // The date the other product's facts were checked, written out ("checked on October 16, 2026"). |
| 106 | expect(p.markdown, p.path).toMatch(/(January|February|March|April|May|June|July|August|September|October|November|December) \d{1,2}, \d{4}/); |
| 107 | expect(faqs(p.markdown).length, `${p.path} questions`).toBeGreaterThan(0); |
| 108 | } |
| 109 | }); |
| 110 | |
| 111 | it.each(pages.map((p) => [p.path, p] as const))("%s: renders for visitors with search tags and structured data", async (path, p) => { |
| 112 | const res = await get(path); |
| 113 | expect(res.status).toBe(200); |
| 114 | expect(res.headers.get("link")).toContain(`<${ORIGIN}${path}.md>; rel="alternate"; type="text/markdown"`); |
| 115 | const html = await res.text(); |
| 116 | expect(html).toContain(`<link rel="canonical" href="${ORIGIN}${path}"/>`); |
| 117 | expect(html).not.toContain('name="robots"'); |
| 118 | expect(html.match(/<h1[ >]/g)?.length).toBe(1); |
| 119 | expect(html).toContain(`<meta name="description" content="`); |
| 120 | expect(html).toContain('href="/features"'); |
| 121 | const g = graph(html); |
| 122 | expect(types(g)).toEqual(expect.arrayContaining(["Organization", "WebSite", "SoftwareApplication", "BreadcrumbList"])); |
| 123 | const page = g.find((x) => x["@id"] === `${ORIGIN}${path}#page`)!; |
| 124 | expect(page.dateModified).toBe(p.updated); |
| 125 | const qs = faqs(p.markdown); |
| 126 | const faq = g.find((x) => x["@type"] === "FAQPage") as { mainEntity: Array<{ name: string }> } | undefined; |
| 127 | expect(faq?.mainEntity.length ?? 0).toBe(qs.length); |
| 128 | }); |
| 129 | |
| 130 | it.each(pages.map((p) => [p.path, p] as const))("%s: is served as Markdown", async (path, p) => { |
| 131 | for (const res of [await get(`${path}.md`), await get(path, { accept: "text/markdown" })]) { |
| 132 | expect(res.status).toBe(200); |
| 133 | expect(res.headers.get("content-type")).toBe("text/markdown; charset=utf-8"); |
| 134 | const md = await res.text(); |
| 135 | expect(md.startsWith(`# ${p.title}\n\n> ${p.description}\n\nSource: ${ORIGIN}${path} (last updated ${p.updated})`)).toBe(true); |
| 136 | expect(md).not.toMatch(/\]\(\/(?!\/)/); |
| 137 | } |
| 138 | }); |
| 139 | |
| 140 | it("marks blog posts up as BlogPostings with the date they went out", async () => { |
| 141 | const post = pages.find((p) => p.path.startsWith("/blog/"))!; |
| 142 | const g = graph(await (await get(post.path)).text()); |
| 143 | const page = g.find((x) => x["@id"] === `${ORIGIN}${post.path}#page`)!; |
| 144 | expect(page["@type"]).toBe("BlogPosting"); |
| 145 | expect(page.datePublished).toBe(post.published ?? post.updated); |
| 146 | const crumbs = g.find((x) => x["@type"] === "BreadcrumbList") as { itemListElement: Array<{ name: string }> }; |
| 147 | expect(crumbs.itemListElement.map((c) => c.name)).toEqual(["Home", "Blog", post.title]); |
| 148 | }); |
| 149 | |
| 150 | it("sends a trailing slash to the page", async () => { |
| 151 | const res = await get("/compare/github/"); |
| 152 | expect(res.status).toBe(301); |
| 153 | expect(res.headers.get("location")).toBe("/compare/github"); |
| 154 | }); |
| 155 | }); |
| 156 | |
| 157 | describe("home page", () => { |
| 158 | it("renders for visitors with the product, its questions and a Markdown version", async () => { |
| 159 | const res = await get("/"); |
| 160 | expect(res.status).toBe(200); |
| 161 | expect(res.headers.get("link")).toContain(`<${ORIGIN}/index.md>; rel="alternate"; type="text/markdown"`); |
| 162 | const html = await res.text(); |
| 163 | expect(html).toContain("<title>Diffinery: open-source GitHub alternative with fast CI, built for AI agents</title>"); |
| 164 | expect(html).toContain("Code hosting and CI for the agentic era</h1>"); |
| 165 | const g = graph(html); |
| 166 | expect(types(g)).toEqual(expect.arrayContaining(["Organization", "WebSite", "SoftwareApplication", "SoftwareSourceCode", "WebPage", "FAQPage"])); |
| 167 | // Open source: the license and the repositories, no pricing, and early access to try it on diffinery.com. |
| 168 | const source = g.find((n) => n["@type"] === "SoftwareSourceCode")!; |
| 169 | expect(source).toMatchObject({ codeRepository: "https://diffinery.com/codician/diffinery", license: "https://www.apache.org/licenses/LICENSE-2.0", sameAs: ["https://github.com/codician-team/diffinery"] }); |
| 170 | expect(g.find((n) => n["@type"] === "SoftwareApplication")).toMatchObject({ license: "https://www.apache.org/licenses/LICENSE-2.0", isAccessibleForFree: true, installUrl: `${ORIGIN}/docs/self-hosting/deploy`, downloadUrl: "https://diffinery.com/codician/diffinery/releases" }); |
| 171 | expect(html).toContain('href="https://diffinery.com/early-access"'); |
| 172 | const faq = g.find((n) => n["@type"] === "FAQPage") as { mainEntity: Array<{ name: string; acceptedAnswer: { text: string } }> }; |
| 173 | expect(faq.mainEntity.find((q) => q.name === "Can I try Diffinery without deploying it?")?.acceptedAnswer.text).toContain("Request early access to diffinery.com"); |
| 174 | expect(html).not.toMatch(/not been announced|hasn't been announced|invite-only while/i); |
| 175 | for (const href of ["/compare/github", "/compare/blacksmith", "/performance", "/agents", "/about", "/llms.txt", "/open-source", "/docs/self-hosting", "https://diffinery.com/codician/diffinery", "https://github.com/codician-team/diffinery"]) expect(html, href).toContain(`href="${href}"`); |
| 176 | }); |
| 177 | |
| 178 | it("doesn't load the Kumo bundle, which only pages with islands use", async () => { |
| 179 | expect(await (await get("/")).text()).not.toContain("/-/kumo/"); |
| 180 | }); |
| 181 | |
| 182 | it("is served as Markdown", async () => { |
| 183 | for (const res of [await get("/index.md"), await get("/", { accept: "text/markdown" })]) { |
| 184 | expect(res.status).toBe(200); |
| 185 | expect(res.headers.get("content-type")).toBe("text/markdown; charset=utf-8"); |
| 186 | const md = await res.text(); |
| 187 | expect(md).toContain("## Frequently asked questions"); |
| 188 | expect(md).toContain(`(${ORIGIN}/compare/github)`); |
| 189 | } |
| 190 | }); |
| 191 | }); |
| 192 | |
| 193 | describe("sign-in page", () => { |
| 194 | // The Deploy workflow checks that /login names both bundle files and that they answer. |
| 195 | it("prefetches the Kumo bundle for the pages after it, without loading it", async () => { |
| 196 | const html = await (await get("/login")).text(); |
| 197 | expect(html).toContain("Sign in with a passkey"); |
| 198 | expect(html).toMatch(/<link rel="prefetch" href="\/-\/kumo\/[a-z0-9]+\.css" as="style"\/?>/); |
| 199 | expect(html).toMatch(/<link rel="prefetch" href="\/-\/kumo\/[a-z0-9]+\.js" as="script"\/?>/); |
| 200 | expect(html).not.toMatch(/<(?:script|link rel="stylesheet")[^>]*\/-\/kumo\//); |
| 201 | }); |
| 202 | }); |
| 203 | |
| 204 | describe("crawlers and AI assistants", () => { |
| 205 | it("lists every site page in llms.txt, before the docs", () => { |
| 206 | const txt = llmsTxt(ORIGIN); |
| 207 | expect(txt).toContain(`(${ORIGIN}/index.md)`); |
| 208 | for (const p of pages) expect(txt).toContain(`(${ORIGIN}${p.path}.md)`); |
| 209 | expect(txt.indexOf(`${ORIGIN}/compare/github.md`)).toBeLessThan(txt.indexOf(`${ORIGIN}/docs/quickstart.md`)); |
| 210 | expect(txt).toContain("request early access at https://diffinery.com/early-access"); |
| 211 | }); |
| 212 | |
| 213 | it("states the Cloudflare claim the same way everywhere, backed by the sourced list of other projects", async () => { |
| 214 | expect(llmsTxt(ORIGIN)).toContain(CLOUDFLARE_CLAIM); |
| 215 | const g = graph(await (await get("/")).text()); |
| 216 | expect((g.find((x) => x["@type"] === "SoftwareApplication")!.featureList as string[]).join(" ")).toContain(CLOUDFLARE_CLAIM); |
| 217 | const page = sitePage("/built-on-cloudflare")!; |
| 218 | expect(page.markdown).toContain("## Other Git and CI projects on Cloudflare"); |
| 219 | expect(page.markdown.split("## Other Git and CI projects on Cloudflare")[1]!.split("\n## ")[0]!.match(/\[source\]\(https:\/\//g)!.length).toBeGreaterThan(5); |
| 220 | }); |
| 221 | |
| 222 | it("holds every site page in llms-full.txt", () => { |
| 223 | const txt = llmsFullTxt(ORIGIN); |
| 224 | expect(txt).toContain(`Source: ${ORIGIN}/ (last updated`); |
| 225 | for (const p of pages) expect(txt).toContain(`# ${p.title}\n\n> ${p.description}\n\nSource: ${ORIGIN}${p.path}`); |
| 226 | }); |
| 227 | |
| 228 | it("lists the site in the sitemap with the date each page was checked", async () => { |
| 229 | const xml = await (await get("/sitemap.xml")).text(); |
| 230 | expect(xml.match(new RegExp(`<loc>${ORIGIN}/</loc>`, "g"))?.length).toBe(1); |
| 231 | for (const p of pages) expect(xml).toContain(`<url><loc>${ORIGIN}${p.path}</loc><lastmod>${p.updated}</lastmod></url>`); |
| 232 | }); |
| 233 | |
| 234 | it("welcomes search engines and AI assistants in robots.txt", async () => { |
| 235 | const txt = await (await get("/robots.txt")).text(); |
| 236 | expect(txt).toContain("User-agent: *\nContent-Signal: search=yes, ai-input=yes, ai-train=yes\nAllow: /"); |
| 237 | expect(txt).toContain(`Sitemap: ${ORIGIN}/sitemap.xml`); |
| 238 | for (const p of pages) expect(txt).not.toMatch(new RegExp(`^Disallow: ${p.path}$`, "m")); |
| 239 | }); |
| 240 | }); |