| 1 | import { describe, expect, it } from "vitest"; |
| 2 | import { diffLineTokens, fenceHighlighter, highlightLines, langForFence, langForPath, LANGS, LIMITS, ready, tokenize, type Lang, type TokLine } from "../../src/web/highlight.js"; |
| 3 | import { fenceLangs, renderMarkdown } from "../../src/web/markdown.js"; |
| 4 | |
| 5 | const text = (line: TokLine | null | undefined) => line?.map((t) => t.text).join("") ?? null; |
| 6 | |
| 7 | describe("language detection", () => { |
| 8 | it("maps file names and extensions", () => { |
| 9 | const cases: Record<string, string | null> = { |
| 10 | "src/app.ts": "typescript", |
| 11 | "src/App.tsx": "tsx", |
| 12 | "lib/x.mjs": "javascript", |
| 13 | "lib/x.cjs": "javascript", |
| 14 | "a/b.jsx": "jsx", |
| 15 | "Dockerfile": "docker", |
| 16 | "deploy/Dockerfile.prod": "docker", |
| 17 | "web.dockerfile": "docker", |
| 18 | "Makefile": "make", |
| 19 | "GNUmakefile": "make", |
| 20 | ".github/workflows/ci.yml": "yaml", |
| 21 | "config.yaml": "yaml", |
| 22 | "inc/x.h": "c", |
| 23 | "inc/x.hpp": "cpp", |
| 24 | "src/x.cc": "cpp", |
| 25 | "run.sh": "shellscript", |
| 26 | "x.bash": "shellscript", |
| 27 | ".zshrc": "shellscript", |
| 28 | ".env": "dotenv", |
| 29 | ".env.local": "dotenv", |
| 30 | ".gitignore": null, |
| 31 | "tsconfig.json": "jsonc", |
| 32 | "tsconfig.build.json": "jsonc", |
| 33 | "package.json": "json", |
| 34 | "Cargo.toml": "toml", |
| 35 | "main.go": "go", |
| 36 | "lib.rs": "rust", |
| 37 | "App.vue": "vue", |
| 38 | "Page.svelte": "svelte", |
| 39 | "main.tf": "terraform", |
| 40 | "nginx.conf": "nginx", |
| 41 | "Gemfile": "ruby", |
| 42 | "build.gradle": "groovy", |
| 43 | "api.proto": "proto", |
| 44 | "fix.patch": "diff", |
| 45 | "x.m": "objective-c", |
| 46 | "core.clj": "clojure", |
| 47 | "README": null, |
| 48 | "LICENSE": null, |
| 49 | "notes.txt": null, |
| 50 | "image.png": null, |
| 51 | }; |
| 52 | for (const [path, lang] of Object.entries(cases)) expect([path, langForPath(path)]).toEqual([path, lang]); |
| 53 | }); |
| 54 | |
| 55 | it("maps fence names and aliases", () => { |
| 56 | expect(langForFence("ts")).toBe("typescript"); |
| 57 | expect(langForFence("TypeScript")).toBe("typescript"); |
| 58 | expect(langForFence("yml")).toBe("yaml"); |
| 59 | expect(langForFence("bash")).toBe("shellscript"); |
| 60 | expect(langForFence("sh")).toBe("shellscript"); |
| 61 | expect(langForFence("console")).toBe("shellscript"); |
| 62 | expect(langForFence("c++")).toBe("cpp"); |
| 63 | expect(langForFence("c#")).toBe("csharp"); |
| 64 | expect(langForFence("golang")).toBe("go"); |
| 65 | expect(langForFence("dockerfile")).toBe("docker"); |
| 66 | expect(langForFence("hcl")).toBe("hcl"); |
| 67 | expect(langForFence("{.python}")).toBe("python"); |
| 68 | expect(langForFence("text")).toBeNull(); |
| 69 | expect(langForFence("mermaid")).toBeNull(); |
| 70 | }); |
| 71 | }); |
| 72 | |
| 73 | describe("tokens", () => { |
| 74 | it("tokenizes with both themes as CSS variables and leaves default text bare", async () => { |
| 75 | // The first tokenization compiles the grammar's regexes, which can use up the |
| 76 | // per-line tokenize time limit on a slow runner (the rest of the line is then left |
| 77 | // plain, by design). Warm the grammar so the assertions below don't race it. |
| 78 | await highlightLines(["const warm = 0;"], "typescript"); |
| 79 | const out = await highlightLines(["const a = 1; // one", "", "export {};"], "typescript"); |
| 80 | expect(out).toHaveLength(3); |
| 81 | expect(out!.map(text)).toEqual(["const a = 1; // one", "", "export {};"]); |
| 82 | const kw = out![0]!.find((t) => t.text === "const")!; |
| 83 | expect(kw.style).toMatch(/^--shiki-light:#[0-9A-F]{6};--shiki-dark:#[0-9A-F]{6}$/i); |
| 84 | expect(out![0]!.some((t) => !t.style)).toBe(true); |
| 85 | expect(out![0]!.every((t) => !t.style || /^(--shiki-[a-z-]+:[^;"<>]+;?)+$/.test(t.style))).toBe(true); |
| 86 | }); |
| 87 | |
| 88 | it("loads every grammar on the JavaScript regex engine", async () => { |
| 89 | for (const lang of LANGS) { |
| 90 | const hl = await ready([lang]); |
| 91 | expect([lang, hl !== null]).toEqual([lang, true]); |
| 92 | const out = tokenize(hl!, "x = 1 # a\n// b\n<a href='c'>d</a>", lang); |
| 93 | expect([lang, out?.length]).toEqual([lang, 3]); |
| 94 | } |
| 95 | }, 60_000); |
| 96 | |
| 97 | it("falls back to plain text for unknown languages and oversized input", async () => { |
| 98 | expect(await highlightLines(["x"], null)).toBeNull(); |
| 99 | expect(await highlightLines(Array.from({ length: LIMITS.lines + 1 }, () => "x"), "typescript")).toBeNull(); |
| 100 | expect(await highlightLines(["x".repeat(LIMITS.bytes + 1)], "typescript")).toBeNull(); |
| 101 | }); |
| 102 | |
| 103 | it("does not tokenize very long lines", async () => { |
| 104 | const long = `const s = "${"a".repeat(LIMITS.lineLength + 10)}";`; |
| 105 | const out = await highlightLines([long, "let b = 2;"], "typescript"); |
| 106 | expect(out!.map(text)).toEqual([long, "let b = 2;"]); |
| 107 | expect(out![0]).toEqual([{ text: long }]); |
| 108 | expect(out![1]!.find((t) => t.text === "let")?.style).toBeTruthy(); |
| 109 | }); |
| 110 | |
| 111 | it("highlights a 1,500-line TypeScript file quickly", async () => { |
| 112 | const chunk = [ |
| 113 | "import { readFile } from \"node:fs/promises\";", |
| 114 | "/** A documented interface. */", |
| 115 | "export interface Item<T extends object = {}> { id: number; name: string; meta?: T }", |
| 116 | "export async function load(path: string): Promise<Item[]> {", |
| 117 | " const raw = await readFile(path, \"utf8\"); // read", |
| 118 | " return JSON.parse(raw).map((x: any, i: number) => ({ id: i, name: `item ${x.name}` }));", |
| 119 | "}", |
| 120 | "class Store { #items = new Map<number, Item>(); get size() { return this.#items.size; } }", |
| 121 | "const re = /^(?:[a-z]+)\\d*$/gi;", |
| 122 | "", |
| 123 | ]; |
| 124 | const lines = Array.from({ length: 1500 }, (_, i) => chunk[i % chunk.length]!); |
| 125 | await ready(["typescript"]); |
| 126 | const t0 = performance.now(); |
| 127 | const out = await highlightLines(lines, "typescript"); |
| 128 | const ms = performance.now() - t0; |
| 129 | console.log(`highlight: 1,500-line TypeScript file in ${ms.toFixed(0)} ms`); |
| 130 | expect(out).toHaveLength(1500); |
| 131 | expect(ms).toBeLessThan(5000); |
| 132 | }); |
| 133 | }); |
| 134 | |
| 135 | describe("paired GitHub themes", () => { |
| 136 | it("match Shiki's own dual-theme output", async () => { |
| 137 | const { createHighlighterCore } = await import("shiki/core"); |
| 138 | const { createJavaScriptRegexEngine } = await import("shiki/engine/javascript"); |
| 139 | const samples: Array<[Lang, string]> = [ |
| 140 | ["typescript", 'import { a } from "b";\n/** doc @param x */\nexport class C<T> extends D { #x = 1; get y(): T { return `v${this.#x}` as T } }\nconst re = /a+/g;'], |
| 141 | ["markdown", "# Title\n\n*em* **bold** ~~gone~~ [link](https://x.y)\n\n> quote\n\n- item"], |
| 142 | ["python", "@dataclass\nclass A:\n def f(self, x: int = 2) -> str:\n return f\"{x!r}\" # c"], |
| 143 | ["diff", "--- a\n+++ b\n@@ -1 +1 @@\n-old\n+new"], |
| 144 | ["yaml", "on:\n push:\n branches: [main] # c\njobs: { a: 1 }"], |
| 145 | ]; |
| 146 | const ref = await createHighlighterCore({ themes: [import("shiki/themes/github-light-default.mjs"), import("shiki/themes/github-dark-default.mjs")], langs: samples.map(([l]) => import(`shiki/langs/${l}.mjs`)), engine: createJavaScriptRegexEngine({ forgiving: true }) }); |
| 147 | const hl = await ready(samples.map(([l]) => l)); |
| 148 | for (const [lang, code] of samples) { |
| 149 | const { tokens: theirs, fg } = ref.codeToTokens(code, { lang, themes: { light: "github-light-default", dark: "github-dark-default" }, defaultColor: false }); |
| 150 | const ours = tokenize(hl!, code, lang)!; |
| 151 | const flat = (lines: Array<Array<{ text: string; style?: string }>>) => lines.map((l) => l.flatMap((t) => [...t.text].map((ch) => `${ch}\t${t.style ?? ""}`))); |
| 152 | // Shiki spells out the default foreground and token backgrounds; ours leaves both to the site. |
| 153 | const style = (s: Record<string, string> = {}) => Object.entries(s).filter(([k]) => !k.endsWith("-bg")).map(([k, v]) => `${k}:${v}`).join(";"); |
| 154 | const expected = theirs.map((l) => l.map((t) => ({ text: t.content, style: style(t.htmlStyle).toLowerCase() === fg!.toLowerCase() ? undefined : style(t.htmlStyle) }))); |
| 155 | expect([lang, flat(ours)]).toEqual([lang, flat(expected)]); |
| 156 | } |
| 157 | }); |
| 158 | }); |
| 159 | |
| 160 | describe("diff mapping", () => { |
| 161 | // Each token records the snippet it was tokenized from. |
| 162 | const fake = (code: string): TokLine[] => code.split("\n").map((l) => [{ text: l, style: code.replace(/\n/g, "|") }]); |
| 163 | |
| 164 | it("maps old and new side tokens back to rows", () => { |
| 165 | const hunk = [" a", "-b", "-c", "+B", " d", "\\ No newline at end of file", "+e"]; |
| 166 | const rows = diffLineTokens(hunk, fake); |
| 167 | expect(rows.map(text)).toEqual(["a", "b", "c", "B", "d", null, "e"]); |
| 168 | expect(rows.map((r) => r?.[0]?.style ?? null)).toEqual(["a|B|d|e", "a|b|c|d", "a|b|c|d", "a|B|d|e", "a|B|d|e", null, "a|B|d|e"]); |
| 169 | }); |
| 170 | |
| 171 | it("treats empty lines as context and fails soft", () => { |
| 172 | expect(diffLineTokens(["", "+x"], fake).map(text)).toEqual(["", "x"]); |
| 173 | expect(diffLineTokens([" a", "+b"], () => null)).toEqual([null, null]); |
| 174 | expect(diffLineTokens([" a", "+b"], () => [])).toEqual([null, null]); |
| 175 | }); |
| 176 | |
| 177 | it("carries grammar state across rows of a hunk", async () => { |
| 178 | const hl = await ready(["typescript"]); |
| 179 | const hunk = [" /* start of a", "-old comment", "+new comment", " end */ const x = 1;"]; |
| 180 | const rows = diffLineTokens(hunk, (code) => tokenize(hl!, code, "typescript")); |
| 181 | expect(rows.map(text)).toEqual(["/* start of a", "old comment", "new comment", "end */ const x = 1;"]); |
| 182 | const comment = rows[0]![0]!.style; |
| 183 | expect(rows[1]![0]!.style).toBe(comment); |
| 184 | expect(rows[2]![0]!.style).toBe(comment); |
| 185 | expect(rows[3]!.find((t) => t.text.includes("const"))?.style).not.toBe(comment); |
| 186 | }); |
| 187 | }); |
| 188 | |
| 189 | describe("markdown fences", () => { |
| 190 | it("highlights named fences and escapes code", async () => { |
| 191 | const md = "# T\n\n```ts\nconst a = \"<b>\" && 1;\n```\n\n```\nplain <i>\n```\n\n```mermaid\ngraph\n```\n"; |
| 192 | expect(fenceLangs(md)).toEqual(["ts", "mermaid"]); |
| 193 | const html = renderMarkdown(md, { highlight: await fenceHighlighter(fenceLangs(md)) }); |
| 194 | expect(html).toContain('<code class="language-ts sh">'); |
| 195 | expect(html).toMatch(/<span style="--shiki-light:#[0-9A-F]{6};--shiki-dark:#[0-9A-F]{6}">const<\/span>/i); |
| 196 | expect(html).toContain(""<b>""); |
| 197 | expect(html).not.toContain("<b>"); |
| 198 | expect(html).toContain("<pre><code>plain <i></code></pre>"); |
| 199 | expect(html).toContain('<code class="language-mermaid">graph</code>'); |
| 200 | }); |
| 201 | |
| 202 | it("renders plain code without a highlighter", async () => { |
| 203 | expect(await fenceHighlighter(["text", "mermaid"])).toBeUndefined(); |
| 204 | expect(renderMarkdown("```ts\nlet x\n```")).toBe('<pre><code class="language-ts">let x</code></pre>'); |
| 205 | }); |
| 206 | }); |