| 1 | /** |
| 2 | * Syntax highlighting with Shiki (JavaScript regex engine, fine-grained grammars |
| 3 | * loaded on first use). Tokens carry both GitHub themes as CSS variables |
| 4 | * (--shiki-light / --shiki-dark); the site stylesheet picks one. Everything here |
| 5 | * fails soft: an unknown language, an oversized input or an error means plain text. |
| 6 | */ |
| 7 | import { createHighlighterCore, type HighlighterCore, type LanguageRegistration, type ThemedToken, type ThemeRegistration } from "shiki/core"; |
| 8 | import { createJavaScriptRegexEngine } from "shiki/engine/javascript"; |
| 9 | import { escapeHtml } from "./markdown.js"; |
| 10 | |
| 11 | type Grammar = () => Promise<{ default: LanguageRegistration[] }>; |
| 12 | |
| 13 | const GRAMMARS = { |
| 14 | astro: () => import("shiki/langs/astro.mjs"), |
| 15 | bat: () => import("shiki/langs/bat.mjs"), |
| 16 | c: () => import("shiki/langs/c.mjs"), |
| 17 | clojure: () => import("shiki/langs/clojure.mjs"), |
| 18 | cmake: () => import("shiki/langs/cmake.mjs"), |
| 19 | cpp: () => import("shiki/langs/cpp.mjs"), |
| 20 | csharp: () => import("shiki/langs/csharp.mjs"), |
| 21 | css: () => import("shiki/langs/css.mjs"), |
| 22 | dart: () => import("shiki/langs/dart.mjs"), |
| 23 | diff: () => import("shiki/langs/diff.mjs"), |
| 24 | docker: () => import("shiki/langs/docker.mjs"), |
| 25 | dotenv: () => import("shiki/langs/dotenv.mjs"), |
| 26 | elixir: () => import("shiki/langs/elixir.mjs"), |
| 27 | erlang: () => import("shiki/langs/erlang.mjs"), |
| 28 | fish: () => import("shiki/langs/fish.mjs"), |
| 29 | fsharp: () => import("shiki/langs/fsharp.mjs"), |
| 30 | go: () => import("shiki/langs/go.mjs"), |
| 31 | graphql: () => import("shiki/langs/graphql.mjs"), |
| 32 | groovy: () => import("shiki/langs/groovy.mjs"), |
| 33 | haskell: () => import("shiki/langs/haskell.mjs"), |
| 34 | hcl: () => import("shiki/langs/hcl.mjs"), |
| 35 | html: () => import("shiki/langs/html.mjs"), |
| 36 | ini: () => import("shiki/langs/ini.mjs"), |
| 37 | java: () => import("shiki/langs/java.mjs"), |
| 38 | javascript: () => import("shiki/langs/javascript.mjs"), |
| 39 | json: () => import("shiki/langs/json.mjs"), |
| 40 | json5: () => import("shiki/langs/json5.mjs"), |
| 41 | jsonc: () => import("shiki/langs/jsonc.mjs"), |
| 42 | jsx: () => import("shiki/langs/jsx.mjs"), |
| 43 | julia: () => import("shiki/langs/julia.mjs"), |
| 44 | kotlin: () => import("shiki/langs/kotlin.mjs"), |
| 45 | less: () => import("shiki/langs/less.mjs"), |
| 46 | lua: () => import("shiki/langs/lua.mjs"), |
| 47 | make: () => import("shiki/langs/make.mjs"), |
| 48 | markdown: () => import("shiki/langs/markdown.mjs"), |
| 49 | nginx: () => import("shiki/langs/nginx.mjs"), |
| 50 | nix: () => import("shiki/langs/nix.mjs"), |
| 51 | "objective-c": () => import("shiki/langs/objective-c.mjs"), |
| 52 | ocaml: () => import("shiki/langs/ocaml.mjs"), |
| 53 | perl: () => import("shiki/langs/perl.mjs"), |
| 54 | php: () => import("shiki/langs/php.mjs"), |
| 55 | powershell: () => import("shiki/langs/powershell.mjs"), |
| 56 | proto: () => import("shiki/langs/proto.mjs"), |
| 57 | python: () => import("shiki/langs/python.mjs"), |
| 58 | r: () => import("shiki/langs/r.mjs"), |
| 59 | ruby: () => import("shiki/langs/ruby.mjs"), |
| 60 | rust: () => import("shiki/langs/rust.mjs"), |
| 61 | sass: () => import("shiki/langs/sass.mjs"), |
| 62 | scala: () => import("shiki/langs/scala.mjs"), |
| 63 | scss: () => import("shiki/langs/scss.mjs"), |
| 64 | shellscript: () => import("shiki/langs/shellscript.mjs"), |
| 65 | sql: () => import("shiki/langs/sql.mjs"), |
| 66 | svelte: () => import("shiki/langs/svelte.mjs"), |
| 67 | swift: () => import("shiki/langs/swift.mjs"), |
| 68 | terraform: () => import("shiki/langs/terraform.mjs"), |
| 69 | toml: () => import("shiki/langs/toml.mjs"), |
| 70 | tsx: () => import("shiki/langs/tsx.mjs"), |
| 71 | typescript: () => import("shiki/langs/typescript.mjs"), |
| 72 | vue: () => import("shiki/langs/vue.mjs"), |
| 73 | xml: () => import("shiki/langs/xml.mjs"), |
| 74 | yaml: () => import("shiki/langs/yaml.mjs"), |
| 75 | zig: () => import("shiki/langs/zig.mjs"), |
| 76 | } satisfies Record<string, Grammar>; |
| 77 | |
| 78 | export type Lang = keyof typeof GRAMMARS; |
| 79 | export const LANGS = Object.keys(GRAMMARS) as Lang[]; |
| 80 | |
| 81 | const EXT: Record<string, Lang> = { |
| 82 | ts: "typescript", mts: "typescript", cts: "typescript", tsx: "tsx", |
| 83 | js: "javascript", mjs: "javascript", cjs: "javascript", jsx: "jsx", |
| 84 | json: "json", webmanifest: "json", geojson: "json", jsonc: "jsonc", json5: "json5", |
| 85 | yaml: "yaml", yml: "yaml", toml: "toml", |
| 86 | md: "markdown", markdown: "markdown", mdown: "markdown", mkd: "markdown", |
| 87 | html: "html", htm: "html", xhtml: "html", |
| 88 | css: "css", scss: "scss", sass: "sass", less: "less", |
| 89 | py: "python", pyi: "python", pyw: "python", go: "go", rs: "rust", java: "java", kt: "kotlin", kts: "kotlin", swift: "swift", |
| 90 | c: "c", h: "c", cc: "cpp", cpp: "cpp", cxx: "cpp", "c++": "cpp", hpp: "cpp", hh: "cpp", hxx: "cpp", ino: "cpp", |
| 91 | cs: "csharp", csx: "csharp", php: "php", phtml: "php", rb: "ruby", rake: "ruby", gemspec: "ruby", ru: "ruby", |
| 92 | sh: "shellscript", bash: "shellscript", zsh: "shellscript", ksh: "shellscript", command: "shellscript", fish: "fish", |
| 93 | ps1: "powershell", psm1: "powershell", psd1: "powershell", bat: "bat", cmd: "bat", |
| 94 | sql: "sql", graphql: "graphql", gql: "graphql", |
| 95 | xml: "xml", xsd: "xml", xsl: "xml", xslt: "xml", svg: "xml", plist: "xml", csproj: "xml", fsproj: "xml", vbproj: "xml", props: "xml", targets: "xml", resx: "xml", xaml: "xml", rss: "xml", atom: "xml", |
| 96 | lua: "lua", dart: "dart", scala: "scala", sc: "scala", sbt: "scala", r: "r", pl: "perl", pm: "perl", |
| 97 | ex: "elixir", exs: "elixir", erl: "erlang", hrl: "erlang", hs: "haskell", |
| 98 | vue: "vue", svelte: "svelte", astro: "astro", tf: "terraform", tfvars: "terraform", hcl: "hcl", nomad: "hcl", |
| 99 | ini: "ini", cfg: "ini", conf: "ini", properties: "ini", env: "dotenv", |
| 100 | mk: "make", mak: "make", cmake: "cmake", diff: "diff", patch: "diff", proto: "proto", zig: "zig", |
| 101 | m: "objective-c", groovy: "groovy", gradle: "groovy", gvy: "groovy", clj: "clojure", cljs: "clojure", cljc: "clojure", edn: "clojure", |
| 102 | ml: "ocaml", mli: "ocaml", fs: "fsharp", fsi: "fsharp", fsx: "fsharp", jl: "julia", nix: "nix", |
| 103 | }; |
| 104 | |
| 105 | const NAMES: Record<string, Lang | null> = { |
| 106 | dockerfile: "docker", containerfile: "docker", makefile: "make", gnumakefile: "make", "cmakelists.txt": "cmake", |
| 107 | gemfile: "ruby", rakefile: "ruby", podfile: "ruby", vagrantfile: "ruby", brewfile: "ruby", guardfile: "ruby", fastfile: "ruby", "gemfile.lock": null, |
| 108 | jenkinsfile: "groovy", "nginx.conf": "nginx", "cargo.lock": "toml", "go.mod": null, "go.sum": null, |
| 109 | ".bashrc": "shellscript", ".bash_profile": "shellscript", ".bash_aliases": "shellscript", ".zshrc": "shellscript", ".zprofile": "shellscript", ".zshenv": "shellscript", ".profile": "shellscript", |
| 110 | ".gitconfig": "ini", ".editorconfig": "ini", ".npmrc": "ini", ".gitmodules": "ini", |
| 111 | ".babelrc": "json", ".eslintrc": "json", ".prettierrc": "json", ".swcrc": "json", |
| 112 | ".gitignore": null, ".dockerignore": null, ".npmignore": null, ".gitattributes": null, ".prettierignore": null, ".eslintignore": null, |
| 113 | }; |
| 114 | |
| 115 | /** Shiki language for a repository path, or null when unknown (rendered as plain text). */ |
| 116 | export function langForPath(path: string): Lang | null { |
| 117 | const name = (path.split("/").pop() ?? "").toLowerCase(); |
| 118 | if (name in NAMES) return NAMES[name]!; |
| 119 | if (name === ".env" || name.startsWith(".env.")) return "dotenv"; |
| 120 | if (name.startsWith("dockerfile.") || name.endsWith(".dockerfile")) return "docker"; |
| 121 | if (name.startsWith("makefile.")) return "make"; |
| 122 | if (/^(tsconfig|jsconfig)(\.[\w-]+)*\.json$/.test(name) || /(^|\/)\.vscode\/[^/]+\.json$/i.test(path) || name === "devcontainer.json") return "jsonc"; |
| 123 | const dot = name.lastIndexOf("."); |
| 124 | return dot >= 0 ? (EXT[name.slice(dot + 1)] ?? null) : null; |
| 125 | } |
| 126 | |
| 127 | const ALIASES: Record<string, Lang> = { |
| 128 | ...EXT, |
| 129 | typescript: "typescript", javascript: "javascript", node: "javascript", python: "python", python3: "python", py3: "python", golang: "go", rust: "rust", |
| 130 | kotlin: "kotlin", csharp: "csharp", "c#": "csharp", cpp: "cpp", "objective-c": "objective-c", objc: "objective-c", objectivec: "objective-c", |
| 131 | ruby: "ruby", shell: "shellscript", shellscript: "shellscript", console: "shellscript", shellsession: "shellscript", powershell: "powershell", pwsh: "powershell", ps: "powershell", |
| 132 | batch: "bat", docker: "docker", dockerfile: "docker", make: "make", makefile: "make", perl: "perl", elixir: "elixir", erlang: "erlang", haskell: "haskell", |
| 133 | terraform: "terraform", nginx: "nginx", dotenv: "dotenv", protobuf: "proto", "f#": "fsharp", fsharp: "fsharp", ocaml: "ocaml", julia: "julia", clojure: "clojure", |
| 134 | postgres: "sql", postgresql: "sql", psql: "sql", mysql: "sql", sqlite: "sql", plpgsql: "sql", jsonl: "json", "objective-cpp": "cpp", |
| 135 | }; |
| 136 | |
| 137 | /** Shiki language for a Markdown fence info string ("ts", "yaml", "c++" …), or null. */ |
| 138 | export function langForFence(info: string): Lang | null { |
| 139 | const name = info.trim().toLowerCase().replace(/^\{?\.?/, "").replace(/\}$/, ""); |
| 140 | return name in GRAMMARS ? (name as Lang) : (ALIASES[name] ?? null); |
| 141 | } |
| 142 | |
| 143 | // ---------------------------------------------------------------- highlighter |
| 144 | |
| 145 | /** Highlighting limits: beyond them, plain text. Workers have little CPU to spare. */ |
| 146 | export const LIMITS = { lines: 2000, bytes: 200_000, lineLength: 2000, lineMs: 500 }; |
| 147 | |
| 148 | export interface Tok { |
| 149 | text: string; |
| 150 | /** CSS custom properties (--shiki-light etc.); absent for the default foreground. */ |
| 151 | style?: string; |
| 152 | } |
| 153 | export type TokLine = Tok[]; |
| 154 | |
| 155 | const PAIR = "github-pair"; |
| 156 | |
| 157 | /** |
| 158 | * GitHub's light and dark default themes share one rule list (same scopes and font styles, |
| 159 | * different colors). Tokenizing once with a theme whose rule colors are |
| 160 | * placeholders, then decoding each placeholder to both palettes, gives exactly |
| 161 | * Shiki's dual-theme output at half the CPU (dual themes tokenize once per theme). |
| 162 | */ |
| 163 | function pairThemes(light: ThemeRegistration, dark: ThemeRegistration): { theme: ThemeRegistration; palette: Map<string, [string, string]> } { |
| 164 | const lr = light.tokenColors ?? []; |
| 165 | const dr = dark.tokenColors ?? []; |
| 166 | const fg = (t: ThemeRegistration) => (t.fg ?? t.colors?.["editor.foreground"] ?? "").toUpperCase(); |
| 167 | const palette = new Map<string, [string, string]>([["#000000", [fg(light), fg(dark)]]]); |
| 168 | if (lr.length !== dr.length || !palette.get("#000000")!.every(Boolean)) throw new Error("themes do not pair"); |
| 169 | const tokenColors = lr.map((r, i) => { |
| 170 | const d = dr[i]!; |
| 171 | if (JSON.stringify(r.scope) !== JSON.stringify(d.scope) || r.settings.fontStyle !== d.settings.fontStyle || !r.settings.foreground !== !d.settings.foreground) throw new Error("themes do not pair"); |
| 172 | const settings: { fontStyle?: string; foreground?: string } = r.settings.fontStyle !== undefined ? { fontStyle: r.settings.fontStyle } : {}; |
| 173 | if (r.settings.foreground) { |
| 174 | settings.foreground = `#${(i + 1).toString(16).padStart(6, "0").toUpperCase()}`; |
| 175 | palette.set(settings.foreground, [r.settings.foreground.toUpperCase(), d.settings.foreground!.toUpperCase()]); |
| 176 | } |
| 177 | return { scope: r.scope, settings }; |
| 178 | }); |
| 179 | return { theme: { name: PAIR, type: "light", fg: "#000000", bg: "#ffffff", colors: { "editor.foreground": "#000000", "editor.background": "#ffffff" }, tokenColors }, palette }; |
| 180 | } |
| 181 | |
| 182 | let core: Promise<HighlighterCore> | null = null; |
| 183 | let palette = new Map<string, [string, string]>(); |
| 184 | const styles = new Map<string, string | undefined>(); |
| 185 | const loading = new Map<Lang, Promise<void>>(); |
| 186 | |
| 187 | function highlighter(): Promise<HighlighterCore> { |
| 188 | core ??= Promise.all([import("shiki/themes/github-light-default.mjs"), import("shiki/themes/github-dark-default.mjs")]) |
| 189 | .then(([l, d]) => { |
| 190 | const pair = pairThemes(l.default, d.default); |
| 191 | palette = pair.palette; |
| 192 | return createHighlighterCore({ themes: [pair.theme], langs: [], engine: createJavaScriptRegexEngine({ forgiving: true }) }); |
| 193 | }) |
| 194 | .catch((e) => { |
| 195 | core = null; |
| 196 | throw e; |
| 197 | }); |
| 198 | return core; |
| 199 | } |
| 200 | |
| 201 | /** The highlighter with these grammars loaded; null when anything fails. */ |
| 202 | export async function ready(langs: Iterable<Lang>): Promise<HighlighterCore | null> { |
| 203 | try { |
| 204 | const hl = await highlighter(); |
| 205 | await Promise.all( |
| 206 | [...new Set(langs)].map((l) => { |
| 207 | let p = loading.get(l); |
| 208 | if (!p) { |
| 209 | p = hl |
| 210 | .loadLanguage(GRAMMARS[l]()) |
| 211 | // Compile the grammar's top-level rules now: on a cold Worker that alone can outlast the |
| 212 | // per-line time limit, which would leave the first line of a file unhighlighted. |
| 213 | .then(() => void hl.codeToTokens("x", { lang: l, theme: PAIR })) |
| 214 | .catch((e) => { |
| 215 | loading.delete(l); |
| 216 | throw e; |
| 217 | }); |
| 218 | loading.set(l, p); |
| 219 | } |
| 220 | return p; |
| 221 | }), |
| 222 | ); |
| 223 | return hl; |
| 224 | } catch (e) { |
| 225 | console.warn("highlight: load failed", e); |
| 226 | return null; |
| 227 | } |
| 228 | } |
| 229 | |
| 230 | /** Shiki's dual-theme custom properties for a placeholder color and font style; undefined for plain text. */ |
| 231 | function styleOf(color: string | undefined, fontStyle: number | undefined): string | undefined { |
| 232 | const key = `${color}/${fontStyle}`; |
| 233 | if (styles.has(key)) return styles.get(key); |
| 234 | const plain = palette.get("#000000")!; |
| 235 | const [l, d] = (color && palette.get(color.toUpperCase())) || plain; |
| 236 | const fs = fontStyle && fontStyle > 0 ? fontStyle : 0; |
| 237 | let style: string | undefined; |
| 238 | if (l !== plain[0] || d !== plain[1] || fs) { |
| 239 | const deco = [fs & 4 ? "underline" : "", fs & 8 ? "line-through" : ""].filter(Boolean).join(" "); |
| 240 | const font = (side: string) => `${fs & 1 ? `;--shiki-${side}-font-style:italic` : ""}${fs & 2 ? `;--shiki-${side}-font-weight:bold` : ""}${deco ? `;--shiki-${side}-text-decoration:${deco}` : ""}`; |
| 241 | style = `--shiki-light:${l}${font("light")};--shiki-dark:${d}${font("dark")}`; |
| 242 | } |
| 243 | styles.set(key, style); |
| 244 | return style; |
| 245 | } |
| 246 | |
| 247 | function toks(line: ThemedToken[]): TokLine { |
| 248 | const out: TokLine = []; |
| 249 | for (const t of line) { |
| 250 | const style = styleOf(t.color, t.fontStyle); |
| 251 | const last = out[out.length - 1]; |
| 252 | if (last && last.style === style) last.text += t.content; |
| 253 | else out.push(style ? { text: t.content, style } : { text: t.content }); |
| 254 | } |
| 255 | return out; |
| 256 | } |
| 257 | |
| 258 | /** Tokenize code with a highlighter that already has `lang` loaded; null when over the limits or on error. */ |
| 259 | export function tokenize(hl: HighlighterCore, code: string, lang: Lang): TokLine[] | null { |
| 260 | if (code.length > LIMITS.bytes) return null; |
| 261 | try { |
| 262 | const { tokens } = hl.codeToTokens(code, { lang, theme: PAIR, tokenizeMaxLineLength: LIMITS.lineLength, tokenizeTimeLimit: LIMITS.lineMs }); |
| 263 | return tokens.map(toks); |
| 264 | } catch (e) { |
| 265 | console.warn("highlight: tokenize failed", e); |
| 266 | return null; |
| 267 | } |
| 268 | } |
| 269 | |
| 270 | /** Highlight whole-file lines (as shown in the file view); null means render plain text. */ |
| 271 | export async function highlightLines(lines: string[], lang: Lang | null): Promise<TokLine[] | null> { |
| 272 | if (!lang || !lines.length || lines.length > LIMITS.lines) return null; |
| 273 | let bytes = 0; |
| 274 | for (const l of lines) bytes += l.length + 1; |
| 275 | if (bytes > LIMITS.bytes) return null; |
| 276 | const hl = await ready([lang]); |
| 277 | const out = hl && tokenize(hl, lines.join("\n"), lang); |
| 278 | return out && out.length === lines.length ? out : null; |
| 279 | } |
| 280 | |
| 281 | /** |
| 282 | * Tokens for each line of a unified-diff hunk. The old side (context and deletions) |
| 283 | * and the new side (context and additions) are tokenized as contiguous snippets so |
| 284 | * grammar state carries from line to line, then mapped back to rows. Rows that are |
| 285 | * not code ("\ No newline at end of file") get null, as does every row on failure. |
| 286 | */ |
| 287 | export function diffLineTokens(lines: string[], tokenizeSide: (code: string) => TokLine[] | null): Array<TokLine | null> { |
| 288 | const olds: string[] = []; |
| 289 | const news: string[] = []; |
| 290 | for (const l of lines) { |
| 291 | const k = l[0]; |
| 292 | if (k === "\\") continue; |
| 293 | if (k !== "+") olds.push(l.slice(1)); |
| 294 | if (k !== "-") news.push(l.slice(1)); |
| 295 | } |
| 296 | const o = olds.length ? tokenizeSide(olds.join("\n")) : []; |
| 297 | const n = news.length ? tokenizeSide(news.join("\n")) : []; |
| 298 | if (!o || !n || o.length !== olds.length || n.length !== news.length) return lines.map(() => null); |
| 299 | let oi = 0; |
| 300 | let ni = 0; |
| 301 | return lines.map((l) => { |
| 302 | const k = l[0]; |
| 303 | if (k === "\\") return null; |
| 304 | if (k === "+") return n[ni++] ?? null; |
| 305 | if (k === "-") return o[oi++] ?? null; |
| 306 | oi++; |
| 307 | return n[ni++] ?? null; |
| 308 | }); |
| 309 | } |
| 310 | |
| 311 | /** Render tokens as HTML (for the Markdown renderer, which builds strings). */ |
| 312 | export const tokensHtml = (lines: TokLine[]): string => |
| 313 | lines.map((l) => l.map((t) => (t.style ? `<span style="${escapeHtml(t.style)}">${escapeHtml(t.text)}</span>` : escapeHtml(t.text))).join("")).join("\n"); |
| 314 | |
| 315 | /** A sync fence highlighter for renderMarkdown, with the grammars the document's fences name already loaded. */ |
| 316 | export async function fenceHighlighter(langs: string[]): Promise<((code: string, info: string) => string | null) | undefined> { |
| 317 | const wanted = [...new Set(langs.map(langForFence).filter((l): l is Lang => l !== null))].slice(0, 8); |
| 318 | if (!wanted.length) return undefined; |
| 319 | const hl = await ready(wanted); |
| 320 | if (!hl) return undefined; |
| 321 | let budget = LIMITS.lines; |
| 322 | return (code, info) => { |
| 323 | const lang = langForFence(info); |
| 324 | if (!lang || !wanted.includes(lang)) return null; |
| 325 | const n = code.split("\n").length; |
| 326 | if (n > budget) return null; |
| 327 | budget -= n; |
| 328 | const t = tokenize(hl, code, lang); |
| 329 | return t ? tokensHtml(t) : null; |
| 330 | }; |
| 331 | } |