Warm each grammar on load so cold Workers highlight the first line (#26)
* Warm each grammar when it loads so cold Workers highlight the first line Diffinery's own CI failed on main: on a slow cold start, compiling the TypeScript grammar inside the first line's 200 ms budget ran out, so that line came back as plain text. Grammars now tokenize a one-character sample right after loading, outside the per-line limit, and the limit is 500 ms. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01QMFviVKD61QaDPxdrh7C3f * Drop the stale 200 ms figure from the highlight test comment Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01QMFviVKD61QaDPxdrh7C3f --------- Co-authored-by: Claude <noreply@anthropic.com>
Canberk Demirci authored · GitHub committed signed1 parent 66dd3facommit 75ee9acaf2ee491f576028189cc401bc5c6cc186Browse files
2 files changed+12−7
modifiedapps/diffinery/src/web/highlight.ts+11−6View file
| @@ -143,7 +143,7 @@ | ||
| 143 | 143 | // ---------------------------------------------------------------- highlighter |
| 144 | 144 | |
| 145 | 145 | /** Highlighting limits: beyond them, plain text. Workers have little CPU to spare. */ |
| 146 | -export const LIMITS = { lines: 2000, bytes: 200_000, lineLength: 2000 }; | |
| 146 | +export const LIMITS = { lines: 2000, bytes: 200_000, lineLength: 2000, lineMs: 500 }; | |
| 147 | 147 | |
| 148 | 148 | export interface Tok { |
| 149 | 149 | text: string; |
| @@ -206,10 +206,15 @@ | ||
| 206 | 206 | [...new Set(langs)].map((l) => { |
| 207 | 207 | let p = loading.get(l); |
| 208 | 208 | if (!p) { |
| 209 | - p = hl.loadLanguage(GRAMMARS[l]()).catch((e) => { | |
| 210 | - loading.delete(l); | |
| 211 | - throw e; | |
| 212 | - }); | |
| 209 | + p = hl | |
| 210 | + .loadLanguage(GRAMMARS[l]()) | |
| 211 | + // Compile the grammar's top-level rules now: on a cold Worker that alone can outlast the | |
| 212 | + // per-line time limit, which would leave the first line of a file unhighlighted. | |
| 213 | + .then(() => void hl.codeToTokens("x", { lang: l, theme: PAIR })) | |
| 214 | + .catch((e) => { | |
| 215 | + loading.delete(l); | |
| 216 | + throw e; | |
| 217 | + }); | |
| 213 | 218 | loading.set(l, p); |
| 214 | 219 | } |
| 215 | 220 | return p; |
| @@ -254,7 +259,7 @@ | ||
| 254 | 259 | export function tokenize(hl: HighlighterCore, code: string, lang: Lang): TokLine[] | null { |
| 255 | 260 | if (code.length > LIMITS.bytes) return null; |
| 256 | 261 | try { |
| 257 | - const { tokens } = hl.codeToTokens(code, { lang, theme: PAIR, tokenizeMaxLineLength: LIMITS.lineLength, tokenizeTimeLimit: 200 }); | |
| 262 | + const { tokens } = hl.codeToTokens(code, { lang, theme: PAIR, tokenizeMaxLineLength: LIMITS.lineLength, tokenizeTimeLimit: LIMITS.lineMs }); | |
| 258 | 263 | return tokens.map(toks); |
| 259 | 264 | } catch (e) { |
| 260 | 265 | console.warn("highlight: tokenize failed", e); |
modifiedapps/diffinery/test/unit/highlight.test.ts+1−1View file
| @@ -73,7 +73,7 @@ | ||
| 73 | 73 | describe("tokens", () => { |
| 74 | 74 | it("tokenizes with both themes as CSS variables and leaves default text bare", async () => { |
| 75 | 75 | // The first tokenization compiles the grammar's regexes, which can use up the |
| 76 | - // 200 ms tokenize time limit on a slow runner (the rest of the line is then left | |
| 76 | + // per-line tokenize time limit on a slow runner (the rest of the line is then left | |
| 77 | 77 | // plain, by design). Warm the grammar so the assertions below don't race it. |
| 78 | 78 | await highlightLines(["const warm = 0;"], "typescript"); |
| 79 | 79 | const out = await highlightLines(["const a = 1; // one", "", "export {};"], "typescript"); |