Improve markdown rendering and syntax highlighting

src/components/code-block-inline.tsx

@@ -3,6 +3,8 @@ import type { Highlighter } from "shiki/bundle/full"
 import { useTheme } from "../lib/theme"
 import { getSharedHighlighter, escapeHtml } from "../lib/markdown"
 
+const inlineLoadedLanguages = new Set<string>()
+
 interface CodeBlockInlineProps {
   code: string
   language?: string
@@ -18,7 +20,7 @@ export function CodeBlockInline(props: CodeBlockInlineProps) {
   onMount(async () => {
     highlighter = await getSharedHighlighter()
     setReady(true)
-    updateHighlight()
+    await updateHighlight()
   })
 
   createEffect(() => {
@@ -26,11 +28,11 @@ export function CodeBlockInline(props: CodeBlockInlineProps) {
       isDark()
       props.code
       props.language
-      updateHighlight()
+      void updateHighlight()
     }
   })
 
-  const updateHighlight = () => {
+  const updateHighlight = async () => {
     if (!highlighter) return
 
     if (!props.language) {
@@ -39,6 +41,11 @@ export function CodeBlockInline(props: CodeBlockInlineProps) {
     }
 
     try {
+      if (!inlineLoadedLanguages.has(props.language)) {
+        await highlighter.loadLanguage(props.language)
+        inlineLoadedLanguages.add(props.language)
+      }
+
       const highlighted = highlighter.codeToHtml(props.code, {
         lang: props.language,
         theme: isDark() ? "github-dark" : "github-light",

src/components/markdown.tsx

@@ -1,5 +1,5 @@
 import { createEffect, createSignal, onMount, onCleanup } from "solid-js"
-import { renderMarkdown } from "../lib/markdown"
+import { renderMarkdown, onLanguagesLoaded } from "../lib/markdown"
 import type { TextPart } from "../types/message"
 
 interface MarkdownProps {
@@ -16,13 +16,13 @@ export function Markdown(props: MarkdownProps) {
     const part = props.part
     const text = part.text || ""
 
+    latestRequestedText = text
+
     if (part.renderCache && part.renderCache.text === text) {
       setHtml(part.renderCache.html)
       return
     }
 
-    latestRequestedText = text
-
     try {
       const rendered = await renderMarkdown(text)
 
@@ -62,8 +62,29 @@ export function Markdown(props: MarkdownProps) {
 
     containerRef?.addEventListener("click", handleClick)
 
+    // Register listener for language loading completion
+    const cleanupLanguageListener = onLanguagesLoaded(async () => {
+      const part = props.part
+      const text = part.text || ""
+
+      if (latestRequestedText !== text) {
+        return
+      }
+
+      try {
+        const rendered = await renderMarkdown(text)
+        if (latestRequestedText === text) {
+          setHtml(rendered)
+          part.renderCache = { text, html: rendered }
+        }
+      } catch (error) {
+        console.error("Failed to re-render markdown after language load:", error)
+      }
+    })
+
     onCleanup(() => {
       containerRef?.removeEventListener("click", handleClick)
+      cleanupLanguageListener()
     })
   })
 

src/lib/markdown.ts

@@ -1,32 +1,42 @@
 import { marked } from "marked"
 import { createHighlighter, type Highlighter, bundledLanguages } from "shiki/bundle/full"
 
-const CORE_LANGUAGES = [
-  "bash",
-  "shell",
-  "sh",
-  "javascript",
-  "typescript",
-  "tsx",
-  "jsx",
-  "json",
-  "yaml",
-  "yml",
-  "markdown",
-  "md",
-  "html",
-  "css",
-  "scss",
-  "python",
-  "go",
-  "rust",
-]
-
 let highlighter: Highlighter | null = null
 let highlighterPromise: Promise<Highlighter> | null = null
 let currentTheme: "light" | "dark" = "light"
 let isInitialized = false
 
+// Track loaded languages and queue for on-demand loading
+const loadedLanguages = new Set<string>()
+const queuedLanguages = new Set<string>()
+const languageLoadQueue: Array<() => Promise<void>> = []
+let isQueueRunning = false
+
+// Pub/sub mechanism for language loading notifications
+const languageListeners: Array<() => void> = []
+
+export function onLanguagesLoaded(callback: () => void): () => void {
+  languageListeners.push(callback)
+
+  // Return cleanup function
+  return () => {
+    const index = languageListeners.indexOf(callback)
+    if (index > -1) {
+      languageListeners.splice(index, 1)
+    }
+  }
+}
+
+function triggerLanguageListeners() {
+  for (const listener of languageListeners) {
+    try {
+      listener()
+    } catch (error) {
+      console.error("Error in language listener:", error)
+    }
+  }
+}
+
 async function getOrCreateHighlighter() {
   if (highlighter) {
     return highlighter
@@ -36,11 +46,10 @@ async function getOrCreateHighlighter() {
     return highlighterPromise
   }
 
-  const filteredLangs = CORE_LANGUAGES.filter((lang) => lang in bundledLanguages)
-
+  // Create highlighter with no preloaded languages
   highlighterPromise = createHighlighter({
     themes: ["github-light", "github-dark"],
-    langs: filteredLangs,
+    langs: [],
   })
 
   highlighter = await highlighterPromise
@@ -48,6 +57,91 @@ async function getOrCreateHighlighter() {
   return highlighter
 }
 
+function normalizeLanguageToken(token: string): string {
+  return token.trim().toLowerCase()
+}
+
+function resolveLanguage(token: string): { canonical: string | null; raw: string } {
+  const normalized = normalizeLanguageToken(token)
+
+  // Check if it's a direct key match
+  if (normalized in bundledLanguages) {
+    return { canonical: normalized, raw: normalized }
+  }
+
+  // Check aliases
+  for (const [key, lang] of Object.entries(bundledLanguages)) {
+    if (lang.aliases?.includes(normalized)) {
+      return { canonical: key, raw: normalized }
+    }
+  }
+
+  return { canonical: null, raw: normalized }
+}
+
+async function ensureLanguages(content: string) {
+  // Parse code fences to extract language tokens
+  const codeBlockRegex = /```(\w*[#.\-+\w]*)/g
+  const foundLanguages = new Set<string>()
+  let match
+
+  while ((match = codeBlockRegex.exec(content)) !== null) {
+    const langToken = match[1]
+    if (langToken) {
+      foundLanguages.add(langToken)
+    }
+  }
+
+  // Queue language loading tasks
+  for (const token of foundLanguages) {
+    const { canonical, raw } = resolveLanguage(token)
+    const langKey = canonical || raw
+
+    // Skip if already loaded or queued
+    if (loadedLanguages.has(langKey) || queuedLanguages.has(langKey)) {
+      continue
+    }
+
+    queuedLanguages.add(langKey)
+
+    // Queue the language loading task
+    languageLoadQueue.push(async () => {
+      try {
+        const h = await getOrCreateHighlighter()
+        await h.loadLanguage(langKey)
+        loadedLanguages.add(langKey)
+        triggerLanguageListeners()
+      } catch {
+        // Quietly ignore errors
+      } finally {
+        queuedLanguages.delete(langKey)
+      }
+    })
+  }
+
+  // Trigger queue runner if not already running
+  if (languageLoadQueue.length > 0 && !isQueueRunning) {
+    runLanguageLoadQueue()
+  }
+}
+
+async function runLanguageLoadQueue() {
+  if (isQueueRunning || languageLoadQueue.length === 0) {
+    return
+  }
+
+  isQueueRunning = true
+
+  while (languageLoadQueue.length > 0) {
+    const task = languageLoadQueue.shift()
+    if (task) {
+      await task()
+    }
+  }
+
+  isQueueRunning = false
+}
+
 function setupRenderer(isDark: boolean) {
   if (!highlighter) return
 
@@ -81,15 +175,24 @@ function setupRenderer(isDark: boolean) {
       return `<div class="markdown-code-block" data-language="${escapedLang}" data-code="${encodedCode}">${header}<pre><code>${escapeHtml(code)}</code></pre></div>`
     }
 
-    try {
-      const html = highlighter.codeToHtml(code, {
-        lang,
-        theme: isDark ? "github-dark" : "github-light",
-      })
-      return `<div class="markdown-code-block" data-language="${escapedLang}" data-code="${encodedCode}">${header}${html}</div>`
-    } catch {
-      return `<div class="markdown-code-block" data-language="${escapedLang}" data-code="${encodedCode}">${header}<pre><code class="language-${escapedLang}">${escapeHtml(code)}</code></pre></div>`
+    // Resolve language and check if it's loaded
+    const { canonical, raw } = resolveLanguage(lang)
+    const langKey = canonical || raw
+
+    // Use highlighting if language is loaded, otherwise fall back to plain code
+    if (loadedLanguages.has(langKey)) {
+      try {
+        const html = highlighter.codeToHtml(code, {
+          lang: langKey,
+          theme: currentTheme === "dark" ? "github-dark" : "github-light",
+        })
+        return `<div class="markdown-code-block" data-language="${escapedLang}" data-code="${encodedCode}">${header}${html}</div>`
+      } catch {
+        // Fall through to plain code if highlighting fails
+      }
     }
+
+    return `<div class="markdown-code-block" data-language="${escapedLang}" data-code="${encodedCode}">${header}<pre><code class="language-${escapedLang}">${escapeHtml(code)}</code></pre></div>`
   }
 
   renderer.link = (href: string, title: string | null | undefined, text: string) => {
@@ -118,6 +221,11 @@ export async function renderMarkdown(content: string): Promise<string> {
   if (!isInitialized) {
     await initMarkdown(currentTheme === "dark")
   }
+
+  // Queue language loading but don't wait for it to complete
+  await ensureLanguages(content)
+
+  // Proceed to parse immediately - highlighting will be available on next render
   return marked.parse(content) as Promise<string>
 }