Add back syntax highlighting and keep bundle small via dynamic script build

embed/.prettierignore

@@ -6,3 +6,4 @@
 
 **/dist
 **/static/**
+src/utils/chat/hljs.js

embed/package.json

@@ -15,6 +15,7 @@
     "@phosphor-icons/react": "^2.0.13",
     "dompurify": "^3.0.8",
     "he": "^1.2.0",
+    "highlight.js": "^11.9.0",
     "lodash.debounce": "^4.0.8",
     "markdown-it": "^13.0.1",
     "react": "^18.2.0",
@@ -39,4 +40,4 @@
     "vite": "^5.0.0",
     "vite-plugin-singlefile": "^0.13.5"
   }
-}
+}

embed/scripts/updateHljs.mjs

@@ -0,0 +1,35 @@
+// What is this script?
+// We want to support code syntax highlighting in the embed modal, but we cannot afford to have the static build
+// be large in size. To prevent HighlightJs from loading all 193+ language stylings and bloating the script, we instead take a large subset that
+// covers most languages and then dynamically build and register each language since HLJS cannot just register with an array of langs.
+// Since the embed is a single script - we need to statically import each library and register that associate language.
+// we can then export this as a custom implementation of HLJS and call it a day and keep the bundle small.
+
+import fs from 'fs'
+
+const SUPPORTED_HIGHLIGHTS = ['apache', 'bash', 'c', 'cpp', 'csharp', 'css', 'diff', 'go', 'graphql', 'ini', 'java', 'javascript', 'json', 'kotlin', 'less', 'lua', 'makefile', 'markdown', 'nginx', 'objectivec', 'perl', 'pgsql', 'php', 'php-template', 'plaintext', 'python', 'python-repl', 'r', 'ruby', 'rust', 'scss', 'shell', 'sql', 'swift', 'typescript', 'vbnet', 'wasm', 'xml', 'yaml'];
+function quickClean(input) {
+  return input.replace(/[^a-zA-Z0-9]/g, '');
+}
+
+let content = `/*
+This is a dynamically generated file to help de-bloat the app since this script is a static bundle.
+You should not modify this file directly. You can regenerate it with "node scripts/updateHljs.mjd" from the embed folder.
+Last generated ${(new Date).toDateString()}
+----------------------
+*/\n\n`
+content += 'import hljs from "highlight.js/lib/core";\n';
+
+SUPPORTED_HIGHLIGHTS.forEach((lang) => {
+  content += `import ${quickClean(lang)}HljsSupport from 'highlight.js/lib/languages/${lang}'\n`;
+});
+
+SUPPORTED_HIGHLIGHTS.forEach((lang) => {
+  content += ` hljs.registerLanguage('${lang}', ${quickClean(lang)}HljsSupport)\n`;
+})
+
+content += `// The above should now register on the languages we wish to support statically.\n`;
+content += `export const staticHljs = hljs;\n`
+
+fs.writeFileSync('src/utils/chat/hljs.js', content, { encoding: 'utf8' })
+console.log(`Static build of HLJS completed - src/utils/chat/hljs.js`)

embed/src/components/ChatWindow/ChatContainer/ChatHistory/HistoricalMessage/index.jsx

@@ -21,7 +21,10 @@ const HistoricalMessage = forwardRef(
               : embedderSettings.AI_BACKGROUND_COLOR
         }`}
       >
-        <div className={`py-2 px-2 w-full flex flex-col`}>
+        <div
+          style={{ wordBreak: "break-word" }}
+          className={`py-2 px-2 w-full flex flex-col`}
+        >
           <div className="flex">
             {error ? (
               <div className="p-2 rounded-lg bg-red-50 text-red-500">

embed/src/components/ChatWindow/ChatContainer/ChatHistory/PromptReply/index.jsx

@@ -46,7 +46,10 @@ const PromptReply = forwardRef(
         ref={ref}
         className={`flex justify-center items-end w-full ${embedderSettings.AI_BACKGROUND_COLOR}`}
       >
-        <div className="py-2 px-2 w-full flex flex-col">
+        <div
+          style={{ wordBreak: "break-word" }}
+          className="py-2 px-2 w-full flex flex-col"
+        >
           <div className="flex gap-x-5">
             <span
               className={`reply whitespace-pre-line text-white font-normal text-sm md:text-sm flex flex-col gap-y-1 mt-2`}

embed/src/utils/chat/hljs.js

@@ -0,0 +1,88 @@
+/*
+This is a dynamically generated file to help de-bloat the app since this script is a static bundle.
+You should not modify this file directly. You can regenerate it with "node scripts/updateHljs.mjd" from the embed folder.
+Last generated Fri Feb 02 2024
+----------------------
+*/
+
+import hljs from "highlight.js/lib/core";
+import apacheHljsSupport from 'highlight.js/lib/languages/apache'
+import bashHljsSupport from 'highlight.js/lib/languages/bash'
+import cHljsSupport from 'highlight.js/lib/languages/c'
+import cppHljsSupport from 'highlight.js/lib/languages/cpp'
+import csharpHljsSupport from 'highlight.js/lib/languages/csharp'
+import cssHljsSupport from 'highlight.js/lib/languages/css'
+import diffHljsSupport from 'highlight.js/lib/languages/diff'
+import goHljsSupport from 'highlight.js/lib/languages/go'
+import graphqlHljsSupport from 'highlight.js/lib/languages/graphql'
+import iniHljsSupport from 'highlight.js/lib/languages/ini'
+import javaHljsSupport from 'highlight.js/lib/languages/java'
+import javascriptHljsSupport from 'highlight.js/lib/languages/javascript'
+import jsonHljsSupport from 'highlight.js/lib/languages/json'
+import kotlinHljsSupport from 'highlight.js/lib/languages/kotlin'
+import lessHljsSupport from 'highlight.js/lib/languages/less'
+import luaHljsSupport from 'highlight.js/lib/languages/lua'
+import makefileHljsSupport from 'highlight.js/lib/languages/makefile'
+import markdownHljsSupport from 'highlight.js/lib/languages/markdown'
+import nginxHljsSupport from 'highlight.js/lib/languages/nginx'
+import objectivecHljsSupport from 'highlight.js/lib/languages/objectivec'
+import perlHljsSupport from 'highlight.js/lib/languages/perl'
+import pgsqlHljsSupport from 'highlight.js/lib/languages/pgsql'
+import phpHljsSupport from 'highlight.js/lib/languages/php'
+import phptemplateHljsSupport from 'highlight.js/lib/languages/php-template'
+import plaintextHljsSupport from 'highlight.js/lib/languages/plaintext'
+import pythonHljsSupport from 'highlight.js/lib/languages/python'
+import pythonreplHljsSupport from 'highlight.js/lib/languages/python-repl'
+import rHljsSupport from 'highlight.js/lib/languages/r'
+import rubyHljsSupport from 'highlight.js/lib/languages/ruby'
+import rustHljsSupport from 'highlight.js/lib/languages/rust'
+import scssHljsSupport from 'highlight.js/lib/languages/scss'
+import shellHljsSupport from 'highlight.js/lib/languages/shell'
+import sqlHljsSupport from 'highlight.js/lib/languages/sql'
+import swiftHljsSupport from 'highlight.js/lib/languages/swift'
+import typescriptHljsSupport from 'highlight.js/lib/languages/typescript'
+import vbnetHljsSupport from 'highlight.js/lib/languages/vbnet'
+import wasmHljsSupport from 'highlight.js/lib/languages/wasm'
+import xmlHljsSupport from 'highlight.js/lib/languages/xml'
+import yamlHljsSupport from 'highlight.js/lib/languages/yaml'
+ hljs.registerLanguage('apache', apacheHljsSupport)
+ hljs.registerLanguage('bash', bashHljsSupport)
+ hljs.registerLanguage('c', cHljsSupport)
+ hljs.registerLanguage('cpp', cppHljsSupport)
+ hljs.registerLanguage('csharp', csharpHljsSupport)
+ hljs.registerLanguage('css', cssHljsSupport)
+ hljs.registerLanguage('diff', diffHljsSupport)
+ hljs.registerLanguage('go', goHljsSupport)
+ hljs.registerLanguage('graphql', graphqlHljsSupport)
+ hljs.registerLanguage('ini', iniHljsSupport)
+ hljs.registerLanguage('java', javaHljsSupport)
+ hljs.registerLanguage('javascript', javascriptHljsSupport)
+ hljs.registerLanguage('json', jsonHljsSupport)
+ hljs.registerLanguage('kotlin', kotlinHljsSupport)
+ hljs.registerLanguage('less', lessHljsSupport)
+ hljs.registerLanguage('lua', luaHljsSupport)
+ hljs.registerLanguage('makefile', makefileHljsSupport)
+ hljs.registerLanguage('markdown', markdownHljsSupport)
+ hljs.registerLanguage('nginx', nginxHljsSupport)
+ hljs.registerLanguage('objectivec', objectivecHljsSupport)
+ hljs.registerLanguage('perl', perlHljsSupport)
+ hljs.registerLanguage('pgsql', pgsqlHljsSupport)
+ hljs.registerLanguage('php', phpHljsSupport)
+ hljs.registerLanguage('php-template', phptemplateHljsSupport)
+ hljs.registerLanguage('plaintext', plaintextHljsSupport)
+ hljs.registerLanguage('python', pythonHljsSupport)
+ hljs.registerLanguage('python-repl', pythonreplHljsSupport)
+ hljs.registerLanguage('r', rHljsSupport)
+ hljs.registerLanguage('ruby', rubyHljsSupport)
+ hljs.registerLanguage('rust', rustHljsSupport)
+ hljs.registerLanguage('scss', scssHljsSupport)
+ hljs.registerLanguage('shell', shellHljsSupport)
+ hljs.registerLanguage('sql', sqlHljsSupport)
+ hljs.registerLanguage('swift', swiftHljsSupport)
+ hljs.registerLanguage('typescript', typescriptHljsSupport)
+ hljs.registerLanguage('vbnet', vbnetHljsSupport)
+ hljs.registerLanguage('wasm', wasmHljsSupport)
+ hljs.registerLanguage('xml', xmlHljsSupport)
+ hljs.registerLanguage('yaml', yamlHljsSupport)
+// The above should now register on the languages we wish to support statically.
+export const staticHljs = hljs;

embed/src/utils/chat/markdown.js

@@ -1,14 +1,31 @@
 import { encode as HTMLEncode } from "he";
 import markdownIt from "markdown-it";
+import { staticHljs as hljs } from "./hljs";
 import { v4 } from "uuid";
 
-// TODO can we add back Hljs without bloating the app in a bad way?
-
 const markdown = markdownIt({
   html: true,
   typographer: true,
-  highlight: function (code) {
+  highlight: function (code, lang) {
     const uuid = v4();
+    if (lang && hljs.getLanguage(lang)) {
+      try {
+        return (
+          `<div class="whitespace-pre-line w-full rounded-lg bg-black-900 pb-4 relative font-mono font-normal text-sm text-slate-200">
+          <div class="w-full flex items-center absolute top-0 left-0 text-slate-200 bg-stone-800 px-4 py-2 text-xs font-sans justify-between rounded-t-md">
+            <div class="flex gap-2"><code class="text-xs">${lang}</code></div>
+            <button data-code-snippet data-code="code-${uuid}" class="flex items-center gap-x-2">
+              <svg stroke="currentColor" fill="none" stroke-width="2" viewBox="0 0 24 24" stroke-linecap="round" stroke-linejoin="round" class="h-4 w-4" height="1em" width="1em" xmlns="http://www.w3.org/2000/svg"><path d="M16 4h2a2 2 0 0 1 2 2v14a2 2 0 0 1-2 2H6a2 2 0 0 1-2-2V6a2 2 0 0 1 2-2h2"></path><rect x="8" y="2" width="8" height="4" rx="1" ry="1"></rect></svg>
+              <p>Copy</p>
+            </button>
+          </div>
+        <pre class="whitespace-pre-wrap px-2">` +
+          hljs.highlight(code, { language: lang, ignoreIllegals: true }).value +
+          "</pre></div>"
+        );
+      } catch (__) {}
+    }
+
     return (
       `<div class="whitespace-pre-line w-full rounded-lg bg-black-900 pb-4 relative font-mono font-normal text-sm text-slate-200">
         <div class="w-full flex items-center absolute top-0 left-0 text-slate-200 bg-stone-800 px-4 py-2 text-xs font-sans justify-between rounded-t-md">

embed/yarn.lock

@@ -1642,6 +1642,11 @@ he@^1.2.0:
   resolved "https://registry.yarnpkg.com/he/-/he-1.2.0.tgz#84ae65fa7eafb165fddb61566ae14baf05664f0f"
   integrity sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==
 
+highlight.js@^11.9.0:
+  version "11.9.0"
+  resolved "https://registry.yarnpkg.com/highlight.js/-/highlight.js-11.9.0.tgz#04ab9ee43b52a41a047432c8103e2158a1b8b5b0"
+  integrity sha512-fJ7cW7fQGCYAkgv4CPfwFHrfd/cLS4Hau96JuJ+ZTOWhjnhoeN1ub1tFmALm/+lW5z4WCAuAV9bm05AP0mS6Gw==
+
 human-signals@^2.1.0:
   version "2.1.0"
   resolved "https://registry.yarnpkg.com/human-signals/-/human-signals-2.1.0.tgz#dc91fcba42e4d06e4abaed33b3e7a3c02f514ea0"