merge with master

README.md

@@ -188,7 +188,7 @@ You can verify these claims by finding all locations `Telemetry.sendTelemetry` i
 
 ---
 
-Copyright © 2023 [Mintplex Labs][profile-link]. <br />
+Copyright © 2024 [Mintplex Labs][profile-link]. <br />
 This project is [MIT](./LICENSE) licensed.
 
 <!-- LINK GROUP -->

docker/.env.example

@@ -66,6 +66,12 @@ GID='1000'
 # GROQ_API_KEY=gsk_abcxyz
 # GROQ_MODEL_PREF=llama2-70b-4096
 
+# LLM_PROVIDER='generic-openai'
+# GENERIC_OPEN_AI_BASE_PATH='http://proxy.url.openai.com/v1'
+# GENERIC_OPEN_AI_MODEL_PREF='gpt-3.5-turbo'
+# GENERIC_OPEN_AI_MODEL_TOKEN_LIMIT=4096
+# GENERIC_OPEN_AI_API_KEY=sk-123abc
+
 ###########################################
 ######## Embedding API SElECTION ##########
 ###########################################

frontend/src/components/LLMSelection/GenericOpenAiOptions/index.jsx

@@ -0,0 +1,66 @@
+export default function GenericOpenAiOptions({ settings }) {
+  return (
+    <div className="flex gap-4 flex-wrap">
+      <div className="flex flex-col w-60">
+        <label className="text-white text-sm font-semibold block mb-4">
+          Base URL
+        </label>
+        <input
+          type="url"
+          name="GenericOpenAiBasePath"
+          className="bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:border-white block w-full p-2.5"
+          placeholder="eg: https://proxy.openai.com"
+          defaultValue={settings?.GenericOpenAiBasePath}
+          required={true}
+          autoComplete="off"
+          spellCheck={false}
+        />
+      </div>
+      <div className="flex flex-col w-60">
+        <label className="text-white text-sm font-semibold block mb-4">
+          API Key
+        </label>
+        <input
+          type="password"
+          name="GenericOpenAiKey"
+          className="bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:border-white block w-full p-2.5"
+          placeholder="Generic service API Key"
+          defaultValue={settings?.GenericOpenAiKey ? "*".repeat(20) : ""}
+          required={false}
+          autoComplete="off"
+          spellCheck={false}
+        />
+      </div>
+      <div className="flex flex-col w-60">
+        <label className="text-white text-sm font-semibold block mb-4">
+          Chat Model Name
+        </label>
+        <input
+          type="text"
+          name="GenericOpenAiModelPref"
+          className="bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:border-white block w-full p-2.5"
+          placeholder="Model id used for chat requests"
+          defaultValue={settings?.GenericOpenAiModelPref}
+          required={true}
+          autoComplete="off"
+        />
+      </div>
+      <div className="flex flex-col w-60">
+        <label className="text-white text-sm font-semibold block mb-4">
+          Token context window
+        </label>
+        <input
+          type="number"
+          name="GenericOpenAiTokenLimit"
+          className="bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:border-white block w-full p-2.5"
+          placeholder="Content window limit (eg: 4096)"
+          min={1}
+          onScroll={(e) => e.target.blur()}
+          defaultValue={settings?.GenericOpenAiTokenLimit}
+          required={true}
+          autoComplete="off"
+        />
+      </div>
+    </div>
+  );
+}

frontend/src/components/LLMSelection/GroqAiOptions/index.jsx

@@ -28,7 +28,13 @@ export default function GroqAiOptions({ settings }) {
             required={true}
             className="bg-zinc-900 border-gray-500 text-white text-sm rounded-lg block w-full p-2.5"
           >
-            {["llama2-70b-4096", "mixtral-8x7b-32768"].map((model) => {
+            {[
+              "llama2-70b-4096",
+              "mixtral-8x7b-32768",
+              "llama3-8b-8192",
+              "llama3-70b-8192",
+              "gemma-7b-it",
+            ].map((model) => {
               return (
                 <option key={model} value={model}>
                   {model}

frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/HistoricalMessage/index.jsx

@@ -44,7 +44,7 @@ const HistoricalMessage = ({
             </div>
           ) : (
             <span
-              className={`flex flex-col gap-y-1 mt-2`}
+              className={`flex flex-col gap-y-1`}
               dangerouslySetInnerHTML={{
                 __html: DOMPurify.sanitize(renderMarkdown(message)),
               }}

frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/PromptReply/index.jsx

@@ -21,7 +21,7 @@ const PromptReply = ({
       <div
         className={`flex justify-center items-end w-full ${assistantBackgroundColor}`}
       >
-        <div className="py-8 px-4 w-full flex gap-x-5 md:max-w-[800px] flex-col">
+        <div className="py-6 px-4 w-full flex gap-x-5 md:max-w-[800px] flex-col">
           <div className="flex gap-x-5">
             <WorkspaceProfileImage workspace={workspace} />
             <div className="mt-3 ml-5 dot-falling"></div>
@@ -36,7 +36,7 @@ const PromptReply = ({
       <div
         className={`flex justify-center items-end w-full ${assistantBackgroundColor}`}
       >
-        <div className="py-8 px-4 w-full flex gap-x-5 md:max-w-[800px] flex-col">
+        <div className="py-6 px-4 w-full flex gap-x-5 md:max-w-[800px] flex-col">
           <div className="flex gap-x-5">
             <WorkspaceProfileImage workspace={workspace} />
             <span
@@ -57,7 +57,7 @@ const PromptReply = ({
       key={uuid}
       className={`flex justify-center items-end w-full ${assistantBackgroundColor}`}
     >
-      <div className="py-8 px-4 w-full flex gap-x-5 md:max-w-[800px] flex-col">
+      <div className="py-6 px-4 w-full flex gap-x-5 md:max-w-[800px] flex-col">
         <div className="flex gap-x-5">
           <WorkspaceProfileImage workspace={workspace} />
           <span

frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/index.jsx

@@ -12,9 +12,39 @@ export default function ChatHistory({ history = [], workspace, sendCommand }) {
   const { showing, showModal, hideModal } = useManageWorkspaceModal();
   const [isAtBottom, setIsAtBottom] = useState(true);
   const chatHistoryRef = useRef(null);
+  const [textSize, setTextSize] = useState("normal");
+
+  const getTextSizeClass = (size) => {
+    switch (size) {
+      case "small":
+        return "text-[12px]";
+      case "large":
+        return "text-[18px]";
+      default:
+        return "text-[14px]";
+    }
+  };
 
   useEffect(() => {
-    scrollToBottom();
+    const storedTextSize = window.localStorage.getItem("anythingllm_text_size");
+    if (storedTextSize) {
+      setTextSize(getTextSizeClass(storedTextSize));
+    }
+
+    const handleTextSizeChange = (event) => {
+      const size = event.detail;
+      setTextSize(getTextSizeClass(size));
+    };
+
+    window.addEventListener("textSizeChange", handleTextSizeChange);
+
+    return () => {
+      window.removeEventListener("textSizeChange", handleTextSizeChange);
+    };
+  }, []);
+
+  useEffect(() => {
+    if (isAtBottom) scrollToBottom();
   }, [history]);
 
   const handleScroll = () => {
@@ -91,7 +121,7 @@ export default function ChatHistory({ history = [], workspace, sendCommand }) {
 
   return (
     <div
-      className="markdown text-white/80 font-light text-sm h-full md:h-[83%] pb-[100px] pt-6 md:pt-0 md:pb-20 md:mx-0 overflow-y-scroll flex flex-col justify-start no-scroll"
+      className={`markdown text-white/80 font-light ${textSize} h-full md:h-[83%] pb-[100px] pt-6 md:pt-0 md:pb-20 md:mx-0 overflow-y-scroll flex flex-col justify-start no-scroll`}
       id="chat-history"
       ref={chatHistoryRef}
     >

frontend/src/components/WorkspaceChat/ChatContainer/PromptInput/TextSizeMenu/index.jsx

@@ -0,0 +1,124 @@
+import { useState, useRef, useEffect } from "react";
+import { TextT } from "@phosphor-icons/react";
+import { Tooltip } from "react-tooltip";
+
+export default function TextSizeButton() {
+  const [showTextSizeMenu, setShowTextSizeMenu] = useState(false);
+  const buttonRef = useRef(null);
+
+  return (
+    <>
+      <div
+        ref={buttonRef}
+        id="text-size-btn"
+        data-tooltip-id="tooltip-text-size-btn"
+        data-tooltip-content="Change text size"
+        aria-label="Change text size"
+        onClick={() => setShowTextSizeMenu(!showTextSizeMenu)}
+        className={`relative flex justify-center items-center opacity-60 hover:opacity-100 cursor-pointer ${
+          showTextSizeMenu ? "!opacity-100" : ""
+        }`}
+      >
+        <TextT
+          weight="fill"
+          className="w-6 h-6 pointer-events-none text-white"
+        />
+        <Tooltip
+          id="tooltip-text-size-btn"
+          place="top"
+          delayShow={300}
+          className="tooltip !text-xs z-99"
+        />
+      </div>
+      <TextSizeMenu
+        showing={showTextSizeMenu}
+        setShowing={setShowTextSizeMenu}
+        buttonRef={buttonRef}
+      />
+    </>
+  );
+}
+
+function TextSizeMenu({ showing, setShowing, buttonRef }) {
+  const formRef = useRef(null);
+  const [selectedSize, setSelectedSize] = useState(
+    window.localStorage.getItem("anythingllm_text_size") || "normal"
+  );
+
+  useEffect(() => {
+    function listenForOutsideClick() {
+      if (!showing || !formRef.current) return false;
+      document.addEventListener("click", closeIfOutside);
+    }
+    listenForOutsideClick();
+  }, [showing, formRef.current]);
+
+  const closeIfOutside = ({ target }) => {
+    if (target.id === "text-size-btn") return;
+    const isOutside = !formRef?.current?.contains(target);
+    if (!isOutside) return;
+    setShowing(false);
+  };
+
+  const handleTextSizeChange = (size) => {
+    setSelectedSize(size);
+    window.localStorage.setItem("anythingllm_text_size", size);
+    window.dispatchEvent(new CustomEvent("textSizeChange", { detail: size }));
+  };
+
+  if (!buttonRef.current) return null;
+
+  return (
+    <div hidden={!showing}>
+      <div
+        ref={formRef}
+        className="absolute bottom-16 -ml-8 w-[140px] p-2 bg-zinc-800 rounded-lg shadow-md flex flex-col justify-center items-start gap-2 z-50"
+      >
+        <button
+          onClick={(e) => {
+            e.preventDefault();
+            setShowing(false);
+            handleTextSizeChange("small");
+          }}
+          className={`w-full hover:cursor-pointer px-2 py-1 rounded-md flex flex-col justify-start group ${
+            selectedSize === "small" ? "bg-zinc-700" : "hover:bg-zinc-700"
+          }`}
+        >
+          <div className="w-full flex-col text-left flex pointer-events-none">
+            <div className="text-white text-xs">Small</div>
+          </div>
+        </button>
+
+        <button
+          onClick={(e) => {
+            e.preventDefault();
+            setShowing(false);
+            handleTextSizeChange("normal");
+          }}
+          className={`w-full hover:cursor-pointer px-2 py-1 rounded-md flex flex-col justify-start group ${
+            selectedSize === "normal" ? "bg-zinc-700" : "hover:bg-zinc-700"
+          }`}
+        >
+          <div className="w-full flex-col text-left flex pointer-events-none">
+            <div className="text-white text-sm">Normal</div>
+          </div>
+        </button>
+
+        <button
+          onClick={(e) => {
+            e.preventDefault();
+            setShowing(false);
+            handleTextSizeChange("large");
+          }}
+          className={`w-full hover:cursor-pointer px-2 py-1 rounded-md flex flex-col justify-start group ${
+            selectedSize === "large" ? "bg-zinc-700" : "hover:bg-zinc-700"
+          }`}
+        >
+          <div className="w-full flex-col text-left flex pointer-events-none">
+            <div className="text-white text-[16px]">Large</div>
+          </div>
+        </button>
+      </div>
+    </div>
+  );
+}

frontend/src/components/WorkspaceChat/ChatContainer/PromptInput/index.jsx

@@ -11,6 +11,7 @@ import AvailableAgentsButton, {
   AvailableAgents,
   useAvailableAgents,
 } from "./AgentMenu";
+import TextSizeButton from "./TextSizeMenu";
 export default function PromptInput({
   message,
   submit,
@@ -137,6 +138,7 @@ export default function PromptInput({
                   showing={showAgents}
                   setShowAgents={setShowAgents}
                 />
+                <TextSizeButton />
               </div>
             </div>
           </div>

frontend/src/hooks/useGetProvidersModels.js

@@ -19,7 +19,13 @@ const PROVIDER_DEFAULT_MODELS = {
   localai: [],
   ollama: [],
   togetherai: [],
-  groq: ["llama2-70b-4096", "mixtral-8x7b-32768"],
+  groq: [
+    "llama2-70b-4096",
+    "mixtral-8x7b-32768",
+    "llama3-8b-8192",
+    "llama3-70b-8192",
+    "gemma-7b-it",
+  ],
   native: [],
 };
 
@@ -34,7 +40,7 @@ function groupModels(models) {
   }, {});
 }
 
-const groupedProviders = ["togetherai", "openai"];
+const groupedProviders = ["togetherai", "openai", "openrouter"];
 export default function useGetProviderModels(provider = null) {
   const [defaultModels, setDefaultModels] = useState([]);
   const [customModels, setCustomModels] = useState([]);

frontend/src/pages/GeneralSettings/EmbeddingPreference/index.jsx

@@ -30,7 +30,7 @@ const EMBEDDERS = [
     logo: AnythingLLMIcon,
     options: (settings) => <NativeEmbeddingOptions settings={settings} />,
     description:
-      "Use the built-in embedding engine for AnythingLLM. Zero setup!",
+      "Use the built-in embedding provider for AnythingLLM. Zero setup!",
   },
   {
     name: "OpenAI",

frontend/src/pages/GeneralSettings/LLMPreference/index.jsx

@@ -4,6 +4,7 @@ import { isMobile } from "react-device-detect";
 import System from "@/models/system";
 import showToast from "@/utils/toast";
 import OpenAiLogo from "@/media/llmprovider/openai.png";
+import GenericOpenAiLogo from "@/media/llmprovider/generic-openai.png";
 import AzureOpenAiLogo from "@/media/llmprovider/azure.png";
 import AnthropicLogo from "@/media/llmprovider/anthropic.png";
 import GeminiLogo from "@/media/llmprovider/gemini.png";
@@ -18,6 +19,7 @@ import OpenRouterLogo from "@/media/llmprovider/openrouter.jpeg";
 import GroqLogo from "@/media/llmprovider/groq.png";
 import PreLoader from "@/components/Preloader";
 import OpenAiOptions from "@/components/LLMSelection/OpenAiOptions";
+import GenericOpenAiOptions from "@/components/LLMSelection/GenericOpenAiOptions";
 import AzureAiOptions from "@/components/LLMSelection/AzureAiOptions";
 import AnthropicAiOptions from "@/components/LLMSelection/AnthropicAiOptions";
 import LMStudioOptions from "@/components/LLMSelection/LMStudioOptions";
@@ -148,6 +150,20 @@ export const AVAILABLE_LLM_PROVIDERS = [
       "The fastest LLM inferencing available for real-time AI applications.",
     requiredConfig: ["GroqApiKey"],
   },
+  {
+    name: "Generic OpenAI",
+    value: "generic-openai",
+    logo: GenericOpenAiLogo,
+    options: (settings) => <GenericOpenAiOptions settings={settings} />,
+    description:
+      "Connect to any OpenAi-compatible service via a custom configuration",
+    requiredConfig: [
+      "GenericOpenAiBasePath",
+      "GenericOpenAiModelPref",
+      "GenericOpenAiTokenLimit",
+      "GenericOpenAiKey",
+    ],
+  },
   // {
   //   name: "Native",
   //   value: "native",

frontend/src/pages/GeneralSettings/PrivacyAndData/index.jsx

@@ -88,7 +88,9 @@ function ThirdParty({ settings }) {
           </ul>
         </div>
         <div className="flex flex-col gap-y-2 border-b border-zinc-500/50 pb-4">
-          <div className="text-white text-base font-bold">Embedding Engine</div>
+          <div className="text-white text-base font-bold">
+            Embedding Preference
+          </div>
           <div className="flex items-center gap-2.5">
             <img
               src={EMBEDDING_ENGINE_PRIVACY[embeddingEngine].logo}

frontend/src/pages/OnboardingFlow/Steps/CustomLogo/index.jsx

@@ -18,7 +18,7 @@ export default function CustomLogo({ setHeader, setForwardBtn, setBackBtn }) {
   }
 
   function handleBack() {
-    navigate(paths.onboarding.vectorDatabase());
+    navigate(paths.onboarding.llmPreference());
   }
 
   useEffect(() => {

frontend/src/pages/OnboardingFlow/Steps/DataHandling/index.jsx

@@ -2,6 +2,7 @@ import PreLoader from "@/components/Preloader";
 import System from "@/models/system";
 import AnythingLLMIcon from "@/media/logo/anything-llm-icon.png";
 import OpenAiLogo from "@/media/llmprovider/openai.png";
+import GenericOpenAiLogo from "@/media/llmprovider/generic-openai.png";
 import AzureOpenAiLogo from "@/media/llmprovider/azure.png";
 import AnthropicLogo from "@/media/llmprovider/anthropic.png";
 import GeminiLogo from "@/media/llmprovider/gemini.png";
@@ -136,6 +137,13 @@ export const LLM_SELECTION_PRIVACY = {
     ],
     logo: GroqLogo,
   },
+  "generic-openai": {
+    name: "Generic OpenAI compatible service",
+    description: [
+      "Data is shared according to the terms of service applicable with your generic endpoint provider.",
+    ],
+    logo: GenericOpenAiLogo,
+  },
 };
 
 export const VECTOR_DB_PRIVACY = {
@@ -305,7 +313,9 @@ export default function DataHandling({ setHeader, setForwardBtn, setBackBtn }) {
           </ul>
         </div>
         <div className="flex flex-col gap-y-2 border-b border-zinc-500/50 pb-4">
-          <div className="text-white text-base font-bold">Embedding Engine</div>
+          <div className="text-white text-base font-bold">
+            Embedding Preference
+          </div>
           <div className="flex items-center gap-2.5">
             <img
               src={EMBEDDING_ENGINE_PRIVACY[embeddingEngine].logo}
@@ -344,6 +354,9 @@ export default function DataHandling({ setHeader, setForwardBtn, setBackBtn }) {
           </ul>
         </div>
       </div>
+      <p className="text-white/60 text-sm font-medium py-1">
+        These settings can be reconfigured at any time in the settings.
+      </p>
     </div>
   );
 }

frontend/src/pages/OnboardingFlow/Steps/EmbeddingPreference/index.jsx

@@ -1,186 +0,0 @@
-import { MagnifyingGlass } from "@phosphor-icons/react";
-import { useEffect, useState, useRef } from "react";
-import AnythingLLMIcon from "@/media/logo/anything-llm-icon.png";
-import OpenAiLogo from "@/media/llmprovider/openai.png";
-import AzureOpenAiLogo from "@/media/llmprovider/azure.png";
-import LocalAiLogo from "@/media/llmprovider/localai.png";
-import OllamaLogo from "@/media/llmprovider/ollama.png";
-import LMStudioLogo from "@/media/llmprovider/lmstudio.png";
-import NativeEmbeddingOptions from "@/components/EmbeddingSelection/NativeEmbeddingOptions";
-import OpenAiOptions from "@/components/EmbeddingSelection/OpenAiOptions";
-import AzureAiOptions from "@/components/EmbeddingSelection/AzureAiOptions";
-import LocalAiOptions from "@/components/EmbeddingSelection/LocalAiOptions";
-import OllamaEmbeddingOptions from "@/components/EmbeddingSelection/OllamaOptions";
-import LMStudioEmbeddingOptions from "@/components/EmbeddingSelection/LMStudioOptions";
-import EmbedderItem from "@/components/EmbeddingSelection/EmbedderItem";
-import System from "@/models/system";
-import paths from "@/utils/paths";
-import showToast from "@/utils/toast";
-import { useNavigate } from "react-router-dom";
-
-const TITLE = "Embedding Preference";
-const DESCRIPTION =
-  "AnythingLLM can work with many embedding models. This will be the model which turns documents into vectors.";
-const EMBEDDERS = [
-  {
-    name: "AnythingLLM Embedder",
-    value: "native",
-    logo: AnythingLLMIcon,
-    options: (settings) => <NativeEmbeddingOptions settings={settings} />,
-    description:
-      "Use the built-in embedding engine for AnythingLLM. Zero setup!",
-  },
-  {
-    name: "OpenAI",
-    value: "openai",
-    logo: OpenAiLogo,
-    options: (settings) => <OpenAiOptions settings={settings} />,
-    description: "The standard option for most non-commercial use.",
-  },
-  {
-    name: "Azure OpenAI",
-    value: "azure",
-    logo: AzureOpenAiLogo,
-    options: (settings) => <AzureAiOptions settings={settings} />,
-    description: "The enterprise option of OpenAI hosted on Azure services.",
-  },
-  {
-    name: "Local AI",
-    value: "localai",
-    logo: LocalAiLogo,
-    options: (settings) => <LocalAiOptions settings={settings} />,
-    description: "Run embedding models locally on your own machine.",
-  },
-  {
-    name: "Ollama",
-    value: "ollama",
-    logo: OllamaLogo,
-    options: (settings) => <OllamaEmbeddingOptions settings={settings} />,
-    description: "Run embedding models locally on your own machine.",
-  },
-  {
-    name: "LM Studio",
-    value: "lmstudio",
-    logo: LMStudioLogo,
-    options: (settings) => <LMStudioEmbeddingOptions settings={settings} />,
-    description:
-      "Discover, download, and run thousands of cutting edge LLMs in a few clicks.",
-  },
-];
-
-export default function EmbeddingPreference({
-  setHeader,
-  setForwardBtn,
-  setBackBtn,
-}) {
-  const [searchQuery, setSearchQuery] = useState("");
-  const [filteredEmbedders, setFilteredEmbedders] = useState([]);
-  const [selectedEmbedder, setSelectedEmbedder] = useState(null);
-  const [settings, setSettings] = useState(null);
-  const formRef = useRef(null);
-  const hiddenSubmitButtonRef = useRef(null);
-  const navigate = useNavigate();
-
-  useEffect(() => {
-    async function fetchKeys() {
-      const _settings = await System.keys();
-      setSettings(_settings);
-      setSelectedEmbedder(_settings?.EmbeddingEngine || "native");
-    }
-    fetchKeys();
-  }, []);
-
-  function handleForward() {
-    if (hiddenSubmitButtonRef.current) {
-      hiddenSubmitButtonRef.current.click();
-    }
-  }
-
-  function handleBack() {
-    navigate(paths.onboarding.llmPreference());
-  }
-
-  const handleSubmit = async (e) => {
-    e.preventDefault();
-    const form = e.target;
-    const data = {};
-    const formData = new FormData(form);
-    data.EmbeddingEngine = selectedEmbedder;
-    for (var [key, value] of formData.entries()) data[key] = value;
-
-    const { error } = await System.updateSystem(data);
-    if (error) {
-      showToast(`Failed to save embedding settings: ${error}`, "error");
-      return;
-    }
-    navigate(paths.onboarding.vectorDatabase());
-  };
-
-  useEffect(() => {
-    setHeader({ title: TITLE, description: DESCRIPTION });
-    setForwardBtn({ showing: true, disabled: false, onClick: handleForward });
-    setBackBtn({ showing: true, disabled: false, onClick: handleBack });
-  }, []);
-
-  useEffect(() => {
-    const filtered = EMBEDDERS.filter((embedder) =>
-      embedder.name.toLowerCase().includes(searchQuery.toLowerCase())
-    );
-    setFilteredEmbedders(filtered);
-  }, [searchQuery, selectedEmbedder]);
-
-  return (
-    <div>
-      <form ref={formRef} onSubmit={handleSubmit} className="w-full">
-        <div className="w-full relative border-slate-300/40 shadow border-2 rounded-lg text-white">
-          <div className="w-full p-4 absolute top-0 rounded-t-lg backdrop-blur-sm">
-            <div className="w-full flex items-center sticky top-0 z-20">
-              <MagnifyingGlass
-                size={16}
-                weight="bold"
-                className="absolute left-4 z-30 text-white"
-              />
-              <input
-                type="text"
-                placeholder="Search Embedding providers"
-                className="bg-zinc-600 z-20 pl-10 h-[38px] rounded-full w-full px-4 py-1 text-sm border-2 border-slate-300/40 outline-none focus:border-white text-white"
-                onChange={(e) => setSearchQuery(e.target.value)}
-                autoComplete="off"
-                onKeyDown={(e) => {
-                  if (e.key === "Enter") e.preventDefault();
-                }}
-              />
-            </div>
-          </div>
-          <div className="px-4 pt-[70px] flex flex-col gap-y-1 max-h-[390px] overflow-y-auto no-scroll pb-4">
-            {filteredEmbedders.map((embedder) => {
-              return (
-                <EmbedderItem
-                  key={embedder.name}
-                  name={embedder.name}
-                  value={embedder.value}
-                  image={embedder.logo}
-                  description={embedder.description}
-                  checked={selectedEmbedder === embedder.value}
-                  onClick={() => setSelectedEmbedder(embedder.value)}
-                />
-              );
-            })}
-          </div>
-        </div>
-        <div className="mt-4 flex flex-col gap-y-1">
-          {selectedEmbedder &&
-            EMBEDDERS.find(
-              (embedder) => embedder.value === selectedEmbedder
-            )?.options(settings)}
-        </div>
-        <button
-          type="submit"
-          ref={hiddenSubmitButtonRef}
-          hidden
-          aria-hidden="true"
-        ></button>
-      </form>
-    </div>
-  );
-}

frontend/src/pages/OnboardingFlow/Steps/LLMPreference/index.jsx

@@ -1,6 +1,7 @@
 import { MagnifyingGlass } from "@phosphor-icons/react";
 import { useEffect, useState, useRef } from "react";
 import OpenAiLogo from "@/media/llmprovider/openai.png";
+import GenericOpenAiLogo from "@/media/llmprovider/generic-openai.png";
 import AzureOpenAiLogo from "@/media/llmprovider/azure.png";
 import AnthropicLogo from "@/media/llmprovider/anthropic.png";
 import GeminiLogo from "@/media/llmprovider/gemini.png";
@@ -14,6 +15,7 @@ import PerplexityLogo from "@/media/llmprovider/perplexity.png";
 import OpenRouterLogo from "@/media/llmprovider/openrouter.jpeg";
 import GroqLogo from "@/media/llmprovider/groq.png";
 import OpenAiOptions from "@/components/LLMSelection/OpenAiOptions";
+import GenericOpenAiOptions from "@/components/LLMSelection/GenericOpenAiOptions";
 import AzureAiOptions from "@/components/LLMSelection/AzureAiOptions";
 import AnthropicAiOptions from "@/components/LLMSelection/AnthropicAiOptions";
 import LMStudioOptions from "@/components/LLMSelection/LMStudioOptions";
@@ -36,6 +38,120 @@ const TITLE = "LLM Preference";
 const DESCRIPTION =
   "AnythingLLM can work with many LLM providers. This will be the service which handles chatting.";
 
+const LLMS = [
+  {
+    name: "OpenAI",
+    value: "openai",
+    logo: OpenAiLogo,
+    options: (settings) => <OpenAiOptions settings={settings} />,
+    description: "The standard option for most non-commercial use.",
+  },
+  {
+    name: "Azure OpenAI",
+    value: "azure",
+    logo: AzureOpenAiLogo,
+    options: (settings) => <AzureAiOptions settings={settings} />,
+    description: "The enterprise option of OpenAI hosted on Azure services.",
+  },
+  {
+    name: "Anthropic",
+    value: "anthropic",
+    logo: AnthropicLogo,
+    options: (settings) => <AnthropicAiOptions settings={settings} />,
+    description: "A friendly AI Assistant hosted by Anthropic.",
+  },
+  {
+    name: "Gemini",
+    value: "gemini",
+    logo: GeminiLogo,
+    options: (settings) => <GeminiLLMOptions settings={settings} />,
+    description: "Google's largest and most capable AI model",
+  },
+  {
+    name: "HuggingFace",
+    value: "huggingface",
+    logo: HuggingFaceLogo,
+    options: (settings) => <HuggingFaceOptions settings={settings} />,
+    description:
+      "Access 150,000+ open-source LLMs and the world's AI community",
+  },
+  {
+    name: "Ollama",
+    value: "ollama",
+    logo: OllamaLogo,
+    options: (settings) => <OllamaLLMOptions settings={settings} />,
+    description: "Run LLMs locally on your own machine.",
+  },
+  {
+    name: "LM Studio",
+    value: "lmstudio",
+    logo: LMStudioLogo,
+    options: (settings) => <LMStudioOptions settings={settings} />,
+    description:
+      "Discover, download, and run thousands of cutting edge LLMs in a few clicks.",
+  },
+  {
+    name: "Local AI",
+    value: "localai",
+    logo: LocalAiLogo,
+    options: (settings) => <LocalAiOptions settings={settings} />,
+    description: "Run LLMs locally on your own machine.",
+  },
+  {
+    name: "Together AI",
+    value: "togetherai",
+    logo: TogetherAILogo,
+    options: (settings) => <TogetherAiOptions settings={settings} />,
+    description: "Run open source models from Together AI.",
+  },
+  {
+    name: "Mistral",
+    value: "mistral",
+    logo: MistralLogo,
+    options: (settings) => <MistralOptions settings={settings} />,
+    description: "Run open source models from Mistral AI.",
+  },
+  {
+    name: "Perplexity AI",
+    value: "perplexity",
+    logo: PerplexityLogo,
+    options: (settings) => <PerplexityOptions settings={settings} />,
+    description:
+      "Run powerful and internet-connected models hosted by Perplexity AI.",
+  },
+  {
+    name: "OpenRouter",
+    value: "openrouter",
+    logo: OpenRouterLogo,
+    options: (settings) => <OpenRouterOptions settings={settings} />,
+    description: "A unified interface for LLMs.",
+  },
+  {
+    name: "Groq",
+    value: "groq",
+    logo: GroqLogo,
+    options: (settings) => <GroqAiOptions settings={settings} />,
+    description:
+      "The fastest LLM inferencing available for real-time AI applications.",
+  },
+  {
+    name: "Generic OpenAI",
+    value: "generic-openai",
+    logo: GenericOpenAiLogo,
+    options: (settings) => <GenericOpenAiOptions settings={settings} />,
+    description:
+      "Connect to any OpenAi-compatible service via a custom configuration",
+  },
+  // {
+  //   name: "Native",
+  //   value: "native",
+  //   logo: AnythingLLMIcon,
+  //   options: (settings) => <NativeLLMOptions settings={settings} />,
+  //   description:
+  //     "Use a downloaded custom Llama model for chatting on this AnythingLLM instance.",
+  // },
+];
+
 export default function LLMPreference({
   setHeader,
   setForwardBtn,
@@ -59,104 +175,6 @@ export default function LLMPreference({
     fetchKeys();
   }, []);
 
-  const LLMS = [
-    {
-      name: "OpenAI",
-      value: "openai",
-      logo: OpenAiLogo,
-      options: <OpenAiOptions settings={settings} />,
-      description: "The standard option for most non-commercial use.",
-    },
-    {
-      name: "Azure OpenAI",
-      value: "azure",
-      logo: AzureOpenAiLogo,
-      options: <AzureAiOptions settings={settings} />,
-      description: "The enterprise option of OpenAI hosted on Azure services.",
-    },
-    {
-      name: "Anthropic",
-      value: "anthropic",
-      logo: AnthropicLogo,
-      options: <AnthropicAiOptions settings={settings} />,
-      description: "A friendly AI Assistant hosted by Anthropic.",
-    },
-    {
-      name: "Gemini",
-      value: "gemini",
-      logo: GeminiLogo,
-      options: <GeminiLLMOptions settings={settings} />,
-      description: "Google's largest and most capable AI model",
-    },
-    {
-      name: "HuggingFace",
-      value: "huggingface",
-      logo: HuggingFaceLogo,
-      options: <HuggingFaceOptions settings={settings} />,
-      description:
-        "Access 150,000+ open-source LLMs and the world's AI community",
-    },
-    {
-      name: "Ollama",
-      value: "ollama",
-      logo: OllamaLogo,
-      options: <OllamaLLMOptions settings={settings} />,
-      description: "Run LLMs locally on your own machine.",
-    },
-    {
-      name: "LM Studio",
-      value: "lmstudio",
-      logo: LMStudioLogo,
-      options: <LMStudioOptions settings={settings} />,
-      description:
-        "Discover, download, and run thousands of cutting edge LLMs in a few clicks.",
-    },
-    {
-      name: "Local AI",
-      value: "localai",
-      logo: LocalAiLogo,
-      options: <LocalAiOptions settings={settings} />,
-      description: "Run LLMs locally on your own machine.",
-    },
-    {
-      name: "Together AI",
-      value: "togetherai",
-      logo: TogetherAILogo,
-      options: <TogetherAiOptions settings={settings} />,
-      description: "Run open source models from Together AI.",
-    },
-    {
-      name: "Mistral",
-      value: "mistral",
-      logo: MistralLogo,
-      options: <MistralOptions settings={settings} />,
-      description: "Run open source models from Mistral AI.",
-    },
-    {
-      name: "Perplexity AI",
-      value: "perplexity",
-      logo: PerplexityLogo,
-      options: <PerplexityOptions settings={settings} />,
-      description:
-        "Run powerful and internet-connected models hosted by Perplexity AI.",
-    },
-    {
-      name: "OpenRouter",
-      value: "openrouter",
-      logo: OpenRouterLogo,
-      options: <OpenRouterOptions settings={settings} />,
-      description: "A unified interface for LLMs.",
-    },
-    {
-      name: "Groq",
-      value: "groq",
-      logo: GroqLogo,
-      options: <GroqAiOptions settings={settings} />,
-      description:
-        "The fastest LLM inferencing available for real-time AI applications.",
-    },
-  ];
-
   function handleForward() {
     if (hiddenSubmitButtonRef.current) {
       hiddenSubmitButtonRef.current.click();
@@ -172,6 +190,9 @@ export default function LLMPreference({
     const data = {};
     const formData = new FormData(form);
     data.LLMProvider = selectedLLM;
+    // Default to AnythingLLM embedder and LanceDB
+    data.EmbeddingEngine = "native";
+    data.VectorDB = "lancedb";
     for (var [key, value] of formData.entries()) data[key] = value;
 
     const { error } = await System.updateSystem(data);
@@ -179,7 +200,7 @@ export default function LLMPreference({
       showToast(`Failed to save LLM settings: ${error}`, "error");
       return;
     }
-    navigate(paths.onboarding.embeddingPreference());
+    navigate(paths.onboarding.customLogo());
   };
 
   useEffect(() => {
@@ -236,7 +257,7 @@ export default function LLMPreference({
         </div>
         <div className="mt-4 flex flex-col gap-y-1">
           {selectedLLM &&
-            LLMS.find((llm) => llm.value === selectedLLM)?.options}
+            LLMS.find((llm) => llm.value === selectedLLM)?.options(settings)}
         </div>
         <button
           type="submit"

frontend/src/pages/OnboardingFlow/Steps/VectorDatabaseConnection/index.jsx

@@ -1,202 +0,0 @@
-import React, { useEffect, useState, useRef } from "react";
-import { MagnifyingGlass } from "@phosphor-icons/react";
-import ChromaLogo from "@/media/vectordbs/chroma.png";
-import PineconeLogo from "@/media/vectordbs/pinecone.png";
-import LanceDbLogo from "@/media/vectordbs/lancedb.png";
-import WeaviateLogo from "@/media/vectordbs/weaviate.png";
-import QDrantLogo from "@/media/vectordbs/qdrant.png";
-import MilvusLogo from "@/media/vectordbs/milvus.png";
-import ZillizLogo from "@/media/vectordbs/zilliz.png";
-import AstraDBLogo from "@/media/vectordbs/astraDB.png";
-import System from "@/models/system";
-import paths from "@/utils/paths";
-import PineconeDBOptions from "@/components/VectorDBSelection/PineconeDBOptions";
-import ChromaDBOptions from "@/components/VectorDBSelection/ChromaDBOptions";
-import QDrantDBOptions from "@/components/VectorDBSelection/QDrantDBOptions";
-import WeaviateDBOptions from "@/components/VectorDBSelection/WeaviateDBOptions";
-import LanceDBOptions from "@/components/VectorDBSelection/LanceDBOptions";
-import MilvusOptions from "@/components/VectorDBSelection/MilvusDBOptions";
-import ZillizCloudOptions from "@/components/VectorDBSelection/ZillizCloudOptions";
-import AstraDBOptions from "@/components/VectorDBSelection/AstraDBOptions";
-import showToast from "@/utils/toast";
-import { useNavigate } from "react-router-dom";
-import VectorDBItem from "@/components/VectorDBSelection/VectorDBItem";
-
-const TITLE = "Vector Database Connection";
-const DESCRIPTION =
-  "These are the credentials and settings for your vector database of choice.";
-
-export default function VectorDatabaseConnection({
-  setHeader,
-  setForwardBtn,
-  setBackBtn,
-}) {
-  const [searchQuery, setSearchQuery] = useState("");
-  const [filteredVDBs, setFilteredVDBs] = useState([]);
-  const [selectedVDB, setSelectedVDB] = useState(null);
-  const [settings, setSettings] = useState(null);
-  const formRef = useRef(null);
-  const hiddenSubmitButtonRef = useRef(null);
-  const navigate = useNavigate();
-
-  useEffect(() => {
-    async function fetchKeys() {
-      const _settings = await System.keys();
-      setSettings(_settings);
-      setSelectedVDB(_settings?.VectorDB || "lancedb");
-    }
-    fetchKeys();
-  }, []);
-
-  const VECTOR_DBS = [
-    {
-      name: "LanceDB",
-      value: "lancedb",
-      logo: LanceDbLogo,
-      options: <LanceDBOptions />,
-      description:
-        "100% local vector DB that runs on the same instance as AnythingLLM.",
-    },
-    {
-      name: "Chroma",
-      value: "chroma",
-      logo: ChromaLogo,
-      options: <ChromaDBOptions settings={settings} />,
-      description:
-        "Open source vector database you can host yourself or on the cloud.",
-    },
-    {
-      name: "Pinecone",
-      value: "pinecone",
-      logo: PineconeLogo,
-      options: <PineconeDBOptions settings={settings} />,
-      description: "100% cloud-based vector database for enterprise use cases.",
-    },
-    {
-      name: "Zilliz Cloud",
-      value: "zilliz",
-      logo: ZillizLogo,
-      options: <ZillizCloudOptions settings={settings} />,
-      description:
-        "Cloud hosted vector database built for enterprise with SOC 2 compliance.",
-    },
-    {
-      name: "QDrant",
-      value: "qdrant",
-      logo: QDrantLogo,
-      options: <QDrantDBOptions settings={settings} />,
-      description: "Open source local and distributed cloud vector database.",
-    },
-    {
-      name: "Weaviate",
-      value: "weaviate",
-      logo: WeaviateLogo,
-      options: <WeaviateDBOptions settings={settings} />,
-      description:
-        "Open source local and cloud hosted multi-modal vector database.",
-    },
-    {
-      name: "Milvus",
-      value: "milvus",
-      logo: MilvusLogo,
-      options: <MilvusOptions settings={settings} />,
-      description: "Open-source, highly scalable, and blazing fast.",
-    },
-    {
-      name: "AstraDB",
-      value: "astra",
-      logo: AstraDBLogo,
-      options: <AstraDBOptions settings={settings} />,
-      description: "Vector Search for Real-world GenAI.",
-    },
-  ];
-
-  function handleForward() {
-    if (hiddenSubmitButtonRef.current) {
-      hiddenSubmitButtonRef.current.click();
-    }
-  }
-
-  function handleBack() {
-    navigate(paths.onboarding.embeddingPreference());
-  }
-
-  const handleSubmit = async (e) => {
-    e.preventDefault();
-    const form = e.target;
-    const data = {};
-    const formData = new FormData(form);
-    data.VectorDB = selectedVDB;
-    for (var [key, value] of formData.entries()) data[key] = value;
-    const { error } = await System.updateSystem(data);
-    if (error) {
-      showToast(`Failed to save Vector Database settings: ${error}`, "error");
-      return;
-    }
-    navigate(paths.onboarding.customLogo());
-  };
-
-  useEffect(() => {
-    setHeader({ title: TITLE, description: DESCRIPTION });
-    setForwardBtn({ showing: true, disabled: false, onClick: handleForward });
-    setBackBtn({ showing: true, disabled: false, onClick: handleBack });
-  }, []);
-
-  useEffect(() => {
-    const filtered = VECTOR_DBS.filter((vdb) =>
-      vdb.name.toLowerCase().includes(searchQuery.toLowerCase())
-    );
-    setFilteredVDBs(filtered);
-  }, [searchQuery, selectedVDB]);
-
-  return (
-    <>
-      <form ref={formRef} onSubmit={handleSubmit} className="w-full">
-        <div className="w-full relative border-slate-300/40 shadow border-2 rounded-lg text-white pb-4">
-          <div className="w-full p-4 absolute top-0 rounded-t-lg backdrop-blur-sm">
-            <div className="w-full flex items-center sticky top-0 z-20">
-              <MagnifyingGlass
-                size={16}
-                weight="bold"
-                className="absolute left-4 z-30 text-white"
-              />
-              <input
-                type="text"
-                placeholder="Search vector databases"
-                className="bg-zinc-600 z-20 pl-10 h-[38px] rounded-full w-full px-4 py-1 text-sm border-2 border-slate-300/40 outline-none focus:border-white text-white"
-                onChange={(e) => setSearchQuery(e.target.value)}
-                autoComplete="off"
-                onKeyDown={(e) => {
-                  if (e.key === "Enter") e.preventDefault();
-                }}
-              />
-            </div>
-          </div>
-          <div className="px-4 pt-[70px] flex flex-col gap-y-1 max-h-[390px] overflow-y-auto no-scroll">
-            {filteredVDBs.map((vdb) => (
-              <VectorDBItem
-                key={vdb.name}
-                name={vdb.name}
-                value={vdb.value}
-                image={vdb.logo}
-                description={vdb.description}
-                checked={selectedVDB === vdb.value}
-                onClick={setSelectedVDB}
-              />
-            ))}
-          </div>
-        </div>
-        <div className="mt-4 flex flex-col gap-y-1">
-          {selectedVDB &&
-            VECTOR_DBS.find((vdb) => vdb.value === selectedVDB)?.options}
-        </div>
-        <button
-          type="submit"
-          ref={hiddenSubmitButtonRef}
-          hidden
-          aria-hidden="true"
-        ></button>
-      </form>
-    </>
-  );
-}

frontend/src/pages/OnboardingFlow/Steps/index.jsx

@@ -3,8 +3,6 @@ import { useState } from "react";
 import { isMobile } from "react-device-detect";
 import Home from "./Home";
 import LLMPreference from "./LLMPreference";
-import EmbeddingPreference from "./EmbeddingPreference";
-import VectorDatabaseConnection from "./VectorDatabaseConnection";
 import CustomLogo from "./CustomLogo";
 import UserSetup from "./UserSetup";
 import DataHandling from "./DataHandling";
@@ -14,8 +12,6 @@ import CreateWorkspace from "./CreateWorkspace";
 const OnboardingSteps = {
   home: Home,
   "llm-preference": LLMPreference,
-  "embedding-preference": EmbeddingPreference,
-  "vector-database": VectorDatabaseConnection,
   "custom-logo": CustomLogo,
   "user-setup": UserSetup,
   "data-handling": DataHandling,

frontend/src/pages/WorkspaceSettings/ChatSettings/WorkspaceLLMSelection/index.jsx

@@ -5,6 +5,9 @@ import { AVAILABLE_LLM_PROVIDERS } from "@/pages/GeneralSettings/LLMPreference";
 import { CaretUpDown, MagnifyingGlass, X } from "@phosphor-icons/react";
 import ChatModelSelection from "../ChatModelSelection";
 
+// Some providers can only be associated with a single model.
+// In that case there is no selection to be made so we can just move on.
+const NO_MODEL_SELECTION = ["default", "huggingface", "generic-openai"];
 const DISABLED_PROVIDERS = ["azure", "lmstudio", "native"];
 const LLM_DEFAULT = {
   name: "System default",
@@ -145,7 +148,7 @@ export default function WorkspaceLLMSelection({
           </button>
         )}
       </div>
-      {selectedLLM !== "default" && (
+      {!NO_MODEL_SELECTION.includes(selectedLLM) && (
         <div className="mt-4 flex flex-col gap-y-1">
           <ChatModelSelection
             provider={selectedLLM}

server/.env.example

@@ -63,6 +63,12 @@ JWT_SECRET="my-random-string-for-seeding" # Please generate random string at lea
 # GROQ_API_KEY=gsk_abcxyz
 # GROQ_MODEL_PREF=llama2-70b-4096
 
+# LLM_PROVIDER='generic-openai'
+# GENERIC_OPEN_AI_BASE_PATH='http://proxy.url.openai.com/v1'
+# GENERIC_OPEN_AI_MODEL_PREF='gpt-3.5-turbo'
+# GENERIC_OPEN_AI_MODEL_TOKEN_LIMIT=4096
+# GENERIC_OPEN_AI_API_KEY=sk-123abc
+
 ###########################################
 ######## Embedding API SElECTION ##########
 ###########################################

server/endpoints/api/workspace/index.js

@@ -436,9 +436,9 @@ function apiWorkspaceEndpoints(app) {
 
         await Document.removeDocuments(currWorkspace, deletes);
         await Document.addDocuments(currWorkspace, adds);
-        const updatedWorkspace = await Workspace.get(
+        const updatedWorkspace = await Workspace.get({
-          `id = ${Number(currWorkspace.id)}`
+          id: Number(currWorkspace.id),
-        );
+        });
         response.status(200).json({ workspace: updatedWorkspace });
       } catch (e) {
         console.log(e.message, e);

server/models/systemSettings.js

@@ -362,6 +362,12 @@ const SystemSettings = {
       HuggingFaceLLMEndpoint: process.env.HUGGING_FACE_LLM_ENDPOINT,
       HuggingFaceLLMAccessToken: !!process.env.HUGGING_FACE_LLM_API_KEY,
       HuggingFaceLLMTokenLimit: process.env.HUGGING_FACE_LLM_TOKEN_LIMIT,
+
+      // Generic OpenAI Keys
+      GenericOpenAiBasePath: process.env.GENERIC_OPEN_AI_BASE_PATH,
+      GenericOpenAiModelPref: process.env.GENERIC_OPEN_AI_MODEL_PREF,
+      GenericOpenAiTokenLimit: process.env.GENERIC_OPEN_AI_MODEL_TOKEN_LIMIT,
+      GenericOpenAiKey: !!process.env.GENERIC_OPEN_AI_API_KEY,
     };
   },
 };

server/models/workspaceAgentInvocation.js

@@ -3,7 +3,9 @@ const { v4: uuidv4 } = require("uuid");
 
 const WorkspaceAgentInvocation = {
   // returns array of strings with their @ handle.
+  // must start with @
   parseAgents: function (promptString) {
+    if (!promptString.startsWith("@")) return [];
     return promptString.split(/\s+/).filter((v) => v.startsWith("@"));
   },
 

server/storage/models/.gitignore

@@ -1,3 +1,4 @@
 Xenova
 downloaded/*
-!downloaded/.placeholder
+!downloaded/.placeholder
+openrouter

server/utils/AiProviders/genericOpenAi/index.js

@@ -0,0 +1,193 @@
+const { NativeEmbedder } = require("../../EmbeddingEngines/native");
+const { chatPrompt } = require("../../chats");
+const { handleDefaultStreamResponse } = require("../../helpers/chat/responses");
+
+class GenericOpenAiLLM {
+  constructor(embedder = null, modelPreference = null) {
+    const { Configuration, OpenAIApi } = require("openai");
+    if (!process.env.GENERIC_OPEN_AI_BASE_PATH)
+      throw new Error(
+        "GenericOpenAI must have a valid base path to use for the api."
+      );
+
+    this.basePath = process.env.GENERIC_OPEN_AI_BASE_PATH;
+    const config = new Configuration({
+      basePath: this.basePath,
+      apiKey: process.env.GENERIC_OPEN_AI_API_KEY ?? null,
+    });
+    this.openai = new OpenAIApi(config);
+    this.model =
+      modelPreference ?? process.env.GENERIC_OPEN_AI_MODEL_PREF ?? null;
+    if (!this.model)
+      throw new Error("GenericOpenAI must have a valid model set.");
+    this.limits = {
+      history: this.promptWindowLimit() * 0.15,
+      system: this.promptWindowLimit() * 0.15,
+      user: this.promptWindowLimit() * 0.7,
+    };
+
+    if (!embedder)
+      console.warn(
+        "No embedding provider defined for GenericOpenAiLLM - falling back to NativeEmbedder for embedding!"
+      );
+    this.embedder = !embedder ? new NativeEmbedder() : embedder;
+    this.defaultTemp = 0.7;
+    this.log(`Inference API: ${this.basePath} Model: ${this.model}`);
+  }
+
+  log(text, ...args) {
+    console.log(`\x1b[36m[${this.constructor.name}]\x1b[0m ${text}`, ...args);
+  }
+
+  #appendContext(contextTexts = []) {
+    if (!contextTexts || !contextTexts.length) return "";
+    return (
+      "\nContext:\n" +
+      contextTexts
+        .map((text, i) => {
+          return `[CONTEXT ${i}]:\n${text}\n[END CONTEXT ${i}]\n\n`;
+        })
+        .join("")
+    );
+  }
+
+  streamingEnabled() {
+    return "streamChat" in this && "streamGetChatCompletion" in this;
+  }
+
+  // Ensure the user set a value for the token limit
+  // and if undefined - assume 4096 window.
+  promptWindowLimit() {
+    const limit = process.env.GENERIC_OPEN_AI_MODEL_TOKEN_LIMIT || 4096;
+    if (!limit || isNaN(Number(limit)))
+      throw new Error("No token context limit was set.");
+    return Number(limit);
+  }
+
+  // Short circuit since we have no idea if the model is valid or not
+  // in pre-flight for generic endpoints
+  isValidChatCompletionModel(_modelName = "") {
+    return true;
+  }
+
+  constructPrompt({
+    systemPrompt = "",
+    contextTexts = [],
+    chatHistory = [],
+    userPrompt = "",
+  }) {
+    const prompt = {
+      role: "system",
+      content: `${systemPrompt}${this.#appendContext(contextTexts)}`,
+    };
+    return [prompt, ...chatHistory, { role: "user", content: userPrompt }];
+  }
+
+  async isSafe(_input = "") {
+    // Not implemented so must be stubbed
+    return { safe: true, reasons: [] };
+  }
+
+  async sendChat(chatHistory = [], prompt, workspace = {}, rawHistory = []) {
+    const textResponse = await this.openai
+      .createChatCompletion({
+        model: this.model,
+        temperature: Number(workspace?.openAiTemp ?? this.defaultTemp),
+        n: 1,
+        messages: await this.compressMessages(
+          {
+            systemPrompt: chatPrompt(workspace),
+            userPrompt: prompt,
+            chatHistory,
+          },
+          rawHistory
+        ),
+      })
+      .then((json) => {
+        const res = json.data;
+        if (!res.hasOwnProperty("choices"))
+          throw new Error("GenericOpenAI chat: No results!");
+        if (res.choices.length === 0)
+          throw new Error("GenericOpenAI chat: No results length!");
+        return res.choices[0].message.content;
+      })
+      .catch((error) => {
+        throw new Error(
+          `GenericOpenAI::createChatCompletion failed with: ${error.message}`
+        );
+      });
+
+    return textResponse;
+  }
+
+  async streamChat(chatHistory = [], prompt, workspace = {}, rawHistory = []) {
+    const streamRequest = await this.openai.createChatCompletion(
+      {
+        model: this.model,
+        stream: true,
+        temperature: Number(workspace?.openAiTemp ?? this.defaultTemp),
+        n: 1,
+        messages: await this.compressMessages(
+          {
+            systemPrompt: chatPrompt(workspace),
+            userPrompt: prompt,
+            chatHistory,
+          },
+          rawHistory
+        ),
+      },
+      { responseType: "stream" }
+    );
+    return streamRequest;
+  }
+
+  async getChatCompletion(messages = null, { temperature = 0.7 }) {
+    const { data } = await this.openai
+      .createChatCompletion({
+        model: this.model,
+        messages,
+        temperature,
+      })
+      .catch((e) => {
+        throw new Error(e.response.data.error.message);
+      });
+
+    if (!data.hasOwnProperty("choices")) return null;
+    return data.choices[0].message.content;
+  }
+
+  async streamGetChatCompletion(messages = null, { temperature = 0.7 }) {
+    const streamRequest = await this.openai.createChatCompletion(
+      {
+        model: this.model,
+        stream: true,
+        messages,
+        temperature,
+      },
+      { responseType: "stream" }
+    );
+    return streamRequest;
+  }
+
+  handleStream(response, stream, responseProps) {
+    return handleDefaultStreamResponse(response, stream, responseProps);
+  }
+
+  // Simple wrapper for dynamic embedder & normalize interface for all LLM implementations
+  async embedTextInput(textInput) {
+    return await this.embedder.embedTextInput(textInput);
+  }
+  async embedChunks(textChunks = []) {
+    return await this.embedder.embedChunks(textChunks);
+  }
+
+  async compressMessages(promptArgs = {}, rawHistory = []) {
+    const { messageArrayCompressor } = require("../../helpers/chat");
+    const messageArray = this.constructPrompt(promptArgs);
+    return await messageArrayCompressor(this, messageArray, rawHistory);
+  }
+}
+
+module.exports = {
+  GenericOpenAiLLM,
+};

server/utils/AiProviders/groq/index.js

@@ -40,20 +40,31 @@ class GroqLLM {
   streamingEnabled() {
     return "streamChat" in this && "streamGetChatCompletion" in this;
   }
-
   promptWindowLimit() {
     switch (this.model) {
       case "llama2-70b-4096":
         return 4096;
       case "mixtral-8x7b-32768":
         return 32_768;
+      case "llama3-8b-8192":
+        return 8192;
+      case "llama3-70b-8192":
+        return 8192;
+      case "gemma-7b-it":
+        return 8192;
       default:
         return 4096;
     }
   }
 
   async isValidChatCompletionModel(modelName = "") {
-    const validModels = ["llama2-70b-4096", "mixtral-8x7b-32768"];
+    const validModels = [
+      "llama2-70b-4096",
+      "mixtral-8x7b-32768",
+      "llama3-8b-8192",
+      "llama3-70b-8192",
+      "gemma-7b-it",
+    ];
     const isPreset = validModels.some((model) => modelName === model);
     if (isPreset) return true;
 

server/utils/AiProviders/openRouter/index.js

@@ -5,11 +5,9 @@ const {
   writeResponseChunk,
   clientAbortedHandler,
 } = require("../../helpers/chat/responses");
-
+const fs = require("fs");
-function openRouterModels() {
+const path = require("path");
-  const { MODELS } = require("./models.js");
+const { safeJsonParse } = require("../../http");
-  return MODELS || {};
-}
 
 class OpenRouterLLM {
   constructor(embedder = null, modelPreference = null) {
@@ -17,8 +15,9 @@ class OpenRouterLLM {
     if (!process.env.OPENROUTER_API_KEY)
       throw new Error("No OpenRouter API key was set.");
 
+    this.basePath = "https://openrouter.ai/api/v1";
     const config = new Configuration({
-      basePath: "https://openrouter.ai/api/v1",
+      basePath: this.basePath,
       apiKey: process.env.OPENROUTER_API_KEY,
       baseOptions: {
         headers: {
@@ -38,6 +37,81 @@ class OpenRouterLLM {
 
     this.embedder = !embedder ? new NativeEmbedder() : embedder;
     this.defaultTemp = 0.7;
+
+    const cacheFolder = path.resolve(
+      process.env.STORAGE_DIR
+        ? path.resolve(process.env.STORAGE_DIR, "models", "openrouter")
+        : path.resolve(__dirname, `../../../storage/models/openrouter`)
+    );
+    fs.mkdirSync(cacheFolder, { recursive: true });
+    this.cacheModelPath = path.resolve(cacheFolder, "models.json");
+    this.cacheAtPath = path.resolve(cacheFolder, ".cached_at");
+  }
+
+  log(text, ...args) {
+    console.log(`\x1b[36m[${this.constructor.name}]\x1b[0m ${text}`, ...args);
+  }
+
+  async init() {
+    await this.#syncModels();
+    return this;
+  }
+
+  // This checks if the .cached_at file has a timestamp that is more than 1Week (in millis)
+  // from the current date. If it is, then we will refetch the API so that all the models are up
+  // to date.
+  #cacheIsStale() {
+    const MAX_STALE = 6.048e8; // 1 Week in MS
+    if (!fs.existsSync(this.cacheAtPath)) return true;
+    const now = Number(new Date());
+    const timestampMs = Number(fs.readFileSync(this.cacheAtPath));
+    return now - timestampMs > MAX_STALE;
+  }
+
+  // The OpenRouter model API has a lot of models, so we cache this locally in the directory
+  // as if the cache directory JSON file is stale or does not exist we will fetch from API and store it.
+  // This might slow down the first request, but we need the proper token context window
+  // for each model and this is a constructor property - so we can really only get it if this cache exists.
+  // We used to have this as a chore, but given there is an API to get the info - this makes little sense.
+  async #syncModels() {
+    if (fs.existsSync(this.cacheModelPath) && !this.#cacheIsStale())
+      return false;
+
+    this.log(
+      "Model cache is not present or stale. Fetching from OpenRouter API."
+    );
+    await fetch(`${this.basePath}/models`, {
+      method: "GET",
+      headers: {
+        "Content-Type": "application/json",
+      },
+    })
+      .then((res) => res.json())
+      .then(({ data = [] }) => {
+        const models = {};
+        data.forEach((model) => {
+          models[model.id] = {
+            id: model.id,
+            name: model.name,
+            organization:
+              model.id.split("/")[0].charAt(0).toUpperCase() +
+              model.id.split("/")[0].slice(1),
+            maxLength: model.context_length,
+          };
+        });
+        fs.writeFileSync(this.cacheModelPath, JSON.stringify(models), {
+          encoding: "utf-8",
+        });
+        fs.writeFileSync(this.cacheAtPath, String(Number(new Date())), {
+          encoding: "utf-8",
+        });
+        return models;
+      })
+      .catch((e) => {
+        console.error(e);
+        return {};
+      });
+    return;
   }
 
   #appendContext(contextTexts = []) {
@@ -52,8 +126,12 @@ class OpenRouterLLM {
     );
   }
 
-  allModelInformation() {
+  models() {
-    return openRouterModels();
+    if (!fs.existsSync(this.cacheModelPath)) return {};
+    return safeJsonParse(
+      fs.readFileSync(this.cacheModelPath, { encoding: "utf-8" }),
+      {}
+    );
   }
 
   streamingEnabled() {
@@ -61,12 +139,13 @@ class OpenRouterLLM {
   }
 
   promptWindowLimit() {
-    const availableModels = this.allModelInformation();
+    const availableModels = this.models();
     return availableModels[this.model]?.maxLength || 4096;
   }
 
   async isValidChatCompletionModel(model = "") {
-    const availableModels = this.allModelInformation();
+    await this.#syncModels();
+    const availableModels = this.models();
     return availableModels.hasOwnProperty(model);
   }
 
@@ -343,5 +422,4 @@ class OpenRouterLLM {
 
 module.exports = {
   OpenRouterLLM,
-  openRouterModels,
 };

server/utils/AiProviders/openRouter/models.js

@@ -1,778 +0,0 @@
-const MODELS = {
-  "openrouter/auto": {
-    id: "openrouter/auto",
-    name: "Auto (best for prompt)",
-    organization: "Openrouter",
-    maxLength: 128000,
-  },
-  "nousresearch/nous-capybara-7b:free": {
-    id: "nousresearch/nous-capybara-7b:free",
-    name: "Nous: Capybara 7B (free)",
-    organization: "Nousresearch",
-    maxLength: 4096,
-  },
-  "mistralai/mistral-7b-instruct:free": {
-    id: "mistralai/mistral-7b-instruct:free",
-    name: "Mistral 7B Instruct (free)",
-    organization: "Mistralai",
-    maxLength: 32768,
-  },
-  "openchat/openchat-7b:free": {
-    id: "openchat/openchat-7b:free",
-    name: "OpenChat 3.5 (free)",
-    organization: "Openchat",
-    maxLength: 8192,
-  },
-  "gryphe/mythomist-7b:free": {
-    id: "gryphe/mythomist-7b:free",
-    name: "MythoMist 7B (free)",
-    organization: "Gryphe",
-    maxLength: 32768,
-  },
-  "undi95/toppy-m-7b:free": {
-    id: "undi95/toppy-m-7b:free",
-    name: "Toppy M 7B (free)",
-    organization: "Undi95",
-    maxLength: 4096,
-  },
-  "openrouter/cinematika-7b:free": {
-    id: "openrouter/cinematika-7b:free",
-    name: "Cinematika 7B (alpha) (free)",
-    organization: "Openrouter",
-    maxLength: 8000,
-  },
-  "google/gemma-7b-it:free": {
-    id: "google/gemma-7b-it:free",
-    name: "Google: Gemma 7B (free)",
-    organization: "Google",
-    maxLength: 8192,
-  },
-  "jebcarter/psyfighter-13b": {
-    id: "jebcarter/psyfighter-13b",
-    name: "Psyfighter 13B",
-    organization: "Jebcarter",
-    maxLength: 4096,
-  },
-  "koboldai/psyfighter-13b-2": {
-    id: "koboldai/psyfighter-13b-2",
-    name: "Psyfighter v2 13B",
-    organization: "Koboldai",
-    maxLength: 4096,
-  },
-  "intel/neural-chat-7b": {
-    id: "intel/neural-chat-7b",
-    name: "Neural Chat 7B v3.1",
-    organization: "Intel",
-    maxLength: 4096,
-  },
-  "haotian-liu/llava-13b": {
-    id: "haotian-liu/llava-13b",
-    name: "Llava 13B",
-    organization: "Haotian-liu",
-    maxLength: 2048,
-  },
-  "nousresearch/nous-hermes-2-vision-7b": {
-    id: "nousresearch/nous-hermes-2-vision-7b",
-    name: "Nous: Hermes 2 Vision 7B (alpha)",
-    organization: "Nousresearch",
-    maxLength: 4096,
-  },
-  "meta-llama/llama-2-13b-chat": {
-    id: "meta-llama/llama-2-13b-chat",
-    name: "Meta: Llama v2 13B Chat",
-    organization: "Meta-llama",
-    maxLength: 4096,
-  },
-  "migtissera/synthia-70b": {
-    id: "migtissera/synthia-70b",
-    name: "Synthia 70B",
-    organization: "Migtissera",
-    maxLength: 8192,
-  },
-  "pygmalionai/mythalion-13b": {
-    id: "pygmalionai/mythalion-13b",
-    name: "Pygmalion: Mythalion 13B",
-    organization: "Pygmalionai",
-    maxLength: 8192,
-  },
-  "xwin-lm/xwin-lm-70b": {
-    id: "xwin-lm/xwin-lm-70b",
-    name: "Xwin 70B",
-    organization: "Xwin-lm",
-    maxLength: 8192,
-  },
-  "alpindale/goliath-120b": {
-    id: "alpindale/goliath-120b",
-    name: "Goliath 120B",
-    organization: "Alpindale",
-    maxLength: 6144,
-  },
-  "neversleep/noromaid-20b": {
-    id: "neversleep/noromaid-20b",
-    name: "Noromaid 20B",
-    organization: "Neversleep",
-    maxLength: 8192,
-  },
-  "gryphe/mythomist-7b": {
-    id: "gryphe/mythomist-7b",
-    name: "MythoMist 7B",
-    organization: "Gryphe",
-    maxLength: 32768,
-  },
-  "sophosympatheia/midnight-rose-70b": {
-    id: "sophosympatheia/midnight-rose-70b",
-    name: "Midnight Rose 70B",
-    organization: "Sophosympatheia",
-    maxLength: 4096,
-  },
-  "undi95/remm-slerp-l2-13b:extended": {
-    id: "undi95/remm-slerp-l2-13b:extended",
-    name: "ReMM SLERP 13B (extended)",
-    organization: "Undi95",
-    maxLength: 6144,
-  },
-  "mancer/weaver": {
-    id: "mancer/weaver",
-    name: "Mancer: Weaver (alpha)",
-    organization: "Mancer",
-    maxLength: 8000,
-  },
-  "nousresearch/nous-hermes-llama2-13b": {
-    id: "nousresearch/nous-hermes-llama2-13b",
-    name: "Nous: Hermes 13B",
-    organization: "Nousresearch",
-    maxLength: 4096,
-  },
-  "nousresearch/nous-capybara-7b": {
-    id: "nousresearch/nous-capybara-7b",
-    name: "Nous: Capybara 7B",
-    organization: "Nousresearch",
-    maxLength: 4096,
-  },
-  "meta-llama/codellama-34b-instruct": {
-    id: "meta-llama/codellama-34b-instruct",
-    name: "Meta: CodeLlama 34B Instruct",
-    organization: "Meta-llama",
-    maxLength: 8192,
-  },
-  "codellama/codellama-70b-instruct": {
-    id: "codellama/codellama-70b-instruct",
-    name: "Meta: CodeLlama 70B Instruct",
-    organization: "Codellama",
-    maxLength: 2048,
-  },
-  "phind/phind-codellama-34b": {
-    id: "phind/phind-codellama-34b",
-    name: "Phind: CodeLlama 34B v2",
-    organization: "Phind",
-    maxLength: 4096,
-  },
-  "teknium/openhermes-2-mistral-7b": {
-    id: "teknium/openhermes-2-mistral-7b",
-    name: "OpenHermes 2 Mistral 7B",
-    organization: "Teknium",
-    maxLength: 4096,
-  },
-  "teknium/openhermes-2.5-mistral-7b": {
-    id: "teknium/openhermes-2.5-mistral-7b",
-    name: "OpenHermes 2.5 Mistral 7B",
-    organization: "Teknium",
-    maxLength: 4096,
-  },
-  "undi95/remm-slerp-l2-13b": {
-    id: "undi95/remm-slerp-l2-13b",
-    name: "ReMM SLERP 13B",
-    organization: "Undi95",
-    maxLength: 4096,
-  },
-  "openrouter/cinematika-7b": {
-    id: "openrouter/cinematika-7b",
-    name: "Cinematika 7B (alpha)",
-    organization: "Openrouter",
-    maxLength: 8000,
-  },
-  "01-ai/yi-34b-chat": {
-    id: "01-ai/yi-34b-chat",
-    name: "Yi 34B Chat",
-    organization: "01-ai",
-    maxLength: 4096,
-  },
-  "01-ai/yi-34b": {
-    id: "01-ai/yi-34b",
-    name: "Yi 34B (base)",
-    organization: "01-ai",
-    maxLength: 4096,
-  },
-  "01-ai/yi-6b": {
-    id: "01-ai/yi-6b",
-    name: "Yi 6B (base)",
-    organization: "01-ai",
-    maxLength: 4096,
-  },
-  "togethercomputer/stripedhyena-nous-7b": {
-    id: "togethercomputer/stripedhyena-nous-7b",
-    name: "StripedHyena Nous 7B",
-    organization: "Togethercomputer",
-    maxLength: 32768,
-  },
-  "togethercomputer/stripedhyena-hessian-7b": {
-    id: "togethercomputer/stripedhyena-hessian-7b",
-    name: "StripedHyena Hessian 7B (base)",
-    organization: "Togethercomputer",
-    maxLength: 32768,
-  },
-  "mistralai/mixtral-8x7b": {
-    id: "mistralai/mixtral-8x7b",
-    name: "Mixtral 8x7B (base)",
-    organization: "Mistralai",
-    maxLength: 32768,
-  },
-  "nousresearch/nous-hermes-yi-34b": {
-    id: "nousresearch/nous-hermes-yi-34b",
-    name: "Nous: Hermes 2 Yi 34B",
-    organization: "Nousresearch",
-    maxLength: 4096,
-  },
-  "nousresearch/nous-hermes-2-mixtral-8x7b-sft": {
-    id: "nousresearch/nous-hermes-2-mixtral-8x7b-sft",
-    name: "Nous: Hermes 2 Mixtral 8x7B SFT",
-    organization: "Nousresearch",
-    maxLength: 32000,
-  },
-  "nousresearch/nous-hermes-2-mistral-7b-dpo": {
-    id: "nousresearch/nous-hermes-2-mistral-7b-dpo",
-    name: "Nous: Hermes 2 Mistral 7B DPO",
-    organization: "Nousresearch",
-    maxLength: 8192,
-  },
-  "open-orca/mistral-7b-openorca": {
-    id: "open-orca/mistral-7b-openorca",
-    name: "Mistral OpenOrca 7B",
-    organization: "Open-orca",
-    maxLength: 8192,
-  },
-  "huggingfaceh4/zephyr-7b-beta": {
-    id: "huggingfaceh4/zephyr-7b-beta",
-    name: "Hugging Face: Zephyr 7B",
-    organization: "Huggingfaceh4",
-    maxLength: 4096,
-  },
-  "openai/gpt-3.5-turbo": {
-    id: "openai/gpt-3.5-turbo",
-    name: "OpenAI: GPT-3.5 Turbo",
-    organization: "Openai",
-    maxLength: 16385,
-  },
-  "openai/gpt-3.5-turbo-0125": {
-    id: "openai/gpt-3.5-turbo-0125",
-    name: "OpenAI: GPT-3.5 Turbo 16k",
-    organization: "Openai",
-    maxLength: 16385,
-  },
-  "openai/gpt-3.5-turbo-1106": {
-    id: "openai/gpt-3.5-turbo-1106",
-    name: "OpenAI: GPT-3.5 Turbo 16k (older v1106)",
-    organization: "Openai",
-    maxLength: 16385,
-  },
-  "openai/gpt-3.5-turbo-0613": {
-    id: "openai/gpt-3.5-turbo-0613",
-    name: "OpenAI: GPT-3.5 Turbo (older v0613)",
-    organization: "Openai",
-    maxLength: 4095,
-  },
-  "openai/gpt-3.5-turbo-0301": {
-    id: "openai/gpt-3.5-turbo-0301",
-    name: "OpenAI: GPT-3.5 Turbo (older v0301)",
-    organization: "Openai",
-    maxLength: 4095,
-  },
-  "openai/gpt-3.5-turbo-16k": {
-    id: "openai/gpt-3.5-turbo-16k",
-    name: "OpenAI: GPT-3.5 Turbo 16k",
-    organization: "Openai",
-    maxLength: 16385,
-  },
-  "openai/gpt-4-turbo": {
-    id: "openai/gpt-4-turbo",
-    name: "OpenAI: GPT-4 Turbo",
-    organization: "Openai",
-    maxLength: 128000,
-  },
-  "openai/gpt-4-turbo-preview": {
-    id: "openai/gpt-4-turbo-preview",
-    name: "OpenAI: GPT-4 Turbo Preview",
-    organization: "Openai",
-    maxLength: 128000,
-  },
-  "openai/gpt-4-1106-preview": {
-    id: "openai/gpt-4-1106-preview",
-    name: "OpenAI: GPT-4 Turbo (older v1106)",
-    organization: "Openai",
-    maxLength: 128000,
-  },
-  "openai/gpt-4": {
-    id: "openai/gpt-4",
-    name: "OpenAI: GPT-4",
-    organization: "Openai",
-    maxLength: 8191,
-  },
-  "openai/gpt-4-0314": {
-    id: "openai/gpt-4-0314",
-    name: "OpenAI: GPT-4 (older v0314)",
-    organization: "Openai",
-    maxLength: 8191,
-  },
-  "openai/gpt-4-32k": {
-    id: "openai/gpt-4-32k",
-    name: "OpenAI: GPT-4 32k",
-    organization: "Openai",
-    maxLength: 32767,
-  },
-  "openai/gpt-4-32k-0314": {
-    id: "openai/gpt-4-32k-0314",
-    name: "OpenAI: GPT-4 32k (older v0314)",
-    organization: "Openai",
-    maxLength: 32767,
-  },
-  "openai/gpt-4-vision-preview": {
-    id: "openai/gpt-4-vision-preview",
-    name: "OpenAI: GPT-4 Vision",
-    organization: "Openai",
-    maxLength: 128000,
-  },
-  "openai/gpt-3.5-turbo-instruct": {
-    id: "openai/gpt-3.5-turbo-instruct",
-    name: "OpenAI: GPT-3.5 Turbo Instruct",
-    organization: "Openai",
-    maxLength: 4095,
-  },
-  "google/palm-2-chat-bison": {
-    id: "google/palm-2-chat-bison",
-    name: "Google: PaLM 2 Chat",
-    organization: "Google",
-    maxLength: 25804,
-  },
-  "google/palm-2-codechat-bison": {
-    id: "google/palm-2-codechat-bison",
-    name: "Google: PaLM 2 Code Chat",
-    organization: "Google",
-    maxLength: 20070,
-  },
-  "google/palm-2-chat-bison-32k": {
-    id: "google/palm-2-chat-bison-32k",
-    name: "Google: PaLM 2 Chat 32k",
-    organization: "Google",
-    maxLength: 91750,
-  },
-  "google/palm-2-codechat-bison-32k": {
-    id: "google/palm-2-codechat-bison-32k",
-    name: "Google: PaLM 2 Code Chat 32k",
-    organization: "Google",
-    maxLength: 91750,
-  },
-  "google/gemini-pro": {
-    id: "google/gemini-pro",
-    name: "Google: Gemini Pro 1.0",
-    organization: "Google",
-    maxLength: 91728,
-  },
-  "google/gemini-pro-vision": {
-    id: "google/gemini-pro-vision",
-    name: "Google: Gemini Pro Vision 1.0",
-    organization: "Google",
-    maxLength: 45875,
-  },
-  "google/gemini-pro-1.5": {
-    id: "google/gemini-pro-1.5",
-    name: "Google: Gemini Pro 1.5 (preview)",
-    organization: "Google",
-    maxLength: 2800000,
-  },
-  "perplexity/pplx-70b-online": {
-    id: "perplexity/pplx-70b-online",
-    name: "Perplexity: PPLX 70B Online",
-    organization: "Perplexity",
-    maxLength: 4096,
-  },
-  "perplexity/pplx-7b-online": {
-    id: "perplexity/pplx-7b-online",
-    name: "Perplexity: PPLX 7B Online",
-    organization: "Perplexity",
-    maxLength: 4096,
-  },
-  "perplexity/pplx-7b-chat": {
-    id: "perplexity/pplx-7b-chat",
-    name: "Perplexity: PPLX 7B Chat",
-    organization: "Perplexity",
-    maxLength: 8192,
-  },
-  "perplexity/pplx-70b-chat": {
-    id: "perplexity/pplx-70b-chat",
-    name: "Perplexity: PPLX 70B Chat",
-    organization: "Perplexity",
-    maxLength: 4096,
-  },
-  "perplexity/sonar-small-chat": {
-    id: "perplexity/sonar-small-chat",
-    name: "Perplexity: Sonar 7B",
-    organization: "Perplexity",
-    maxLength: 16384,
-  },
-  "perplexity/sonar-medium-chat": {
-    id: "perplexity/sonar-medium-chat",
-    name: "Perplexity: Sonar 8x7B",
-    organization: "Perplexity",
-    maxLength: 16384,
-  },
-  "perplexity/sonar-small-online": {
-    id: "perplexity/sonar-small-online",
-    name: "Perplexity: Sonar 7B Online",
-    organization: "Perplexity",
-    maxLength: 12000,
-  },
-  "perplexity/sonar-medium-online": {
-    id: "perplexity/sonar-medium-online",
-    name: "Perplexity: Sonar 8x7B Online",
-    organization: "Perplexity",
-    maxLength: 12000,
-  },
-  "fireworks/mixtral-8x22b-instruct-preview": {
-    id: "fireworks/mixtral-8x22b-instruct-preview",
-    name: "Fireworks Mixtral 8x22B Instruct OH (preview)",
-    organization: "Fireworks",
-    maxLength: 8192,
-  },
-  "anthropic/claude-3-opus": {
-    id: "anthropic/claude-3-opus",
-    name: "Anthropic: Claude 3 Opus",
-    organization: "Anthropic",
-    maxLength: 200000,
-  },
-  "anthropic/claude-3-sonnet": {
-    id: "anthropic/claude-3-sonnet",
-    name: "Anthropic: Claude 3 Sonnet",
-    organization: "Anthropic",
-    maxLength: 200000,
-  },
-  "anthropic/claude-3-haiku": {
-    id: "anthropic/claude-3-haiku",
-    name: "Anthropic: Claude 3 Haiku",
-    organization: "Anthropic",
-    maxLength: 200000,
-  },
-  "anthropic/claude-3-opus:beta": {
-    id: "anthropic/claude-3-opus:beta",
-    name: "Anthropic: Claude 3 Opus (self-moderated)",
-    organization: "Anthropic",
-    maxLength: 200000,
-  },
-  "anthropic/claude-3-sonnet:beta": {
-    id: "anthropic/claude-3-sonnet:beta",
-    name: "Anthropic: Claude 3 Sonnet (self-moderated)",
-    organization: "Anthropic",
-    maxLength: 200000,
-  },
-  "anthropic/claude-3-haiku:beta": {
-    id: "anthropic/claude-3-haiku:beta",
-    name: "Anthropic: Claude 3 Haiku (self-moderated)",
-    organization: "Anthropic",
-    maxLength: 200000,
-  },
-  "meta-llama/llama-2-70b-chat": {
-    id: "meta-llama/llama-2-70b-chat",
-    name: "Meta: Llama v2 70B Chat",
-    organization: "Meta-llama",
-    maxLength: 4096,
-  },
-  "nousresearch/nous-capybara-34b": {
-    id: "nousresearch/nous-capybara-34b",
-    name: "Nous: Capybara 34B",
-    organization: "Nousresearch",
-    maxLength: 32768,
-  },
-  "jondurbin/airoboros-l2-70b": {
-    id: "jondurbin/airoboros-l2-70b",
-    name: "Airoboros 70B",
-    organization: "Jondurbin",
-    maxLength: 4096,
-  },
-  "jondurbin/bagel-34b": {
-    id: "jondurbin/bagel-34b",
-    name: "Bagel 34B v0.2",
-    organization: "Jondurbin",
-    maxLength: 8000,
-  },
-  "austism/chronos-hermes-13b": {
-    id: "austism/chronos-hermes-13b",
-    name: "Chronos Hermes 13B v2",
-    organization: "Austism",
-    maxLength: 4096,
-  },
-  "mistralai/mistral-7b-instruct": {
-    id: "mistralai/mistral-7b-instruct",
-    name: "Mistral 7B Instruct",
-    organization: "Mistralai",
-    maxLength: 32768,
-  },
-  "gryphe/mythomax-l2-13b": {
-    id: "gryphe/mythomax-l2-13b",
-    name: "MythoMax 13B",
-    organization: "Gryphe",
-    maxLength: 4096,
-  },
-  "openchat/openchat-7b": {
-    id: "openchat/openchat-7b",
-    name: "OpenChat 3.5",
-    organization: "Openchat",
-    maxLength: 8192,
-  },
-  "undi95/toppy-m-7b": {
-    id: "undi95/toppy-m-7b",
-    name: "Toppy M 7B",
-    organization: "Undi95",
-    maxLength: 4096,
-  },
-  "lizpreciatior/lzlv-70b-fp16-hf": {
-    id: "lizpreciatior/lzlv-70b-fp16-hf",
-    name: "lzlv 70B",
-    organization: "Lizpreciatior",
-    maxLength: 4096,
-  },
-  "mistralai/mixtral-8x7b-instruct": {
-    id: "mistralai/mixtral-8x7b-instruct",
-    name: "Mixtral 8x7B Instruct",
-    organization: "Mistralai",
-    maxLength: 32768,
-  },
-  "cognitivecomputations/dolphin-mixtral-8x7b": {
-    id: "cognitivecomputations/dolphin-mixtral-8x7b",
-    name: "Dolphin 2.6 Mixtral 8x7B 🐬",
-    organization: "Cognitivecomputations",
-    maxLength: 32000,
-  },
-  "neversleep/noromaid-mixtral-8x7b-instruct": {
-    id: "neversleep/noromaid-mixtral-8x7b-instruct",
-    name: "Noromaid Mixtral 8x7B Instruct",
-    organization: "Neversleep",
-    maxLength: 8000,
-  },
-  "nousresearch/nous-hermes-2-mixtral-8x7b-dpo": {
-    id: "nousresearch/nous-hermes-2-mixtral-8x7b-dpo",
-    name: "Nous: Hermes 2 Mixtral 8x7B DPO",
-    organization: "Nousresearch",
-    maxLength: 32000,
-  },
-  "rwkv/rwkv-5-world-3b": {
-    id: "rwkv/rwkv-5-world-3b",
-    name: "RWKV v5 World 3B",
-    organization: "Rwkv",
-    maxLength: 10000,
-  },
-  "recursal/rwkv-5-3b-ai-town": {
-    id: "recursal/rwkv-5-3b-ai-town",
-    name: "RWKV v5 3B AI Town",
-    organization: "Recursal",
-    maxLength: 10000,
-  },
-  "recursal/eagle-7b": {
-    id: "recursal/eagle-7b",
-    name: "RWKV v5: Eagle 7B",
-    organization: "Recursal",
-    maxLength: 10000,
-  },
-  "google/gemma-7b-it": {
-    id: "google/gemma-7b-it",
-    name: "Google: Gemma 7B",
-    organization: "Google",
-    maxLength: 8192,
-  },
-  "databricks/dbrx-instruct": {
-    id: "databricks/dbrx-instruct",
-    name: "Databricks: DBRX 132B Instruct",
-    organization: "Databricks",
-    maxLength: 32768,
-  },
-  "huggingfaceh4/zephyr-orpo-141b-a35b": {
-    id: "huggingfaceh4/zephyr-orpo-141b-a35b",
-    name: "Zephyr 141B-A35B",
-    organization: "Huggingfaceh4",
-    maxLength: 65536,
-  },
-  "anthropic/claude-2": {
-    id: "anthropic/claude-2",
-    name: "Anthropic: Claude v2",
-    organization: "Anthropic",
-    maxLength: 200000,
-  },
-  "anthropic/claude-2.1": {
-    id: "anthropic/claude-2.1",
-    name: "Anthropic: Claude v2.1",
-    organization: "Anthropic",
-    maxLength: 200000,
-  },
-  "anthropic/claude-2.0": {
-    id: "anthropic/claude-2.0",
-    name: "Anthropic: Claude v2.0",
-    organization: "Anthropic",
-    maxLength: 100000,
-  },
-  "anthropic/claude-instant-1": {
-    id: "anthropic/claude-instant-1",
-    name: "Anthropic: Claude Instant v1",
-    organization: "Anthropic",
-    maxLength: 100000,
-  },
-  "anthropic/claude-instant-1.2": {
-    id: "anthropic/claude-instant-1.2",
-    name: "Anthropic: Claude Instant v1.2",
-    organization: "Anthropic",
-    maxLength: 100000,
-  },
-  "anthropic/claude-1": {
-    id: "anthropic/claude-1",
-    name: "Anthropic: Claude v1",
-    organization: "Anthropic",
-    maxLength: 100000,
-  },
-  "anthropic/claude-1.2": {
-    id: "anthropic/claude-1.2",
-    name: "Anthropic: Claude (older v1)",
-    organization: "Anthropic",
-    maxLength: 100000,
-  },
-  "anthropic/claude-instant-1.0": {
-    id: "anthropic/claude-instant-1.0",
-    name: "Anthropic: Claude Instant (older v1)",
-    organization: "Anthropic",
-    maxLength: 100000,
-  },
-  "anthropic/claude-instant-1.1": {
-    id: "anthropic/claude-instant-1.1",
-    name: "Anthropic: Claude Instant (older v1.1)",
-    organization: "Anthropic",
-    maxLength: 100000,
-  },
-  "anthropic/claude-2:beta": {
-    id: "anthropic/claude-2:beta",
-    name: "Anthropic: Claude v2 (self-moderated)",
-    organization: "Anthropic",
-    maxLength: 200000,
-  },
-  "anthropic/claude-2.1:beta": {
-    id: "anthropic/claude-2.1:beta",
-    name: "Anthropic: Claude v2.1 (self-moderated)",
-    organization: "Anthropic",
-    maxLength: 200000,
-  },
-  "anthropic/claude-2.0:beta": {
-    id: "anthropic/claude-2.0:beta",
-    name: "Anthropic: Claude v2.0 (self-moderated)",
-    organization: "Anthropic",
-    maxLength: 100000,
-  },
-  "anthropic/claude-instant-1:beta": {
-    id: "anthropic/claude-instant-1:beta",
-    name: "Anthropic: Claude Instant v1 (self-moderated)",
-    organization: "Anthropic",
-    maxLength: 100000,
-  },
-  "mistralai/mixtral-8x22b": {
-    id: "mistralai/mixtral-8x22b",
-    name: "Mistral: Mixtral 8x22B (base)",
-    organization: "Mistralai",
-    maxLength: 65536,
-  },
-  "huggingfaceh4/zephyr-7b-beta:free": {
-    id: "huggingfaceh4/zephyr-7b-beta:free",
-    name: "Hugging Face: Zephyr 7B (free)",
-    organization: "Huggingfaceh4",
-    maxLength: 4096,
-  },
-  "mistralai/mixtral-8x7b-instruct:nitro": {
-    id: "mistralai/mixtral-8x7b-instruct:nitro",
-    name: "Mixtral 8x7B Instruct (nitro)",
-    organization: "Mistralai",
-    maxLength: 32768,
-  },
-  "meta-llama/llama-2-70b-chat:nitro": {
-    id: "meta-llama/llama-2-70b-chat:nitro",
-    name: "Meta: Llama v2 70B Chat (nitro)",
-    organization: "Meta-llama",
-    maxLength: 4096,
-  },
-  "gryphe/mythomax-l2-13b:nitro": {
-    id: "gryphe/mythomax-l2-13b:nitro",
-    name: "MythoMax 13B (nitro)",
-    organization: "Gryphe",
-    maxLength: 4096,
-  },
-  "mistralai/mistral-7b-instruct:nitro": {
-    id: "mistralai/mistral-7b-instruct:nitro",
-    name: "Mistral 7B Instruct (nitro)",
-    organization: "Mistralai",
-    maxLength: 32768,
-  },
-  "google/gemma-7b-it:nitro": {
-    id: "google/gemma-7b-it:nitro",
-    name: "Google: Gemma 7B (nitro)",
-    organization: "Google",
-    maxLength: 8192,
-  },
-  "databricks/dbrx-instruct:nitro": {
-    id: "databricks/dbrx-instruct:nitro",
-    name: "Databricks: DBRX 132B Instruct (nitro)",
-    organization: "Databricks",
-    maxLength: 32768,
-  },
-  "gryphe/mythomax-l2-13b:extended": {
-    id: "gryphe/mythomax-l2-13b:extended",
-    name: "MythoMax 13B (extended)",
-    organization: "Gryphe",
-    maxLength: 8192,
-  },
-  "mistralai/mistral-tiny": {
-    id: "mistralai/mistral-tiny",
-    name: "Mistral Tiny",
-    organization: "Mistralai",
-    maxLength: 32000,
-  },
-  "mistralai/mistral-small": {
-    id: "mistralai/mistral-small",
-    name: "Mistral Small",
-    organization: "Mistralai",
-    maxLength: 32000,
-  },
-  "mistralai/mistral-medium": {
-    id: "mistralai/mistral-medium",
-    name: "Mistral Medium",
-    organization: "Mistralai",
-    maxLength: 32000,
-  },
-  "mistralai/mistral-large": {
-    id: "mistralai/mistral-large",
-    name: "Mistral Large",
-    organization: "Mistralai",
-    maxLength: 32000,
-  },
-  "cohere/command": {
-    id: "cohere/command",
-    name: "Cohere: Command",
-    organization: "Cohere",
-    maxLength: 4096,
-  },
-  "cohere/command-r": {
-    id: "cohere/command-r",
-    name: "Cohere: Command R",
-    organization: "Cohere",
-    maxLength: 128000,
-  },
-  "cohere/command-r-plus": {
-    id: "cohere/command-r-plus",
-    name: "Cohere: Command R+",
-    organization: "Cohere",
-    maxLength: 128000,
-  },
-};
-
-module.exports.MODELS = MODELS;

server/utils/AiProviders/openRouter/scripts/.gitignore

@@ -1 +0,0 @@
-*.json

server/utils/AiProviders/openRouter/scripts/parse.mjs

@@ -1,37 +0,0 @@
-// OpenRouter has lots of models we can use so we use this script
-// to cache all the models. We can see the list of all the models
-// here: https://openrouter.ai/docs#models
-
-// To run, cd into this directory and run `node parse.mjs`
-// copy outputs into the export in ../models.js
-
-// Update the date below if you run this again because OpenRouter added new models.
-// Last Collected: Apr 14, 2024
-
-import fs from "fs";
-
-async function parseChatModels() {
-  const models = {};
-  const response = await fetch("https://openrouter.ai/api/v1/models");
-  const data = await response.json();
-  data.data.forEach((model) => {
-    models[model.id] = {
-      id: model.id,
-      name: model.name,
-      // capitalize first letter
-      organization:
-        model.id.split("/")[0].charAt(0).toUpperCase() +
-        model.id.split("/")[0].slice(1),
-      maxLength: model.context_length,
-    };
-  });
-
-  fs.writeFileSync(
-    "chat_models.json",
-    JSON.stringify(models, null, 2),
-    "utf-8"
-  );
-  return models;
-}
-
-parseChatModels();

server/utils/AiProviders/perplexity/models.js

@@ -19,6 +19,16 @@ const MODELS = {
     name: "sonar-medium-online",
     maxLength: 12000,
   },
+  "llama-3-8b-instruct": {
+    id: "llama-3-8b-instruct",
+    name: "llama-3-8b-instruct",
+    maxLength: 8192,
+  },
+  "llama-3-70b-instruct": {
+    id: "llama-3-70b-instruct",
+    name: "llama-3-70b-instruct",
+    maxLength: 8192,
+  },
   "codellama-70b-instruct": {
     id: "codellama-70b-instruct",
     name: "codellama-70b-instruct",
@@ -34,6 +44,11 @@ const MODELS = {
     name: "mixtral-8x7b-instruct",
     maxLength: 16384,
   },
+  "mixtral-8x22b-instruct": {
+    id: "mixtral-8x22b-instruct",
+    name: "mixtral-8x22b-instruct",
+    maxLength: 16384,
+  },
 };
 
 module.exports.MODELS = MODELS;

server/utils/AiProviders/perplexity/scripts/chat_models.txt

@@ -4,6 +4,9 @@
 | `sonar-small-online`  | 7B              | 12000          | Chat Completion |
 | `sonar-medium-chat`   | 8x7B            | 16384          | Chat Completion |
 | `sonar-medium-online` | 8x7B            | 12000          | Chat Completion |
+| `llama-3-8b-instruct`     | 8B              | 8192           | Chat Completion |
+| `llama-3-70b-instruct`    | 70B             | 8192           | Chat Completion |
 | `codellama-70b-instruct`  | 70B             | 16384          | Chat Completion |
 | `mistral-7b-instruct` [1] | 7B              | 16384          | Chat Completion |
-| `mixtral-8x7b-instruct`   | 8x7B            | 16384          | Chat Completion |
+| `mixtral-8x7b-instruct`   | 8x7B            | 16384          | Chat Completion |
+| `mixtral-8x22b-instruct`  | 8x22B           | 16384          | Chat Completion |

server/utils/AiProviders/perplexity/scripts/parse.mjs

@@ -8,7 +8,7 @@
 // copy outputs into the export in ../models.js
 
 // Update the date below if you run this again because Perplexity added new models.
-// Last Collected: Apr 14, 2024
+// Last Collected: Apr 25, 2024
 
 import fs from "fs";
 

server/utils/helpers/customModels.js

@@ -1,4 +1,4 @@
-const { openRouterModels } = require("../AiProviders/openRouter");
+const { OpenRouterLLM } = require("../AiProviders/openRouter");
 const { perplexityModels } = require("../AiProviders/perplexity");
 const { togetherAiModels } = require("../AiProviders/togetherAi");
 const SUPPORT_CUSTOM_MODELS = [
@@ -232,7 +232,8 @@ async function getPerplexityModels() {
 }
 
 async function getOpenRouterModels() {
-  const knownModels = await openRouterModels();
+  const openrouter = await new OpenRouterLLM().init();
+  const knownModels = openrouter.models();
   if (!Object.keys(knownModels).length === 0)
     return { models: [], error: null };
 

server/utils/helpers/index.js

@@ -77,8 +77,13 @@ function getLLMProvider({ provider = null, model = null } = {}) {
     case "groq":
       const { GroqLLM } = require("../AiProviders/groq");
       return new GroqLLM(embedder, model);
+    case "generic-openai":
+      const { GenericOpenAiLLM } = require("../AiProviders/genericOpenAi");
+      return new GenericOpenAiLLM(embedder, model);
     default:
-      throw new Error("ENV: No LLM_PROVIDER value found in environment!");
+      throw new Error(
+        `ENV: No valid LLM_PROVIDER value found in environment! Using ${process.env.LLM_PROVIDER}`
+      );
   }
 }
 

server/utils/helpers/updateENV.js

@@ -132,6 +132,24 @@ const KEY_MAPPING = {
     checks: [nonZero],
   },
 
+  // Generic OpenAI InferenceSettings
+  GenericOpenAiBasePath: {
+    envKey: "GENERIC_OPEN_AI_BASE_PATH",
+    checks: [isValidURL],
+  },
+  GenericOpenAiModelPref: {
+    envKey: "GENERIC_OPEN_AI_MODEL_PREF",
+    checks: [isNotEmpty],
+  },
+  GenericOpenAiTokenLimit: {
+    envKey: "GENERIC_OPEN_AI_MODEL_TOKEN_LIMIT",
+    checks: [nonZero],
+  },
+  GenericOpenAiKey: {
+    envKey: "GENERIC_OPEN_AI_API_KEY",
+    checks: [],
+  },
+
   EmbeddingEngine: {
     envKey: "EMBEDDING_ENGINE",
     checks: [supportedEmbeddingModel],
@@ -375,6 +393,7 @@ function supportedLLM(input = "") {
     "perplexity",
     "openrouter",
     "groq",
+    "generic-openai",
   ].includes(input);
   return validSelection ? null : `${input} is not a valid LLM provider.`;
 }