kamp/anything-llm
1
0
Fork 0
mirror of https://github.com/Mintplex-Labs/anything-llm.git synced 2024-06-04 14:50:09 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into 1086-feat-implement-normalized-input-fields

Browse Source
This commit is contained in:
shatfield4 2024-04-23 15:06:41 -07:00
commit a9d5f462c6
34 changed files with 862 additions and 1340 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
README.md
View File

@ -85,7 +85,7 @@ Some cool features of AnythingLLM
- [Azure OpenAI](https://azure.microsoft.com/en-us/products/ai-services/openai-service)
- [LocalAi (all)](https://localai.io/)
- [Ollama (all)](https://ollama.ai/)
<!-- - [LM Studio (all)](https://lmstudio.ai) -->
- [LM Studio (all)](https://lmstudio.ai)
**Supported Transcription models:**
@ -188,7 +188,7 @@ You can verify these claims by finding all locations `Telemetry.sendTelemetry` i
---
Copyright © 2023 [Mintplex Labs][profile-link]. <br />
Copyright © 2024 [Mintplex Labs][profile-link]. <br />
This project is [MIT](./LICENSE) licensed.
<!-- LINK GROUP -->

2
collector/processSingleFile/convert/asPDF.js
View File

@ -22,7 +22,7 @@ async function asPDF({ fullFilePath = "", filename = "" }) {
doc.metadata?.loc?.pageNumber || "unknown"
} --`
);
if (!doc.pageContent.length) continue;
if (!doc.pageContent || !doc.pageContent.length) continue;
pageContent.push(doc.pageContent);
}

13
docker/.env.example
View File

@ -66,6 +66,12 @@ GID='1000'
# GROQ_API_KEY=gsk_abcxyz
# GROQ_MODEL_PREF=llama2-70b-4096
# LLM_PROVIDER='generic-openai'
# GENERIC_OPEN_AI_BASE_PATH='http://proxy.url.openai.com/v1'
# GENERIC_OPEN_AI_MODEL_PREF='gpt-3.5-turbo'
# GENERIC_OPEN_AI_MODEL_TOKEN_LIMIT=4096
# GENERIC_OPEN_AI_API_KEY=sk-123abc
###########################################
######## Embedding API SElECTION ##########
###########################################
@ -85,10 +91,15 @@ GID='1000'
# EMBEDDING_MODEL_MAX_CHUNK_LENGTH=1000 # The max chunk size in chars a string to embed can be
# EMBEDDING_ENGINE='ollama'
# EMBEDDING_BASE_PATH='http://127.0.0.1:11434'
# EMBEDDING_BASE_PATH='http://host.docker.internal:11434'
# EMBEDDING_MODEL_PREF='nomic-embed-text:latest'
# EMBEDDING_MODEL_MAX_CHUNK_LENGTH=8192
# EMBEDDING_ENGINE='lmstudio'
# EMBEDDING_BASE_PATH='https://host.docker.internal:1234/v1'
# EMBEDDING_MODEL_PREF='nomic-ai/nomic-embed-text-v1.5-GGUF/nomic-embed-text-v1.5.Q4_0.gguf'
# EMBEDDING_MODEL_MAX_CHUNK_LENGTH=8192
###########################################
######## Vector Database Selection ########
###########################################

120
frontend/src/components/EmbeddingSelection/LMStudioOptions/index.jsx Normal file
View File

@ -0,0 +1,120 @@
import React, { useEffect, useState } from "react";
import System from "@/models/system";
export default function LMStudioEmbeddingOptions({ settings }) {
const [basePathValue, setBasePathValue] = useState(
settings?.EmbeddingBasePath
);
const [basePath, setBasePath] = useState(settings?.EmbeddingBasePath);
return (
<div className="w-full flex flex-col gap-y-4">
<div className="w-full flex items-center gap-4">
<div className="flex flex-col w-60">
<label className="text-white text-sm font-semibold block mb-4">
LMStudio Base URL
</label>
<input
type="url"
name="EmbeddingBasePath"
className="bg-zinc-900 text-white placeholder-white/20 text-sm rounded-lg focus:border-white block w-full p-2.5"
placeholder="http://localhost:1234/v1"
defaultValue={settings?.EmbeddingBasePath}
onChange={(e) => setBasePathValue(e.target.value)}
onBlur={() => setBasePath(basePathValue)}
required={true}
autoComplete="off"
spellCheck={false}
/>
</div>
<LMStudioModelSelection settings={settings} basePath={basePath} />
<div className="flex flex-col w-60">
<label className="text-white text-sm font-semibold block mb-4">
Max embedding chunk length
</label>
<input
type="number"
name="EmbeddingModelMaxChunkLength"
className="bg-zinc-900 text-white placeholder-white/20 text-sm rounded-lg focus:border-white block w-full p-2.5"
placeholder="8192"
min={1}
onScroll={(e) => e.target.blur()}
defaultValue={settings?.EmbeddingModelMaxChunkLength}
required={false}
autoComplete="off"
/>
</div>
</div>
</div>
);
}
function LMStudioModelSelection({ settings, basePath = null }) {
const [customModels, setCustomModels] = useState([]);
const [loading, setLoading] = useState(true);
useEffect(() => {
async function findCustomModels() {
if (!basePath || !basePath.includes("/v1")) {
setCustomModels([]);
setLoading(false);
return;
}
setLoading(true);
const { models } = await System.customModels("lmstudio", null, basePath);
setCustomModels(models || []);
setLoading(false);
}
findCustomModels();
}, [basePath]);
if (loading || customModels.length == 0) {
return (
<div className="flex flex-col w-60">
<label className="text-white text-sm font-semibold block mb-4">
Chat Model Selection
</label>
<select
name="EmbeddingModelPref"
disabled={true}
className="bg-zinc-900 border-gray-500 text-white text-sm rounded-lg block w-full p-2.5"
>
<option disabled={true} selected={true}>
{basePath?.includes("/v1")
? "-- loading available models --"
: "-- waiting for URL --"}
</option>
</select>
</div>
);
}
return (
<div className="flex flex-col w-60">
<label className="text-white text-sm font-semibold block mb-4">
Chat Model Selection
</label>
<select
name="EmbeddingModelPref"
required={true}
className="bg-zinc-900 border-gray-500 text-white text-sm rounded-lg block w-full p-2.5"
>
{customModels.length > 0 && (
<optgroup label="Your loaded models">
{customModels.map((model) => {
return (
<option
key={model.id}
value={model.id}
selected={settings.EmbeddingModelPref === model.id}
>
{model.id}
</option>
);
})}
</optgroup>
)}
</select>
</div>
);
}

66
frontend/src/components/LLMSelection/GenericOpenAiOptions/index.jsx Normal file
View File

@ -0,0 +1,66 @@
export default function GenericOpenAiOptions({ settings }) {
return (
<div className="flex gap-4 flex-wrap">
<div className="flex flex-col w-60">
<label className="text-white text-sm font-semibold block mb-4">
Base URL
</label>
<input
type="url"
name="GenericOpenAiBasePath"
className="bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:border-white block w-full p-2.5"
placeholder="eg: https://proxy.openai.com"
defaultValue={settings?.GenericOpenAiBasePath}
required={true}
autoComplete="off"
spellCheck={false}
/>
</div>
<div className="flex flex-col w-60">
<label className="text-white text-sm font-semibold block mb-4">
API Key
</label>
<input
type="password"
name="GenericOpenAiKey"
className="bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:border-white block w-full p-2.5"
placeholder="Generic service API Key"
defaultValue={settings?.GenericOpenAiKey ? "*".repeat(20) : ""}
required={false}
autoComplete="off"
spellCheck={false}
/>
</div>
<div className="flex flex-col w-60">
<label className="text-white text-sm font-semibold block mb-4">
Chat Model Name
</label>
<input
type="text"
name="GenericOpenAiModelPref"
className="bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:border-white block w-full p-2.5"
placeholder="Model id used for chat requests"
defaultValue={settings?.GenericOpenAiModelPref}
required={true}
autoComplete="off"
/>
</div>
<div className="flex flex-col w-60">
<label className="text-white text-sm font-semibold block mb-4">
Token context window
</label>
<input
type="number"
name="GenericOpenAiTokenLimit"
className="bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:border-white block w-full p-2.5"
placeholder="Content window limit (eg: 4096)"
min={1}
onScroll={(e) => e.target.blur()}
defaultValue={settings?.GenericOpenAiTokenLimit}
required={true}
autoComplete="off"
/>
</div>
</div>
);
}

8
frontend/src/components/LLMSelection/GroqAiOptions/index.jsx
View File

@ -26,7 +26,13 @@ export default function GroqAiOptions({ settings }) {
required={true}
className="bg-zinc-900 border-gray-500 text-white text-sm rounded-lg block w-full p-2.5"
>
{["llama2-70b-4096", "mixtral-8x7b-32768"].map((model) => {
{[
"llama2-70b-4096",
"mixtral-8x7b-32768",
"llama3-8b-8192",
"llama3-70b-8192",
"gemma-7b-it",
].map((model) => {
return (
<option key={model} value={model}>
{model}

2
frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/index.jsx
View File

@ -14,7 +14,7 @@ export default function ChatHistory({ history = [], workspace, sendCommand }) {
const chatHistoryRef = useRef(null);
useEffect(() => {
scrollToBottom();
if (isAtBottom) scrollToBottom();
}, [history]);
const handleScroll = () => {

10
frontend/src/hooks/useGetProvidersModels.js
View File

@ -19,7 +19,13 @@ const PROVIDER_DEFAULT_MODELS = {
localai: [],
ollama: [],
togetherai: [],
groq: ["llama2-70b-4096", "mixtral-8x7b-32768"],
groq: [
"llama2-70b-4096",
"mixtral-8x7b-32768",
"llama3-8b-8192",
"llama3-70b-8192",
"gemma-7b-it",
],
native: [],
};
@ -34,7 +40,7 @@ function groupModels(models) {
}, {});
}
const groupedProviders = ["togetherai", "openai"];
const groupedProviders = ["togetherai", "openai", "openrouter"];
export default function useGetProviderModels(provider = null) {
const [defaultModels, setDefaultModels] = useState([]);
const [customModels, setCustomModels] = useState([]);

BIN
frontend/src/media/llmprovider/generic-openai.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 29 KiB

12
frontend/src/pages/GeneralSettings/EmbeddingPreference/index.jsx
View File

@ -8,6 +8,7 @@ import OpenAiLogo from "@/media/llmprovider/openai.png";
import AzureOpenAiLogo from "@/media/llmprovider/azure.png";
import LocalAiLogo from "@/media/llmprovider/localai.png";
import OllamaLogo from "@/media/llmprovider/ollama.png";
import LMStudioLogo from "@/media/llmprovider/lmstudio.png";
import PreLoader from "@/components/Preloader";
import ChangeWarningModal from "@/components/ChangeWarning";
import OpenAiOptions from "@/components/EmbeddingSelection/OpenAiOptions";
@ -15,6 +16,7 @@ import AzureAiOptions from "@/components/EmbeddingSelection/AzureAiOptions";
import LocalAiOptions from "@/components/EmbeddingSelection/LocalAiOptions";
import NativeEmbeddingOptions from "@/components/EmbeddingSelection/NativeEmbeddingOptions";
import OllamaEmbeddingOptions from "@/components/EmbeddingSelection/OllamaOptions";
import LMStudioEmbeddingOptions from "@/components/EmbeddingSelection/LMStudioOptions";
import EmbedderItem from "@/components/EmbeddingSelection/EmbedderItem";
import { CaretUpDown, MagnifyingGlass, X } from "@phosphor-icons/react";
import { useModal } from "@/hooks/useModal";
@ -28,7 +30,7 @@ const EMBEDDERS = [
logo: AnythingLLMIcon,
options: (settings) => <NativeEmbeddingOptions settings={settings} />,
description:
"Use the built-in embedding engine for AnythingLLM. Zero setup!",
"Use the built-in embedding provider for AnythingLLM. Zero setup!",
},
{
name: "OpenAI",
@ -58,6 +60,14 @@ const EMBEDDERS = [
options: (settings) => <OllamaEmbeddingOptions settings={settings} />,
description: "Run embedding models locally on your own machine.",
},
{
name: "LM Studio",
value: "lmstudio",
logo: LMStudioLogo,
options: (settings) => <LMStudioEmbeddingOptions settings={settings} />,
description:
"Discover, download, and run thousands of cutting edge LLMs in a few clicks.",
},
];
export default function GeneralEmbeddingPreference() {

16
frontend/src/pages/GeneralSettings/LLMPreference/index.jsx
View File

@ -5,6 +5,7 @@ import System from "@/models/system";
import showToast from "@/utils/toast";
import AnythingLLMIcon from "@/media/logo/anything-llm-icon.png";
import OpenAiLogo from "@/media/llmprovider/openai.png";
import GenericOpenAiLogo from "@/media/llmprovider/generic-openai.png";
import AzureOpenAiLogo from "@/media/llmprovider/azure.png";
import AnthropicLogo from "@/media/llmprovider/anthropic.png";
import GeminiLogo from "@/media/llmprovider/gemini.png";
@ -19,6 +20,7 @@ import OpenRouterLogo from "@/media/llmprovider/openrouter.jpeg";
import GroqLogo from "@/media/llmprovider/groq.png";
import PreLoader from "@/components/Preloader";
import OpenAiOptions from "@/components/LLMSelection/OpenAiOptions";
import GenericOpenAiOptions from "@/components/LLMSelection/GenericOpenAiOptions";
import AzureAiOptions from "@/components/LLMSelection/AzureAiOptions";
import AnthropicAiOptions from "@/components/LLMSelection/AnthropicAiOptions";
import LMStudioOptions from "@/components/LLMSelection/LMStudioOptions";
@ -150,6 +152,20 @@ export const AVAILABLE_LLM_PROVIDERS = [
"The fastest LLM inferencing available for real-time AI applications.",
requiredConfig: ["GroqApiKey"],
},
{
name: "Generic OpenAI",
value: "generic-openai",
logo: GenericOpenAiLogo,
options: (settings) => <GenericOpenAiOptions settings={settings} />,
description:
"Connect to any OpenAi-compatible service via a custom configuration",
requiredConfig: [
"GenericOpenAiBasePath",
"GenericOpenAiModelPref",
"GenericOpenAiTokenLimit",
"GenericOpenAiKey",
],
},
{
name: "Native",
value: "native",

4
frontend/src/pages/GeneralSettings/PrivacyAndData/index.jsx
View File

@ -88,7 +88,9 @@ function ThirdParty({ settings }) {
</ul>
</div>
<div className="flex flex-col gap-y-2 border-b border-zinc-500/50 pb-4">
<div className="text-white text-base font-bold">Embedding Engine</div>
<div className="text-white text-base font-bold">
Embedding Preference
</div>
<div className="flex items-center gap-2.5">
<img
src={EMBEDDING_ENGINE_PRIVACY[embeddingEngine].logo}

2
frontend/src/pages/OnboardingFlow/Steps/CustomLogo/index.jsx
View File

@ -18,7 +18,7 @@ export default function CustomLogo({ setHeader, setForwardBtn, setBackBtn }) {
}
function handleBack() {
navigate(paths.onboarding.vectorDatabase());
navigate(paths.onboarding.llmPreference());
}
useEffect(() => {

22
frontend/src/pages/OnboardingFlow/Steps/DataHandling/index.jsx
View File

@ -2,6 +2,7 @@ import PreLoader from "@/components/Preloader";
import System from "@/models/system";
import AnythingLLMIcon from "@/media/logo/anything-llm-icon.png";
import OpenAiLogo from "@/media/llmprovider/openai.png";
import GenericOpenAiLogo from "@/media/llmprovider/generic-openai.png";
import AzureOpenAiLogo from "@/media/llmprovider/azure.png";
import AnthropicLogo from "@/media/llmprovider/anthropic.png";
import GeminiLogo from "@/media/llmprovider/gemini.png";
@ -136,6 +137,13 @@ export const LLM_SELECTION_PRIVACY = {
],
logo: GroqLogo,
},
"generic-openai": {
name: "Generic OpenAI compatible service",
description: [
"Data is shared according to the terms of service applicable with your generic endpoint provider.",
],
logo: GenericOpenAiLogo,
},
};
export const VECTOR_DB_PRIVACY = {
@ -237,6 +245,13 @@ export const EMBEDDING_ENGINE_PRIVACY = {
],
logo: OllamaLogo,
},
lmstudio: {
name: "LMStudio",
description: [
"Your document text is embedded privately on the server running LMStudio",
],
logo: LMStudioLogo,
},
};
export default function DataHandling({ setHeader, setForwardBtn, setBackBtn }) {
@ -298,7 +313,9 @@ export default function DataHandling({ setHeader, setForwardBtn, setBackBtn }) {
</ul>
</div>
<div className="flex flex-col gap-y-2 border-b border-zinc-500/50 pb-4">
<div className="text-white text-base font-bold">Embedding Engine</div>
<div className="text-white text-base font-bold">
Embedding Preference
</div>
<div className="flex items-center gap-2.5">
<img
src={EMBEDDING_ENGINE_PRIVACY[embeddingEngine].logo}
@ -337,6 +354,9 @@ export default function DataHandling({ setHeader, setForwardBtn, setBackBtn }) {
</ul>
</div>
</div>
<p className="text-white/60 text-sm font-medium py-1">
These settings can be reconfigured at any time in the settings.
</p>
</div>
);
}

176
frontend/src/pages/OnboardingFlow/Steps/EmbeddingPreference/index.jsx
View File

@ -1,176 +0,0 @@
import { MagnifyingGlass } from "@phosphor-icons/react";
import { useEffect, useState, useRef } from "react";
import AnythingLLMIcon from "@/media/logo/anything-llm-icon.png";
import OpenAiLogo from "@/media/llmprovider/openai.png";
import AzureOpenAiLogo from "@/media/llmprovider/azure.png";
import LocalAiLogo from "@/media/llmprovider/localai.png";
import OllamaLogo from "@/media/llmprovider/ollama.png";
import NativeEmbeddingOptions from "@/components/EmbeddingSelection/NativeEmbeddingOptions";
import OpenAiOptions from "@/components/EmbeddingSelection/OpenAiOptions";
import AzureAiOptions from "@/components/EmbeddingSelection/AzureAiOptions";
import LocalAiOptions from "@/components/EmbeddingSelection/LocalAiOptions";
import OllamaEmbeddingOptions from "@/components/EmbeddingSelection/OllamaOptions";
import EmbedderItem from "@/components/EmbeddingSelection/EmbedderItem";
import System from "@/models/system";
import paths from "@/utils/paths";
import showToast from "@/utils/toast";
import { useNavigate } from "react-router-dom";
const TITLE = "Embedding Preference";
const DESCRIPTION =
"AnythingLLM can work with many embedding models. This will be the model which turns documents into vectors.";
export default function EmbeddingPreference({
setHeader,
setForwardBtn,
setBackBtn,
}) {
const [searchQuery, setSearchQuery] = useState("");
const [filteredEmbedders, setFilteredEmbedders] = useState([]);
const [selectedEmbedder, setSelectedEmbedder] = useState(null);
const [settings, setSettings] = useState(null);
const formRef = useRef(null);
const hiddenSubmitButtonRef = useRef(null);
const navigate = useNavigate();
useEffect(() => {
async function fetchKeys() {
const _settings = await System.keys();
setSettings(_settings);
setSelectedEmbedder(_settings?.EmbeddingEngine || "native");
}
fetchKeys();
}, []);
const EMBEDDERS = [
{
name: "AnythingLLM Embedder",
value: "native",
logo: AnythingLLMIcon,
options: <NativeEmbeddingOptions settings={settings} />,
description:
"Use the built-in embedding engine for AnythingLLM. Zero setup!",
},
{
name: "OpenAI",
value: "openai",
logo: OpenAiLogo,
options: <OpenAiOptions settings={settings} />,
description: "The standard option for most non-commercial use.",
},
{
name: "Azure OpenAI",
value: "azure",
logo: AzureOpenAiLogo,
options: <AzureAiOptions settings={settings} />,
description: "The enterprise option of OpenAI hosted on Azure services.",
},
{
name: "Local AI",
value: "localai",
logo: LocalAiLogo,
options: <LocalAiOptions settings={settings} />,
description: "Run embedding models locally on your own machine.",
},
{
name: "Ollama",
value: "ollama",
logo: OllamaLogo,
options: <OllamaEmbeddingOptions settings={settings} />,
description: "Run embedding models locally on your own machine.",
},
];
function handleForward() {
if (hiddenSubmitButtonRef.current) {
hiddenSubmitButtonRef.current.click();
}
}
function handleBack() {
navigate(paths.onboarding.llmPreference());
}
const handleSubmit = async (e) => {
e.preventDefault();
const form = e.target;
const data = {};
const formData = new FormData(form);
data.EmbeddingEngine = selectedEmbedder;
for (var [key, value] of formData.entries()) data[key] = value;
const { error } = await System.updateSystem(data);
if (error) {
showToast(`Failed to save embedding settings: ${error}`, "error");
return;
}
navigate(paths.onboarding.vectorDatabase());
};
useEffect(() => {
setHeader({ title: TITLE, description: DESCRIPTION });
setForwardBtn({ showing: true, disabled: false, onClick: handleForward });
setBackBtn({ showing: true, disabled: false, onClick: handleBack });
}, []);
useEffect(() => {
const filtered = EMBEDDERS.filter((embedder) =>
embedder.name.toLowerCase().includes(searchQuery.toLowerCase())
);
setFilteredEmbedders(filtered);
}, [searchQuery, selectedEmbedder]);
return (
<div>
<form ref={formRef} onSubmit={handleSubmit} className="w-full">
<div className="w-full relative border-slate-300/40 shadow border-2 rounded-lg text-white">
<div className="w-full p-4 absolute top-0 rounded-t-lg backdrop-blur-sm">
<div className="w-full flex items-center sticky top-0 z-20">
<MagnifyingGlass
size={16}
weight="bold"
className="absolute left-4 z-30 text-white"
/>
<input
type="text"
placeholder="Search Embedding providers"
className="bg-zinc-600 z-20 pl-10 h-[38px] rounded-full w-full px-4 py-1 text-sm border-2 border-slate-300/40 outline-none focus:border-white text-white"
onChange={(e) => setSearchQuery(e.target.value)}
autoComplete="off"
onKeyDown={(e) => {
if (e.key === "Enter") e.preventDefault();
}}
/>
</div>
</div>
<div className="px-4 pt-[70px] flex flex-col gap-y-1 max-h-[390px] overflow-y-auto no-scroll pb-4">
{filteredEmbedders.map((embedder) => {
return (
<EmbedderItem
key={embedder.name}
name={embedder.name}
value={embedder.value}
image={embedder.logo}
description={embedder.description}
checked={selectedEmbedder === embedder.value}
onClick={() => setSelectedEmbedder(embedder.value)}
/>
);
})}
</div>
</div>
<div className="mt-4 flex flex-col gap-y-1">
{selectedEmbedder &&
EMBEDDERS.find((embedder) => embedder.value === selectedEmbedder)
?.options}
</div>
<button
type="submit"
ref={hiddenSubmitButtonRef}
hidden
aria-hidden="true"
></button>
</form>
</div>
);
}

229
frontend/src/pages/OnboardingFlow/Steps/LLMPreference/index.jsx
View File

@ -1,6 +1,7 @@
import { MagnifyingGlass } from "@phosphor-icons/react";
import { useEffect, useState, useRef } from "react";
import OpenAiLogo from "@/media/llmprovider/openai.png";
import GenericOpenAiLogo from "@/media/llmprovider/generic-openai.png";
import AzureOpenAiLogo from "@/media/llmprovider/azure.png";
import AnthropicLogo from "@/media/llmprovider/anthropic.png";
import GeminiLogo from "@/media/llmprovider/gemini.png";
@ -15,6 +16,7 @@ import PerplexityLogo from "@/media/llmprovider/perplexity.png";
import OpenRouterLogo from "@/media/llmprovider/openrouter.jpeg";
import GroqLogo from "@/media/llmprovider/groq.png";
import OpenAiOptions from "@/components/LLMSelection/OpenAiOptions";
import GenericOpenAiOptions from "@/components/LLMSelection/GenericOpenAiOptions";
import AzureAiOptions from "@/components/LLMSelection/AzureAiOptions";
import AnthropicAiOptions from "@/components/LLMSelection/AnthropicAiOptions";
import LMStudioOptions from "@/components/LLMSelection/LMStudioOptions";
@ -38,6 +40,120 @@ const TITLE = "LLM Preference";
const DESCRIPTION =
"AnythingLLM can work with many LLM providers. This will be the service which handles chatting.";
const LLMS = [
{
name: "OpenAI",
value: "openai",
logo: OpenAiLogo,
options: (settings) => <OpenAiOptions settings={settings} />,
description: "The standard option for most non-commercial use.",
},
{
name: "Azure OpenAI",
value: "azure",
logo: AzureOpenAiLogo,
options: (settings) => <AzureAiOptions settings={settings} />,
description: "The enterprise option of OpenAI hosted on Azure services.",
},
{
name: "Anthropic",
value: "anthropic",
logo: AnthropicLogo,
options: (settings) => <AnthropicAiOptions settings={settings} />,
description: "A friendly AI Assistant hosted by Anthropic.",
},
{
name: "Gemini",
value: "gemini",
logo: GeminiLogo,
options: (settings) => <GeminiLLMOptions settings={settings} />,
description: "Google's largest and most capable AI model",
},
{
name: "HuggingFace",
value: "huggingface",
logo: HuggingFaceLogo,
options: (settings) => <HuggingFaceOptions settings={settings} />,
description:
"Access 150,000+ open-source LLMs and the world's AI community",
},
{
name: "Ollama",
value: "ollama",
logo: OllamaLogo,
options: (settings) => <OllamaLLMOptions settings={settings} />,
description: "Run LLMs locally on your own machine.",
},
{
name: "LM Studio",
value: "lmstudio",
logo: LMStudioLogo,
options: (settings) => <LMStudioOptions settings={settings} />,
description:
"Discover, download, and run thousands of cutting edge LLMs in a few clicks.",
},
{
name: "Local AI",
value: "localai",
logo: LocalAiLogo,
options: (settings) => <LocalAiOptions settings={settings} />,
description: "Run LLMs locally on your own machine.",
},
{
name: "Together AI",
value: "togetherai",
logo: TogetherAILogo,
options: (settings) => <TogetherAiOptions settings={settings} />,
description: "Run open source models from Together AI.",
},
{
name: "Mistral",
value: "mistral",
logo: MistralLogo,
options: (settings) => <MistralOptions settings={settings} />,
description: "Run open source models from Mistral AI.",
},
{
name: "Perplexity AI",
value: "perplexity",
logo: PerplexityLogo,
options: (settings) => <PerplexityOptions settings={settings} />,
description:
"Run powerful and internet-connected models hosted by Perplexity AI.",
},
{
name: "OpenRouter",
value: "openrouter",
logo: OpenRouterLogo,
options: (settings) => <OpenRouterOptions settings={settings} />,
description: "A unified interface for LLMs.",
},
{
name: "Groq",
value: "groq",
logo: GroqLogo,
options: (settings) => <GroqAiOptions settings={settings} />,
description:
"The fastest LLM inferencing available for real-time AI applications.",
},
{
name: "Generic OpenAI",
value: "generic-openai",
logo: GenericOpenAiLogo,
options: (settings) => <GenericOpenAiOptions settings={settings} />,
description:
"Connect to any OpenAi-compatible service via a custom configuration",
},
{
name: "Native",
value: "native",
logo: AnythingLLMIcon,
options: (settings) => <NativeLLMOptions settings={settings} />,
description:
"Use a downloaded custom Llama model for chatting on this AnythingLLM instance.",
},
];
export default function LLMPreference({
setHeader,
setForwardBtn,
@ -61,112 +177,6 @@ export default function LLMPreference({
fetchKeys();
}, []);
const LLMS = [
{
name: "OpenAI",
value: "openai",
logo: OpenAiLogo,
options: <OpenAiOptions settings={settings} />,
description: "The standard option for most non-commercial use.",
},
{
name: "Azure OpenAI",
value: "azure",
logo: AzureOpenAiLogo,
options: <AzureAiOptions settings={settings} />,
description: "The enterprise option of OpenAI hosted on Azure services.",
},
{
name: "Anthropic",
value: "anthropic",
logo: AnthropicLogo,
options: <AnthropicAiOptions settings={settings} />,
description: "A friendly AI Assistant hosted by Anthropic.",
},
{
name: "Gemini",
value: "gemini",
logo: GeminiLogo,
options: <GeminiLLMOptions settings={settings} />,
description: "Google's largest and most capable AI model",
},
{
name: "HuggingFace",
value: "huggingface",
logo: HuggingFaceLogo,
options: <HuggingFaceOptions settings={settings} />,
description:
"Access 150,000+ open-source LLMs and the world's AI community",
},
{
name: "Ollama",
value: "ollama",
logo: OllamaLogo,
options: <OllamaLLMOptions settings={settings} />,
description: "Run LLMs locally on your own machine.",
},
{
name: "LM Studio",
value: "lmstudio",
logo: LMStudioLogo,
options: <LMStudioOptions settings={settings} />,
description:
"Discover, download, and run thousands of cutting edge LLMs in a few clicks.",
},
{
name: "Local AI",
value: "localai",
logo: LocalAiLogo,
options: <LocalAiOptions settings={settings} />,
description: "Run LLMs locally on your own machine.",
},
{
name: "Together AI",
value: "togetherai",
logo: TogetherAILogo,
options: <TogetherAiOptions settings={settings} />,
description: "Run open source models from Together AI.",
},
{
name: "Mistral",
value: "mistral",
logo: MistralLogo,
options: <MistralOptions settings={settings} />,
description: "Run open source models from Mistral AI.",
},
{
name: "Perplexity AI",
value: "perplexity",
logo: PerplexityLogo,
options: <PerplexityOptions settings={settings} />,
description:
"Run powerful and internet-connected models hosted by Perplexity AI.",
},
{
name: "OpenRouter",
value: "openrouter",
logo: OpenRouterLogo,
options: <OpenRouterOptions settings={settings} />,
description: "A unified interface for LLMs.",
},
{
name: "Groq",
value: "groq",
logo: GroqLogo,
options: <GroqAiOptions settings={settings} />,
description:
"The fastest LLM inferencing available for real-time AI applications.",
},
{
name: "Native",
value: "native",
logo: AnythingLLMIcon,
options: <NativeLLMOptions settings={settings} />,
description:
"Use a downloaded custom Llama model for chatting on this AnythingLLM instance.",
},
];
function handleForward() {
if (hiddenSubmitButtonRef.current) {
hiddenSubmitButtonRef.current.click();
@ -183,6 +193,9 @@ export default function LLMPreference({
const data = {};
const formData = new FormData(form);
data.LLMProvider = selectedLLM;
// Default to AnythingLLM embedder and LanceDB
data.EmbeddingEngine = "native";
data.VectorDB = "lancedb";
for (var [key, value] of formData.entries()) data[key] = value;
const { error } = await System.updateSystem(data);
@ -190,7 +203,7 @@ export default function LLMPreference({
showToast(`Failed to save LLM settings: ${error}`, "error");
return;
}
navigate(paths.onboarding.embeddingPreference());
navigate(paths.onboarding.customLogo());
};
useEffect(() => {
@ -248,7 +261,7 @@ export default function LLMPreference({
</div>
<div className="mt-4 flex flex-col gap-y-1">
{selectedLLM &&
LLMS.find((llm) => llm.value === selectedLLM)?.options}
LLMS.find((llm) => llm.value === selectedLLM)?.options(settings)}
</div>
<button
type="submit"

202
frontend/src/pages/OnboardingFlow/Steps/VectorDatabaseConnection/index.jsx
View File

@ -1,202 +0,0 @@
import React, { useEffect, useState, useRef } from "react";
import { MagnifyingGlass } from "@phosphor-icons/react";
import ChromaLogo from "@/media/vectordbs/chroma.png";
import PineconeLogo from "@/media/vectordbs/pinecone.png";
import LanceDbLogo from "@/media/vectordbs/lancedb.png";
import WeaviateLogo from "@/media/vectordbs/weaviate.png";
import QDrantLogo from "@/media/vectordbs/qdrant.png";
import MilvusLogo from "@/media/vectordbs/milvus.png";
import ZillizLogo from "@/media/vectordbs/zilliz.png";
import AstraDBLogo from "@/media/vectordbs/astraDB.png";
import System from "@/models/system";
import paths from "@/utils/paths";
import PineconeDBOptions from "@/components/VectorDBSelection/PineconeDBOptions";
import ChromaDBOptions from "@/components/VectorDBSelection/ChromaDBOptions";
import QDrantDBOptions from "@/components/VectorDBSelection/QDrantDBOptions";
import WeaviateDBOptions from "@/components/VectorDBSelection/WeaviateDBOptions";
import LanceDBOptions from "@/components/VectorDBSelection/LanceDBOptions";
import MilvusOptions from "@/components/VectorDBSelection/MilvusDBOptions";
import ZillizCloudOptions from "@/components/VectorDBSelection/ZillizCloudOptions";
import AstraDBOptions from "@/components/VectorDBSelection/AstraDBOptions";
import showToast from "@/utils/toast";
import { useNavigate } from "react-router-dom";
import VectorDBItem from "@/components/VectorDBSelection/VectorDBItem";
const TITLE = "Vector Database Connection";
const DESCRIPTION =
"These are the credentials and settings for your vector database of choice.";
export default function VectorDatabaseConnection({
setHeader,
setForwardBtn,
setBackBtn,
}) {
const [searchQuery, setSearchQuery] = useState("");
const [filteredVDBs, setFilteredVDBs] = useState([]);
const [selectedVDB, setSelectedVDB] = useState(null);
const [settings, setSettings] = useState(null);
const formRef = useRef(null);
const hiddenSubmitButtonRef = useRef(null);
const navigate = useNavigate();
useEffect(() => {
async function fetchKeys() {
const _settings = await System.keys();
setSettings(_settings);
setSelectedVDB(_settings?.VectorDB || "lancedb");
}
fetchKeys();
}, []);
const VECTOR_DBS = [
{
name: "LanceDB",
value: "lancedb",
logo: LanceDbLogo,
options: <LanceDBOptions />,
description:
"100% local vector DB that runs on the same instance as AnythingLLM.",
},
{
name: "Chroma",
value: "chroma",
logo: ChromaLogo,
options: <ChromaDBOptions settings={settings} />,
description:
"Open source vector database you can host yourself or on the cloud.",
},
{
name: "Pinecone",
value: "pinecone",
logo: PineconeLogo,
options: <PineconeDBOptions settings={settings} />,
description: "100% cloud-based vector database for enterprise use cases.",
},
{
name: "Zilliz Cloud",
value: "zilliz",
logo: ZillizLogo,
options: <ZillizCloudOptions settings={settings} />,
description:
"Cloud hosted vector database built for enterprise with SOC 2 compliance.",
},
{
name: "QDrant",
value: "qdrant",
logo: QDrantLogo,
options: <QDrantDBOptions settings={settings} />,
description: "Open source local and distributed cloud vector database.",
},
{
name: "Weaviate",
value: "weaviate",
logo: WeaviateLogo,
options: <WeaviateDBOptions settings={settings} />,
description:
"Open source local and cloud hosted multi-modal vector database.",
},
{
name: "Milvus",
value: "milvus",
logo: MilvusLogo,
options: <MilvusOptions settings={settings} />,
description: "Open-source, highly scalable, and blazing fast.",
},
{
name: "AstraDB",
value: "astra",
logo: AstraDBLogo,
options: <AstraDBOptions settings={settings} />,
description: "Vector Search for Real-world GenAI.",
},
];
function handleForward() {
if (hiddenSubmitButtonRef.current) {
hiddenSubmitButtonRef.current.click();
}
}
function handleBack() {
navigate(paths.onboarding.embeddingPreference());
}
const handleSubmit = async (e) => {
e.preventDefault();
const form = e.target;
const data = {};
const formData = new FormData(form);
data.VectorDB = selectedVDB;
for (var [key, value] of formData.entries()) data[key] = value;
const { error } = await System.updateSystem(data);
if (error) {
showToast(`Failed to save Vector Database settings: ${error}`, "error");
return;
}
navigate(paths.onboarding.customLogo());
};
useEffect(() => {
setHeader({ title: TITLE, description: DESCRIPTION });
setForwardBtn({ showing: true, disabled: false, onClick: handleForward });
setBackBtn({ showing: true, disabled: false, onClick: handleBack });
}, []);
useEffect(() => {
const filtered = VECTOR_DBS.filter((vdb) =>
vdb.name.toLowerCase().includes(searchQuery.toLowerCase())
);
setFilteredVDBs(filtered);
}, [searchQuery, selectedVDB]);
return (
<>
<form ref={formRef} onSubmit={handleSubmit} className="w-full">
<div className="w-full relative border-slate-300/40 shadow border-2 rounded-lg text-white pb-4">
<div className="w-full p-4 absolute top-0 rounded-t-lg backdrop-blur-sm">
<div className="w-full flex items-center sticky top-0 z-20">
<MagnifyingGlass
size={16}
weight="bold"
className="absolute left-4 z-30 text-white"
/>
<input
type="text"
placeholder="Search vector databases"
className="bg-zinc-600 z-20 pl-10 h-[38px] rounded-full w-full px-4 py-1 text-sm border-2 border-slate-300/40 outline-none focus:border-white text-white"
onChange={(e) => setSearchQuery(e.target.value)}
autoComplete="off"
onKeyDown={(e) => {
if (e.key === "Enter") e.preventDefault();
}}
/>
</div>
</div>
<div className="px-4 pt-[70px] flex flex-col gap-y-1 max-h-[390px] overflow-y-auto no-scroll">
{filteredVDBs.map((vdb) => (
<VectorDBItem
key={vdb.name}
name={vdb.name}
value={vdb.value}
image={vdb.logo}
description={vdb.description}
checked={selectedVDB === vdb.value}
onClick={setSelectedVDB}
/>
))}
</div>
</div>
<div className="mt-4 flex flex-col gap-y-1">
{selectedVDB &&
VECTOR_DBS.find((vdb) => vdb.value === selectedVDB)?.options}
</div>
<button
type="submit"
ref={hiddenSubmitButtonRef}
hidden
aria-hidden="true"
></button>
</form>
</>
);
}

4
frontend/src/pages/OnboardingFlow/Steps/index.jsx
View File

@ -3,8 +3,6 @@ import { useState } from "react";
import { isMobile } from "react-device-detect";
import Home from "./Home";
import LLMPreference from "./LLMPreference";
import EmbeddingPreference from "./EmbeddingPreference";
import VectorDatabaseConnection from "./VectorDatabaseConnection";
import CustomLogo from "./CustomLogo";
import UserSetup from "./UserSetup";
import DataHandling from "./DataHandling";
@ -14,8 +12,6 @@ import CreateWorkspace from "./CreateWorkspace";
const OnboardingSteps = {
home: Home,
"llm-preference": LLMPreference,
"embedding-preference": EmbeddingPreference,
"vector-database": VectorDatabaseConnection,
"custom-logo": CustomLogo,
"user-setup": UserSetup,
"data-handling": DataHandling,

5
frontend/src/pages/WorkspaceSettings/ChatSettings/WorkspaceLLMSelection/index.jsx
View File

@ -5,6 +5,9 @@ import { AVAILABLE_LLM_PROVIDERS } from "@/pages/GeneralSettings/LLMPreference";
import { CaretUpDown, MagnifyingGlass, X } from "@phosphor-icons/react";
import ChatModelSelection from "../ChatModelSelection";
// Some providers can only be associated with a single model.
// In that case there is no selection to be made so we can just move on.
const NO_MODEL_SELECTION = ["default", "huggingface", "generic-openai"];
const DISABLED_PROVIDERS = ["azure", "lmstudio", "native"];
const LLM_DEFAULT = {
name: "System default",
@ -145,7 +148,7 @@ export default function WorkspaceLLMSelection({
</button>
)}
</div>
{selectedLLM !== "default" && (
{!NO_MODEL_SELECTION.includes(selectedLLM) && (
<div className="mt-4 flex flex-col gap-y-1">
<ChatModelSelection
provider={selectedLLM}

11
server/.env.example
View File

@ -63,6 +63,12 @@ JWT_SECRET="my-random-string-for-seeding" # Please generate random string at lea
# GROQ_API_KEY=gsk_abcxyz
# GROQ_MODEL_PREF=llama2-70b-4096
# LLM_PROVIDER='generic-openai'
# GENERIC_OPEN_AI_BASE_PATH='http://proxy.url.openai.com/v1'
# GENERIC_OPEN_AI_MODEL_PREF='gpt-3.5-turbo'
# GENERIC_OPEN_AI_MODEL_TOKEN_LIMIT=4096
# GENERIC_OPEN_AI_API_KEY=sk-123abc
###########################################
######## Embedding API SElECTION ##########
###########################################
@ -86,6 +92,11 @@ JWT_SECRET="my-random-string-for-seeding" # Please generate random string at lea
# EMBEDDING_MODEL_PREF='nomic-embed-text:latest'
# EMBEDDING_MODEL_MAX_CHUNK_LENGTH=8192
# EMBEDDING_ENGINE='lmstudio'
# EMBEDDING_BASE_PATH='https://localhost:1234/v1'
# EMBEDDING_MODEL_PREF='nomic-ai/nomic-embed-text-v1.5-GGUF/nomic-embed-text-v1.5.Q4_0.gguf'
# EMBEDDING_MODEL_MAX_CHUNK_LENGTH=8192
###########################################
######## Vector Database Selection ########
###########################################

6
server/endpoints/api/workspace/index.js
View File

@ -436,9 +436,9 @@ function apiWorkspaceEndpoints(app) {
await Document.removeDocuments(currWorkspace, deletes);
await Document.addDocuments(currWorkspace, adds);
const updatedWorkspace = await Workspace.get(
`id = ${Number(currWorkspace.id)}`
);
const updatedWorkspace = await Workspace.get({
id: Number(currWorkspace.id),
});
response.status(200).json({ workspace: updatedWorkspace });
} catch (e) {
console.log(e.message, e);

6
server/models/systemSettings.js
View File

@ -358,6 +358,12 @@ const SystemSettings = {
HuggingFaceLLMEndpoint: process.env.HUGGING_FACE_LLM_ENDPOINT,
HuggingFaceLLMAccessToken: !!process.env.HUGGING_FACE_LLM_API_KEY,
HuggingFaceLLMTokenLimit: process.env.HUGGING_FACE_LLM_TOKEN_LIMIT,
// Generic OpenAI Keys
GenericOpenAiBasePath: process.env.GENERIC_OPEN_AI_BASE_PATH,
GenericOpenAiModelPref: process.env.GENERIC_OPEN_AI_MODEL_PREF,
GenericOpenAiTokenLimit: process.env.GENERIC_OPEN_AI_MODEL_TOKEN_LIMIT,
GenericOpenAiKey: !!process.env.GENERIC_OPEN_AI_API_KEY,
};
},
};

2
server/models/workspaceAgentInvocation.js
View File

@ -3,7 +3,9 @@ const { v4: uuidv4 } = require("uuid");
const WorkspaceAgentInvocation = {
// returns array of strings with their @ handle.
// must start with @
parseAgents: function (promptString) {
if (!promptString.startsWith("@")) return [];
return promptString.split(/\s+/).filter((v) => v.startsWith("@"));
},

3
server/storage/models/.gitignore vendored
View File

@ -1,3 +1,4 @@
Xenova
downloaded/*
!downloaded/.placeholder
!downloaded/.placeholder
openrouter

193
server/utils/AiProviders/genericOpenAi/index.js Normal file
View File

@ -0,0 +1,193 @@
const { NativeEmbedder } = require("../../EmbeddingEngines/native");
const { chatPrompt } = require("../../chats");
const { handleDefaultStreamResponse } = require("../../helpers/chat/responses");
class GenericOpenAiLLM {
constructor(embedder = null, modelPreference = null) {
const { Configuration, OpenAIApi } = require("openai");
if (!process.env.GENERIC_OPEN_AI_BASE_PATH)
throw new Error(
"GenericOpenAI must have a valid base path to use for the api."
);
this.basePath = process.env.GENERIC_OPEN_AI_BASE_PATH;
const config = new Configuration({
basePath: this.basePath,
apiKey: process.env.GENERIC_OPEN_AI_API_KEY ?? null,
});
this.openai = new OpenAIApi(config);
this.model =
modelPreference ?? process.env.GENERIC_OPEN_AI_MODEL_PREF ?? null;
if (!this.model)
throw new Error("GenericOpenAI must have a valid model set.");
this.limits = {
history: this.promptWindowLimit() * 0.15,
system: this.promptWindowLimit() * 0.15,
user: this.promptWindowLimit() * 0.7,
};
if (!embedder)
console.warn(
"No embedding provider defined for GenericOpenAiLLM - falling back to NativeEmbedder for embedding!"
);
this.embedder = !embedder ? new NativeEmbedder() : embedder;
this.defaultTemp = 0.7;
this.log(`Inference API: ${this.basePath} Model: ${this.model}`);
}
log(text, ...args) {
console.log(`\x1b[36m[${this.constructor.name}]\x1b[0m ${text}`, ...args);
}
#appendContext(contextTexts = []) {
if (!contextTexts || !contextTexts.length) return "";
return (
"\nContext:\n" +
contextTexts
.map((text, i) => {
return `[CONTEXT ${i}]:\n${text}\n[END CONTEXT ${i}]\n\n`;
})
.join("")
);
}
streamingEnabled() {
return "streamChat" in this && "streamGetChatCompletion" in this;
}
// Ensure the user set a value for the token limit
// and if undefined - assume 4096 window.
promptWindowLimit() {
const limit = process.env.GENERIC_OPEN_AI_MODEL_TOKEN_LIMIT || 4096;
if (!limit || isNaN(Number(limit)))
throw new Error("No token context limit was set.");
return Number(limit);
}
// Short circuit since we have no idea if the model is valid or not
// in pre-flight for generic endpoints
isValidChatCompletionModel(_modelName = "") {
return true;
}
constructPrompt({
systemPrompt = "",
contextTexts = [],
chatHistory = [],
userPrompt = "",
}) {
const prompt = {
role: "system",
content: `${systemPrompt}${this.#appendContext(contextTexts)}`,
};
return [prompt, ...chatHistory, { role: "user", content: userPrompt }];
}
async isSafe(_input = "") {
// Not implemented so must be stubbed
return { safe: true, reasons: [] };
}
async sendChat(chatHistory = [], prompt, workspace = {}, rawHistory = []) {
const textResponse = await this.openai
.createChatCompletion({
model: this.model,
temperature: Number(workspace?.openAiTemp ?? this.defaultTemp),
n: 1,
messages: await this.compressMessages(
{
systemPrompt: chatPrompt(workspace),
userPrompt: prompt,
chatHistory,
},
rawHistory
),
})
.then((json) => {
const res = json.data;
if (!res.hasOwnProperty("choices"))
throw new Error("GenericOpenAI chat: No results!");
if (res.choices.length === 0)
throw new Error("GenericOpenAI chat: No results length!");
return res.choices[0].message.content;
})
.catch((error) => {
throw new Error(
`GenericOpenAI::createChatCompletion failed with: ${error.message}`
);
});
return textResponse;
}
async streamChat(chatHistory = [], prompt, workspace = {}, rawHistory = []) {
const streamRequest = await this.openai.createChatCompletion(
{
model: this.model,
stream: true,
temperature: Number(workspace?.openAiTemp ?? this.defaultTemp),
n: 1,
messages: await this.compressMessages(
{
systemPrompt: chatPrompt(workspace),
userPrompt: prompt,
chatHistory,
},
rawHistory
),
},
{ responseType: "stream" }
);
return streamRequest;
}
async getChatCompletion(messages = null, { temperature = 0.7 }) {
const { data } = await this.openai
.createChatCompletion({
model: this.model,
messages,
temperature,
})
.catch((e) => {
throw new Error(e.response.data.error.message);
});
if (!data.hasOwnProperty("choices")) return null;
return data.choices[0].message.content;
}
async streamGetChatCompletion(messages = null, { temperature = 0.7 }) {
const streamRequest = await this.openai.createChatCompletion(
{
model: this.model,
stream: true,
messages,
temperature,
},
{ responseType: "stream" }
);
return streamRequest;
}
handleStream(response, stream, responseProps) {
return handleDefaultStreamResponse(response, stream, responseProps);
}
// Simple wrapper for dynamic embedder & normalize interface for all LLM implementations
async embedTextInput(textInput) {
return await this.embedder.embedTextInput(textInput);
}
async embedChunks(textChunks = []) {
return await this.embedder.embedChunks(textChunks);
}
async compressMessages(promptArgs = {}, rawHistory = []) {
const { messageArrayCompressor } = require("../../helpers/chat");
const messageArray = this.constructPrompt(promptArgs);
return await messageArrayCompressor(this, messageArray, rawHistory);
}
}
module.exports = {
GenericOpenAiLLM,
};

15
server/utils/AiProviders/groq/index.js
View File

@ -40,20 +40,31 @@ class GroqLLM {
streamingEnabled() {
return "streamChat" in this && "streamGetChatCompletion" in this;
}
promptWindowLimit() {
switch (this.model) {
case "llama2-70b-4096":
return 4096;
case "mixtral-8x7b-32768":
return 32_768;
case "llama3-8b-8192":
return 8192;
case "llama3-70b-8192":
return 8192;
case "gemma-7b-it":
return 8192;
default:
return 4096;
}
}
async isValidChatCompletionModel(modelName = "") {
const validModels = ["llama2-70b-4096", "mixtral-8x7b-32768"];
const validModels = [
"llama2-70b-4096",
"mixtral-8x7b-32768",
"llama3-8b-8192",
"llama3-70b-8192",
"gemma-7b-it",
];
const isPreset = validModels.some((model) => modelName === model);
if (isPreset) return true;

100
server/utils/AiProviders/openRouter/index.js
View File

@ -5,11 +5,9 @@ const {
writeResponseChunk,
clientAbortedHandler,
} = require("../../helpers/chat/responses");
function openRouterModels() {
const { MODELS } = require("./models.js");
return MODELS || {};
}
const fs = require("fs");
const path = require("path");
const { safeJsonParse } = require("../../http");
class OpenRouterLLM {
constructor(embedder = null, modelPreference = null) {
@ -17,8 +15,9 @@ class OpenRouterLLM {
if (!process.env.OPENROUTER_API_KEY)
throw new Error("No OpenRouter API key was set.");
this.basePath = "https://openrouter.ai/api/v1";
const config = new Configuration({
basePath: "https://openrouter.ai/api/v1",
basePath: this.basePath,
apiKey: process.env.OPENROUTER_API_KEY,
baseOptions: {
headers: {
@ -38,6 +37,81 @@ class OpenRouterLLM {
this.embedder = !embedder ? new NativeEmbedder() : embedder;
this.defaultTemp = 0.7;
const cacheFolder = path.resolve(
process.env.STORAGE_DIR
? path.resolve(process.env.STORAGE_DIR, "models", "openrouter")
: path.resolve(__dirname, `../../../storage/models/openrouter`)
);
fs.mkdirSync(cacheFolder, { recursive: true });
this.cacheModelPath = path.resolve(cacheFolder, "models.json");
this.cacheAtPath = path.resolve(cacheFolder, ".cached_at");
}
log(text, ...args) {
console.log(`\x1b[36m[${this.constructor.name}]\x1b[0m ${text}`, ...args);
}
async init() {
await this.#syncModels();
return this;
}
// This checks if the .cached_at file has a timestamp that is more than 1Week (in millis)
// from the current date. If it is, then we will refetch the API so that all the models are up
// to date.
#cacheIsStale() {
const MAX_STALE = 6.048e8; // 1 Week in MS
if (!fs.existsSync(this.cacheAtPath)) return true;
const now = Number(new Date());
const timestampMs = Number(fs.readFileSync(this.cacheAtPath));
return now - timestampMs > MAX_STALE;
}
// The OpenRouter model API has a lot of models, so we cache this locally in the directory
// as if the cache directory JSON file is stale or does not exist we will fetch from API and store it.
// This might slow down the first request, but we need the proper token context window
// for each model and this is a constructor property - so we can really only get it if this cache exists.
// We used to have this as a chore, but given there is an API to get the info - this makes little sense.
async #syncModels() {
if (fs.existsSync(this.cacheModelPath) && !this.#cacheIsStale())
return false;
this.log(
"Model cache is not present or stale. Fetching from OpenRouter API."
);
await fetch(`${this.basePath}/models`, {
method: "GET",
headers: {
"Content-Type": "application/json",
},
})
.then((res) => res.json())
.then(({ data = [] }) => {
const models = {};
data.forEach((model) => {
models[model.id] = {
id: model.id,
name: model.name,
organization:
model.id.split("/")[0].charAt(0).toUpperCase() +
model.id.split("/")[0].slice(1),
maxLength: model.context_length,
};
});
fs.writeFileSync(this.cacheModelPath, JSON.stringify(models), {
encoding: "utf-8",
});
fs.writeFileSync(this.cacheAtPath, String(Number(new Date())), {
encoding: "utf-8",
});
return models;
})
.catch((e) => {
console.error(e);
return {};
});
return;
}
#appendContext(contextTexts = []) {
@ -52,8 +126,12 @@ class OpenRouterLLM {
);
}
allModelInformation() {
return openRouterModels();
models() {
if (!fs.existsSync(this.cacheModelPath)) return {};
return safeJsonParse(
fs.readFileSync(this.cacheModelPath, { encoding: "utf-8" }),
{}
);
}
streamingEnabled() {
@ -61,12 +139,13 @@ class OpenRouterLLM {
}
promptWindowLimit() {
const availableModels = this.allModelInformation();
const availableModels = this.models();
return availableModels[this.model]?.maxLength || 4096;
}
async isValidChatCompletionModel(model = "") {
const availableModels = this.allModelInformation();
await this.#syncModels();
const availableModels = this.models();
return availableModels.hasOwnProperty(model);
}
@ -343,5 +422,4 @@ class OpenRouterLLM {
module.exports = {
OpenRouterLLM,
openRouterModels,
};

778
server/utils/AiProviders/openRouter/models.js
View File

@ -1,778 +0,0 @@
const MODELS = {
"openrouter/auto": {
id: "openrouter/auto",
name: "Auto (best for prompt)",
organization: "Openrouter",
maxLength: 128000,
},
"nousresearch/nous-capybara-7b:free": {
id: "nousresearch/nous-capybara-7b:free",
name: "Nous: Capybara 7B (free)",
organization: "Nousresearch",
maxLength: 4096,
},
"mistralai/mistral-7b-instruct:free": {
id: "mistralai/mistral-7b-instruct:free",
name: "Mistral 7B Instruct (free)",
organization: "Mistralai",
maxLength: 32768,
},
"openchat/openchat-7b:free": {
id: "openchat/openchat-7b:free",
name: "OpenChat 3.5 (free)",
organization: "Openchat",
maxLength: 8192,
},
"gryphe/mythomist-7b:free": {
id: "gryphe/mythomist-7b:free",
name: "MythoMist 7B (free)",
organization: "Gryphe",
maxLength: 32768,
},
"undi95/toppy-m-7b:free": {
id: "undi95/toppy-m-7b:free",
name: "Toppy M 7B (free)",
organization: "Undi95",
maxLength: 4096,
},
"openrouter/cinematika-7b:free": {
id: "openrouter/cinematika-7b:free",
name: "Cinematika 7B (alpha) (free)",
organization: "Openrouter",
maxLength: 8000,
},
"google/gemma-7b-it:free": {
id: "google/gemma-7b-it:free",
name: "Google: Gemma 7B (free)",
organization: "Google",
maxLength: 8192,
},
"jebcarter/psyfighter-13b": {
id: "jebcarter/psyfighter-13b",
name: "Psyfighter 13B",
organization: "Jebcarter",
maxLength: 4096,
},
"koboldai/psyfighter-13b-2": {
id: "koboldai/psyfighter-13b-2",
name: "Psyfighter v2 13B",
organization: "Koboldai",
maxLength: 4096,
},
"intel/neural-chat-7b": {
id: "intel/neural-chat-7b",
name: "Neural Chat 7B v3.1",
organization: "Intel",
maxLength: 4096,
},
"haotian-liu/llava-13b": {
id: "haotian-liu/llava-13b",
name: "Llava 13B",
organization: "Haotian-liu",
maxLength: 2048,
},
"nousresearch/nous-hermes-2-vision-7b": {
id: "nousresearch/nous-hermes-2-vision-7b",
name: "Nous: Hermes 2 Vision 7B (alpha)",
organization: "Nousresearch",
maxLength: 4096,
},
"meta-llama/llama-2-13b-chat": {
id: "meta-llama/llama-2-13b-chat",
name: "Meta: Llama v2 13B Chat",
organization: "Meta-llama",
maxLength: 4096,
},
"migtissera/synthia-70b": {
id: "migtissera/synthia-70b",
name: "Synthia 70B",
organization: "Migtissera",
maxLength: 8192,
},
"pygmalionai/mythalion-13b": {
id: "pygmalionai/mythalion-13b",
name: "Pygmalion: Mythalion 13B",
organization: "Pygmalionai",
maxLength: 8192,
},
"xwin-lm/xwin-lm-70b": {
id: "xwin-lm/xwin-lm-70b",
name: "Xwin 70B",
organization: "Xwin-lm",
maxLength: 8192,
},
"alpindale/goliath-120b": {
id: "alpindale/goliath-120b",
name: "Goliath 120B",
organization: "Alpindale",
maxLength: 6144,
},
"neversleep/noromaid-20b": {
id: "neversleep/noromaid-20b",
name: "Noromaid 20B",
organization: "Neversleep",
maxLength: 8192,
},
"gryphe/mythomist-7b": {
id: "gryphe/mythomist-7b",
name: "MythoMist 7B",
organization: "Gryphe",
maxLength: 32768,
},
"sophosympatheia/midnight-rose-70b": {
id: "sophosympatheia/midnight-rose-70b",
name: "Midnight Rose 70B",
organization: "Sophosympatheia",
maxLength: 4096,
},
"undi95/remm-slerp-l2-13b:extended": {
id: "undi95/remm-slerp-l2-13b:extended",
name: "ReMM SLERP 13B (extended)",
organization: "Undi95",
maxLength: 6144,
},
"mancer/weaver": {
id: "mancer/weaver",
name: "Mancer: Weaver (alpha)",
organization: "Mancer",
maxLength: 8000,
},
"nousresearch/nous-hermes-llama2-13b": {
id: "nousresearch/nous-hermes-llama2-13b",
name: "Nous: Hermes 13B",
organization: "Nousresearch",
maxLength: 4096,
},
"nousresearch/nous-capybara-7b": {
id: "nousresearch/nous-capybara-7b",
name: "Nous: Capybara 7B",
organization: "Nousresearch",
maxLength: 4096,
},
"meta-llama/codellama-34b-instruct": {
id: "meta-llama/codellama-34b-instruct",
name: "Meta: CodeLlama 34B Instruct",
organization: "Meta-llama",
maxLength: 8192,
},
"codellama/codellama-70b-instruct": {
id: "codellama/codellama-70b-instruct",
name: "Meta: CodeLlama 70B Instruct",
organization: "Codellama",
maxLength: 2048,
},
"phind/phind-codellama-34b": {
id: "phind/phind-codellama-34b",
name: "Phind: CodeLlama 34B v2",
organization: "Phind",
maxLength: 4096,
},
"teknium/openhermes-2-mistral-7b": {
id: "teknium/openhermes-2-mistral-7b",
name: "OpenHermes 2 Mistral 7B",
organization: "Teknium",
maxLength: 4096,
},
"teknium/openhermes-2.5-mistral-7b": {
id: "teknium/openhermes-2.5-mistral-7b",
name: "OpenHermes 2.5 Mistral 7B",
organization: "Teknium",
maxLength: 4096,
},
"undi95/remm-slerp-l2-13b": {
id: "undi95/remm-slerp-l2-13b",
name: "ReMM SLERP 13B",
organization: "Undi95",
maxLength: 4096,
},
"openrouter/cinematika-7b": {
id: "openrouter/cinematika-7b",
name: "Cinematika 7B (alpha)",
organization: "Openrouter",
maxLength: 8000,
},
"01-ai/yi-34b-chat": {
id: "01-ai/yi-34b-chat",
name: "Yi 34B Chat",
organization: "01-ai",
maxLength: 4096,
},
"01-ai/yi-34b": {
id: "01-ai/yi-34b",
name: "Yi 34B (base)",
organization: "01-ai",
maxLength: 4096,
},
"01-ai/yi-6b": {
id: "01-ai/yi-6b",
name: "Yi 6B (base)",
organization: "01-ai",
maxLength: 4096,
},
"togethercomputer/stripedhyena-nous-7b": {
id: "togethercomputer/stripedhyena-nous-7b",
name: "StripedHyena Nous 7B",
organization: "Togethercomputer",
maxLength: 32768,
},
"togethercomputer/stripedhyena-hessian-7b": {
id: "togethercomputer/stripedhyena-hessian-7b",
name: "StripedHyena Hessian 7B (base)",
organization: "Togethercomputer",
maxLength: 32768,
},
"mistralai/mixtral-8x7b": {
id: "mistralai/mixtral-8x7b",
name: "Mixtral 8x7B (base)",
organization: "Mistralai",
maxLength: 32768,
},
"nousresearch/nous-hermes-yi-34b": {
id: "nousresearch/nous-hermes-yi-34b",
name: "Nous: Hermes 2 Yi 34B",
organization: "Nousresearch",
maxLength: 4096,
},
"nousresearch/nous-hermes-2-mixtral-8x7b-sft": {
id: "nousresearch/nous-hermes-2-mixtral-8x7b-sft",
name: "Nous: Hermes 2 Mixtral 8x7B SFT",
organization: "Nousresearch",
maxLength: 32000,
},
"nousresearch/nous-hermes-2-mistral-7b-dpo": {
id: "nousresearch/nous-hermes-2-mistral-7b-dpo",
name: "Nous: Hermes 2 Mistral 7B DPO",
organization: "Nousresearch",
maxLength: 8192,
},
"open-orca/mistral-7b-openorca": {
id: "open-orca/mistral-7b-openorca",
name: "Mistral OpenOrca 7B",
organization: "Open-orca",
maxLength: 8192,
},
"huggingfaceh4/zephyr-7b-beta": {
id: "huggingfaceh4/zephyr-7b-beta",
name: "Hugging Face: Zephyr 7B",
organization: "Huggingfaceh4",
maxLength: 4096,
},
"openai/gpt-3.5-turbo": {
id: "openai/gpt-3.5-turbo",
name: "OpenAI: GPT-3.5 Turbo",
organization: "Openai",
maxLength: 16385,
},
"openai/gpt-3.5-turbo-0125": {
id: "openai/gpt-3.5-turbo-0125",
name: "OpenAI: GPT-3.5 Turbo 16k",
organization: "Openai",
maxLength: 16385,
},
"openai/gpt-3.5-turbo-1106": {
id: "openai/gpt-3.5-turbo-1106",
name: "OpenAI: GPT-3.5 Turbo 16k (older v1106)",
organization: "Openai",
maxLength: 16385,
},
"openai/gpt-3.5-turbo-0613": {
id: "openai/gpt-3.5-turbo-0613",
name: "OpenAI: GPT-3.5 Turbo (older v0613)",
organization: "Openai",
maxLength: 4095,
},
"openai/gpt-3.5-turbo-0301": {
id: "openai/gpt-3.5-turbo-0301",
name: "OpenAI: GPT-3.5 Turbo (older v0301)",
organization: "Openai",
maxLength: 4095,
},
"openai/gpt-3.5-turbo-16k": {
id: "openai/gpt-3.5-turbo-16k",
name: "OpenAI: GPT-3.5 Turbo 16k",
organization: "Openai",
maxLength: 16385,
},
"openai/gpt-4-turbo": {
id: "openai/gpt-4-turbo",
name: "OpenAI: GPT-4 Turbo",
organization: "Openai",
maxLength: 128000,
},
"openai/gpt-4-turbo-preview": {
id: "openai/gpt-4-turbo-preview",
name: "OpenAI: GPT-4 Turbo Preview",
organization: "Openai",
maxLength: 128000,
},
"openai/gpt-4-1106-preview": {
id: "openai/gpt-4-1106-preview",
name: "OpenAI: GPT-4 Turbo (older v1106)",
organization: "Openai",
maxLength: 128000,
},
"openai/gpt-4": {
id: "openai/gpt-4",
name: "OpenAI: GPT-4",
organization: "Openai",
maxLength: 8191,
},
"openai/gpt-4-0314": {
id: "openai/gpt-4-0314",
name: "OpenAI: GPT-4 (older v0314)",
organization: "Openai",
maxLength: 8191,
},
"openai/gpt-4-32k": {
id: "openai/gpt-4-32k",
name: "OpenAI: GPT-4 32k",
organization: "Openai",
maxLength: 32767,
},
"openai/gpt-4-32k-0314": {
id: "openai/gpt-4-32k-0314",
name: "OpenAI: GPT-4 32k (older v0314)",
organization: "Openai",
maxLength: 32767,
},
"openai/gpt-4-vision-preview": {
id: "openai/gpt-4-vision-preview",
name: "OpenAI: GPT-4 Vision",
organization: "Openai",
maxLength: 128000,
},
"openai/gpt-3.5-turbo-instruct": {
id: "openai/gpt-3.5-turbo-instruct",
name: "OpenAI: GPT-3.5 Turbo Instruct",
organization: "Openai",
maxLength: 4095,
},
"google/palm-2-chat-bison": {
id: "google/palm-2-chat-bison",
name: "Google: PaLM 2 Chat",
organization: "Google",
maxLength: 25804,
},
"google/palm-2-codechat-bison": {
id: "google/palm-2-codechat-bison",
name: "Google: PaLM 2 Code Chat",
organization: "Google",
maxLength: 20070,
},
"google/palm-2-chat-bison-32k": {
id: "google/palm-2-chat-bison-32k",
name: "Google: PaLM 2 Chat 32k",
organization: "Google",
maxLength: 91750,
},
"google/palm-2-codechat-bison-32k": {
id: "google/palm-2-codechat-bison-32k",
name: "Google: PaLM 2 Code Chat 32k",
organization: "Google",
maxLength: 91750,
},
"google/gemini-pro": {
id: "google/gemini-pro",
name: "Google: Gemini Pro 1.0",
organization: "Google",
maxLength: 91728,
},
"google/gemini-pro-vision": {
id: "google/gemini-pro-vision",
name: "Google: Gemini Pro Vision 1.0",
organization: "Google",
maxLength: 45875,
},
"google/gemini-pro-1.5": {
id: "google/gemini-pro-1.5",
name: "Google: Gemini Pro 1.5 (preview)",
organization: "Google",
maxLength: 2800000,
},
"perplexity/pplx-70b-online": {
id: "perplexity/pplx-70b-online",
name: "Perplexity: PPLX 70B Online",
organization: "Perplexity",
maxLength: 4096,
},
"perplexity/pplx-7b-online": {
id: "perplexity/pplx-7b-online",
name: "Perplexity: PPLX 7B Online",
organization: "Perplexity",
maxLength: 4096,
},
"perplexity/pplx-7b-chat": {
id: "perplexity/pplx-7b-chat",
name: "Perplexity: PPLX 7B Chat",
organization: "Perplexity",
maxLength: 8192,
},
"perplexity/pplx-70b-chat": {
id: "perplexity/pplx-70b-chat",
name: "Perplexity: PPLX 70B Chat",
organization: "Perplexity",
maxLength: 4096,
},
"perplexity/sonar-small-chat": {
id: "perplexity/sonar-small-chat",
name: "Perplexity: Sonar 7B",
organization: "Perplexity",
maxLength: 16384,
},
"perplexity/sonar-medium-chat": {
id: "perplexity/sonar-medium-chat",
name: "Perplexity: Sonar 8x7B",
organization: "Perplexity",
maxLength: 16384,
},
"perplexity/sonar-small-online": {
id: "perplexity/sonar-small-online",
name: "Perplexity: Sonar 7B Online",
organization: "Perplexity",
maxLength: 12000,
},
"perplexity/sonar-medium-online": {
id: "perplexity/sonar-medium-online",
name: "Perplexity: Sonar 8x7B Online",
organization: "Perplexity",
maxLength: 12000,
},
"fireworks/mixtral-8x22b-instruct-preview": {
id: "fireworks/mixtral-8x22b-instruct-preview",
name: "Fireworks Mixtral 8x22B Instruct OH (preview)",
organization: "Fireworks",
maxLength: 8192,
},
"anthropic/claude-3-opus": {
id: "anthropic/claude-3-opus",
name: "Anthropic: Claude 3 Opus",
organization: "Anthropic",
maxLength: 200000,
},
"anthropic/claude-3-sonnet": {
id: "anthropic/claude-3-sonnet",
name: "Anthropic: Claude 3 Sonnet",
organization: "Anthropic",
maxLength: 200000,
},
"anthropic/claude-3-haiku": {
id: "anthropic/claude-3-haiku",
name: "Anthropic: Claude 3 Haiku",
organization: "Anthropic",
maxLength: 200000,
},
"anthropic/claude-3-opus:beta": {
id: "anthropic/claude-3-opus:beta",
name: "Anthropic: Claude 3 Opus (self-moderated)",
organization: "Anthropic",
maxLength: 200000,
},
"anthropic/claude-3-sonnet:beta": {
id: "anthropic/claude-3-sonnet:beta",
name: "Anthropic: Claude 3 Sonnet (self-moderated)",
organization: "Anthropic",
maxLength: 200000,
},
"anthropic/claude-3-haiku:beta": {
id: "anthropic/claude-3-haiku:beta",
name: "Anthropic: Claude 3 Haiku (self-moderated)",
organization: "Anthropic",
maxLength: 200000,
},
"meta-llama/llama-2-70b-chat": {
id: "meta-llama/llama-2-70b-chat",
name: "Meta: Llama v2 70B Chat",
organization: "Meta-llama",
maxLength: 4096,
},
"nousresearch/nous-capybara-34b": {
id: "nousresearch/nous-capybara-34b",
name: "Nous: Capybara 34B",
organization: "Nousresearch",
maxLength: 32768,
},
"jondurbin/airoboros-l2-70b": {
id: "jondurbin/airoboros-l2-70b",
name: "Airoboros 70B",
organization: "Jondurbin",
maxLength: 4096,
},
"jondurbin/bagel-34b": {
id: "jondurbin/bagel-34b",
name: "Bagel 34B v0.2",
organization: "Jondurbin",
maxLength: 8000,
},
"austism/chronos-hermes-13b": {
id: "austism/chronos-hermes-13b",
name: "Chronos Hermes 13B v2",
organization: "Austism",
maxLength: 4096,
},
"mistralai/mistral-7b-instruct": {
id: "mistralai/mistral-7b-instruct",
name: "Mistral 7B Instruct",
organization: "Mistralai",
maxLength: 32768,
},
"gryphe/mythomax-l2-13b": {
id: "gryphe/mythomax-l2-13b",
name: "MythoMax 13B",
organization: "Gryphe",
maxLength: 4096,
},
"openchat/openchat-7b": {
id: "openchat/openchat-7b",
name: "OpenChat 3.5",
organization: "Openchat",
maxLength: 8192,
},
"undi95/toppy-m-7b": {
id: "undi95/toppy-m-7b",
name: "Toppy M 7B",
organization: "Undi95",
maxLength: 4096,
},
"lizpreciatior/lzlv-70b-fp16-hf": {
id: "lizpreciatior/lzlv-70b-fp16-hf",
name: "lzlv 70B",
organization: "Lizpreciatior",
maxLength: 4096,
},
"mistralai/mixtral-8x7b-instruct": {
id: "mistralai/mixtral-8x7b-instruct",
name: "Mixtral 8x7B Instruct",
organization: "Mistralai",
maxLength: 32768,
},
"cognitivecomputations/dolphin-mixtral-8x7b": {
id: "cognitivecomputations/dolphin-mixtral-8x7b",
name: "Dolphin 2.6 Mixtral 8x7B 🐬",
organization: "Cognitivecomputations",
maxLength: 32000,
},
"neversleep/noromaid-mixtral-8x7b-instruct": {
id: "neversleep/noromaid-mixtral-8x7b-instruct",
name: "Noromaid Mixtral 8x7B Instruct",
organization: "Neversleep",
maxLength: 8000,
},
"nousresearch/nous-hermes-2-mixtral-8x7b-dpo": {
id: "nousresearch/nous-hermes-2-mixtral-8x7b-dpo",
name: "Nous: Hermes 2 Mixtral 8x7B DPO",
organization: "Nousresearch",
maxLength: 32000,
},
"rwkv/rwkv-5-world-3b": {
id: "rwkv/rwkv-5-world-3b",
name: "RWKV v5 World 3B",
organization: "Rwkv",
maxLength: 10000,
},
"recursal/rwkv-5-3b-ai-town": {
id: "recursal/rwkv-5-3b-ai-town",
name: "RWKV v5 3B AI Town",
organization: "Recursal",
maxLength: 10000,
},
"recursal/eagle-7b": {
id: "recursal/eagle-7b",
name: "RWKV v5: Eagle 7B",
organization: "Recursal",
maxLength: 10000,
},
"google/gemma-7b-it": {
id: "google/gemma-7b-it",
name: "Google: Gemma 7B",
organization: "Google",
maxLength: 8192,
},
"databricks/dbrx-instruct": {
id: "databricks/dbrx-instruct",
name: "Databricks: DBRX 132B Instruct",
organization: "Databricks",
maxLength: 32768,
},
"huggingfaceh4/zephyr-orpo-141b-a35b": {
id: "huggingfaceh4/zephyr-orpo-141b-a35b",
name: "Zephyr 141B-A35B",
organization: "Huggingfaceh4",
maxLength: 65536,
},
"anthropic/claude-2": {
id: "anthropic/claude-2",
name: "Anthropic: Claude v2",
organization: "Anthropic",
maxLength: 200000,
},
"anthropic/claude-2.1": {
id: "anthropic/claude-2.1",
name: "Anthropic: Claude v2.1",
organization: "Anthropic",
maxLength: 200000,
},
"anthropic/claude-2.0": {
id: "anthropic/claude-2.0",
name: "Anthropic: Claude v2.0",
organization: "Anthropic",
maxLength: 100000,
},
"anthropic/claude-instant-1": {
id: "anthropic/claude-instant-1",
name: "Anthropic: Claude Instant v1",
organization: "Anthropic",
maxLength: 100000,
},
"anthropic/claude-instant-1.2": {
id: "anthropic/claude-instant-1.2",
name: "Anthropic: Claude Instant v1.2",
organization: "Anthropic",
maxLength: 100000,
},
"anthropic/claude-1": {
id: "anthropic/claude-1",
name: "Anthropic: Claude v1",
organization: "Anthropic",
maxLength: 100000,
},
"anthropic/claude-1.2": {
id: "anthropic/claude-1.2",
name: "Anthropic: Claude (older v1)",
organization: "Anthropic",
maxLength: 100000,
},
"anthropic/claude-instant-1.0": {
id: "anthropic/claude-instant-1.0",
name: "Anthropic: Claude Instant (older v1)",
organization: "Anthropic",
maxLength: 100000,
},
"anthropic/claude-instant-1.1": {
id: "anthropic/claude-instant-1.1",
name: "Anthropic: Claude Instant (older v1.1)",
organization: "Anthropic",
maxLength: 100000,
},
"anthropic/claude-2:beta": {
id: "anthropic/claude-2:beta",
name: "Anthropic: Claude v2 (self-moderated)",
organization: "Anthropic",
maxLength: 200000,
},
"anthropic/claude-2.1:beta": {
id: "anthropic/claude-2.1:beta",
name: "Anthropic: Claude v2.1 (self-moderated)",
organization: "Anthropic",
maxLength: 200000,
},
"anthropic/claude-2.0:beta": {
id: "anthropic/claude-2.0:beta",
name: "Anthropic: Claude v2.0 (self-moderated)",
organization: "Anthropic",
maxLength: 100000,
},
"anthropic/claude-instant-1:beta": {
id: "anthropic/claude-instant-1:beta",
name: "Anthropic: Claude Instant v1 (self-moderated)",
organization: "Anthropic",
maxLength: 100000,
},
"mistralai/mixtral-8x22b": {
id: "mistralai/mixtral-8x22b",
name: "Mistral: Mixtral 8x22B (base)",
organization: "Mistralai",
maxLength: 65536,
},
"huggingfaceh4/zephyr-7b-beta:free": {
id: "huggingfaceh4/zephyr-7b-beta:free",
name: "Hugging Face: Zephyr 7B (free)",
organization: "Huggingfaceh4",
maxLength: 4096,
},
"mistralai/mixtral-8x7b-instruct:nitro": {
id: "mistralai/mixtral-8x7b-instruct:nitro",
name: "Mixtral 8x7B Instruct (nitro)",
organization: "Mistralai",
maxLength: 32768,
},
"meta-llama/llama-2-70b-chat:nitro": {
id: "meta-llama/llama-2-70b-chat:nitro",
name: "Meta: Llama v2 70B Chat (nitro)",
organization: "Meta-llama",
maxLength: 4096,
},
"gryphe/mythomax-l2-13b:nitro": {
id: "gryphe/mythomax-l2-13b:nitro",
name: "MythoMax 13B (nitro)",
organization: "Gryphe",
maxLength: 4096,
},
"mistralai/mistral-7b-instruct:nitro": {
id: "mistralai/mistral-7b-instruct:nitro",
name: "Mistral 7B Instruct (nitro)",
organization: "Mistralai",
maxLength: 32768,
},
"google/gemma-7b-it:nitro": {
id: "google/gemma-7b-it:nitro",
name: "Google: Gemma 7B (nitro)",
organization: "Google",
maxLength: 8192,
},
"databricks/dbrx-instruct:nitro": {
id: "databricks/dbrx-instruct:nitro",
name: "Databricks: DBRX 132B Instruct (nitro)",
organization: "Databricks",
maxLength: 32768,
},
"gryphe/mythomax-l2-13b:extended": {
id: "gryphe/mythomax-l2-13b:extended",
name: "MythoMax 13B (extended)",
organization: "Gryphe",
maxLength: 8192,
},
"mistralai/mistral-tiny": {
id: "mistralai/mistral-tiny",
name: "Mistral Tiny",
organization: "Mistralai",
maxLength: 32000,
},
"mistralai/mistral-small": {
id: "mistralai/mistral-small",
name: "Mistral Small",
organization: "Mistralai",
maxLength: 32000,
},
"mistralai/mistral-medium": {
id: "mistralai/mistral-medium",
name: "Mistral Medium",
organization: "Mistralai",
maxLength: 32000,
},
"mistralai/mistral-large": {
id: "mistralai/mistral-large",
name: "Mistral Large",
organization: "Mistralai",
maxLength: 32000,
},
"cohere/command": {
id: "cohere/command",
name: "Cohere: Command",
organization: "Cohere",
maxLength: 4096,
},
"cohere/command-r": {
id: "cohere/command-r",
name: "Cohere: Command R",
organization: "Cohere",
maxLength: 128000,
},
"cohere/command-r-plus": {
id: "cohere/command-r-plus",
name: "Cohere: Command R+",
organization: "Cohere",
maxLength: 128000,
},
};
module.exports.MODELS = MODELS;

1
server/utils/AiProviders/openRouter/scripts/.gitignore vendored
View File

@ -1 +0,0 @@
*.json

37
server/utils/AiProviders/openRouter/scripts/parse.mjs
View File

@ -1,37 +0,0 @@
// OpenRouter has lots of models we can use so we use this script
// to cache all the models. We can see the list of all the models
// here: https://openrouter.ai/docs#models
// To run, cd into this directory and run `node parse.mjs`
// copy outputs into the export in ../models.js
// Update the date below if you run this again because OpenRouter added new models.
// Last Collected: Apr 14, 2024
import fs from "fs";
async function parseChatModels() {
const models = {};
const response = await fetch("https://openrouter.ai/api/v1/models");
const data = await response.json();
data.data.forEach((model) => {
models[model.id] = {
id: model.id,
name: model.name,
// capitalize first letter
organization:
model.id.split("/")[0].charAt(0).toUpperCase() +
model.id.split("/")[0].slice(1),
maxLength: model.context_length,
};
});
fs.writeFileSync(
"chat_models.json",
JSON.stringify(models, null, 2),
"utf-8"
);
return models;
}
parseChatModels();

110
server/utils/EmbeddingEngines/lmstudio/index.js Normal file
View File

@ -0,0 +1,110 @@
const { maximumChunkLength } = require("../../helpers");
class LMStudioEmbedder {
constructor() {
if (!process.env.EMBEDDING_BASE_PATH)
throw new Error("No embedding base path was set.");
if (!process.env.EMBEDDING_MODEL_PREF)
throw new Error("No embedding model was set.");
this.basePath = `${process.env.EMBEDDING_BASE_PATH}/embeddings`;
this.model = process.env.EMBEDDING_MODEL_PREF;
// Limit of how many strings we can process in a single pass to stay with resource or network limits
// Limit of how many strings we can process in a single pass to stay with resource or network limits
this.maxConcurrentChunks = 1;
this.embeddingMaxChunkLength = maximumChunkLength();
}
log(text, ...args) {
console.log(`\x1b[36m[${this.constructor.name}]\x1b[0m ${text}`, ...args);
}
async #isAlive() {
return await fetch(`${this.basePath}/models`, {
method: "HEAD",
})
.then((res) => res.ok)
.catch((e) => {
this.log(e.message);
return false;
});
}
async embedTextInput(textInput) {
const result = await this.embedChunks(textInput);
return result?.[0] || [];
}
async embedChunks(textChunks = []) {
if (!(await this.#isAlive()))
throw new Error(
`LMStudio service could not be reached. Is LMStudio running?`
);
this.log(
`Embedding ${textChunks.length} chunks of text with ${this.model}.`
);
// LMStudio will drop all queued requests now? So if there are many going on
// we need to do them sequentially or else only the first resolves and the others
// get dropped or go unanswered >:(
let results = [];
let hasError = false;
for (const chunk of textChunks) {
if (hasError) break; // If an error occurred don't continue and exit early.
results.push(
await fetch(this.basePath, {
method: "POST",
headers: {
"Content-Type": "application/json",
},
body: JSON.stringify({
model: this.model,
input: chunk,
}),
})
.then((res) => res.json())
.then((json) => {
const embedding = json.data[0].embedding;
if (!Array.isArray(embedding) || !embedding.length)
throw {
type: "EMPTY_ARR",
message: "The embedding was empty from LMStudio",
};
return { data: embedding, error: null };
})
.catch((error) => {
hasError = true;
return { data: [], error };
})
);
}
// Accumulate errors from embedding.
// If any are present throw an abort error.
const errors = results
.filter((res) => !!res.error)
.map((res) => res.error)
.flat();
if (errors.length > 0) {
let uniqueErrors = new Set();
console.log(errors);
errors.map((error) =>
uniqueErrors.add(`[${error.type}]: ${error.message}`)
);
if (errors.length > 0)
throw new Error(
`LMStudio Failed to embed: ${Array.from(uniqueErrors).join(", ")}`
);
}
const data = results.map((res) => res?.data || []);
return data.length > 0 ? data : null;
}
}
module.exports = {
LMStudioEmbedder,
};

5
server/utils/helpers/customModels.js
View File

@ -1,4 +1,4 @@
const { openRouterModels } = require("../AiProviders/openRouter");
const { OpenRouterLLM } = require("../AiProviders/openRouter");
const { perplexityModels } = require("../AiProviders/perplexity");
const { togetherAiModels } = require("../AiProviders/togetherAi");
const SUPPORT_CUSTOM_MODELS = [
@ -232,7 +232,8 @@ async function getPerplexityModels() {
}
async function getOpenRouterModels() {
const knownModels = await openRouterModels();
const openrouter = await new OpenRouterLLM().init();
const knownModels = openrouter.models();
if (!Object.keys(knownModels).length === 0)
return { models: [], error: null };

10
server/utils/helpers/index.js
View File

@ -77,8 +77,13 @@ function getLLMProvider({ provider = null, model = null } = {}) {
case "groq":
const { GroqLLM } = require("../AiProviders/groq");
return new GroqLLM(embedder, model);
case "generic-openai":
const { GenericOpenAiLLM } = require("../AiProviders/genericOpenAi");
return new GenericOpenAiLLM(embedder, model);
default:
throw new Error("ENV: No LLM_PROVIDER value found in environment!");
throw new Error(
`ENV: No valid LLM_PROVIDER value found in environment! Using ${process.env.LLM_PROVIDER}`
);
}
}
@ -102,6 +107,9 @@ function getEmbeddingEngineSelection() {
case "native":
const { NativeEmbedder } = require("../EmbeddingEngines/native");
return new NativeEmbedder();
case "lmstudio":
const { LMStudioEmbedder } = require("../EmbeddingEngines/lmstudio");
return new LMStudioEmbedder();
default:
return null;
}

28
server/utils/helpers/updateENV.js
View File

@ -132,6 +132,24 @@ const KEY_MAPPING = {
checks: [nonZero],
},
// Generic OpenAI InferenceSettings
GenericOpenAiBasePath: {
envKey: "GENERIC_OPEN_AI_BASE_PATH",
checks: [isValidURL],
},
GenericOpenAiModelPref: {
envKey: "GENERIC_OPEN_AI_MODEL_PREF",
checks: [isNotEmpty],
},
GenericOpenAiTokenLimit: {
envKey: "GENERIC_OPEN_AI_MODEL_TOKEN_LIMIT",
checks: [nonZero],
},
GenericOpenAiKey: {
envKey: "GENERIC_OPEN_AI_API_KEY",
checks: [],
},
EmbeddingEngine: {
envKey: "EMBEDDING_ENGINE",
checks: [supportedEmbeddingModel],
@ -375,6 +393,7 @@ function supportedLLM(input = "") {
"perplexity",
"openrouter",
"groq",
"generic-openai",
].includes(input);
return validSelection ? null : `${input} is not a valid LLM provider.`;
}
@ -408,7 +427,14 @@ function validAnthropicModel(input = "") {
}
function supportedEmbeddingModel(input = "") {
const supported = ["openai", "azure", "localai", "native", "ollama"];
const supported = [
"openai",
"azure",
"localai",
"native",
"ollama",
"lmstudio",
];
return supported.includes(input)
? null
: `Invalid Embedding model type. Must be one of ${supported.join(", ")}.`;