diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 00000000..450dd779 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,5 @@ +{ + "cSpell.words": [ + "openai" + ] +} \ No newline at end of file diff --git a/docker/.env.example b/docker/.env.example index e44acd02..6b9791eb 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -1,8 +1,24 @@ SERVER_PORT=3001 -OPEN_AI_KEY= -OPEN_MODEL_PREF='gpt-3.5-turbo' CACHE_VECTORS="true" +# JWT_SECRET="my-random-string-for-seeding" # Only needed if AUTH_TOKEN is set. Please generate random string at least 12 chars long. +########################################### +######## LLM API SElECTION ################ +########################################### +LLM_PROVIDER='openai' +# OPEN_AI_KEY= +OPEN_MODEL_PREF='gpt-3.5-turbo' + +# LLM_PROVIDER='azure' +# AZURE_OPENAI_ENDPOINT= +# AZURE_OPENAI_KEY= +# OPEN_MODEL_PREF='my-gpt35-deployment' # This is the "deployment" on Azure you want to use. Not the base model. +# EMBEDDING_MODEL_PREF='embedder-model' # This is the "deployment" on Azure you want to use for embeddings. Not the base model. Valid base model is text-embedding-ada-002 + + +########################################### +######## Vector Database Selection ######## +########################################### # Enable all below if you are using vector database: Chroma. # VECTOR_DB="chroma" # CHROMA_ENDPOINT='http://localhost:8000' @@ -18,7 +34,6 @@ PINECONE_INDEX= # CLOUD DEPLOYMENT VARIRABLES ONLY # AUTH_TOKEN="hunter2" # This is the password to your application if remote hosting. -# JWT_SECRET="my-random-string-for-seeding" # Only needed if AUTH_TOKEN is set. Please generate random string at least 12 chars long. # NO_DEBUG="true" STORAGE_DIR="./server/storage" GOOGLE_APIS_KEY= diff --git a/frontend/src/components/Modals/Settings/Keys/index.jsx b/frontend/src/components/Modals/Settings/Keys/index.jsx deleted file mode 100644 index 84b75373..00000000 --- a/frontend/src/components/Modals/Settings/Keys/index.jsx +++ /dev/null @@ -1,220 +0,0 @@ -import React, { useState } from "react"; -import { AlertCircle, Loader } from "react-feather"; -import System from "../../../../models/system"; - -const noop = () => false; -export default function SystemKeys({ hideModal = noop, user, settings = {} }) { - const canDebug = settings.MultiUserMode - ? settings?.CanDebug && user?.role === "admin" - : settings?.CanDebug; - function validSettings(settings) { - return ( - settings?.OpenAiKey && - !!settings?.OpenAiModelPref && - !!settings?.VectorDB && - (settings?.VectorDB === "chroma" ? !!settings?.ChromaEndpoint : true) && - (settings?.VectorDB === "pinecone" - ? !!settings?.PineConeKey && - !!settings?.PineConeEnvironment && - !!settings?.PineConeIndex - : true) - ); - } - - return ( -
-
-
-

- These are the credentials and settings for how your AnythingLLM - instance will function. Its important these keys are current and - correct. -

-
-
-
- {!validSettings(settings) && ( -
- -

- Ensure all fields are green before attempting to use - AnythingLLM or it may not function as expected! -

-
- )} - - -
-
-
- -
-
-
- ); -} - -function ShowKey({ name, env, value, valid, allowDebug = true }) { - const [isValid, setIsValid] = useState(valid); - const [debug, setDebug] = useState(false); - const [saving, setSaving] = useState(false); - const handleSubmit = async (e) => { - e.preventDefault(); - setSaving(true); - const data = {}; - const form = new FormData(e.target); - for (var [key, value] of form.entries()) data[key] = value; - const { error } = await System.updateSystem(data); - if (!!error) { - alert(error); - setSaving(false); - setIsValid(false); - return; - } - - setSaving(false); - setDebug(false); - setIsValid(true); - }; - - if (!isValid) { - return ( -
-
- - -
-

- Need setup in .env file. -

- {allowDebug && ( - <> - {debug ? ( -
- {saving ? ( - <> - - - ) : ( - <> - - - - )} -
- ) : ( - - )} - - )} -
-
-
- ); - } - - return ( -
-
- - - {allowDebug && ( -
- {debug ? ( -
- {saving ? ( - <> - - - ) : ( - <> - - - - )} -
- ) : ( - - )} -
- )} -
-
- ); -} diff --git a/frontend/src/components/Modals/Settings/LLMSelection/index.jsx b/frontend/src/components/Modals/Settings/LLMSelection/index.jsx new file mode 100644 index 00000000..94b75ace --- /dev/null +++ b/frontend/src/components/Modals/Settings/LLMSelection/index.jsx @@ -0,0 +1,281 @@ +import React, { useState } from "react"; +import System from "../../../../models/system"; +import OpenAiLogo from "../../../../media/llmprovider/openai.png"; +import AzureOpenAiLogo from "../../../../media/llmprovider/azure.png"; +import AnthropicLogo from "../../../../media/llmprovider/anthropic.png"; + +const noop = () => false; +export default function LLMSelection({ + hideModal = noop, + user, + settings = {}, +}) { + const [hasChanges, setHasChanges] = useState(false); + const [llmChoice, setLLMChoice] = useState(settings?.LLMProvider || "openai"); + const [saving, setSaving] = useState(false); + const [error, setError] = useState(null); + const canDebug = settings.MultiUserMode + ? settings?.CanDebug && user?.role === "admin" + : settings?.CanDebug; + + function updateLLMChoice(selection) { + if (!canDebug || selection === llmChoice) return false; + setHasChanges(true); + setLLMChoice(selection); + } + + const handleSubmit = async (e) => { + e.preventDefault(); + setSaving(true); + setError(null); + const data = {}; + const form = new FormData(e.target); + for (var [key, value] of form.entries()) data[key] = value; + const { error } = await System.updateSystem(data); + setError(error); + setSaving(false); + setHasChanges(!!error ? true : false); + }; + return ( +
+
+
+

+ These are the credentials and settings for your preferred LLM chat & + embedding provider. Its important these keys are current and correct + or else AnythingLLM will not function properly. +

+
+ + {!!error && ( +
+

{error}

+
+ )} + +
setHasChanges(true)}> +
+
+

+ LLM providers +

+
+ + + + +
+ {llmChoice === "openai" && ( + <> +
+ + +
+ +
+ + +
+ + )} + + {llmChoice === "azure" && ( + <> +
+ + +
+ +
+ + +
+ +
+ + +
+ +
+ + +
+ + )} + + {llmChoice === "anthropic-claude-2" && ( +
+

+ This provider is unavailable and cannot be used in + AnythingLLM currently. +

+
+ )} +
+
+
+ +
+
+
+ +
+
+
+ ); +} + +const LLMProviderOption = ({ + name, + link, + description, + value, + image, + checked = false, + onClick, +}) => { + return ( +
onClick(value)}> + + +
+ ); +}; diff --git a/frontend/src/components/Modals/Settings/VectorDbs/index.jsx b/frontend/src/components/Modals/Settings/VectorDbs/index.jsx index 0c4f5a38..c4ad0aec 100644 --- a/frontend/src/components/Modals/Settings/VectorDbs/index.jsx +++ b/frontend/src/components/Modals/Settings/VectorDbs/index.jsx @@ -57,7 +57,7 @@ export default function VectorDBSelection({

- Vector database provider + Vector database providers

@@ -96,7 +96,7 @@ export default function VectorDBSelection({ Pinecone DB API Key false; export default function SystemSettingsModal({ hideModal = noop }) { const { user } = useUser(); const [loading, setLoading] = useState(true); - const [selectedTab, setSelectedTab] = useState("keys"); + const [selectedTab, setSelectedTab] = useState("llm"); const [settings, setSettings] = useState(null); - const Component = TABS[selectedTab || "keys"]; + const Component = TABS[selectedTab || "llm"]; useEffect(() => { async function fetchKeys() { @@ -87,10 +94,10 @@ function SettingTabs({ selectedTab, changeTab, settings, user }) { return (
    } + active={selectedTab === "llm"} + displayName="LLM Choice" + tabName="llm" + icon={} onClick={changeTab} /> { try { + const llmProvider = process.env.LLM_PROVIDER || "openai"; const vectorDB = process.env.VECTOR_DB || "pinecone"; const results = { CanDebug: !!!process.env.NO_DEBUG, RequiresAuth: !!process.env.AUTH_TOKEN, - VectorDB: vectorDB, - OpenAiKey: !!process.env.OPEN_AI_KEY, - OpenAiModelPref: process.env.OPEN_MODEL_PREF || "gpt-3.5-turbo", AuthToken: !!process.env.AUTH_TOKEN, JWTSecret: !!process.env.JWT_SECRET, StorageDir: process.env.STORAGE_DIR, MultiUserMode: await SystemSettings.isMultiUserMode(), + VectorDB: vectorDB, ...(vectorDB === "pinecone" ? { PineConeEnvironment: process.env.PINECONE_ENVIRONMENT, @@ -61,6 +60,22 @@ function systemEndpoints(app) { ChromaEndpoint: process.env.CHROMA_ENDPOINT, } : {}), + LLMProvider: llmProvider, + ...(llmProvider === "openai" + ? { + OpenAiKey: !!process.env.OPEN_AI_KEY, + OpenAiModelPref: process.env.OPEN_MODEL_PREF || "gpt-3.5-turbo", + } + : {}), + + ...(llmProvider === "azure" + ? { + AzureOpenAiEndpoint: process.env.AZURE_OPENAI_ENDPOINT, + AzureOpenAiKey: !!process.env.AZURE_OPENAI_KEY, + AzureOpenAiModelPref: process.env.OPEN_MODEL_PREF, + AzureOpenAiEmbeddingModelPref: process.env.EMBEDDING_MODEL_PREF, + } + : {}), }; response.status(200).json({ results }); } catch (e) { diff --git a/server/package.json b/server/package.json index 24b84210..3d8ec2b8 100644 --- a/server/package.json +++ b/server/package.json @@ -15,6 +15,7 @@ "lint": "yarn prettier --write ./endpoints ./models ./utils index.js" }, "dependencies": { + "@azure/openai": "^1.0.0-beta.3", "@googleapis/youtube": "^9.0.0", "@pinecone-database/pinecone": "^0.1.6", "archiver": "^5.3.1", @@ -43,4 +44,4 @@ "nodemon": "^2.0.22", "prettier": "^2.4.1" } -} \ No newline at end of file +} diff --git a/server/utils/AiProviders/azureOpenAi/index.js b/server/utils/AiProviders/azureOpenAi/index.js new file mode 100644 index 00000000..8743cc8a --- /dev/null +++ b/server/utils/AiProviders/azureOpenAi/index.js @@ -0,0 +1,99 @@ +class AzureOpenAi { + constructor() { + const { OpenAIClient, AzureKeyCredential } = require("@azure/openai"); + const openai = new OpenAIClient( + process.env.AZURE_OPENAI_ENDPOINT, + new AzureKeyCredential(process.env.AZURE_OPENAI_KEY) + ); + this.openai = openai; + } + + isValidChatModel(_modelName = "") { + // The Azure user names their "models" as deployments and they can be any name + // so we rely on the user to put in the correct deployment as only they would + // know it. + return true; + } + + async isSafe(_input = "") { + // Not implemented by Azure OpenAI so must be stubbed + return { safe: true, reasons: [] }; + } + + async sendChat(chatHistory = [], prompt, workspace = {}) { + const model = process.env.OPEN_MODEL_PREF; + if (!model) + throw new Error( + "No OPEN_MODEL_PREF ENV defined. This must the name of a deployment on your Azure account for an LLM chat model like GPT-3.5." + ); + + const textResponse = await this.openai + .getChatCompletions( + model, + [ + { role: "system", content: "" }, + ...chatHistory, + { role: "user", content: prompt }, + ], + { + temperature: Number(workspace?.openAiTemp ?? 0.7), + n: 1, + } + ) + .then((res) => { + if (!res.hasOwnProperty("choices")) + throw new Error("OpenAI chat: No results!"); + if (res.choices.length === 0) + throw new Error("OpenAI chat: No results length!"); + return res.choices[0].message.content; + }) + .catch((error) => { + console.log(error); + throw new Error( + `AzureOpenAI::getChatCompletions failed with: ${error.message}` + ); + }); + return textResponse; + } + + async getChatCompletion(messages = [], { temperature = 0.7 }) { + const model = process.env.OPEN_MODEL_PREF; + if (!model) + throw new Error( + "No OPEN_MODEL_PREF ENV defined. This must the name of a deployment on your Azure account for an LLM chat model like GPT-3.5." + ); + + const data = await this.openai.getChatCompletions(model, messages, { + temperature, + }); + if (!data.hasOwnProperty("choices")) return null; + return data.choices[0].message.content; + } + + async embedTextInput(textInput) { + const result = await this.embedChunks(textInput); + return result?.[0] || []; + } + + async embedChunks(textChunks = []) { + const textEmbeddingModel = + process.env.EMBEDDING_MODEL_PREF || "text-embedding-ada-002"; + if (!textEmbeddingModel) + throw new Error( + "No EMBEDDING_MODEL_PREF ENV defined. This must the name of a deployment on your Azure account for an embedding model." + ); + + const { data = [] } = await this.openai.getEmbeddings( + textEmbeddingModel, + textChunks + ); + return data.length > 0 && + data.every((embd) => embd.hasOwnProperty("embedding")) + ? data.map((embd) => embd.embedding) + : null; + } +} + +module.exports = { + AzureOpenAi, +}; diff --git a/server/utils/AiProviders/openAi/index.js b/server/utils/AiProviders/openAi/index.js index 64b64bdc..bacc56da 100644 --- a/server/utils/AiProviders/openAi/index.js +++ b/server/utils/AiProviders/openAi/index.js @@ -1,7 +1,6 @@ -const { Configuration, OpenAIApi } = require("openai"); - class OpenAi { constructor() { + const { Configuration, OpenAIApi } = require("openai"); const config = new Configuration({ apiKey: process.env.OPEN_AI_KEY, }); diff --git a/server/utils/chats/index.js b/server/utils/chats/index.js index b6f74c15..a3a96bc2 100644 --- a/server/utils/chats/index.js +++ b/server/utils/chats/index.js @@ -3,7 +3,8 @@ const { OpenAi } = require("../AiProviders/openAi"); const { WorkspaceChats } = require("../../models/workspaceChats"); const { resetMemory } = require("./commands/reset"); const moment = require("moment"); -const { getVectorDbClass } = require("../helpers"); +const { getVectorDbClass, getLLMProvider } = require("../helpers"); +const { AzureOpenAi } = require("../AiProviders/azureOpenAi"); function convertToChatHistory(history = []) { const formattedHistory = []; @@ -66,7 +67,7 @@ async function chatWithWorkspace( user = null ) { const uuid = uuidv4(); - const openai = new OpenAi(); + const LLMConnector = getLLMProvider(); const VectorDb = getVectorDbClass(); const command = grepCommand(message); @@ -74,7 +75,7 @@ async function chatWithWorkspace( return await VALID_COMMANDS[command](workspace, message, uuid, user); } - const { safe, reasons = [] } = await openai.isSafe(message); + const { safe, reasons = [] } = await LLMConnector.isSafe(message); if (!safe) { return { id: uuid, @@ -93,7 +94,11 @@ async function chatWithWorkspace( if (!hasVectorizedSpace || embeddingsCount === 0) { const rawHistory = await WorkspaceChats.forWorkspace(workspace.id); const chatHistory = convertToPromptHistory(rawHistory); - const response = await openai.sendChat(chatHistory, message, workspace); + const response = await LLMConnector.sendChat( + chatHistory, + message, + workspace + ); const data = { text: response, sources: [], type: "chat" }; await WorkspaceChats.new({ diff --git a/server/utils/helpers/index.js b/server/utils/helpers/index.js index 1a5aac86..5be56507 100644 --- a/server/utils/helpers/index.js +++ b/server/utils/helpers/index.js @@ -1,21 +1,34 @@ function getVectorDbClass() { - const { Pinecone } = require("../vectorDbProviders/pinecone"); - const { Chroma } = require("../vectorDbProviders/chroma"); - const { LanceDb } = require("../vectorDbProviders/lance"); - const vectorSelection = process.env.VECTOR_DB || "pinecone"; switch (vectorSelection) { case "pinecone": + const { Pinecone } = require("../vectorDbProviders/pinecone"); return Pinecone; case "chroma": + const { Chroma } = require("../vectorDbProviders/chroma"); return Chroma; case "lancedb": + const { LanceDb } = require("../vectorDbProviders/lance"); return LanceDb; default: throw new Error("ENV: No VECTOR_DB value found in environment!"); } } +function getLLMProvider() { + const vectorSelection = process.env.LLM_PROVIDER || "openai"; + switch (vectorSelection) { + case "openai": + const { OpenAi } = require("../AiProviders/openAi"); + return new OpenAi(); + case "azure": + const { AzureOpenAi } = require("../AiProviders/azureOpenAi"); + return new AzureOpenAi(); + default: + throw new Error("ENV: No LLM_PROVIDER value found in environment!"); + } +} + function toChunks(arr, size) { return Array.from({ length: Math.ceil(arr.length / size) }, (_v, i) => arr.slice(i * size, i * size + size) @@ -24,5 +37,6 @@ function toChunks(arr, size) { module.exports = { getVectorDbClass, + getLLMProvider, toChunks, }; diff --git a/server/utils/helpers/updateENV.js b/server/utils/helpers/updateENV.js index 0ff95fa4..64c91988 100644 --- a/server/utils/helpers/updateENV.js +++ b/server/utils/helpers/updateENV.js @@ -1,4 +1,9 @@ const KEY_MAPPING = { + LLMProvider: { + envKey: "LLM_PROVIDER", + checks: [isNotEmpty, supportedLLM], + }, + // OpenAI Settings OpenAiKey: { envKey: "OPEN_AI_KEY", checks: [isNotEmpty, validOpenAIKey], @@ -7,6 +12,25 @@ const KEY_MAPPING = { envKey: "OPEN_MODEL_PREF", checks: [isNotEmpty, validOpenAIModel], }, + // Azure OpenAI Settings + AzureOpenAiEndpoint: { + envKey: "AZURE_OPENAI_ENDPOINT", + checks: [isNotEmpty, validAzureURL], + }, + AzureOpenAiKey: { + envKey: "AZURE_OPENAI_KEY", + checks: [isNotEmpty], + }, + AzureOpenAiModelPref: { + envKey: "OPEN_MODEL_PREF", + checks: [isNotEmpty], + }, + AzureOpenAiEmbeddingModelPref: { + envKey: "EMBEDDING_MODEL_PREF", + checks: [isNotEmpty], + }, + + // Vector Database Selection Settings VectorDB: { envKey: "VECTOR_DB", checks: [isNotEmpty, supportedVectorDB], @@ -27,6 +51,8 @@ const KEY_MAPPING = { envKey: "PINECONE_INDEX", checks: [], }, + + // System Settings AuthToken: { envKey: "AUTH_TOKEN", checks: [], @@ -56,6 +82,10 @@ function validOpenAIKey(input = "") { return input.startsWith("sk-") ? null : "OpenAI Key must start with sk-"; } +function supportedLLM(input = "") { + return ["openai", "azure"].includes(input); +} + function validOpenAIModel(input = "") { const validModels = [ "gpt-4", @@ -85,6 +115,17 @@ function validChromaURL(input = "") { : null; } +function validAzureURL(input = "") { + try { + new URL(input); + if (!input.includes("openai.azure.com")) + return "URL must include openai.azure.com"; + return null; + } catch { + return "Not a valid URL"; + } +} + // This will force update .env variables which for any which reason were not able to be parsed or // read from an ENV file as this seems to be a complicating step for many so allowing people to write // to the process will at least alleviate that issue. It does not perform comprehensive validity checks or sanity checks diff --git a/server/utils/vectorDbProviders/chroma/index.js b/server/utils/vectorDbProviders/chroma/index.js index ee8ac485..4a527ac7 100644 --- a/server/utils/vectorDbProviders/chroma/index.js +++ b/server/utils/vectorDbProviders/chroma/index.js @@ -1,14 +1,9 @@ -const { ChromaClient, OpenAIEmbeddingFunction } = require("chromadb"); -const { Chroma: ChromaStore } = require("langchain/vectorstores/chroma"); -const { OpenAI } = require("langchain/llms/openai"); -const { VectorDBQAChain } = require("langchain/chains"); -const { OpenAIEmbeddings } = require("langchain/embeddings/openai"); +const { ChromaClient } = require("chromadb"); const { RecursiveCharacterTextSplitter } = require("langchain/text_splitter"); const { storeVectorResult, cachedVectorInformation } = require("../../files"); const { v4: uuidv4 } = require("uuid"); -const { toChunks } = require("../../helpers"); +const { toChunks, getLLMProvider } = require("../../helpers"); const { chatPrompt } = require("../../chats"); -const { OpenAi } = require("../../AiProviders/openAi"); const Chroma = { name: "Chroma", @@ -49,22 +44,6 @@ const Chroma = { const namespace = await this.namespace(client, _namespace); return namespace?.vectorCount || 0; }, - embeddingFunc: function () { - return new OpenAIEmbeddingFunction({ - openai_api_key: process.env.OPEN_AI_KEY, - }); - }, - embedder: function () { - return new OpenAIEmbeddings({ openAIApiKey: process.env.OPEN_AI_KEY }); - }, - llm: function ({ temperature = 0.7 }) { - const model = process.env.OPEN_MODEL_PREF || "gpt-3.5-turbo"; - return new OpenAI({ - openAIApiKey: process.env.OPEN_AI_KEY, - modelName: model, - temperature, - }); - }, similarityResponse: async function (client, namespace, queryVector) { const collection = await client.getCollection({ name: namespace }); const result = { @@ -131,7 +110,6 @@ const Chroma = { const collection = await client.getOrCreateCollection({ name: namespace, metadata: { "hnsw:space": "cosine" }, - embeddingFunction: this.embeddingFunc(), }); const { chunks } = cacheResult; const documentVectors = []; @@ -176,10 +154,10 @@ const Chroma = { const textChunks = await textSplitter.splitText(pageContent); console.log("Chunks created from document:", textChunks.length); - const openAiConnector = new OpenAi(); + const LLMConnector = getLLMProvider(); const documentVectors = []; const vectors = []; - const vectorValues = await openAiConnector.embedChunks(textChunks); + const vectorValues = await LLMConnector.embedChunks(textChunks); const submission = { ids: [], embeddings: [], @@ -216,7 +194,6 @@ const Chroma = { const collection = await client.getOrCreateCollection({ name: namespace, metadata: { "hnsw:space": "cosine" }, - embeddingFunction: this.embeddingFunc(), }); if (vectors.length > 0) { @@ -245,7 +222,6 @@ const Chroma = { if (!(await this.namespaceExists(client, namespace))) return; const collection = await client.getCollection({ name: namespace, - embeddingFunction: this.embeddingFunc(), }); const knownDocuments = await DocumentVectors.where(`docId = '${docId}'`); @@ -271,22 +247,36 @@ const Chroma = { }; } - const vectorStore = await ChromaStore.fromExistingCollection( - this.embedder(), - { collectionName: namespace, url: process.env.CHROMA_ENDPOINT } + const LLMConnector = getLLMProvider(); + const queryVector = await LLMConnector.embedTextInput(input); + const { contextTexts, sourceDocuments } = await this.similarityResponse( + client, + namespace, + queryVector ); - const model = this.llm({ + const prompt = { + role: "system", + content: `${chatPrompt(workspace)} + Context: + ${contextTexts + .map((text, i) => { + return `[CONTEXT ${i}]:\n${text}\n[END CONTEXT ${i}]\n\n`; + }) + .join("")}`, + }; + const memory = [prompt, { role: "user", content: input }]; + const responseText = await LLMConnector.getChatCompletion(memory, { temperature: workspace?.openAiTemp ?? 0.7, }); - const chain = VectorDBQAChain.fromLLM(model, vectorStore, { - k: 5, - returnSourceDocuments: true, + // When we roll out own response we have separate metadata and texts, + // so for source collection we need to combine them. + const sources = sourceDocuments.map((metadata, i) => { + return { metadata: { ...metadata, text: contextTexts[i] } }; }); - const response = await chain.call({ query: input }); return { - response: response.text, - sources: this.curateSources(response.sourceDocuments), + response: responseText, + sources: this.curateSources(sources), message: false, }; }, @@ -312,8 +302,8 @@ const Chroma = { }; } - const openAiConnector = new OpenAi(); - const queryVector = await openAiConnector.embedTextInput(input); + const LLMConnector = getLLMProvider(); + const queryVector = await LLMConnector.embedTextInput(input); const { contextTexts, sourceDocuments } = await this.similarityResponse( client, namespace, @@ -330,7 +320,7 @@ const Chroma = { .join("")}`, }; const memory = [prompt, ...chatHistory, { role: "user", content: input }]; - const responseText = await openAiConnector.getChatCompletion(memory, { + const responseText = await LLMConnector.getChatCompletion(memory, { temperature: workspace?.openAiTemp ?? 0.7, }); diff --git a/server/utils/vectorDbProviders/lance/index.js b/server/utils/vectorDbProviders/lance/index.js index 38b30af9..aeb33534 100644 --- a/server/utils/vectorDbProviders/lance/index.js +++ b/server/utils/vectorDbProviders/lance/index.js @@ -1,11 +1,10 @@ const lancedb = require("vectordb"); -const { toChunks } = require("../../helpers"); +const { toChunks, getLLMProvider } = require("../../helpers"); const { OpenAIEmbeddings } = require("langchain/embeddings/openai"); const { RecursiveCharacterTextSplitter } = require("langchain/text_splitter"); const { storeVectorResult, cachedVectorInformation } = require("../../files"); const { v4: uuidv4 } = require("uuid"); const { chatPrompt } = require("../../chats"); -const { OpenAi } = require("../../AiProviders/openAi"); const LanceDb = { uri: `${ @@ -169,11 +168,11 @@ const LanceDb = { const textChunks = await textSplitter.splitText(pageContent); console.log("Chunks created from document:", textChunks.length); - const openAiConnector = new OpenAi(); + const LLMConnector = getLLMProvider(); const documentVectors = []; const vectors = []; const submissions = []; - const vectorValues = await openAiConnector.embedChunks(textChunks); + const vectorValues = await LLMConnector.embedChunks(textChunks); if (!!vectorValues && vectorValues.length > 0) { for (const [i, vector] of vectorValues.entries()) { @@ -230,9 +229,8 @@ const LanceDb = { }; } - // LanceDB does not have langchainJS support so we roll our own here. - const openAiConnector = new OpenAi(); - const queryVector = await openAiConnector.embedTextInput(input); + const LLMConnector = getLLMProvider(); + const queryVector = await LLMConnector.embedTextInput(input); const { contextTexts, sourceDocuments } = await this.similarityResponse( client, namespace, @@ -249,7 +247,7 @@ const LanceDb = { .join("")}`, }; const memory = [prompt, { role: "user", content: input }]; - const responseText = await openAiConnector.getChatCompletion(memory, { + const responseText = await LLMConnector.getChatCompletion(memory, { temperature: workspace?.openAiTemp ?? 0.7, }); @@ -281,8 +279,8 @@ const LanceDb = { }; } - const openAiConnector = new OpenAi(); - const queryVector = await openAiConnector.embedTextInput(input); + const LLMConnector = getLLMProvider(); + const queryVector = await LLMConnector.embedTextInput(input); const { contextTexts, sourceDocuments } = await this.similarityResponse( client, namespace, @@ -299,7 +297,7 @@ const LanceDb = { .join("")}`, }; const memory = [prompt, ...chatHistory, { role: "user", content: input }]; - const responseText = await openAiConnector.getChatCompletion(memory, { + const responseText = await LLMConnector.getChatCompletion(memory, { temperature: workspace?.openAiTemp ?? 0.7, }); diff --git a/server/utils/vectorDbProviders/pinecone/index.js b/server/utils/vectorDbProviders/pinecone/index.js index 6021cfa3..91d97578 100644 --- a/server/utils/vectorDbProviders/pinecone/index.js +++ b/server/utils/vectorDbProviders/pinecone/index.js @@ -1,14 +1,9 @@ const { PineconeClient } = require("@pinecone-database/pinecone"); -const { PineconeStore } = require("langchain/vectorstores/pinecone"); -const { OpenAI } = require("langchain/llms/openai"); -const { VectorDBQAChain } = require("langchain/chains"); -const { OpenAIEmbeddings } = require("langchain/embeddings/openai"); const { RecursiveCharacterTextSplitter } = require("langchain/text_splitter"); const { storeVectorResult, cachedVectorInformation } = require("../../files"); const { v4: uuidv4 } = require("uuid"); -const { toChunks } = require("../../helpers"); +const { toChunks, getLLMProvider } = require("../../helpers"); const { chatPrompt } = require("../../chats"); -const { OpenAi } = require("../../AiProviders/openAi"); const Pinecone = { name: "Pinecone", @@ -29,17 +24,6 @@ const Pinecone = { if (!status.ready) throw new Error("Pinecode::Index not ready."); return { client, pineconeIndex, indexName: process.env.PINECONE_INDEX }; }, - embedder: function () { - return new OpenAIEmbeddings({ openAIApiKey: process.env.OPEN_AI_KEY }); - }, - llm: function ({ temperature = 0.7 }) { - const model = process.env.OPEN_MODEL_PREF || "gpt-3.5-turbo"; - return new OpenAI({ - openAIApiKey: process.env.OPEN_AI_KEY, - modelName: model, - temperature, - }); - }, totalIndicies: async function () { const { pineconeIndex } = await this.connect(); const { namespaces } = await pineconeIndex.describeIndexStats1(); @@ -144,10 +128,10 @@ const Pinecone = { const textChunks = await textSplitter.splitText(pageContent); console.log("Chunks created from document:", textChunks.length); - const openAiConnector = new OpenAi(); + const LLMConnector = getLLMProvider(); const documentVectors = []; const vectors = []; - const vectorValues = await openAiConnector.embedChunks(textChunks); + const vectorValues = await LLMConnector.embedChunks(textChunks); if (!!vectorValues && vectorValues.length > 0) { for (const [i, vector] of vectorValues.entries()) { @@ -246,22 +230,32 @@ const Pinecone = { }; } - const vectorStore = await PineconeStore.fromExistingIndex(this.embedder(), { + const LLMConnector = getLLMProvider(); + const queryVector = await LLMConnector.embedTextInput(input); + const { contextTexts, sourceDocuments } = await this.similarityResponse( pineconeIndex, namespace, - }); + queryVector + ); + const prompt = { + role: "system", + content: `${chatPrompt(workspace)} + Context: + ${contextTexts + .map((text, i) => { + return `[CONTEXT ${i}]:\n${text}\n[END CONTEXT ${i}]\n\n`; + }) + .join("")}`, + }; - const model = this.llm({ + const memory = [prompt, { role: "user", content: input }]; + const responseText = await LLMConnector.getChatCompletion(memory, { temperature: workspace?.openAiTemp ?? 0.7, }); - const chain = VectorDBQAChain.fromLLM(model, vectorStore, { - k: 5, - returnSourceDocuments: true, - }); - const response = await chain.call({ query: input }); + return { - response: response.text, - sources: this.curateSources(response.sourceDocuments), + response: responseText, + sources: this.curateSources(sourceDocuments), message: false, }; }, @@ -284,8 +278,8 @@ const Pinecone = { "Invalid namespace - has it been collected and seeded yet?" ); - const openAiConnector = new OpenAi(); - const queryVector = await openAiConnector.embedTextInput(input); + const LLMConnector = getLLMProvider(); + const queryVector = await LLMConnector.embedTextInput(input); const { contextTexts, sourceDocuments } = await this.similarityResponse( pineconeIndex, namespace, @@ -303,7 +297,7 @@ const Pinecone = { }; const memory = [prompt, ...chatHistory, { role: "user", content: input }]; - const responseText = await openAiConnector.getChatCompletion(memory, { + const responseText = await LLMConnector.getChatCompletion(memory, { temperature: workspace?.openAiTemp ?? 0.7, }); diff --git a/server/yarn.lock b/server/yarn.lock index 1a82497e..cd1514e7 100644 --- a/server/yarn.lock +++ b/server/yarn.lock @@ -26,6 +26,93 @@ pad-left "^2.1.0" tslib "^2.5.0" +"@azure-rest/core-client@^1.1.3": + version "1.1.4" + resolved "https://registry.yarnpkg.com/@azure-rest/core-client/-/core-client-1.1.4.tgz#628381c3653f6dbae584ca6f2ae5f74a5c015526" + integrity sha512-RUIQOA8T0WcbNlddr8hjl2MuC5GVRqmMwPXqBVsgvdKesLy+eg3y/6nf3qe2fvcJMI1gF6VtgU5U4hRaR4w4ag== + dependencies: + "@azure/abort-controller" "^1.1.0" + "@azure/core-auth" "^1.3.0" + "@azure/core-rest-pipeline" "^1.5.0" + "@azure/core-tracing" "^1.0.1" + "@azure/core-util" "^1.0.0" + tslib "^2.2.0" + +"@azure/abort-controller@^1.0.0", "@azure/abort-controller@^1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@azure/abort-controller/-/abort-controller-1.1.0.tgz#788ee78457a55af8a1ad342acb182383d2119249" + integrity sha512-TrRLIoSQVzfAJX9H1JeFjzAoDGcoK1IYX1UImfceTZpsyYfWr09Ss1aHW1y5TrrR3iq6RZLBwJ3E24uwPhwahw== + dependencies: + tslib "^2.2.0" + +"@azure/core-auth@^1.3.0", "@azure/core-auth@^1.4.0": + version "1.5.0" + resolved "https://registry.yarnpkg.com/@azure/core-auth/-/core-auth-1.5.0.tgz#a41848c5c31cb3b7c84c409885267d55a2c92e44" + integrity sha512-udzoBuYG1VBoHVohDTrvKjyzel34zt77Bhp7dQntVGGD0ehVq48owENbBG8fIgkHRNUBQH5k1r0hpoMu5L8+kw== + dependencies: + "@azure/abort-controller" "^1.0.0" + "@azure/core-util" "^1.1.0" + tslib "^2.2.0" + +"@azure/core-lro@^2.5.3": + version "2.5.4" + resolved "https://registry.yarnpkg.com/@azure/core-lro/-/core-lro-2.5.4.tgz#b21e2bcb8bd9a8a652ff85b61adeea51a8055f90" + integrity sha512-3GJiMVH7/10bulzOKGrrLeG/uCBH/9VtxqaMcB9lIqAeamI/xYQSHJL/KcsLDuH+yTjYpro/u6D/MuRe4dN70Q== + dependencies: + "@azure/abort-controller" "^1.0.0" + "@azure/core-util" "^1.2.0" + "@azure/logger" "^1.0.0" + tslib "^2.2.0" + +"@azure/core-rest-pipeline@^1.10.2", "@azure/core-rest-pipeline@^1.5.0": + version "1.12.0" + resolved "https://registry.yarnpkg.com/@azure/core-rest-pipeline/-/core-rest-pipeline-1.12.0.tgz#a36dd361807494845522824532c076daa27c2786" + integrity sha512-+MnSB0vGZjszSzr5AW8z93/9fkDu2RLtWmAN8gskURq7EW2sSwqy8jZa0V26rjuBVkwhdA3Hw8z3VWoeBUOw+A== + dependencies: + "@azure/abort-controller" "^1.0.0" + "@azure/core-auth" "^1.4.0" + "@azure/core-tracing" "^1.0.1" + "@azure/core-util" "^1.3.0" + "@azure/logger" "^1.0.0" + form-data "^4.0.0" + http-proxy-agent "^5.0.0" + https-proxy-agent "^5.0.0" + tslib "^2.2.0" + +"@azure/core-tracing@^1.0.1": + version "1.0.1" + resolved "https://registry.yarnpkg.com/@azure/core-tracing/-/core-tracing-1.0.1.tgz#352a38cbea438c4a83c86b314f48017d70ba9503" + integrity sha512-I5CGMoLtX+pI17ZdiFJZgxMJApsK6jjfm85hpgp3oazCdq5Wxgh4wMr7ge/TTWW1B5WBuvIOI1fMU/FrOAMKrw== + dependencies: + tslib "^2.2.0" + +"@azure/core-util@^1.0.0", "@azure/core-util@^1.1.0", "@azure/core-util@^1.2.0", "@azure/core-util@^1.3.0": + version "1.4.0" + resolved "https://registry.yarnpkg.com/@azure/core-util/-/core-util-1.4.0.tgz#c120a56b3e48a9e4d20619a0b00268ae9de891c7" + integrity sha512-eGAyJpm3skVQoLiRqm/xPa+SXi/NPDdSHMxbRAz2lSprd+Zs+qrpQGQQ2VQ3Nttu+nSZR4XoYQC71LbEI7jsig== + dependencies: + "@azure/abort-controller" "^1.0.0" + tslib "^2.2.0" + +"@azure/logger@^1.0.0", "@azure/logger@^1.0.3": + version "1.0.4" + resolved "https://registry.yarnpkg.com/@azure/logger/-/logger-1.0.4.tgz#28bc6d0e5b3c38ef29296b32d35da4e483593fa1" + integrity sha512-ustrPY8MryhloQj7OWGe+HrYx+aoiOxzbXTtgblbV3xwCqpzUK36phH3XNHQKj3EPonyFUuDTfR3qFhTEAuZEg== + dependencies: + tslib "^2.2.0" + +"@azure/openai@^1.0.0-beta.3": + version "1.0.0-beta.3" + resolved "https://registry.yarnpkg.com/@azure/openai/-/openai-1.0.0-beta.3.tgz#bf4f5ec0a5644b3a9ce4372620856a65e7721e24" + integrity sha512-gW4odbuy/X/W34SdvXomj/JzR09MyMHCY5Kd2ZxJkQo3IUGqJXz1rEv6QER7IAGgBFgNawE97K6UuJfMmoT0rw== + dependencies: + "@azure-rest/core-client" "^1.1.3" + "@azure/core-auth" "^1.4.0" + "@azure/core-lro" "^2.5.3" + "@azure/core-rest-pipeline" "^1.10.2" + "@azure/logger" "^1.0.3" + tslib "^2.4.0" + "@fortaine/fetch-event-source@^3.0.6": version "3.0.6" resolved "https://registry.yarnpkg.com/@fortaine/fetch-event-source/-/fetch-event-source-3.0.6.tgz#b8552a2ca2c5202f5699b93a92be0188d422b06e" @@ -86,6 +173,11 @@ resolved "https://registry.yarnpkg.com/@tootallnate/once/-/once-1.1.2.tgz#ccb91445360179a04e7fe6aff78c00ffc1eeaf82" integrity sha512-RbzJvlNzmRq5c3O09UipeuXno4tA1FE6ikOjxZK0tuxVv3412l64l5t1W5pj4+rJq9vpkm/kwiR07aZXnsKPxw== +"@tootallnate/once@2": + version "2.0.0" + resolved "https://registry.yarnpkg.com/@tootallnate/once/-/once-2.0.0.tgz#f544a148d3ab35801c1f633a7441fd87c2e484bf" + integrity sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A== + "@types/command-line-args@5.2.0": version "5.2.0" resolved "https://registry.yarnpkg.com/@types/command-line-args/-/command-line-args-5.2.0.tgz#adbb77980a1cc376bb208e3f4142e907410430f6" @@ -1128,6 +1220,15 @@ http-proxy-agent@^4.0.1: agent-base "6" debug "4" +http-proxy-agent@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/http-proxy-agent/-/http-proxy-agent-5.0.0.tgz#5129800203520d434f142bc78ff3c170800f2b43" + integrity sha512-n2hY8YdoRE1i7r6M0w9DIw5GgZN0G25P8zLCRQ8rjXtTU3vsNFBI/vWK/UIeE6g5MUUz6avwAPXmL6Fy9D/90w== + dependencies: + "@tootallnate/once" "2" + agent-base "6" + debug "4" + https-proxy-agent@^5.0.0: version "5.0.1" resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz#c59ef224a04fe8b754f3db0063a25ea30d0005d6" @@ -2301,6 +2402,11 @@ tr46@~0.0.3: resolved "https://registry.yarnpkg.com/tr46/-/tr46-0.0.3.tgz#8184fd347dac9cdc185992f3a6622e14b9d9ab6a" integrity sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw== +tslib@^2.2.0, tslib@^2.4.0: + version "2.6.1" + resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.6.1.tgz#fd8c9a0ff42590b25703c0acb3de3d3f4ede0410" + integrity sha512-t0hLfiEKfMUoqhG+U1oid7Pva4bbDPHYfJNiB7BiIjRkj1pyC++4N3huJfqY6aRH6VTB0rvtzQwjM4K6qpfOig== + tslib@^2.5.0: version "2.6.0" resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.6.0.tgz#b295854684dbda164e181d259a22cd779dcd7bc3"