diff --git a/server/utils/helpers/index.js b/server/utils/helpers/index.js index dde8d7ab..d9a1ba09 100644 --- a/server/utils/helpers/index.js +++ b/server/utils/helpers/index.js @@ -100,6 +100,7 @@ function getLLMProvider({ provider = null, model = null } = {}) { } function getEmbeddingEngineSelection() { + const { NativeEmbedder } = require("../EmbeddingEngines/native"); const engineSelection = process.env.EMBEDDING_ENGINE; switch (engineSelection) { case "openai": @@ -117,7 +118,6 @@ function getEmbeddingEngineSelection() { const { OllamaEmbedder } = require("../EmbeddingEngines/ollama"); return new OllamaEmbedder(); case "native": - const { NativeEmbedder } = require("../EmbeddingEngines/native"); return new NativeEmbedder(); case "lmstudio": const { LMStudioEmbedder } = require("../EmbeddingEngines/lmstudio"); @@ -126,7 +126,7 @@ function getEmbeddingEngineSelection() { const { CohereEmbedder } = require("../EmbeddingEngines/cohere"); return new CohereEmbedder(); default: - return null; + return new NativeEmbedder(); } } diff --git a/server/utils/vectorDbProviders/astra/index.js b/server/utils/vectorDbProviders/astra/index.js index 5f0b086f..50e8ba34 100644 --- a/server/utils/vectorDbProviders/astra/index.js +++ b/server/utils/vectorDbProviders/astra/index.js @@ -3,11 +3,7 @@ const { TextSplitter } = require("../../TextSplitter"); const { SystemSettings } = require("../../../models/systemSettings"); const { storeVectorResult, cachedVectorInformation } = require("../../files"); const { v4: uuidv4 } = require("uuid"); -const { - toChunks, - getLLMProvider, - getEmbeddingEngineSelection, -} = require("../../helpers"); +const { toChunks, getEmbeddingEngineSelection } = require("../../helpers"); const { sourceIdentifier } = require("../../chats"); const AstraDB = { @@ -149,12 +145,13 @@ const AstraDB = { return { vectorized: true, error: null }; } + const EmbedderEngine = getEmbeddingEngineSelection(); const textSplitter = new TextSplitter({ chunkSize: TextSplitter.determineMaxChunkSize( await SystemSettings.getValueOrFallback({ label: "text_splitter_chunk_size", }), - getEmbeddingEngineSelection()?.embeddingMaxChunkLength + EmbedderEngine?.embeddingMaxChunkLength ), chunkOverlap: await SystemSettings.getValueOrFallback( { label: "text_splitter_chunk_overlap" }, @@ -164,10 +161,9 @@ const AstraDB = { const textChunks = await textSplitter.splitText(pageContent); console.log("Chunks created from document:", textChunks.length); - const LLMConnector = getLLMProvider(); const documentVectors = []; const vectors = []; - const vectorValues = await LLMConnector.embedChunks(textChunks); + const vectorValues = await EmbedderEngine.embedChunks(textChunks); if (!!vectorValues && vectorValues.length > 0) { for (const [i, vector] of vectorValues.entries()) { diff --git a/server/utils/vectorDbProviders/chroma/index.js b/server/utils/vectorDbProviders/chroma/index.js index d87b3aad..d17883b7 100644 --- a/server/utils/vectorDbProviders/chroma/index.js +++ b/server/utils/vectorDbProviders/chroma/index.js @@ -3,11 +3,7 @@ const { TextSplitter } = require("../../TextSplitter"); const { SystemSettings } = require("../../../models/systemSettings"); const { storeVectorResult, cachedVectorInformation } = require("../../files"); const { v4: uuidv4 } = require("uuid"); -const { - toChunks, - getLLMProvider, - getEmbeddingEngineSelection, -} = require("../../helpers"); +const { toChunks, getEmbeddingEngineSelection } = require("../../helpers"); const { parseAuthHeader } = require("../../http"); const { sourceIdentifier } = require("../../chats"); @@ -192,12 +188,13 @@ const Chroma = { // We have to do this manually as opposed to using LangChains `Chroma.fromDocuments` // because we then cannot atomically control our namespace to granularly find/remove documents // from vectordb. + const EmbedderEngine = getEmbeddingEngineSelection(); const textSplitter = new TextSplitter({ chunkSize: TextSplitter.determineMaxChunkSize( await SystemSettings.getValueOrFallback({ label: "text_splitter_chunk_size", }), - getEmbeddingEngineSelection()?.embeddingMaxChunkLength + EmbedderEngine?.embeddingMaxChunkLength ), chunkOverlap: await SystemSettings.getValueOrFallback( { label: "text_splitter_chunk_overlap" }, @@ -207,10 +204,9 @@ const Chroma = { const textChunks = await textSplitter.splitText(pageContent); console.log("Chunks created from document:", textChunks.length); - const LLMConnector = getLLMProvider(); const documentVectors = []; const vectors = []; - const vectorValues = await LLMConnector.embedChunks(textChunks); + const vectorValues = await EmbedderEngine.embedChunks(textChunks); const submission = { ids: [], embeddings: [], diff --git a/server/utils/vectorDbProviders/lance/index.js b/server/utils/vectorDbProviders/lance/index.js index 7efb6aa0..db266295 100644 --- a/server/utils/vectorDbProviders/lance/index.js +++ b/server/utils/vectorDbProviders/lance/index.js @@ -1,9 +1,5 @@ const lancedb = require("vectordb"); -const { - toChunks, - getLLMProvider, - getEmbeddingEngineSelection, -} = require("../../helpers"); +const { toChunks, getEmbeddingEngineSelection } = require("../../helpers"); const { TextSplitter } = require("../../TextSplitter"); const { SystemSettings } = require("../../../models/systemSettings"); const { storeVectorResult, cachedVectorInformation } = require("../../files"); @@ -190,12 +186,13 @@ const LanceDb = { // We have to do this manually as opposed to using LangChains `xyz.fromDocuments` // because we then cannot atomically control our namespace to granularly find/remove documents // from vectordb. + const EmbedderEngine = getEmbeddingEngineSelection(); const textSplitter = new TextSplitter({ chunkSize: TextSplitter.determineMaxChunkSize( await SystemSettings.getValueOrFallback({ label: "text_splitter_chunk_size", }), - getEmbeddingEngineSelection()?.embeddingMaxChunkLength + EmbedderEngine?.embeddingMaxChunkLength ), chunkOverlap: await SystemSettings.getValueOrFallback( { label: "text_splitter_chunk_overlap" }, @@ -205,11 +202,10 @@ const LanceDb = { const textChunks = await textSplitter.splitText(pageContent); console.log("Chunks created from document:", textChunks.length); - const LLMConnector = getLLMProvider(); const documentVectors = []; const vectors = []; const submissions = []; - const vectorValues = await LLMConnector.embedChunks(textChunks); + const vectorValues = await EmbedderEngine.embedChunks(textChunks); if (!!vectorValues && vectorValues.length > 0) { for (const [i, vector] of vectorValues.entries()) { diff --git a/server/utils/vectorDbProviders/milvus/index.js b/server/utils/vectorDbProviders/milvus/index.js index c4c91c22..27309233 100644 --- a/server/utils/vectorDbProviders/milvus/index.js +++ b/server/utils/vectorDbProviders/milvus/index.js @@ -8,11 +8,7 @@ const { TextSplitter } = require("../../TextSplitter"); const { SystemSettings } = require("../../../models/systemSettings"); const { v4: uuidv4 } = require("uuid"); const { storeVectorResult, cachedVectorInformation } = require("../../files"); -const { - toChunks, - getLLMProvider, - getEmbeddingEngineSelection, -} = require("../../helpers"); +const { toChunks, getEmbeddingEngineSelection } = require("../../helpers"); const { sourceIdentifier } = require("../../chats"); const Milvus = { @@ -184,12 +180,13 @@ const Milvus = { return { vectorized: true, error: null }; } + const EmbedderEngine = getEmbeddingEngineSelection(); const textSplitter = new TextSplitter({ chunkSize: TextSplitter.determineMaxChunkSize( await SystemSettings.getValueOrFallback({ label: "text_splitter_chunk_size", }), - getEmbeddingEngineSelection()?.embeddingMaxChunkLength + EmbedderEngine?.embeddingMaxChunkLength ), chunkOverlap: await SystemSettings.getValueOrFallback( { label: "text_splitter_chunk_overlap" }, @@ -199,10 +196,9 @@ const Milvus = { const textChunks = await textSplitter.splitText(pageContent); console.log("Chunks created from document:", textChunks.length); - const LLMConnector = getLLMProvider(); const documentVectors = []; const vectors = []; - const vectorValues = await LLMConnector.embedChunks(textChunks); + const vectorValues = await EmbedderEngine.embedChunks(textChunks); if (!!vectorValues && vectorValues.length > 0) { for (const [i, vector] of vectorValues.entries()) { diff --git a/server/utils/vectorDbProviders/pinecone/index.js b/server/utils/vectorDbProviders/pinecone/index.js index cf71c893..9b68ef1b 100644 --- a/server/utils/vectorDbProviders/pinecone/index.js +++ b/server/utils/vectorDbProviders/pinecone/index.js @@ -3,11 +3,7 @@ const { TextSplitter } = require("../../TextSplitter"); const { SystemSettings } = require("../../../models/systemSettings"); const { storeVectorResult, cachedVectorInformation } = require("../../files"); const { v4: uuidv4 } = require("uuid"); -const { - toChunks, - getLLMProvider, - getEmbeddingEngineSelection, -} = require("../../helpers"); +const { toChunks, getEmbeddingEngineSelection } = require("../../helpers"); const { sourceIdentifier } = require("../../chats"); const PineconeDB = { @@ -135,12 +131,13 @@ const PineconeDB = { // because we then cannot atomically control our namespace to granularly find/remove documents // from vectordb. // https://github.com/hwchase17/langchainjs/blob/2def486af734c0ca87285a48f1a04c057ab74bdf/langchain/src/vectorstores/pinecone.ts#L167 + const EmbedderEngine = getEmbeddingEngineSelection(); const textSplitter = new TextSplitter({ chunkSize: TextSplitter.determineMaxChunkSize( await SystemSettings.getValueOrFallback({ label: "text_splitter_chunk_size", }), - getEmbeddingEngineSelection()?.embeddingMaxChunkLength + EmbedderEngine?.embeddingMaxChunkLength ), chunkOverlap: await SystemSettings.getValueOrFallback( { label: "text_splitter_chunk_overlap" }, @@ -150,10 +147,9 @@ const PineconeDB = { const textChunks = await textSplitter.splitText(pageContent); console.log("Chunks created from document:", textChunks.length); - const LLMConnector = getLLMProvider(); const documentVectors = []; const vectors = []; - const vectorValues = await LLMConnector.embedChunks(textChunks); + const vectorValues = await EmbedderEngine.embedChunks(textChunks); if (!!vectorValues && vectorValues.length > 0) { for (const [i, vector] of vectorValues.entries()) { diff --git a/server/utils/vectorDbProviders/qdrant/index.js b/server/utils/vectorDbProviders/qdrant/index.js index 2497c3f3..e8511d0b 100644 --- a/server/utils/vectorDbProviders/qdrant/index.js +++ b/server/utils/vectorDbProviders/qdrant/index.js @@ -3,11 +3,7 @@ const { TextSplitter } = require("../../TextSplitter"); const { SystemSettings } = require("../../../models/systemSettings"); const { storeVectorResult, cachedVectorInformation } = require("../../files"); const { v4: uuidv4 } = require("uuid"); -const { - toChunks, - getLLMProvider, - getEmbeddingEngineSelection, -} = require("../../helpers"); +const { toChunks, getEmbeddingEngineSelection } = require("../../helpers"); const { sourceIdentifier } = require("../../chats"); const QDrant = { @@ -209,12 +205,13 @@ const QDrant = { // We have to do this manually as opposed to using LangChains `Qdrant.fromDocuments` // because we then cannot atomically control our namespace to granularly find/remove documents // from vectordb. + const EmbedderEngine = getEmbeddingEngineSelection(); const textSplitter = new TextSplitter({ chunkSize: TextSplitter.determineMaxChunkSize( await SystemSettings.getValueOrFallback({ label: "text_splitter_chunk_size", }), - getEmbeddingEngineSelection()?.embeddingMaxChunkLength + EmbedderEngine?.embeddingMaxChunkLength ), chunkOverlap: await SystemSettings.getValueOrFallback( { label: "text_splitter_chunk_overlap" }, @@ -224,10 +221,9 @@ const QDrant = { const textChunks = await textSplitter.splitText(pageContent); console.log("Chunks created from document:", textChunks.length); - const LLMConnector = getLLMProvider(); const documentVectors = []; const vectors = []; - const vectorValues = await LLMConnector.embedChunks(textChunks); + const vectorValues = await EmbedderEngine.embedChunks(textChunks); const submission = { ids: [], vectors: [], diff --git a/server/utils/vectorDbProviders/weaviate/index.js b/server/utils/vectorDbProviders/weaviate/index.js index 9e784967..f19329a4 100644 --- a/server/utils/vectorDbProviders/weaviate/index.js +++ b/server/utils/vectorDbProviders/weaviate/index.js @@ -3,11 +3,7 @@ const { TextSplitter } = require("../../TextSplitter"); const { SystemSettings } = require("../../../models/systemSettings"); const { storeVectorResult, cachedVectorInformation } = require("../../files"); const { v4: uuidv4 } = require("uuid"); -const { - toChunks, - getLLMProvider, - getEmbeddingEngineSelection, -} = require("../../helpers"); +const { toChunks, getEmbeddingEngineSelection } = require("../../helpers"); const { camelCase } = require("../../helpers/camelcase"); const { sourceIdentifier } = require("../../chats"); @@ -251,12 +247,13 @@ const Weaviate = { // We have to do this manually as opposed to using LangChains `Chroma.fromDocuments` // because we then cannot atomically control our namespace to granularly find/remove documents // from vectordb. + const EmbedderEngine = getEmbeddingEngineSelection(); const textSplitter = new TextSplitter({ chunkSize: TextSplitter.determineMaxChunkSize( await SystemSettings.getValueOrFallback({ label: "text_splitter_chunk_size", }), - getEmbeddingEngineSelection()?.embeddingMaxChunkLength + EmbedderEngine?.embeddingMaxChunkLength ), chunkOverlap: await SystemSettings.getValueOrFallback( { label: "text_splitter_chunk_overlap" }, @@ -266,10 +263,9 @@ const Weaviate = { const textChunks = await textSplitter.splitText(pageContent); console.log("Chunks created from document:", textChunks.length); - const LLMConnector = getLLMProvider(); const documentVectors = []; const vectors = []; - const vectorValues = await LLMConnector.embedChunks(textChunks); + const vectorValues = await EmbedderEngine.embedChunks(textChunks); const submission = { ids: [], vectors: [], diff --git a/server/utils/vectorDbProviders/zilliz/index.js b/server/utils/vectorDbProviders/zilliz/index.js index 0efe6996..a7ee0438 100644 --- a/server/utils/vectorDbProviders/zilliz/index.js +++ b/server/utils/vectorDbProviders/zilliz/index.js @@ -8,11 +8,7 @@ const { TextSplitter } = require("../../TextSplitter"); const { SystemSettings } = require("../../../models/systemSettings"); const { v4: uuidv4 } = require("uuid"); const { storeVectorResult, cachedVectorInformation } = require("../../files"); -const { - toChunks, - getLLMProvider, - getEmbeddingEngineSelection, -} = require("../../helpers"); +const { toChunks, getEmbeddingEngineSelection } = require("../../helpers"); const { sourceIdentifier } = require("../../chats"); // Zilliz is basically a copy of Milvus DB class with a different constructor @@ -185,12 +181,13 @@ const Zilliz = { return { vectorized: true, error: null }; } + const EmbedderEngine = getEmbeddingEngineSelection(); const textSplitter = new TextSplitter({ chunkSize: TextSplitter.determineMaxChunkSize( await SystemSettings.getValueOrFallback({ label: "text_splitter_chunk_size", }), - getEmbeddingEngineSelection()?.embeddingMaxChunkLength + EmbedderEngine?.embeddingMaxChunkLength ), chunkOverlap: await SystemSettings.getValueOrFallback( { label: "text_splitter_chunk_overlap" }, @@ -200,10 +197,9 @@ const Zilliz = { const textChunks = await textSplitter.splitText(pageContent); console.log("Chunks created from document:", textChunks.length); - const LLMConnector = getLLMProvider(); const documentVectors = []; const vectors = []; - const vectorValues = await LLMConnector.embedChunks(textChunks); + const vectorValues = await EmbedderEngine.embedChunks(textChunks); if (!!vectorValues && vectorValues.length > 0) { for (const [i, vector] of vectorValues.entries()) {