From f0fd91db6f0af98d4ee994b5e516255d08d7352f Mon Sep 17 00:00:00 2001 From: timothycarambat Date: Thu, 8 Jun 2023 18:58:26 -0700 Subject: [PATCH] Reorg some files for clarity --- .../src/components/Modals/ManageWorkspace.jsx | 537 ------------------ .../CannotRemoveModal/index.jsx | 37 ++ .../ConfirmationModal/index.jsx | 89 +++ .../Modals/MangeWorkspace/Directory/index.jsx | 150 +++++ .../Modals/MangeWorkspace/index.jsx | 266 +++++++++ .../Sidebar/ActiveWorkspaces/index.jsx | 7 +- server/utils/helpers/index.js | 6 +- .../chroma/CHROMA_SETUP.md | 0 .../{ => vectorDbProviders}/chroma/index.js | 8 +- .../lance}/index.js | 6 +- .../{ => vectorDbProviders}/pinecone/index.js | 8 +- 11 files changed, 559 insertions(+), 555 deletions(-) delete mode 100644 frontend/src/components/Modals/ManageWorkspace.jsx create mode 100644 frontend/src/components/Modals/MangeWorkspace/CannotRemoveModal/index.jsx create mode 100644 frontend/src/components/Modals/MangeWorkspace/ConfirmationModal/index.jsx create mode 100644 frontend/src/components/Modals/MangeWorkspace/Directory/index.jsx create mode 100644 frontend/src/components/Modals/MangeWorkspace/index.jsx rename server/utils/{ => vectorDbProviders}/chroma/CHROMA_SETUP.md (100%) rename server/utils/{ => vectorDbProviders}/chroma/index.js (98%) rename server/utils/{lancedb => vectorDbProviders/lance}/index.js (98%) rename server/utils/{ => vectorDbProviders}/pinecone/index.js (98%) diff --git a/frontend/src/components/Modals/ManageWorkspace.jsx b/frontend/src/components/Modals/ManageWorkspace.jsx deleted file mode 100644 index 4d64bb41..00000000 --- a/frontend/src/components/Modals/ManageWorkspace.jsx +++ /dev/null @@ -1,537 +0,0 @@ -import React, { useState, useEffect } from "react"; -import { - FileMinus, - FilePlus, - Folder, - FolderMinus, - FolderPlus, - X, - Zap, -} from "react-feather"; -import System from "../../models/system"; -import Workspace from "../../models/workspace"; -import { nFormatter } from "../../utils/numbers"; -import { dollarFormat } from "../../utils/numbers"; -import paths from "../../utils/paths"; -import { useParams } from "react-router-dom"; -import { titleCase } from "text-case"; - -const noop = () => false; -export default function ManageWorkspace({ hideModal = noop, workspace }) { - const { slug } = useParams(); - const [loading, setLoading] = useState(true); - const [saving, setSaving] = useState(false); - const [showConfirmation, setShowConfirmation] = useState(false); - const [directories, setDirectories] = useState(null); - const [originalDocuments, setOriginalDocuments] = useState([]); - const [selectedFiles, setSelectFiles] = useState([]); - const [vectordb, setVectorDB] = useState(null); - const [showingNoRemovalModal, setShowingNoRemovalModal] = useState(false); - - useEffect(() => { - async function fetchKeys() { - const _workspace = await Workspace.bySlug(workspace.slug); - const localFiles = await System.localFiles(); - const settings = await System.keys(); - const originalDocs = _workspace.documents.map((doc) => doc.docpath) || []; - setDirectories(localFiles); - setOriginalDocuments([...originalDocs]); - setSelectFiles([...originalDocs]); - setVectorDB(settings?.VectorDB); - setLoading(false); - } - fetchKeys(); - }, []); - - const deleteWorkspace = async () => { - if ( - !window.confirm( - `You are about to delete your entire ${workspace.name} workspace. This will remove all vector embeddings on your vector database.\n\nThe original source files will remain untouched. This action is irreversible.` - ) - ) - return false; - await Workspace.delete(workspace.slug); - workspace.slug === slug - ? (window.location = paths.home()) - : window.location.reload(); - }; - - const docChanges = () => { - const changes = { - adds: [], - deletes: [], - }; - - selectedFiles.map((doc) => { - const inOriginal = !!originalDocuments.find((oDoc) => oDoc === doc); - if (!inOriginal) { - changes.adds.push(doc); - } - }); - - originalDocuments.map((doc) => { - const selected = !!selectedFiles.find((oDoc) => oDoc === doc); - if (!selected) { - changes.deletes.push(doc); - } - }); - - return changes; - }; - - const confirmChanges = (e) => { - e.preventDefault(); - const changes = docChanges(); - changes.adds.length > 0 ? setShowConfirmation(true) : updateWorkspace(e); - }; - - const updateWorkspace = async (e) => { - e.preventDefault(); - setSaving(true); - setShowConfirmation(false); - const changes = docChanges(); - await Workspace.modifyEmbeddings(workspace.slug, changes); - setSaving(false); - window.location.reload(); - }; - - const isSelected = (filepath) => { - const isFolder = !filepath.includes("/"); - return isFolder - ? selectedFiles.some((doc) => doc.includes(filepath.split("/")[0])) - : selectedFiles.some((doc) => doc.includes(filepath)); - }; - - const isOriginalDoc = (filepath) => { - const isFolder = !filepath.includes("/"); - return isFolder - ? originalDocuments.some((doc) => doc.includes(filepath.split("/")[0])) - : originalDocuments.some((doc) => doc.includes(filepath)); - }; - - const toggleSelection = (filepath) => { - const isFolder = !filepath.includes("/"); - const parent = isFolder ? filepath : filepath.split("/")[0]; - - if (isSelected(filepath)) { - // Certain vector DBs do not contain the ability to delete vectors - // so we cannot remove from these. The user will have to clear the entire workspace. - if (["lancedb"].includes(vectordb) && isOriginalDoc(filepath)) { - setShowingNoRemovalModal(true); - return false; - } - - const updatedDocs = isFolder - ? selectedFiles.filter((doc) => !doc.includes(parent)) - : selectedFiles.filter((doc) => !doc.includes(filepath)); - setSelectFiles([...new Set(updatedDocs)]); - } else { - var newDocs = []; - if (isFolder) { - const folderItems = directories.items.find( - (item) => item.name === parent - ).items; - newDocs = folderItems.map((item) => parent + "/" + item.name); - } else { - newDocs = [filepath]; - } - - const combined = [...selectedFiles, ...newDocs]; - setSelectFiles([...new Set(combined)]); - } - }; - - if (loading) { - return ( -
-
-
-
-
-

- {workspace.name} Settings -

- -
-
-
-

- loading workspace files -

-
-
-
-
-
-
- ); - } - - return ( - <> - {showConfirmation && ( - setShowConfirmation(false)} - additions={docChanges().adds} - updateWorkspace={updateWorkspace} - /> - )} - {showingNoRemovalModal && ( - setShowingNoRemovalModal(false)} - vectordb={vectordb} - /> - )} -
-
-
-
-
-

- "{workspace.name}" workspace settings -

- -
-
-
-
-

- Select folders to add or remove from workspace. -

-

- {selectedFiles.length} documents in workspace selected. -

-
-
- {!!directories && ( - - )} -
-
-
- -
- -
- -
-
-
-
-
- - ); -} - -function Directory({ - files, - parent = null, - nested = 0, - toggleSelection, - isSelected, -}) { - const [isExpanded, toggleExpanded] = useState(false); - const [showDetails, toggleDetails] = useState(false); - const [showZap, setShowZap] = useState(false); - - if (files.type === "folder") { - return ( -
-
- {files.items.some((files) => files.type === "folder") ? ( - - ) : ( - - )} - -
toggleExpanded(!isExpanded)} - > -

{files.name}

- {files.items.some((files) => files.type === "folder") ? ( -

{files.items.length} folders

- ) : ( -

- {files.items.length} documents |{" "} - {nFormatter( - files.items.reduce((a, b) => a + b.token_count_estimate, 0) - )}{" "} - tokens -

- )} -
-
- {isExpanded && - files.items.map((item) => ( - - ))} -
- ); - } - - const { name, type: _type, ...meta } = files; - return ( -
-
- {meta?.cached && ( - - )} - {showZap && ( - -
-
-

- What does{" "} - {" "} - mean? -

-

- This symbol indicates that you have embed this document before - and will not have to pay to re-embed this document. -

-
- -
-
-
- - )} - -
- -
toggleDetails(!showDetails)} - > -

{name}

-
-
-
-
- {showDetails && ( -
- {Object.entries(meta).map(([key, value]) => { - if (key === "cached") return null; - return ( -

- {key}: {value} -

- ); - })} -
- )} -
- ); -} - -function ConfirmationModal({ - directories, - hideConfirm, - additions, - updateWorkspace, -}) { - function estimateCosts() { - const cachedTokens = additions.map((filepath) => { - const [parent, filename] = filepath.split("/"); - const details = directories.items - .find((folder) => folder.name === parent) - .items.find((file) => file.name === filename); - - const { token_count_estimate = 0, cached = false } = details; - return cached ? token_count_estimate : 0; - }); - const tokenEstimates = additions.map((filepath) => { - const [parent, filename] = filepath.split("/"); - const details = directories.items - .find((folder) => folder.name === parent) - .items.find((file) => file.name === filename); - - const { token_count_estimate = 0 } = details; - return token_count_estimate; - }); - - const totalTokens = tokenEstimates.reduce((a, b) => a + b, 0); - const cachedTotal = cachedTokens.reduce((a, b) => a + b, 0); - const dollarValue = 0.0004 * ((totalTokens - cachedTotal) / 1_000); - - return { - dollarValue, - dollarText: - dollarValue < 0.01 ? "< $0.01" : `about ${dollarFormat(dollarValue)}`, - }; - } - - const { dollarValue, dollarText } = estimateCosts(); - return ( - -
-
-

- Are you sure you want to embed these documents? -

- -
- {dollarValue <= 0 ? ( -

- You will be embedding {additions.length} new documents into this - workspace. -
- This will not incur any costs for OpenAI credits. -

- ) : ( -

- You will be embedding {additions.length} new documents into this - workspace.
- This will cost {dollarText} in OpenAI credits. -

- )} -
- -
- - -
-
-
- - ); -} - -function CannotRemoveModal({ hideModal, vectordb }) { - return ( - -
-
-

- You cannot remove this document! -

- -
-

- {titleCase(vectordb)} does not support atomic removal of - documents. -
- Unfortunately, you will have to delete the entire workspace to - remove this document from being referenced. -

-
-
- -
-
-
- - ); -} - -export function useManageWorkspaceModal() { - const [showing, setShowing] = useState(false); - const showModal = () => { - setShowing(true); - }; - const hideModal = () => { - setShowing(false); - }; - - return { showing, showModal, hideModal }; -} diff --git a/frontend/src/components/Modals/MangeWorkspace/CannotRemoveModal/index.jsx b/frontend/src/components/Modals/MangeWorkspace/CannotRemoveModal/index.jsx new file mode 100644 index 00000000..a66442fd --- /dev/null +++ b/frontend/src/components/Modals/MangeWorkspace/CannotRemoveModal/index.jsx @@ -0,0 +1,37 @@ +import React from "react"; +import { titleCase } from "text-case"; + +export default function CannotRemoveModal({ + hideModal, + vectordb, +}) { + return ( + +
+
+

+ You cannot remove this document! +

+ +
+

+ {titleCase(vectordb)} does not support atomic removal of documents.
Unfortunately, you will have to delete the entire workspace to remove this document from being referenced. +

+
+
+ +
+
+
+ + ); +} diff --git a/frontend/src/components/Modals/MangeWorkspace/ConfirmationModal/index.jsx b/frontend/src/components/Modals/MangeWorkspace/ConfirmationModal/index.jsx new file mode 100644 index 00000000..6dce4235 --- /dev/null +++ b/frontend/src/components/Modals/MangeWorkspace/ConfirmationModal/index.jsx @@ -0,0 +1,89 @@ +import React from "react"; +import { dollarFormat } from "../../../../utils/numbers"; + +export default function ConfirmationModal({ + directories, + hideConfirm, + additions, + updateWorkspace, +}) { + function estimateCosts() { + const cachedTokens = additions.map((filepath) => { + const [parent, filename] = filepath.split("/"); + const details = directories.items + .find((folder) => folder.name === parent) + .items.find((file) => file.name === filename); + + const { token_count_estimate = 0, cached = false } = details; + return cached ? token_count_estimate : 0; + }); + const tokenEstimates = additions.map((filepath) => { + const [parent, filename] = filepath.split("/"); + const details = directories.items + .find((folder) => folder.name === parent) + .items.find((file) => file.name === filename); + + const { token_count_estimate = 0 } = details; + return token_count_estimate; + }); + + const totalTokens = tokenEstimates.reduce((a, b) => a + b, 0); + const cachedTotal = cachedTokens.reduce((a, b) => a + b, 0); + const dollarValue = 0.0004 * ((totalTokens - cachedTotal) / 1_000); + + return { + dollarValue, + dollarText: + dollarValue < 0.01 ? "< $0.01" : `about ${dollarFormat(dollarValue)}`, + }; + } + + const { dollarValue, dollarText } = estimateCosts(); + return ( + +
+
+

+ Are you sure you want to embed these documents? +

+ +
+ {dollarValue <= 0 ? ( +

+ You will be embedding {additions.length} new documents into this + workspace. +
+ This will not incur any costs for OpenAI credits. +

+ ) : ( +

+ You will be embedding {additions.length} new documents into this + workspace.
+ This will cost {dollarText} in OpenAI credits. +

+ )} +
+ +
+ + +
+
+
+ + ); +} \ No newline at end of file diff --git a/frontend/src/components/Modals/MangeWorkspace/Directory/index.jsx b/frontend/src/components/Modals/MangeWorkspace/Directory/index.jsx new file mode 100644 index 00000000..b2eb4125 --- /dev/null +++ b/frontend/src/components/Modals/MangeWorkspace/Directory/index.jsx @@ -0,0 +1,150 @@ +import React, { useState } from "react"; +import { + FileMinus, + FilePlus, + Folder, + FolderMinus, + FolderPlus, + Zap, +} from "react-feather"; +import { nFormatter } from "../../../../utils/numbers"; + +export default function Directory({ + files, + parent = null, + nested = 0, + toggleSelection, + isSelected, +}) { + const [isExpanded, toggleExpanded] = useState(false); + const [showDetails, toggleDetails] = useState(false); + const [showZap, setShowZap] = useState(false); + + if (files.type === "folder") { + return ( +
+
+ {files.items.some((files) => files.type === "folder") ? ( + + ) : ( + + )} + +
toggleExpanded(!isExpanded)} + > +

{files.name}

+ {files.items.some((files) => files.type === "folder") ? ( +

{files.items.length} folders

+ ) : ( +

+ {files.items.length} documents |{" "} + {nFormatter( + files.items.reduce((a, b) => a + b.token_count_estimate, 0) + )}{" "} + tokens +

+ )} +
+
+ {isExpanded && + files.items.map((item) => ( + + ))} +
+ ); + } + + const { name, type: _type, ...meta } = files; + return ( +
+
+ {meta?.cached && ( + + )} + {showZap && ( + +
+
+

+ What does{" "} + {" "} + mean? +

+

+ This symbol indicates that you have embed this document before + and will not have to pay to re-embed this document. +

+
+ +
+
+
+ + )} + +
+ +
toggleDetails(!showDetails)} + > +

{name}

+
+
+
+
+ {showDetails && ( +
+ {Object.entries(meta).map(([key, value]) => { + if (key === "cached") return null; + return ( +

+ {key}: {value} +

+ ); + })} +
+ )} +
+ ); +} \ No newline at end of file diff --git a/frontend/src/components/Modals/MangeWorkspace/index.jsx b/frontend/src/components/Modals/MangeWorkspace/index.jsx new file mode 100644 index 00000000..4c3cde12 --- /dev/null +++ b/frontend/src/components/Modals/MangeWorkspace/index.jsx @@ -0,0 +1,266 @@ +import React, { useState, useEffect } from "react"; +import { X } from 'react-feather'; +import System from "../../../models/system"; +import Workspace from "../../../models/workspace"; +import paths from "../../../utils/paths"; +import { useParams } from "react-router-dom"; +import Directory from "./Directory"; +import ConfirmationModal from "./ConfirmationModal"; +import CannotRemoveModal from "./CannotRemoveModal"; + +const noop = () => false; +export default function ManageWorkspace({ hideModal = noop, workspace }) { + const { slug } = useParams(); + const [loading, setLoading] = useState(true); + const [saving, setSaving] = useState(false); + const [showConfirmation, setShowConfirmation] = useState(false); + const [directories, setDirectories] = useState(null); + const [originalDocuments, setOriginalDocuments] = useState([]); + const [selectedFiles, setSelectFiles] = useState([]); + const [vectordb, setVectorDB] = useState(null); + const [showingNoRemovalModal, setShowingNoRemovalModal] = useState(false) + + useEffect(() => { + async function fetchKeys() { + const _workspace = await Workspace.bySlug(workspace.slug); + const localFiles = await System.localFiles(); + const settings = await System.keys(); + const originalDocs = _workspace.documents.map((doc) => doc.docpath) || []; + setDirectories(localFiles); + setOriginalDocuments([...originalDocs]); + setSelectFiles([...originalDocs]); + setVectorDB(settings?.VectorDB) + setLoading(false); + } + fetchKeys(); + }, []); + + const deleteWorkspace = async () => { + if ( + !window.confirm( + `You are about to delete your entire ${workspace.name} workspace. This will remove all vector embeddings on your vector database.\n\nThe original source files will remain untouched. This action is irreversible.` + ) + ) + return false; + await Workspace.delete(workspace.slug); + workspace.slug === slug + ? (window.location = paths.home()) + : window.location.reload(); + }; + + const docChanges = () => { + const changes = { + adds: [], + deletes: [], + }; + + selectedFiles.map((doc) => { + const inOriginal = !!originalDocuments.find((oDoc) => oDoc === doc); + if (!inOriginal) { + changes.adds.push(doc); + } + }); + + originalDocuments.map((doc) => { + const selected = !!selectedFiles.find((oDoc) => oDoc === doc); + if (!selected) { + changes.deletes.push(doc); + } + }); + + return changes; + }; + + const confirmChanges = (e) => { + e.preventDefault(); + const changes = docChanges(); + changes.adds.length > 0 ? setShowConfirmation(true) : updateWorkspace(e); + }; + + const updateWorkspace = async (e) => { + e.preventDefault(); + setSaving(true); + setShowConfirmation(false); + const changes = docChanges(); + await Workspace.modifyEmbeddings(workspace.slug, changes); + setSaving(false); + window.location.reload(); + }; + + const isSelected = (filepath) => { + const isFolder = !filepath.includes("/"); + return isFolder + ? selectedFiles.some((doc) => doc.includes(filepath.split("/")[0])) + : selectedFiles.some((doc) => doc.includes(filepath)); + }; + + const isOriginalDoc = (filepath) => { + const isFolder = !filepath.includes("/"); + return isFolder + ? originalDocuments.some((doc) => doc.includes(filepath.split("/")[0])) + : originalDocuments.some((doc) => doc.includes(filepath)); + } + + const toggleSelection = (filepath) => { + const isFolder = !filepath.includes("/"); + const parent = isFolder ? filepath : filepath.split("/")[0]; + + if (isSelected(filepath)) { + // Certain vector DBs do not contain the ability to delete vectors + // so we cannot remove from these. The user will have to clear the entire workspace. + if (['lancedb'].includes(vectordb) && isOriginalDoc(filepath)) { + setShowingNoRemovalModal(true); + return false; + } + + const updatedDocs = isFolder + ? selectedFiles.filter((doc) => !doc.includes(parent)) + : selectedFiles.filter((doc) => !doc.includes(filepath)); + setSelectFiles([...new Set(updatedDocs)]); + } else { + var newDocs = []; + if (isFolder) { + const folderItems = directories.items.find( + (item) => item.name === parent + ).items; + newDocs = folderItems.map((item) => parent + "/" + item.name); + } else { + newDocs = [filepath]; + } + + const combined = [...selectedFiles, ...newDocs]; + setSelectFiles([...new Set(combined)]); + } + }; + + if (loading) { + return ( +
+
+
+
+
+

+ {workspace.name} Settings +

+ +
+
+
+

+ loading workspace files +

+
+
+
+
+
+
+ ); + } + + return ( + <> + {showConfirmation && ( + setShowConfirmation(false)} + additions={docChanges().adds} + updateWorkspace={updateWorkspace} + /> + )} + {showingNoRemovalModal && ( + setShowingNoRemovalModal(false)} + vectordb={vectordb} + /> + )} +
+
+
+
+
+

+ "{workspace.name}" workspace settings +

+ +
+
+
+
+

+ Select folders to add or remove from workspace. +

+

+ {selectedFiles.length} documents in workspace selected. +

+
+
+ {!!directories && ( + + )} +
+
+
+ +
+ +
+ +
+
+
+
+
+ + ); +} + +export function useManageWorkspaceModal() { + const [showing, setShowing] = useState(false); + const showModal = () => { + setShowing(true); + }; + const hideModal = () => { + setShowing(false); + }; + + return { showing, showModal, hideModal }; +} diff --git a/frontend/src/components/Sidebar/ActiveWorkspaces/index.jsx b/frontend/src/components/Sidebar/ActiveWorkspaces/index.jsx index af9ddefb..67da8086 100644 --- a/frontend/src/components/Sidebar/ActiveWorkspaces/index.jsx +++ b/frontend/src/components/Sidebar/ActiveWorkspaces/index.jsx @@ -5,7 +5,7 @@ import "react-loading-skeleton/dist/skeleton.css"; import Workspace from "../../../models/workspace"; import ManageWorkspace, { useManageWorkspaceModal, -} from "../../Modals/ManageWorkspace"; +} from "../../Modals/MangeWorkspace"; import paths from "../../../utils/paths"; import { useParams } from "react-router-dom"; @@ -51,11 +51,10 @@ export default function ActiveWorkspaces() { >

diff --git a/server/utils/helpers/index.js b/server/utils/helpers/index.js index 08a84cc8..6c9ea2cb 100644 --- a/server/utils/helpers/index.js +++ b/server/utils/helpers/index.js @@ -1,7 +1,7 @@ function getVectorDbClass() { - const { Pinecone } = require("../pinecone"); - const { Chroma } = require("../chroma"); - const { LanceDb } = require("../lancedb"); + const { Pinecone } = require("../vectorDbProviders/pinecone"); + const { Chroma } = require("../vectorDbProviders/chroma"); + const { LanceDb } = require("../vectorDbProviders/lance"); const vectorSelection = process.env.VECTOR_DB || "pinecone"; switch (vectorSelection) { diff --git a/server/utils/chroma/CHROMA_SETUP.md b/server/utils/vectorDbProviders/chroma/CHROMA_SETUP.md similarity index 100% rename from server/utils/chroma/CHROMA_SETUP.md rename to server/utils/vectorDbProviders/chroma/CHROMA_SETUP.md diff --git a/server/utils/chroma/index.js b/server/utils/vectorDbProviders/chroma/index.js similarity index 98% rename from server/utils/chroma/index.js rename to server/utils/vectorDbProviders/chroma/index.js index f6fee744..9584df47 100644 --- a/server/utils/chroma/index.js +++ b/server/utils/vectorDbProviders/chroma/index.js @@ -5,10 +5,10 @@ const { ChatOpenAI } = require("langchain/chat_models/openai"); const { VectorDBQAChain } = require("langchain/chains"); const { OpenAIEmbeddings } = require("langchain/embeddings/openai"); const { RecursiveCharacterTextSplitter } = require("langchain/text_splitter"); -const { storeVectorResult, cachedVectorInformation } = require("../files"); +const { storeVectorResult, cachedVectorInformation } = require("../../files"); const { Configuration, OpenAIApi } = require("openai"); const { v4: uuidv4 } = require("uuid"); -const { toChunks, curateSources } = require("../helpers"); +const { toChunks, curateSources } = require("../../helpers"); const Chroma = { name: "Chroma", @@ -112,7 +112,7 @@ const Chroma = { documentData = {}, fullFilePath = null ) { - const { DocumentVectors } = require("../../models/vectors"); + const { DocumentVectors } = require("../../../models/vectors"); try { const { pageContent, docId, ...metadata } = documentData; if (!pageContent || pageContent.length == 0) return false; @@ -235,7 +235,7 @@ const Chroma = { } }, deleteDocumentFromNamespace: async function (namespace, docId) { - const { DocumentVectors } = require("../../models/vectors"); + const { DocumentVectors } = require("../../../models/vectors"); const { client } = await this.connect(); if (!(await this.namespaceExists(client, namespace))) return; const collection = await client.getCollection({ diff --git a/server/utils/lancedb/index.js b/server/utils/vectorDbProviders/lance/index.js similarity index 98% rename from server/utils/lancedb/index.js rename to server/utils/vectorDbProviders/lance/index.js index f2851d32..8617a423 100644 --- a/server/utils/lancedb/index.js +++ b/server/utils/vectorDbProviders/lance/index.js @@ -1,8 +1,8 @@ const lancedb = require("vectordb"); -const { toChunks } = require("../helpers"); +const { toChunks } = require("../../helpers"); const { OpenAIEmbeddings } = require("langchain/embeddings/openai"); const { RecursiveCharacterTextSplitter } = require("langchain/text_splitter"); -const { storeVectorResult, cachedVectorInformation } = require("../files"); +const { storeVectorResult, cachedVectorInformation } = require("../../files"); const { Configuration, OpenAIApi } = require("openai"); const { v4: uuidv4 } = require("uuid"); @@ -126,7 +126,7 @@ const LanceDb = { documentData = {}, fullFilePath = null ) { - const { DocumentVectors } = require("../../models/vectors"); + const { DocumentVectors } = require("../../../models/vectors"); try { const { pageContent, docId, ...metadata } = documentData; if (!pageContent || pageContent.length == 0) return false; diff --git a/server/utils/pinecone/index.js b/server/utils/vectorDbProviders/pinecone/index.js similarity index 98% rename from server/utils/pinecone/index.js rename to server/utils/vectorDbProviders/pinecone/index.js index e57c34a8..9167b790 100644 --- a/server/utils/pinecone/index.js +++ b/server/utils/vectorDbProviders/pinecone/index.js @@ -7,10 +7,10 @@ const { OpenAIEmbeddings } = require("langchain/embeddings/openai"); const { VectorStoreRetrieverMemory } = require("langchain/memory"); const { PromptTemplate } = require("langchain/prompts"); const { RecursiveCharacterTextSplitter } = require("langchain/text_splitter"); -const { storeVectorResult, cachedVectorInformation } = require("../files"); +const { storeVectorResult, cachedVectorInformation } = require("../../files"); const { Configuration, OpenAIApi } = require("openai"); const { v4: uuidv4 } = require("uuid"); -const { toChunks, curateSources } = require("../helpers"); +const { toChunks, curateSources } = require("../../helpers"); const Pinecone = { name: "Pinecone", @@ -98,7 +98,7 @@ const Pinecone = { documentData = {}, fullFilePath = null ) { - const { DocumentVectors } = require("../../models/vectors"); + const { DocumentVectors } = require("../../../models/vectors"); try { const { pageContent, docId, ...metadata } = documentData; if (!pageContent || pageContent.length == 0) return false; @@ -192,7 +192,7 @@ const Pinecone = { } }, deleteDocumentFromNamespace: async function (namespace, docId) { - const { DocumentVectors } = require("../../models/vectors"); + const { DocumentVectors } = require("../../../models/vectors"); const { pineconeIndex } = await this.connect(); if (!(await this.namespaceExists(pineconeIndex, namespace))) return;