diff --git a/frontend/src/components/Modals/ManageWorkspace.jsx b/frontend/src/components/Modals/ManageWorkspace.jsx deleted file mode 100644 index 4d64bb41..00000000 --- a/frontend/src/components/Modals/ManageWorkspace.jsx +++ /dev/null @@ -1,537 +0,0 @@ -import React, { useState, useEffect } from "react"; -import { - FileMinus, - FilePlus, - Folder, - FolderMinus, - FolderPlus, - X, - Zap, -} from "react-feather"; -import System from "../../models/system"; -import Workspace from "../../models/workspace"; -import { nFormatter } from "../../utils/numbers"; -import { dollarFormat } from "../../utils/numbers"; -import paths from "../../utils/paths"; -import { useParams } from "react-router-dom"; -import { titleCase } from "text-case"; - -const noop = () => false; -export default function ManageWorkspace({ hideModal = noop, workspace }) { - const { slug } = useParams(); - const [loading, setLoading] = useState(true); - const [saving, setSaving] = useState(false); - const [showConfirmation, setShowConfirmation] = useState(false); - const [directories, setDirectories] = useState(null); - const [originalDocuments, setOriginalDocuments] = useState([]); - const [selectedFiles, setSelectFiles] = useState([]); - const [vectordb, setVectorDB] = useState(null); - const [showingNoRemovalModal, setShowingNoRemovalModal] = useState(false); - - useEffect(() => { - async function fetchKeys() { - const _workspace = await Workspace.bySlug(workspace.slug); - const localFiles = await System.localFiles(); - const settings = await System.keys(); - const originalDocs = _workspace.documents.map((doc) => doc.docpath) || []; - setDirectories(localFiles); - setOriginalDocuments([...originalDocs]); - setSelectFiles([...originalDocs]); - setVectorDB(settings?.VectorDB); - setLoading(false); - } - fetchKeys(); - }, []); - - const deleteWorkspace = async () => { - if ( - !window.confirm( - `You are about to delete your entire ${workspace.name} workspace. This will remove all vector embeddings on your vector database.\n\nThe original source files will remain untouched. This action is irreversible.` - ) - ) - return false; - await Workspace.delete(workspace.slug); - workspace.slug === slug - ? (window.location = paths.home()) - : window.location.reload(); - }; - - const docChanges = () => { - const changes = { - adds: [], - deletes: [], - }; - - selectedFiles.map((doc) => { - const inOriginal = !!originalDocuments.find((oDoc) => oDoc === doc); - if (!inOriginal) { - changes.adds.push(doc); - } - }); - - originalDocuments.map((doc) => { - const selected = !!selectedFiles.find((oDoc) => oDoc === doc); - if (!selected) { - changes.deletes.push(doc); - } - }); - - return changes; - }; - - const confirmChanges = (e) => { - e.preventDefault(); - const changes = docChanges(); - changes.adds.length > 0 ? setShowConfirmation(true) : updateWorkspace(e); - }; - - const updateWorkspace = async (e) => { - e.preventDefault(); - setSaving(true); - setShowConfirmation(false); - const changes = docChanges(); - await Workspace.modifyEmbeddings(workspace.slug, changes); - setSaving(false); - window.location.reload(); - }; - - const isSelected = (filepath) => { - const isFolder = !filepath.includes("/"); - return isFolder - ? selectedFiles.some((doc) => doc.includes(filepath.split("/")[0])) - : selectedFiles.some((doc) => doc.includes(filepath)); - }; - - const isOriginalDoc = (filepath) => { - const isFolder = !filepath.includes("/"); - return isFolder - ? originalDocuments.some((doc) => doc.includes(filepath.split("/")[0])) - : originalDocuments.some((doc) => doc.includes(filepath)); - }; - - const toggleSelection = (filepath) => { - const isFolder = !filepath.includes("/"); - const parent = isFolder ? filepath : filepath.split("/")[0]; - - if (isSelected(filepath)) { - // Certain vector DBs do not contain the ability to delete vectors - // so we cannot remove from these. The user will have to clear the entire workspace. - if (["lancedb"].includes(vectordb) && isOriginalDoc(filepath)) { - setShowingNoRemovalModal(true); - return false; - } - - const updatedDocs = isFolder - ? selectedFiles.filter((doc) => !doc.includes(parent)) - : selectedFiles.filter((doc) => !doc.includes(filepath)); - setSelectFiles([...new Set(updatedDocs)]); - } else { - var newDocs = []; - if (isFolder) { - const folderItems = directories.items.find( - (item) => item.name === parent - ).items; - newDocs = folderItems.map((item) => parent + "/" + item.name); - } else { - newDocs = [filepath]; - } - - const combined = [...selectedFiles, ...newDocs]; - setSelectFiles([...new Set(combined)]); - } - }; - - if (loading) { - return ( -
- loading workspace files -
-- Select folders to add or remove from workspace. -
-- {selectedFiles.length} documents in workspace selected. -
-{files.items.length} folders
- ) : ( -- {files.items.length} documents |{" "} - {nFormatter( - files.items.reduce((a, b) => a + b.token_count_estimate, 0) - )}{" "} - tokens -
- )} -- {key}: {value} -
- ); - })} -{files.items.length} folders
+ ) : ( ++ {files.items.length} documents |{" "} + {nFormatter( + files.items.reduce((a, b) => a + b.token_count_estimate, 0) + )}{" "} + tokens +
+ )} ++ {key}: {value} +
+ ); + })} ++ loading workspace files +
++ Select folders to add or remove from workspace. +
++ {selectedFiles.length} documents in workspace selected. +
+diff --git a/server/utils/helpers/index.js b/server/utils/helpers/index.js index 08a84cc8..6c9ea2cb 100644 --- a/server/utils/helpers/index.js +++ b/server/utils/helpers/index.js @@ -1,7 +1,7 @@ function getVectorDbClass() { - const { Pinecone } = require("../pinecone"); - const { Chroma } = require("../chroma"); - const { LanceDb } = require("../lancedb"); + const { Pinecone } = require("../vectorDbProviders/pinecone"); + const { Chroma } = require("../vectorDbProviders/chroma"); + const { LanceDb } = require("../vectorDbProviders/lance"); const vectorSelection = process.env.VECTOR_DB || "pinecone"; switch (vectorSelection) { diff --git a/server/utils/chroma/CHROMA_SETUP.md b/server/utils/vectorDbProviders/chroma/CHROMA_SETUP.md similarity index 100% rename from server/utils/chroma/CHROMA_SETUP.md rename to server/utils/vectorDbProviders/chroma/CHROMA_SETUP.md diff --git a/server/utils/chroma/index.js b/server/utils/vectorDbProviders/chroma/index.js similarity index 98% rename from server/utils/chroma/index.js rename to server/utils/vectorDbProviders/chroma/index.js index f6fee744..9584df47 100644 --- a/server/utils/chroma/index.js +++ b/server/utils/vectorDbProviders/chroma/index.js @@ -5,10 +5,10 @@ const { ChatOpenAI } = require("langchain/chat_models/openai"); const { VectorDBQAChain } = require("langchain/chains"); const { OpenAIEmbeddings } = require("langchain/embeddings/openai"); const { RecursiveCharacterTextSplitter } = require("langchain/text_splitter"); -const { storeVectorResult, cachedVectorInformation } = require("../files"); +const { storeVectorResult, cachedVectorInformation } = require("../../files"); const { Configuration, OpenAIApi } = require("openai"); const { v4: uuidv4 } = require("uuid"); -const { toChunks, curateSources } = require("../helpers"); +const { toChunks, curateSources } = require("../../helpers"); const Chroma = { name: "Chroma", @@ -112,7 +112,7 @@ const Chroma = { documentData = {}, fullFilePath = null ) { - const { DocumentVectors } = require("../../models/vectors"); + const { DocumentVectors } = require("../../../models/vectors"); try { const { pageContent, docId, ...metadata } = documentData; if (!pageContent || pageContent.length == 0) return false; @@ -235,7 +235,7 @@ const Chroma = { } }, deleteDocumentFromNamespace: async function (namespace, docId) { - const { DocumentVectors } = require("../../models/vectors"); + const { DocumentVectors } = require("../../../models/vectors"); const { client } = await this.connect(); if (!(await this.namespaceExists(client, namespace))) return; const collection = await client.getCollection({ diff --git a/server/utils/lancedb/index.js b/server/utils/vectorDbProviders/lance/index.js similarity index 98% rename from server/utils/lancedb/index.js rename to server/utils/vectorDbProviders/lance/index.js index f2851d32..8617a423 100644 --- a/server/utils/lancedb/index.js +++ b/server/utils/vectorDbProviders/lance/index.js @@ -1,8 +1,8 @@ const lancedb = require("vectordb"); -const { toChunks } = require("../helpers"); +const { toChunks } = require("../../helpers"); const { OpenAIEmbeddings } = require("langchain/embeddings/openai"); const { RecursiveCharacterTextSplitter } = require("langchain/text_splitter"); -const { storeVectorResult, cachedVectorInformation } = require("../files"); +const { storeVectorResult, cachedVectorInformation } = require("../../files"); const { Configuration, OpenAIApi } = require("openai"); const { v4: uuidv4 } = require("uuid"); @@ -126,7 +126,7 @@ const LanceDb = { documentData = {}, fullFilePath = null ) { - const { DocumentVectors } = require("../../models/vectors"); + const { DocumentVectors } = require("../../../models/vectors"); try { const { pageContent, docId, ...metadata } = documentData; if (!pageContent || pageContent.length == 0) return false; diff --git a/server/utils/pinecone/index.js b/server/utils/vectorDbProviders/pinecone/index.js similarity index 98% rename from server/utils/pinecone/index.js rename to server/utils/vectorDbProviders/pinecone/index.js index e57c34a8..9167b790 100644 --- a/server/utils/pinecone/index.js +++ b/server/utils/vectorDbProviders/pinecone/index.js @@ -7,10 +7,10 @@ const { OpenAIEmbeddings } = require("langchain/embeddings/openai"); const { VectorStoreRetrieverMemory } = require("langchain/memory"); const { PromptTemplate } = require("langchain/prompts"); const { RecursiveCharacterTextSplitter } = require("langchain/text_splitter"); -const { storeVectorResult, cachedVectorInformation } = require("../files"); +const { storeVectorResult, cachedVectorInformation } = require("../../files"); const { Configuration, OpenAIApi } = require("openai"); const { v4: uuidv4 } = require("uuid"); -const { toChunks, curateSources } = require("../helpers"); +const { toChunks, curateSources } = require("../../helpers"); const Pinecone = { name: "Pinecone", @@ -98,7 +98,7 @@ const Pinecone = { documentData = {}, fullFilePath = null ) { - const { DocumentVectors } = require("../../models/vectors"); + const { DocumentVectors } = require("../../../models/vectors"); try { const { pageContent, docId, ...metadata } = documentData; if (!pageContent || pageContent.length == 0) return false; @@ -192,7 +192,7 @@ const Pinecone = { } }, deleteDocumentFromNamespace: async function (namespace, docId) { - const { DocumentVectors } = require("../../models/vectors"); + const { DocumentVectors } = require("../../../models/vectors"); const { pineconeIndex } = await this.connect(); if (!(await this.namespaceExists(pineconeIndex, namespace))) return;