add ability to purge document from custom documents as well as cleanup its associated cache file (#113)

* add ability to purge document from custom documents as well as cleanup its assoicated cache file

* update alert text

frontend/src/components/Modals/MangeWorkspace/Documents/Directory/index.jsx

@@ -8,6 +8,7 @@ import {
   Zap,
 } from "react-feather";
 import { nFormatter } from "../../../../../utils/numbers";
+import System from "../../../../../models/system";
 
 export default function Directory({
   files,
@@ -19,6 +20,16 @@ export default function Directory({
   const [isExpanded, toggleExpanded] = useState(false);
   const [showDetails, toggleDetails] = useState(false);
   const [showZap, setShowZap] = useState(false);
+  const handleDelete = async (name, meta) => {
+    if (
+      !window.confirm(
+        "Are you sure you want to delete this document?\nThis will require you to re-upload and re-embed it.\nThis document will be removed from any workspace that is currently referencing it.\nThis action is not reversible."
+      )
+    )
+      return false;
+    document?.getElementById(meta?.id)?.remove();
+    await System.deleteDocument(name, meta);
+  };
 
   if (files.type === "folder") {
     return (
@@ -73,7 +84,7 @@ export default function Directory({
 
   const { name, type: _type, ...meta } = files;
   return (
-    <div className="ml-[20px] my-2">
+    <div className="ml-[20px] my-2" id={meta.id}>
       <div className="flex items-center">
         {meta?.cached && (
           <button
@@ -134,6 +145,7 @@ export default function Directory({
         </div>
       </div>
       {showDetails && (
+        <div className="w-full flex flex-col">
           <div className="ml-[20px] flex flex-col gap-y-1 my-1 p-2 rounded-md bg-slate-200 font-mono text-sm overflow-x-scroll">
             {Object.entries(meta).map(([key, value], i) => {
               if (key === "cached") return null;
@@ -144,6 +156,15 @@ export default function Directory({
               );
             })}
           </div>
+          <div
+            onClick={() => handleDelete(`${parent}/${name}`, meta)}
+            className="flex items-center justify-end w-full"
+          >
+            <button className="text-sm text-slate-400 dark:text-stone-500 hover:text-red-500">
+              Purge Document
+            </button>
+          </div>
+        </div>
       )}
     </div>
   );

frontend/src/models/system.js

@@ -86,6 +86,18 @@ const System = {
         return { newValues: null, error: e.message };
       });
   },
+  deleteDocument: async (name, meta) => {
+    return await fetch(`${API_BASE}/system/remove-document`, {
+      method: "DELETE",
+      headers: baseHeaders(),
+      body: JSON.stringify({ name, meta }),
+    })
+      .then((res) => res.ok)
+      .catch((e) => {
+        console.error(e);
+        return false;
+      });
+  },
 };
 
 export default System;

server/endpoints/system.js

@@ -7,6 +7,7 @@ const {
   checkPythonAppAlive,
   acceptedFileTypes,
 } = require("../utils/files/documentProcessor");
+const { purgeDocument } = require("../utils/files/purgeDocument");
 const { getVectorDbClass } = require("../utils/helpers");
 const { updateENV } = require("../utils/helpers/updateENV");
 const { reqBody, makeJWT } = require("../utils/http");
@@ -94,6 +95,17 @@ function systemEndpoints(app) {
     }
   });
 
+  app.delete("/system/remove-document", async (request, response) => {
+    try {
+      const { name, meta } = reqBody(request);
+      await purgeDocument(name, meta);
+      response.sendStatus(200).end();
+    } catch (e) {
+      console.log(e.message, e);
+      response.sendStatus(500).end();
+    }
+  });
+
   app.get("/system/local-files", async (_, response) => {
     try {
       const localFiles = await viewLocalFiles();

server/utils/files/index.js

@@ -141,10 +141,51 @@ async function storeVectorResult(vectorData = [], filename = null) {
   return;
 }
 
+// Purges a file from the documents/ folder.
+async function purgeSourceDocument(filename = null) {
+  if (!filename) return;
+  console.log(`Purging document of ${filename}.`);
+
+  const filePath =
+    process.env.NODE_ENV === "development"
+      ? path.resolve(__dirname, `../../storage/documents`, filename)
+      : path.resolve(process.env.STORAGE_DIR, `documents`, filename);
+
+  if (!fs.existsSync(filePath)) {
+    console.log(`Could not located cachefile for ${filename}`, filePath);
+    return;
+  }
+
+  fs.rmSync(filePath);
+  return;
+}
+
+// Purges a vector-cache file from the vector-cache/ folder.
+async function purgeVectorCache(filename = null) {
+  if (!filename) return;
+  console.log(`Purging cached vectorized results of ${filename}.`);
+
+  const digest = uuidv5(filename, uuidv5.URL);
+  const filePath =
+    process.env.NODE_ENV === "development"
+      ? path.resolve(__dirname, `../../storage/vector-cache`, `${digest}.json`)
+      : path.resolve(process.env.STORAGE_DIR, `vector-cache`, `${digest}.json`);
+
+  if (!fs.existsSync(filePath)) {
+    console.log(`Could not located cache file for ${filename}`, filePath);
+    return;
+  }
+
+  fs.rmSync(filePath);
+  return;
+}
+
 module.exports = {
   cachedVectorInformation,
   collectDocumentData,
   viewLocalFiles,
+  purgeSourceDocument,
+  purgeVectorCache,
   storeVectorResult,
   fileData,
 };

server/utils/files/purgeDocument.js

@@ -0,0 +1,17 @@
+const { purgeVectorCache, purgeSourceDocument } = require(".");
+const { Document } = require("../../models/documents");
+const { Workspace } = require("../../models/workspace");
+
+async function purgeDocument(filename, meta) {
+  const workspaces = await Workspace.where();
+  for (const workspace of workspaces) {
+    await Document.removeDocuments(workspace, [filename]);
+  }
+  await purgeVectorCache(filename);
+  await purgeSourceDocument(filename);
+  return;
+}
+
+module.exports = {
+  purgeDocument,
+};