mirror of
https://github.com/Mintplex-Labs/anything-llm.git
synced 2024-10-02 08:50:11 +02:00
add ability to purge document from custom documents as well as cleanup its associated cache file (#113)
* add ability to purge document from custom documents as well as cleanup its assoicated cache file * update alert text
This commit is contained in:
parent
9d0becb2ee
commit
60a00843df
@ -8,6 +8,7 @@ import {
|
||||
Zap,
|
||||
} from "react-feather";
|
||||
import { nFormatter } from "../../../../../utils/numbers";
|
||||
import System from "../../../../../models/system";
|
||||
|
||||
export default function Directory({
|
||||
files,
|
||||
@ -19,6 +20,16 @@ export default function Directory({
|
||||
const [isExpanded, toggleExpanded] = useState(false);
|
||||
const [showDetails, toggleDetails] = useState(false);
|
||||
const [showZap, setShowZap] = useState(false);
|
||||
const handleDelete = async (name, meta) => {
|
||||
if (
|
||||
!window.confirm(
|
||||
"Are you sure you want to delete this document?\nThis will require you to re-upload and re-embed it.\nThis document will be removed from any workspace that is currently referencing it.\nThis action is not reversible."
|
||||
)
|
||||
)
|
||||
return false;
|
||||
document?.getElementById(meta?.id)?.remove();
|
||||
await System.deleteDocument(name, meta);
|
||||
};
|
||||
|
||||
if (files.type === "folder") {
|
||||
return (
|
||||
@ -73,7 +84,7 @@ export default function Directory({
|
||||
|
||||
const { name, type: _type, ...meta } = files;
|
||||
return (
|
||||
<div className="ml-[20px] my-2">
|
||||
<div className="ml-[20px] my-2" id={meta.id}>
|
||||
<div className="flex items-center">
|
||||
{meta?.cached && (
|
||||
<button
|
||||
@ -134,6 +145,7 @@ export default function Directory({
|
||||
</div>
|
||||
</div>
|
||||
{showDetails && (
|
||||
<div className="w-full flex flex-col">
|
||||
<div className="ml-[20px] flex flex-col gap-y-1 my-1 p-2 rounded-md bg-slate-200 font-mono text-sm overflow-x-scroll">
|
||||
{Object.entries(meta).map(([key, value], i) => {
|
||||
if (key === "cached") return null;
|
||||
@ -144,6 +156,15 @@ export default function Directory({
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
<div
|
||||
onClick={() => handleDelete(`${parent}/${name}`, meta)}
|
||||
className="flex items-center justify-end w-full"
|
||||
>
|
||||
<button className="text-sm text-slate-400 dark:text-stone-500 hover:text-red-500">
|
||||
Purge Document
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
|
@ -86,6 +86,18 @@ const System = {
|
||||
return { newValues: null, error: e.message };
|
||||
});
|
||||
},
|
||||
deleteDocument: async (name, meta) => {
|
||||
return await fetch(`${API_BASE}/system/remove-document`, {
|
||||
method: "DELETE",
|
||||
headers: baseHeaders(),
|
||||
body: JSON.stringify({ name, meta }),
|
||||
})
|
||||
.then((res) => res.ok)
|
||||
.catch((e) => {
|
||||
console.error(e);
|
||||
return false;
|
||||
});
|
||||
},
|
||||
};
|
||||
|
||||
export default System;
|
||||
|
@ -7,6 +7,7 @@ const {
|
||||
checkPythonAppAlive,
|
||||
acceptedFileTypes,
|
||||
} = require("../utils/files/documentProcessor");
|
||||
const { purgeDocument } = require("../utils/files/purgeDocument");
|
||||
const { getVectorDbClass } = require("../utils/helpers");
|
||||
const { updateENV } = require("../utils/helpers/updateENV");
|
||||
const { reqBody, makeJWT } = require("../utils/http");
|
||||
@ -94,6 +95,17 @@ function systemEndpoints(app) {
|
||||
}
|
||||
});
|
||||
|
||||
app.delete("/system/remove-document", async (request, response) => {
|
||||
try {
|
||||
const { name, meta } = reqBody(request);
|
||||
await purgeDocument(name, meta);
|
||||
response.sendStatus(200).end();
|
||||
} catch (e) {
|
||||
console.log(e.message, e);
|
||||
response.sendStatus(500).end();
|
||||
}
|
||||
});
|
||||
|
||||
app.get("/system/local-files", async (_, response) => {
|
||||
try {
|
||||
const localFiles = await viewLocalFiles();
|
||||
|
@ -141,10 +141,51 @@ async function storeVectorResult(vectorData = [], filename = null) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Purges a file from the documents/ folder.
|
||||
async function purgeSourceDocument(filename = null) {
|
||||
if (!filename) return;
|
||||
console.log(`Purging document of ${filename}.`);
|
||||
|
||||
const filePath =
|
||||
process.env.NODE_ENV === "development"
|
||||
? path.resolve(__dirname, `../../storage/documents`, filename)
|
||||
: path.resolve(process.env.STORAGE_DIR, `documents`, filename);
|
||||
|
||||
if (!fs.existsSync(filePath)) {
|
||||
console.log(`Could not located cachefile for ${filename}`, filePath);
|
||||
return;
|
||||
}
|
||||
|
||||
fs.rmSync(filePath);
|
||||
return;
|
||||
}
|
||||
|
||||
// Purges a vector-cache file from the vector-cache/ folder.
|
||||
async function purgeVectorCache(filename = null) {
|
||||
if (!filename) return;
|
||||
console.log(`Purging cached vectorized results of ${filename}.`);
|
||||
|
||||
const digest = uuidv5(filename, uuidv5.URL);
|
||||
const filePath =
|
||||
process.env.NODE_ENV === "development"
|
||||
? path.resolve(__dirname, `../../storage/vector-cache`, `${digest}.json`)
|
||||
: path.resolve(process.env.STORAGE_DIR, `vector-cache`, `${digest}.json`);
|
||||
|
||||
if (!fs.existsSync(filePath)) {
|
||||
console.log(`Could not located cache file for ${filename}`, filePath);
|
||||
return;
|
||||
}
|
||||
|
||||
fs.rmSync(filePath);
|
||||
return;
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
cachedVectorInformation,
|
||||
collectDocumentData,
|
||||
viewLocalFiles,
|
||||
purgeSourceDocument,
|
||||
purgeVectorCache,
|
||||
storeVectorResult,
|
||||
fileData,
|
||||
};
|
||||
|
17
server/utils/files/purgeDocument.js
Normal file
17
server/utils/files/purgeDocument.js
Normal file
@ -0,0 +1,17 @@
|
||||
const { purgeVectorCache, purgeSourceDocument } = require(".");
|
||||
const { Document } = require("../../models/documents");
|
||||
const { Workspace } = require("../../models/workspace");
|
||||
|
||||
async function purgeDocument(filename, meta) {
|
||||
const workspaces = await Workspace.where();
|
||||
for (const workspace of workspaces) {
|
||||
await Document.removeDocuments(workspace, [filename]);
|
||||
}
|
||||
await purgeVectorCache(filename);
|
||||
await purgeSourceDocument(filename);
|
||||
return;
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
purgeDocument,
|
||||
};
|
Loading…
Reference in New Issue
Block a user