mirror of
https://github.com/Mintplex-Labs/anything-llm.git
synced 2024-10-04 01:40:12 +02:00
add ability to purge document from custom documents as well as cleanup its associated cache file (#113)
* add ability to purge document from custom documents as well as cleanup its assoicated cache file * update alert text
This commit is contained in:
parent
9d0becb2ee
commit
60a00843df
@ -8,6 +8,7 @@ import {
|
|||||||
Zap,
|
Zap,
|
||||||
} from "react-feather";
|
} from "react-feather";
|
||||||
import { nFormatter } from "../../../../../utils/numbers";
|
import { nFormatter } from "../../../../../utils/numbers";
|
||||||
|
import System from "../../../../../models/system";
|
||||||
|
|
||||||
export default function Directory({
|
export default function Directory({
|
||||||
files,
|
files,
|
||||||
@ -19,6 +20,16 @@ export default function Directory({
|
|||||||
const [isExpanded, toggleExpanded] = useState(false);
|
const [isExpanded, toggleExpanded] = useState(false);
|
||||||
const [showDetails, toggleDetails] = useState(false);
|
const [showDetails, toggleDetails] = useState(false);
|
||||||
const [showZap, setShowZap] = useState(false);
|
const [showZap, setShowZap] = useState(false);
|
||||||
|
const handleDelete = async (name, meta) => {
|
||||||
|
if (
|
||||||
|
!window.confirm(
|
||||||
|
"Are you sure you want to delete this document?\nThis will require you to re-upload and re-embed it.\nThis document will be removed from any workspace that is currently referencing it.\nThis action is not reversible."
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return false;
|
||||||
|
document?.getElementById(meta?.id)?.remove();
|
||||||
|
await System.deleteDocument(name, meta);
|
||||||
|
};
|
||||||
|
|
||||||
if (files.type === "folder") {
|
if (files.type === "folder") {
|
||||||
return (
|
return (
|
||||||
@ -73,7 +84,7 @@ export default function Directory({
|
|||||||
|
|
||||||
const { name, type: _type, ...meta } = files;
|
const { name, type: _type, ...meta } = files;
|
||||||
return (
|
return (
|
||||||
<div className="ml-[20px] my-2">
|
<div className="ml-[20px] my-2" id={meta.id}>
|
||||||
<div className="flex items-center">
|
<div className="flex items-center">
|
||||||
{meta?.cached && (
|
{meta?.cached && (
|
||||||
<button
|
<button
|
||||||
@ -134,6 +145,7 @@ export default function Directory({
|
|||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
{showDetails && (
|
{showDetails && (
|
||||||
|
<div className="w-full flex flex-col">
|
||||||
<div className="ml-[20px] flex flex-col gap-y-1 my-1 p-2 rounded-md bg-slate-200 font-mono text-sm overflow-x-scroll">
|
<div className="ml-[20px] flex flex-col gap-y-1 my-1 p-2 rounded-md bg-slate-200 font-mono text-sm overflow-x-scroll">
|
||||||
{Object.entries(meta).map(([key, value], i) => {
|
{Object.entries(meta).map(([key, value], i) => {
|
||||||
if (key === "cached") return null;
|
if (key === "cached") return null;
|
||||||
@ -144,6 +156,15 @@ export default function Directory({
|
|||||||
);
|
);
|
||||||
})}
|
})}
|
||||||
</div>
|
</div>
|
||||||
|
<div
|
||||||
|
onClick={() => handleDelete(`${parent}/${name}`, meta)}
|
||||||
|
className="flex items-center justify-end w-full"
|
||||||
|
>
|
||||||
|
<button className="text-sm text-slate-400 dark:text-stone-500 hover:text-red-500">
|
||||||
|
Purge Document
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
)}
|
)}
|
||||||
</div>
|
</div>
|
||||||
);
|
);
|
||||||
|
@ -86,6 +86,18 @@ const System = {
|
|||||||
return { newValues: null, error: e.message };
|
return { newValues: null, error: e.message };
|
||||||
});
|
});
|
||||||
},
|
},
|
||||||
|
deleteDocument: async (name, meta) => {
|
||||||
|
return await fetch(`${API_BASE}/system/remove-document`, {
|
||||||
|
method: "DELETE",
|
||||||
|
headers: baseHeaders(),
|
||||||
|
body: JSON.stringify({ name, meta }),
|
||||||
|
})
|
||||||
|
.then((res) => res.ok)
|
||||||
|
.catch((e) => {
|
||||||
|
console.error(e);
|
||||||
|
return false;
|
||||||
|
});
|
||||||
|
},
|
||||||
};
|
};
|
||||||
|
|
||||||
export default System;
|
export default System;
|
||||||
|
@ -7,6 +7,7 @@ const {
|
|||||||
checkPythonAppAlive,
|
checkPythonAppAlive,
|
||||||
acceptedFileTypes,
|
acceptedFileTypes,
|
||||||
} = require("../utils/files/documentProcessor");
|
} = require("../utils/files/documentProcessor");
|
||||||
|
const { purgeDocument } = require("../utils/files/purgeDocument");
|
||||||
const { getVectorDbClass } = require("../utils/helpers");
|
const { getVectorDbClass } = require("../utils/helpers");
|
||||||
const { updateENV } = require("../utils/helpers/updateENV");
|
const { updateENV } = require("../utils/helpers/updateENV");
|
||||||
const { reqBody, makeJWT } = require("../utils/http");
|
const { reqBody, makeJWT } = require("../utils/http");
|
||||||
@ -94,6 +95,17 @@ function systemEndpoints(app) {
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
app.delete("/system/remove-document", async (request, response) => {
|
||||||
|
try {
|
||||||
|
const { name, meta } = reqBody(request);
|
||||||
|
await purgeDocument(name, meta);
|
||||||
|
response.sendStatus(200).end();
|
||||||
|
} catch (e) {
|
||||||
|
console.log(e.message, e);
|
||||||
|
response.sendStatus(500).end();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
app.get("/system/local-files", async (_, response) => {
|
app.get("/system/local-files", async (_, response) => {
|
||||||
try {
|
try {
|
||||||
const localFiles = await viewLocalFiles();
|
const localFiles = await viewLocalFiles();
|
||||||
|
@ -141,10 +141,51 @@ async function storeVectorResult(vectorData = [], filename = null) {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Purges a file from the documents/ folder.
|
||||||
|
async function purgeSourceDocument(filename = null) {
|
||||||
|
if (!filename) return;
|
||||||
|
console.log(`Purging document of ${filename}.`);
|
||||||
|
|
||||||
|
const filePath =
|
||||||
|
process.env.NODE_ENV === "development"
|
||||||
|
? path.resolve(__dirname, `../../storage/documents`, filename)
|
||||||
|
: path.resolve(process.env.STORAGE_DIR, `documents`, filename);
|
||||||
|
|
||||||
|
if (!fs.existsSync(filePath)) {
|
||||||
|
console.log(`Could not located cachefile for ${filename}`, filePath);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
fs.rmSync(filePath);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Purges a vector-cache file from the vector-cache/ folder.
|
||||||
|
async function purgeVectorCache(filename = null) {
|
||||||
|
if (!filename) return;
|
||||||
|
console.log(`Purging cached vectorized results of ${filename}.`);
|
||||||
|
|
||||||
|
const digest = uuidv5(filename, uuidv5.URL);
|
||||||
|
const filePath =
|
||||||
|
process.env.NODE_ENV === "development"
|
||||||
|
? path.resolve(__dirname, `../../storage/vector-cache`, `${digest}.json`)
|
||||||
|
: path.resolve(process.env.STORAGE_DIR, `vector-cache`, `${digest}.json`);
|
||||||
|
|
||||||
|
if (!fs.existsSync(filePath)) {
|
||||||
|
console.log(`Could not located cache file for ${filename}`, filePath);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
fs.rmSync(filePath);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
module.exports = {
|
module.exports = {
|
||||||
cachedVectorInformation,
|
cachedVectorInformation,
|
||||||
collectDocumentData,
|
collectDocumentData,
|
||||||
viewLocalFiles,
|
viewLocalFiles,
|
||||||
|
purgeSourceDocument,
|
||||||
|
purgeVectorCache,
|
||||||
storeVectorResult,
|
storeVectorResult,
|
||||||
fileData,
|
fileData,
|
||||||
};
|
};
|
||||||
|
17
server/utils/files/purgeDocument.js
Normal file
17
server/utils/files/purgeDocument.js
Normal file
@ -0,0 +1,17 @@
|
|||||||
|
const { purgeVectorCache, purgeSourceDocument } = require(".");
|
||||||
|
const { Document } = require("../../models/documents");
|
||||||
|
const { Workspace } = require("../../models/workspace");
|
||||||
|
|
||||||
|
async function purgeDocument(filename, meta) {
|
||||||
|
const workspaces = await Workspace.where();
|
||||||
|
for (const workspace of workspaces) {
|
||||||
|
await Document.removeDocuments(workspace, [filename]);
|
||||||
|
}
|
||||||
|
await purgeVectorCache(filename);
|
||||||
|
await purgeSourceDocument(filename);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = {
|
||||||
|
purgeDocument,
|
||||||
|
};
|
Loading…
Reference in New Issue
Block a user