mirror of
https://github.com/Mintplex-Labs/anything-llm.git
synced 2024-09-11 23:03:25 +02:00
* WIP converted all sqlite models into prisma calls * modify db setup and fix ApiKey model calls in admin.js * renaming function params to be consistent * converted adminEndpoints to utilize prisma orm * converted chatEndpoints to utilize prisma orm * converted inviteEndpoints to utilize prisma orm * converted systemEndpoints to utilize prisma orm * converted workspaceEndpoints to utilize prisma orm * converting sql queries to prisma calls * fixed default param bug for orderBy and limit * fixed typo for workspace chats * fixed order of deletion to account for sql relations * fix invite CRUD and workspace management CRUD * fixed CRUD for api keys * created prisma setup scripts/docs for understanding how to use prisma * prisma dependency change * removing unneeded console.logs * removing unneeded sql escape function * linting and creating migration script * migration from depreciated sqlite script update * removing unneeded migrations in prisma folder * create backup of old sqlite db and use transactions to ensure all operations complete successfully * adding migrations to gitignore * updated PRISMA.md docs for info on how to use sqlite migration script * comment changes * adding back migrations folder to repo * Reviewing SQL and prisma integraiton on fresh repo * update inline key replacement * ensure migration script executes and maps foreign_keys regardless of db ordering * run migration endpoint * support new prisma backend * bump version * change migration call --------- Co-authored-by: timothycarambat <rambat1010@gmail.com>
111 lines
2.9 KiB
JavaScript
111 lines
2.9 KiB
JavaScript
const { fileData } = require("../utils/files");
|
|
const { v4: uuidv4 } = require("uuid");
|
|
const { getVectorDbClass } = require("../utils/helpers");
|
|
const prisma = require("../utils/prisma");
|
|
const { Telemetry } = require("./telemetry");
|
|
|
|
const Document = {
|
|
forWorkspace: async function (workspaceId = null) {
|
|
if (!workspaceId) return [];
|
|
return await prisma.workspace_documents.findMany({
|
|
where: { workspaceId },
|
|
});
|
|
},
|
|
|
|
delete: async function (clause = {}) {
|
|
try {
|
|
await prisma.workspace_documents.deleteMany({ where: clause });
|
|
return true;
|
|
} catch (error) {
|
|
console.error(error.message);
|
|
return false;
|
|
}
|
|
},
|
|
|
|
firstWhere: async function (clause = {}) {
|
|
try {
|
|
const document = await prisma.workspace_documents.findFirst({
|
|
where: clause,
|
|
});
|
|
return document || null;
|
|
} catch (error) {
|
|
console.error(error.message);
|
|
return null;
|
|
}
|
|
},
|
|
|
|
addDocuments: async function (workspace, additions = []) {
|
|
const VectorDb = getVectorDbClass();
|
|
if (additions.length === 0) return;
|
|
|
|
for (const path of additions) {
|
|
const data = await fileData(path);
|
|
if (!data) continue;
|
|
|
|
const docId = uuidv4();
|
|
const { pageContent, ...metadata } = data;
|
|
const newDoc = {
|
|
docId,
|
|
filename: path.split("/")[1],
|
|
docpath: path,
|
|
workspaceId: workspace.id,
|
|
metadata: JSON.stringify(metadata),
|
|
};
|
|
const vectorized = await VectorDb.addDocumentToNamespace(
|
|
workspace.slug,
|
|
{ ...data, docId },
|
|
path
|
|
);
|
|
if (!vectorized) {
|
|
console.error("Failed to vectorize", path);
|
|
continue;
|
|
}
|
|
|
|
try {
|
|
await prisma.workspace_documents.create({ data: newDoc });
|
|
} catch (error) {
|
|
console.error(error.message);
|
|
}
|
|
}
|
|
|
|
await Telemetry.sendTelemetry("documents_embedded_in_workspace", {
|
|
LLMSelection: process.env.LLM_PROVIDER || "openai",
|
|
VectorDbSelection: process.env.VECTOR_DB || "pinecone",
|
|
});
|
|
return;
|
|
},
|
|
|
|
removeDocuments: async function (workspace, removals = []) {
|
|
const VectorDb = getVectorDbClass();
|
|
if (removals.length === 0) return;
|
|
|
|
for (const path of removals) {
|
|
const document = await this.firstWhere({
|
|
docpath: path,
|
|
workspaceId: workspace.id,
|
|
});
|
|
if (!document) continue;
|
|
await VectorDb.deleteDocumentFromNamespace(
|
|
workspace.slug,
|
|
document.docId
|
|
);
|
|
|
|
try {
|
|
await prisma.workspace_documents.delete({
|
|
where: { id: document.id, workspaceId: workspace.id },
|
|
});
|
|
} catch (error) {
|
|
console.error(error.message);
|
|
}
|
|
}
|
|
|
|
await Telemetry.sendTelemetry("documents_removed_in_workspace", {
|
|
LLMSelection: process.env.LLM_PROVIDER || "openai",
|
|
VectorDbSelection: process.env.VECTOR_DB || "pinecone",
|
|
});
|
|
return true;
|
|
},
|
|
};
|
|
|
|
module.exports = { Document };
|