kamp/anything-llm
1
0
Fork 0
mirror of https://github.com/Mintplex-Labs/anything-llm.git synced 2024-11-19 20:50:09 +01:00
Code Issues Packages Projects Releases Wiki Activity

improve bulkd write speeds

Browse Source
This commit is contained in:
timothycarambat 2023-07-26 18:06:53 -07:00
parent 91f5f94200
commit 24ca64ce46
3 changed files with 62 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
server/models/documents.js
View File

@ -69,11 +69,8 @@ const Document = {
addDocuments: async function (workspace, additions = []) { addDocuments: async function (workspace, additions = []) {
const VectorDb = getVectorDbClass(); const VectorDb = getVectorDbClass();
if (additions.length === 0) return; if (additions.length === 0) return;
const insertParams = [];
const db = await this.db();
const stmt = await db.prepare(
`INSERT INTO ${this.tablename} (docId, filename, docpath, workspaceId, metadata) VALUES (?,?,?,?,?)`
);
for (const path of additions) { for (const path of additions) {
const data = await fileData(path); const data = await fileData(path);
if (!data) continue; if (!data) continue;
@ -96,7 +93,8 @@ const Document = {
console.error("Failed to vectorize", path); console.error("Failed to vectorize", path);
continue; continue;
} }
stmt.run([
insertParams.push([
docId, docId,
newDoc.filename, newDoc.filename,
newDoc.docpath, newDoc.docpath,
@ -104,19 +102,31 @@ const Document = {
newDoc.metadata, newDoc.metadata,
]); ]);
} }
const db = await this.db();
const stmt = await db.prepare(
`INSERT INTO ${this.tablename} (docId, filename, docpath, workspaceId, metadata) VALUES (?,?,?,?,?)`
);
await db.exec("BEGIN TRANSACTION");
try {
for (const params of insertParams) {
await stmt.run(params);
}
await db.exec("COMMIT");
} catch {
await db.exec("ROLLBACK");
}
stmt.finalize(); stmt.finalize();
db.close(); db.close();
return; return;
}, },
removeDocuments: async function (workspace, removals = []) { removeDocuments: async function (workspace, removals = []) {
const VectorDb = getVectorDbClass(); const VectorDb = getVectorDbClass();
const deleteParams = [];
if (removals.length === 0) return; if (removals.length === 0) return;
const db = await this.db();
const stmt = await db.prepare(
`DELETE FROM ${this.tablename} WHERE docpath = ? AND workspaceId = ?`
);
for (const path of removals) { for (const path of removals) {
const document = await this.firstWhere( const document = await this.firstWhere(
`docPath = '${path}' AND workspaceId = ${workspace.id}` `docPath = '${path}' AND workspaceId = ${workspace.id}`
@ -126,8 +136,24 @@ const Document = {
workspace.slug, workspace.slug,
document.docId document.docId
); );
stmt.run([path, workspace.id]); deleteParams.push([path, workspace.id]);
} }
const db = await this.db();
const stmt = await db.prepare(
`DELETE FROM ${this.tablename} WHERE docpath = ? AND workspaceId = ?`
);
await db.exec("BEGIN TRANSACTION");
try {
for (const params of deleteParams) {
await stmt.run(params);
}
await db.exec("COMMIT");
} catch {
await db.exec("ROLLBACK");
}
stmt.finalize(); stmt.finalize();
db.close(); db.close();
return true; return true;

9
server/models/vectors.js
View File

@ -58,7 +58,14 @@ const DocumentVectors = {
[] []
); );
stmt.run(values); await db.exec("BEGIN TRANSACTION");
try {
await stmt.run(values);
await db.exec("COMMIT");
} catch {
await db.exec("ROLLBACK");
}
stmt.finalize(); stmt.finalize();
db.close(); db.close();

16
server/models/workspaceUsers.js
View File

@ -46,8 +46,14 @@ const WorkspaceUser = {
`INSERT INTO ${this.tablename} (user_id, workspace_id) VALUES (?,?)` `INSERT INTO ${this.tablename} (user_id, workspace_id) VALUES (?,?)`
); );
await db.exec("BEGIN TRANSACTION");
try {
for (const workspaceId of workspaceIds) { for (const workspaceId of workspaceIds) {
stmt.run([userId, workspaceId]); await stmt.run([userId, workspaceId]);
}
await db.exec("COMMIT");
} catch {
await db.exec("ROLLBACK");
} }
stmt.finalize(); stmt.finalize();
@ -61,8 +67,14 @@ const WorkspaceUser = {
`INSERT INTO ${this.tablename} (user_id, workspace_id) VALUES (?,?)` `INSERT INTO ${this.tablename} (user_id, workspace_id) VALUES (?,?)`
); );
await db.exec("BEGIN TRANSACTION");
try {
for (const userId of userIds) { for (const userId of userIds) {
stmt.run([userId, workspaceId]); await stmt.run([userId, workspaceId]);
}
await db.exec("COMMIT");
} catch {
await db.exec("ROLLBACK");
} }
stmt.finalize(); stmt.finalize();