anything-llm/server/utils/prisma/migrateFromSqlite.js

282 lines
8.0 KiB
JavaScript
Raw Normal View History

Replace custom sqlite dbms with prisma (#239) * WIP converted all sqlite models into prisma calls * modify db setup and fix ApiKey model calls in admin.js * renaming function params to be consistent * converted adminEndpoints to utilize prisma orm * converted chatEndpoints to utilize prisma orm * converted inviteEndpoints to utilize prisma orm * converted systemEndpoints to utilize prisma orm * converted workspaceEndpoints to utilize prisma orm * converting sql queries to prisma calls * fixed default param bug for orderBy and limit * fixed typo for workspace chats * fixed order of deletion to account for sql relations * fix invite CRUD and workspace management CRUD * fixed CRUD for api keys * created prisma setup scripts/docs for understanding how to use prisma * prisma dependency change * removing unneeded console.logs * removing unneeded sql escape function * linting and creating migration script * migration from depreciated sqlite script update * removing unneeded migrations in prisma folder * create backup of old sqlite db and use transactions to ensure all operations complete successfully * adding migrations to gitignore * updated PRISMA.md docs for info on how to use sqlite migration script * comment changes * adding back migrations folder to repo * Reviewing SQL and prisma integraiton on fresh repo * update inline key replacement * ensure migration script executes and maps foreign_keys regardless of db ordering * run migration endpoint * support new prisma backend * bump version * change migration call --------- Co-authored-by: timothycarambat <rambat1010@gmail.com>
2023-09-28 23:00:03 +02:00
const { PrismaClient } = require("@prisma/client");
const execSync = require("child_process").execSync;
const fs = require("fs");
const path = require("path");
require("dotenv").config();
const DATABASE_PATH = process.env.DB_URL || "../../storage/anythingllm.db";
const BACKUP_PATH = path.join(
path.dirname(DATABASE_PATH),
"anythingllm_backup.db"
);
// Backup the database before migrating data
function backupDatabase() {
try {
fs.copyFileSync(DATABASE_PATH, BACKUP_PATH);
console.log("Database backup created successfully.");
} catch (error) {
console.error("Failed to create database backup:", error);
}
}
backupDatabase();
const prisma = new PrismaClient();
// Reset the prisma database and prepare it for migration of data from sqlite
function resetAndMigrateDatabase() {
try {
console.log("Resetting and migrating the database...");
execSync("cd ../.. && npx prisma migrate reset --skip-seed --force", {
stdio: "inherit",
});
execSync("cd ../.. && npx prisma migrate dev --name init", {
stdio: "inherit",
});
console.log("Database reset and initial migration completed successfully");
} catch (error) {
console.error("Failed to reset and migrate the database:", error);
}
}
resetAndMigrateDatabase();
// Migrate data from sqlite to prisma
async function migrateData() {
try {
console.log("Starting data migration...");
var legacyMap = {
users: {
count: 0,
},
workspaces: {
count: 0,
},
};
// Step 1: Migrate system_settings table
await migrateTable("system_settings", (row) => {
return prisma.system_settings.create({
data: {
label: row.label,
value: row.value,
createdAt: new Date(row.createdAt),
lastUpdatedAt: new Date(row.lastUpdatedAt),
},
});
});
// Step 2: Migrate users table
await migrateTable("users", (row) => {
legacyMap.users[`user_${row.id}`] = legacyMap.users.count + 1;
legacyMap.users.count++;
return prisma.users.create({
data: {
username: row.username,
password: row.password,
role: row.role,
suspended: row.suspended,
createdAt: new Date(row.createdAt),
lastUpdatedAt: new Date(row.lastUpdatedAt),
},
});
});
// Step 3: Migrate workspaces table
await migrateTable("workspaces", (row) => {
legacyMap.workspaces[`workspace_${row.id}`] =
legacyMap.workspaces.count + 1;
legacyMap.workspaces.count++;
return prisma.workspaces.create({
data: {
name: row.name,
slug: row.slug,
vectorTag: row.vectorTag,
openAiTemp: Number(row.openAiTemp) || 0.7,
openAiHistory: Number(row.openAiHistory) || 20,
openAiPrompt: row.openAiPrompt,
createdAt: new Date(row.createdAt),
lastUpdatedAt: new Date(row.lastUpdatedAt),
},
});
});
// Step 4: Migrate api_keys table
await migrateTable("api_keys", async (row) => {
const legacyUserId = row.createdBy
? legacyMap.users?.[`user_${row.createdBy}`]
: null;
return prisma.api_keys.create({
data: {
secret: row.secret,
...(legacyUserId
? { createdBy: Number(legacyUserId) }
: { createdBy: Number(row.createdBy) }),
createdAt: new Date(row.createdAt),
lastUpdatedAt: new Date(row.lastUpdatedAt),
},
});
});
// Step 5: Migrate invites table
await migrateTable("invites", async (row) => {
const legacyCreatedUserId = row.createdBy
? legacyMap.users?.[`user_${row.createdBy}`]
: null;
const legacyClaimedUserId = row.claimedBy
? legacyMap.users?.[`user_${row.claimedBy}`]
: null;
return prisma.invites.create({
data: {
code: row.code,
status: row.status,
...(legacyClaimedUserId
? { claimedBy: Number(legacyClaimedUserId) }
: { claimedBy: Number(row.claimedBy) }),
...(legacyCreatedUserId
? { createdBy: Number(legacyCreatedUserId) }
: { createdBy: Number(row.createdBy) }),
createdAt: new Date(row.createdAt),
lastUpdatedAt: new Date(row.lastUpdatedAt),
},
});
});
// Step 6: Migrate workspace_documents table
await migrateTable("workspace_documents", async (row) => {
const legacyWorkspaceId = row.workspaceId
? legacyMap.workspaces?.[`workspace_${row.workspaceId}`]
: null;
return prisma.workspace_documents.create({
data: {
docId: row.docId,
filename: row.filename,
docpath: row.docpath,
...(legacyWorkspaceId
? { workspaceId: Number(legacyWorkspaceId) }
: {}),
metadata: row.metadata,
createdAt: new Date(row.createdAt),
lastUpdatedAt: new Date(row.lastUpdatedAt),
},
});
});
// Step 7: Migrate document_vectors table
await migrateTable("document_vectors", (row) => {
return prisma.document_vectors.create({
data: {
docId: row.docId,
vectorId: row.vectorId,
createdAt: new Date(row.createdAt),
lastUpdatedAt: new Date(row.lastUpdatedAt),
},
});
});
// Step 8: Migrate welcome_messages table
await migrateTable("welcome_messages", (row) => {
return prisma.welcome_messages.create({
data: {
user: row.user,
response: row.response,
orderIndex: row.orderIndex,
createdAt: new Date(row.createdAt),
},
});
});
// Step 9: Migrate workspace_users table
await migrateTable("workspace_users", async (row) => {
const legacyUserId = row.user_id
? legacyMap.users?.[`user_${row.user_id}`]
: null;
const legacyWorkspaceId = row.workspace_id
? legacyMap.workspaces?.[`workspace_${row.workspace_id}`]
: null;
if (!legacyUserId || !legacyWorkspaceId) return;
return prisma.workspace_users.create({
data: {
user_id: Number(legacyUserId),
workspace_id: Number(legacyWorkspaceId),
createdAt: new Date(row.createdAt),
lastUpdatedAt: new Date(row.lastUpdatedAt),
},
});
});
// Step 10: Migrate workspace_chats table
await migrateTable("workspace_chats", async (row) => {
const legacyUserId = row.user_id
? legacyMap.users?.[`user_${row.user_id}`]
: null;
const legacyWorkspaceId = row.workspaceId
? legacyMap.workspaces?.[`workspace_${row.workspaceId}`]
: null;
return prisma.workspace_chats.create({
data: {
workspaceId: Number(legacyWorkspaceId),
prompt: row.prompt,
response: row.response,
include: row.include === 1,
...(legacyUserId ? { user_id: Number(legacyUserId) } : {}),
createdAt: new Date(row.createdAt),
lastUpdatedAt: new Date(row.lastUpdatedAt),
},
});
});
console.log("Data migration completed successfully");
} catch (error) {
console.error("Data migration failed:", error);
} finally {
await prisma.$disconnect();
}
}
async function migrateTable(tableName, migrateRowFunc) {
const sqlite3 = require("sqlite3").verbose();
const { open } = require("sqlite");
const db = await open({
filename: BACKUP_PATH,
driver: sqlite3.Database,
});
2023-09-28 23:08:36 +02:00
// Check table exists
2023-09-28 23:12:54 +02:00
const { count } = await db.get(
`SELECT COUNT(*) as count FROM sqlite_master WHERE name='${tableName}'`
);
2023-09-28 23:12:54 +02:00
if (count === 0) {
2023-09-28 23:08:36 +02:00
console.log(
`${tableName} does not exist in legacy DB - nothing to migrate - skipping.`
);
return;
}
Replace custom sqlite dbms with prisma (#239) * WIP converted all sqlite models into prisma calls * modify db setup and fix ApiKey model calls in admin.js * renaming function params to be consistent * converted adminEndpoints to utilize prisma orm * converted chatEndpoints to utilize prisma orm * converted inviteEndpoints to utilize prisma orm * converted systemEndpoints to utilize prisma orm * converted workspaceEndpoints to utilize prisma orm * converting sql queries to prisma calls * fixed default param bug for orderBy and limit * fixed typo for workspace chats * fixed order of deletion to account for sql relations * fix invite CRUD and workspace management CRUD * fixed CRUD for api keys * created prisma setup scripts/docs for understanding how to use prisma * prisma dependency change * removing unneeded console.logs * removing unneeded sql escape function * linting and creating migration script * migration from depreciated sqlite script update * removing unneeded migrations in prisma folder * create backup of old sqlite db and use transactions to ensure all operations complete successfully * adding migrations to gitignore * updated PRISMA.md docs for info on how to use sqlite migration script * comment changes * adding back migrations folder to repo * Reviewing SQL and prisma integraiton on fresh repo * update inline key replacement * ensure migration script executes and maps foreign_keys regardless of db ordering * run migration endpoint * support new prisma backend * bump version * change migration call --------- Co-authored-by: timothycarambat <rambat1010@gmail.com>
2023-09-28 23:00:03 +02:00
const upserts = [];
const rows = await db.all(`SELECT * FROM ${tableName}`);
try {
for (const row of rows) {
await migrateRowFunc(row);
upserts.push(row);
}
} catch (e) {
console.error(e);
console.log({ tableName, upserts });
} finally {
await db.close();
}
return;
}
migrateData();