anything-llm/server/utils/files/data.js

198 lines
6.2 KiB
JavaScript
Raw Normal View History

const fs = require("fs");
const path = require("path");
const { v4 } = require("uuid");
async function exportData() {
const uid = `anythingllm-export-${new Date()
.toJSON()
.slice(0, 10)}-${new Date().toJSON().slice(11, 19)}`;
const folder =
process.env.NODE_ENV === "development"
? path.resolve(__dirname, `../../storage/exports/${uid}`)
: path.resolve(process.env.STORAGE_DIR, `exports/${uid}`);
const storageBase =
process.env.NODE_ENV === "development"
? path.resolve(__dirname, `../../storage`)
: path.resolve(process.env.STORAGE_DIR);
try {
fs.mkdirSync(folder, { recursive: true });
if (fs.existsSync(path.resolve(storageBase, `documents`))) {
console.log("\x1b[34m[EXPORTING DATA]\x1b[0m Copying documents!");
fs.cpSync(
path.resolve(storageBase, `documents`),
path.resolve(folder, "documents"),
{ recursive: true }
);
}
if (fs.existsSync(path.resolve(storageBase, `lancedb`))) {
console.log("\x1b[34m[EXPORTING DATA]\x1b[0m Copying LanceDB data!");
fs.cpSync(
path.resolve(storageBase, `lancedb`),
path.resolve(folder, "lancedb"),
{ recursive: true }
);
}
if (fs.existsSync(path.resolve(storageBase, `vector-cache`))) {
console.log("\x1b[34m[EXPORTING DATA]\x1b[0m Copying vector cache!");
fs.cpSync(
path.resolve(storageBase, `vector-cache`),
path.resolve(folder, "vector-cache"),
{ recursive: true }
);
}
if (fs.existsSync(path.resolve(storageBase, `anythingllm.db`))) {
console.log(
"\x1b[34m[EXPORTING DATA]\x1b[0m Copying anythingllm database!"
);
fs.cpSync(
path.resolve(storageBase, `anythingllm.db`),
path.resolve(folder, "anythingllm.db")
);
}
await zipDirectory(folder, path.resolve(folder, `../${uid}.zip`));
fs.rmSync(folder, { recursive: true, force: true });
return { filename: `${uid}.zip`, error: null };
} catch (e) {
// If anything goes wrong - abort and clean up
console.error(e);
if (fs.existsSync(folder))
fs.rmSync(folder, { recursive: true, force: true });
return { filename: null, error: e.message };
}
}
async function unpackAndOverwriteImport(importFilename) {
const importFilepath =
process.env.NODE_ENV === "development"
? path.resolve(__dirname, `../../storage/imports/${importFilename}`)
: path.resolve(process.env.STORAGE_DIR, `imports/${importFilename}`);
if (!fs.existsSync(importFilepath))
return { success: false, error: "Import file does not exist." };
const uid = v4();
const outDir =
process.env.NODE_ENV === "development"
? path.resolve(__dirname, `../../storage/imports/${uid}`)
: path.resolve(process.env.STORAGE_DIR, `imports/${uid}`);
const storageBase =
process.env.NODE_ENV === "development"
? path.resolve(__dirname, `../../storage`)
: path.resolve(process.env.STORAGE_DIR);
try {
console.log(
"\x1b[34m[EXTRACTING DATA]\x1b[0m Extracting data from zip into storage!"
);
const unzipProc = await unzipDirectory(importFilepath, outDir);
if (!unzipProc.success) return unzipProc;
if (fs.existsSync(path.resolve(outDir, `documents`))) {
console.log(
"\x1b[34m[OVERWRITE & IMPORT DATA]\x1b[0m Importing documents!"
);
if (fs.existsSync(path.resolve(storageBase, `documents`)))
fs.rmSync(path.resolve(storageBase, `documents`), {
recursive: true,
force: true,
});
fs.cpSync(
path.resolve(outDir, `documents`),
path.resolve(storageBase, "documents"),
{ recursive: true }
);
}
if (fs.existsSync(path.resolve(outDir, `lancedb`))) {
console.log(
"\x1b[34m[OVERWRITE & IMPORT DATA]\x1b[0m Importing LanceDb!"
);
if (fs.existsSync(path.resolve(storageBase, `lancedb`)))
fs.rmSync(path.resolve(storageBase, `lancedb`), {
recursive: true,
force: true,
});
fs.cpSync(
path.resolve(outDir, `lancedb`),
path.resolve(storageBase, "lancedb"),
{ recursive: true }
);
}
if (fs.existsSync(path.resolve(outDir, `vector-cache`))) {
console.log(
"\x1b[34m[OVERWRITE & IMPORT DATA]\x1b[0m Importing Vector Cache!"
);
if (fs.existsSync(path.resolve(storageBase, `vector-cache`)))
fs.rmSync(path.resolve(storageBase, `vector-cache`), {
recursive: true,
force: true,
});
fs.cpSync(
path.resolve(outDir, `vector-cache`),
path.resolve(storageBase, "vector-cache"),
{ recursive: true }
);
}
if (fs.existsSync(path.resolve(outDir, `anythingllm.db`))) {
console.log(
"\x1b[34m[OVERWRITE & IMPORT DATA]\x1b[0m Importing AnythingLLM DB!"
);
if (fs.existsSync(path.resolve(storageBase, `anythingllm.db`)))
fs.rmSync(path.resolve(storageBase, `anythingllm.db`), { force: true });
fs.cpSync(
path.resolve(outDir, `anythingllm.db`),
path.resolve(storageBase, "anythingllm.db")
);
}
fs.rmSync(outDir, { recursive: true, force: true });
fs.rmSync(importFilepath, { force: true });
return { success: true, error: null };
} catch (e) {
console.error(e);
if (fs.existsSync(outDir))
fs.rmSync(outDir, { recursive: true, force: true });
if (fs.existsSync(importFilepath)) fs.rmSync(importFilepath);
return { success: false, error: e.message };
}
}
function zipDirectory(sourceDir, outPath) {
const archiver = require("archiver");
const archive = archiver("zip", { zlib: { level: 9 } });
const stream = fs.createWriteStream(outPath);
return new Promise((resolve, reject) => {
archive
.directory(sourceDir, false)
.on("error", (err) => reject(err))
.pipe(stream);
stream.on("close", () => resolve());
archive.finalize();
});
}
async function unzipDirectory(sourcePath, outDir) {
const extract = require("extract-zip");
try {
await extract(sourcePath, { dir: outDir });
return { success: true, error: null };
} catch (e) {
console.error("unzipToDirectory", e);
return { success: false, error: e.message };
}
}
module.exports = {
exportData,
unpackAndOverwriteImport,
};