anything-llm/server/endpoints/system.js
Timothy Carambat bbe3c0a5d6
Bulk remove files from file picker (#830)
Do bulk deletion on backend
2024-02-27 11:53:42 -08:00

984 lines
28 KiB
JavaScript

process.env.NODE_ENV === "development"
? require("dotenv").config({ path: `.env.${process.env.NODE_ENV}` })
: require("dotenv").config();
const { viewLocalFiles, normalizePath } = require("../utils/files");
const { purgeDocument, purgeFolder } = require("../utils/files/purgeDocument");
const { getVectorDbClass } = require("../utils/helpers");
const { updateENV, dumpENV } = require("../utils/helpers/updateENV");
const {
reqBody,
makeJWT,
userFromSession,
multiUserMode,
queryParams,
} = require("../utils/http");
const { setupLogoUploads, setupPfpUploads } = require("../utils/files/multer");
const { v4 } = require("uuid");
const { SystemSettings } = require("../models/systemSettings");
const { User } = require("../models/user");
const { validatedRequest } = require("../utils/middleware/validatedRequest");
const { handleLogoUploads } = setupLogoUploads();
const { handlePfpUploads } = setupPfpUploads();
const fs = require("fs");
const path = require("path");
const {
getDefaultFilename,
determineLogoFilepath,
fetchLogo,
validFilename,
renameLogoFile,
removeCustomLogo,
LOGO_FILENAME,
} = require("../utils/files/logo");
const { Telemetry } = require("../models/telemetry");
const { WelcomeMessages } = require("../models/welcomeMessages");
const { ApiKey } = require("../models/apiKeys");
const { getCustomModels } = require("../utils/helpers/customModels");
const { WorkspaceChats } = require("../models/workspaceChats");
const {
flexUserRoleValid,
ROLES,
} = require("../utils/middleware/multiUserProtected");
const { fetchPfp, determinePfpFilepath } = require("../utils/files/pfp");
const {
prepareWorkspaceChatsForExport,
exportChatsAsType,
} = require("../utils/helpers/chat/convertTo");
const { EventLogs } = require("../models/eventLogs");
const { CollectorApi } = require("../utils/collectorApi");
function systemEndpoints(app) {
if (!app) return;
app.get("/ping", (_, response) => {
response.status(200).json({ online: true });
});
app.get("/migrate", async (_, response) => {
const execSync = require("child_process").execSync;
execSync("npx prisma migrate deploy --schema=./prisma/schema.prisma", {
stdio: "inherit",
});
response.sendStatus(200);
});
app.get("/env-dump", async (_, response) => {
if (process.env.NODE_ENV !== "production")
return response.sendStatus(200).end();
await dumpENV();
response.sendStatus(200).end();
});
app.get("/setup-complete", async (_, response) => {
try {
const results = await SystemSettings.currentSettings();
response.status(200).json({ results });
} catch (e) {
console.log(e.message, e);
response.sendStatus(500).end();
}
});
app.get(
"/system/check-token",
[validatedRequest],
async (request, response) => {
try {
if (multiUserMode(response)) {
const user = await userFromSession(request, response);
if (!user || user.suspended) {
response.sendStatus(403).end();
return;
}
response.sendStatus(200).end();
return;
}
response.sendStatus(200).end();
} catch (e) {
console.log(e.message, e);
response.sendStatus(500).end();
}
}
);
app.post("/request-token", async (request, response) => {
try {
const bcrypt = require("bcrypt");
if (await SystemSettings.isMultiUserMode()) {
const { username, password } = reqBody(request);
const existingUser = await User.get({ username });
if (!existingUser) {
await EventLogs.logEvent(
"failed_login_invalid_username",
{
ip: request.ip || "Unknown IP",
username: username || "Unknown user",
},
existingUser?.id
);
response.status(200).json({
user: null,
valid: false,
token: null,
message: "[001] Invalid login credentials.",
});
return;
}
if (!bcrypt.compareSync(password, existingUser.password)) {
await EventLogs.logEvent(
"failed_login_invalid_password",
{
ip: request.ip || "Unknown IP",
username: username || "Unknown user",
},
existingUser?.id
);
response.status(200).json({
user: null,
valid: false,
token: null,
message: "[002] Invalid login credentials.",
});
return;
}
if (existingUser.suspended) {
await EventLogs.logEvent(
"failed_login_account_suspended",
{
ip: request.ip || "Unknown IP",
username: username || "Unknown user",
},
existingUser?.id
);
response.status(200).json({
user: null,
valid: false,
token: null,
message: "[004] Account suspended by admin.",
});
return;
}
await Telemetry.sendTelemetry(
"login_event",
{ multiUserMode: false },
existingUser?.id
);
await EventLogs.logEvent(
"login_event",
{
ip: request.ip || "Unknown IP",
username: existingUser.username || "Unknown user",
},
existingUser?.id
);
response.status(200).json({
valid: true,
user: existingUser,
token: makeJWT(
{ id: existingUser.id, username: existingUser.username },
"30d"
),
message: null,
});
return;
} else {
const { password } = reqBody(request);
if (
!bcrypt.compareSync(
password,
bcrypt.hashSync(process.env.AUTH_TOKEN, 10)
)
) {
await EventLogs.logEvent("failed_login_invalid_password", {
ip: request.ip || "Unknown IP",
multiUserMode: false,
});
response.status(401).json({
valid: false,
token: null,
message: "[003] Invalid password provided",
});
return;
}
await Telemetry.sendTelemetry("login_event", { multiUserMode: false });
await EventLogs.logEvent("login_event", {
ip: request.ip || "Unknown IP",
multiUserMode: false,
});
response.status(200).json({
valid: true,
token: makeJWT({ p: password }, "30d"),
message: null,
});
}
} catch (e) {
console.log(e.message, e);
response.sendStatus(500).end();
}
});
app.get(
"/system/system-vectors",
[validatedRequest, flexUserRoleValid([ROLES.admin, ROLES.manager])],
async (request, response) => {
try {
const query = queryParams(request);
const VectorDb = getVectorDbClass();
const vectorCount = !!query.slug
? await VectorDb.namespaceCount(query.slug)
: await VectorDb.totalVectors();
response.status(200).json({ vectorCount });
} catch (e) {
console.log(e.message, e);
response.sendStatus(500).end();
}
}
);
app.delete(
"/system/remove-document",
[validatedRequest, flexUserRoleValid([ROLES.admin, ROLES.manager])],
async (request, response) => {
try {
const { name } = reqBody(request);
await purgeDocument(name);
response.sendStatus(200).end();
} catch (e) {
console.log(e.message, e);
response.sendStatus(500).end();
}
}
);
app.delete(
"/system/remove-documents",
[validatedRequest, flexUserRoleValid([ROLES.admin, ROLES.manager])],
async (request, response) => {
try {
const { names } = reqBody(request);
for await (const name of names) await purgeDocument(name);
response.sendStatus(200).end();
} catch (e) {
console.log(e.message, e);
response.sendStatus(500).end();
}
}
);
app.delete(
"/system/remove-folder",
[validatedRequest, flexUserRoleValid([ROLES.admin, ROLES.manager])],
async (request, response) => {
try {
const { name } = reqBody(request);
await purgeFolder(name);
response.sendStatus(200).end();
} catch (e) {
console.log(e.message, e);
response.sendStatus(500).end();
}
}
);
app.get(
"/system/local-files",
[validatedRequest, flexUserRoleValid([ROLES.admin, ROLES.manager])],
async (_, response) => {
try {
const localFiles = await viewLocalFiles();
response.status(200).json({ localFiles });
} catch (e) {
console.log(e.message, e);
response.sendStatus(500).end();
}
}
);
app.get(
"/system/document-processing-status",
[validatedRequest],
async (_, response) => {
try {
const online = await new CollectorApi().online();
response.sendStatus(online ? 200 : 503);
} catch (e) {
console.log(e.message, e);
response.sendStatus(500).end();
}
}
);
app.get(
"/system/accepted-document-types",
[validatedRequest],
async (_, response) => {
try {
const types = await new CollectorApi().acceptedFileTypes();
if (!types) {
response.sendStatus(404).end();
return;
}
response.status(200).json({ types });
} catch (e) {
console.log(e.message, e);
response.sendStatus(500).end();
}
}
);
app.post(
"/system/update-env",
[validatedRequest, flexUserRoleValid([ROLES.admin])],
async (request, response) => {
try {
const body = reqBody(request);
const { newValues, error } = await updateENV(
body,
false,
response?.locals?.user?.id
);
if (process.env.NODE_ENV === "production") await dumpENV();
response.status(200).json({ newValues, error });
} catch (e) {
console.log(e.message, e);
response.sendStatus(500).end();
}
}
);
app.post(
"/system/update-password",
[validatedRequest],
async (request, response) => {
try {
// Cannot update password in multi - user mode.
if (multiUserMode(response)) {
response.sendStatus(401).end();
return;
}
const { usePassword, newPassword } = reqBody(request);
const { error } = await updateENV(
{
AuthToken: usePassword ? newPassword : "",
JWTSecret: usePassword ? v4() : "",
},
true
);
if (process.env.NODE_ENV === "production") await dumpENV();
response.status(200).json({ success: !error, error });
} catch (e) {
console.log(e.message, e);
response.sendStatus(500).end();
}
}
);
app.post(
"/system/enable-multi-user",
[validatedRequest],
async (request, response) => {
try {
const { username, password } = reqBody(request);
const multiUserModeEnabled = await SystemSettings.isMultiUserMode();
if (multiUserModeEnabled) {
response.status(200).json({
success: false,
error: "Multi-user mode is already enabled.",
});
return;
}
const { user, error } = await User.create({
username,
password,
role: ROLES.admin,
});
await SystemSettings.updateSettings({
multi_user_mode: true,
users_can_delete_workspaces: false,
limit_user_messages: false,
message_limit: 25,
});
await updateENV(
{
AuthToken: "",
JWTSecret: process.env.JWT_SECRET || v4(),
},
true
);
if (process.env.NODE_ENV === "production") await dumpENV();
await Telemetry.sendTelemetry("enabled_multi_user_mode", {
multiUserMode: true,
});
await EventLogs.logEvent("multi_user_mode_enabled", {}, user?.id);
response.status(200).json({ success: !!user, error });
} catch (e) {
await User.delete({});
await SystemSettings.updateSettings({
multi_user_mode: false,
});
console.log(e.message, e);
response.sendStatus(500).end();
}
}
);
app.get("/system/multi-user-mode", async (_, response) => {
try {
const multiUserMode = await SystemSettings.isMultiUserMode();
response.status(200).json({ multiUserMode });
} catch (e) {
console.log(e.message, e);
response.sendStatus(500).end();
}
});
app.get("/system/logo", async function (_, response) {
try {
const defaultFilename = getDefaultFilename();
const logoPath = await determineLogoFilepath(defaultFilename);
const { found, buffer, size, mime } = fetchLogo(logoPath);
if (!found) {
response.sendStatus(204).end();
return;
}
response.writeHead(200, {
"Content-Type": mime || "image/png",
"Content-Disposition": `attachment; filename=${path.basename(
logoPath
)}`,
"Content-Length": size,
});
response.end(Buffer.from(buffer, "base64"));
return;
} catch (error) {
console.error("Error processing the logo request:", error);
response.status(500).json({ message: "Internal server error" });
}
});
app.get("/system/footer-data", [validatedRequest], async (_, response) => {
try {
const footerData =
(await SystemSettings.get({ label: "footer_data" }))?.value ??
JSON.stringify([]);
response.status(200).json({ footerData: footerData });
} catch (error) {
console.error("Error fetching footer data:", error);
response.status(500).json({ message: "Internal server error" });
}
});
app.get("/system/support-email", [validatedRequest], async (_, response) => {
try {
const supportEmail =
(
await SystemSettings.get({
label: "support_email",
})
)?.value ?? null;
response.status(200).json({ supportEmail: supportEmail });
} catch (error) {
console.error("Error fetching support email:", error);
response.status(500).json({ message: "Internal server error" });
}
});
app.get(
"/system/pfp/:id",
[validatedRequest, flexUserRoleValid([ROLES.all])],
async function (request, response) {
try {
const { id } = request.params;
const pfpPath = await determinePfpFilepath(id);
if (!pfpPath) {
response.sendStatus(204).end();
return;
}
const { found, buffer, size, mime } = fetchPfp(pfpPath);
if (!found) {
response.sendStatus(204).end();
return;
}
response.writeHead(200, {
"Content-Type": mime || "image/png",
"Content-Disposition": `attachment; filename=${path.basename(
pfpPath
)}`,
"Content-Length": size,
});
response.end(Buffer.from(buffer, "base64"));
return;
} catch (error) {
console.error("Error processing the logo request:", error);
response.status(500).json({ message: "Internal server error" });
}
}
);
app.post(
"/system/upload-pfp",
[validatedRequest, flexUserRoleValid([ROLES.all])],
handlePfpUploads.single("file"),
async function (request, response) {
try {
const user = await userFromSession(request, response);
const uploadedFileName = request.randomFileName;
if (!uploadedFileName) {
return response.status(400).json({ message: "File upload failed." });
}
const userRecord = await User.get({ id: user.id });
const oldPfpFilename = userRecord.pfpFilename;
console.log("oldPfpFilename", oldPfpFilename);
if (oldPfpFilename) {
const oldPfpPath = path.join(
__dirname,
`../storage/assets/pfp/${normalizePath(userRecord.pfpFilename)}`
);
if (fs.existsSync(oldPfpPath)) fs.unlinkSync(oldPfpPath);
}
const { success, error } = await User.update(user.id, {
pfpFilename: uploadedFileName,
});
return response.status(success ? 200 : 500).json({
message: success
? "Profile picture uploaded successfully."
: error || "Failed to update with new profile picture.",
});
} catch (error) {
console.error("Error processing the profile picture upload:", error);
response.status(500).json({ message: "Internal server error" });
}
}
);
app.delete(
"/system/remove-pfp",
[validatedRequest, flexUserRoleValid([ROLES.all])],
async function (request, response) {
try {
const user = await userFromSession(request, response);
const userRecord = await User.get({ id: user.id });
const oldPfpFilename = userRecord.pfpFilename;
console.log("oldPfpFilename", oldPfpFilename);
if (oldPfpFilename) {
const oldPfpPath = path.join(
__dirname,
`../storage/assets/pfp/${normalizePath(oldPfpFilename)}`
);
if (fs.existsSync(oldPfpPath)) fs.unlinkSync(oldPfpPath);
}
const { success, error } = await User.update(user.id, {
pfpFilename: null,
});
return response.status(success ? 200 : 500).json({
message: success
? "Profile picture removed successfully."
: error || "Failed to remove profile picture.",
});
} catch (error) {
console.error("Error processing the profile picture removal:", error);
response.status(500).json({ message: "Internal server error" });
}
}
);
app.post(
"/system/upload-logo",
[validatedRequest, flexUserRoleValid([ROLES.admin, ROLES.manager])],
handleLogoUploads.single("logo"),
async (request, response) => {
if (!request.file || !request.file.originalname) {
return response.status(400).json({ message: "No logo file provided." });
}
if (!validFilename(request.file.originalname)) {
return response.status(400).json({
message: "Invalid file name. Please choose a different file.",
});
}
try {
const newFilename = await renameLogoFile(request.file.originalname);
const existingLogoFilename = await SystemSettings.currentLogoFilename();
await removeCustomLogo(existingLogoFilename);
const { success, error } = await SystemSettings.updateSettings({
logo_filename: newFilename,
});
return response.status(success ? 200 : 500).json({
message: success
? "Logo uploaded successfully."
: error || "Failed to update with new logo.",
});
} catch (error) {
console.error("Error processing the logo upload:", error);
response.status(500).json({ message: "Error uploading the logo." });
}
}
);
app.get("/system/is-default-logo", async (_, response) => {
try {
const currentLogoFilename = await SystemSettings.currentLogoFilename();
const isDefaultLogo = currentLogoFilename === LOGO_FILENAME;
response.status(200).json({ isDefaultLogo });
} catch (error) {
console.error("Error processing the logo request:", error);
response.status(500).json({ message: "Internal server error" });
}
});
app.get(
"/system/remove-logo",
[validatedRequest, flexUserRoleValid([ROLES.admin, ROLES.manager])],
async (_request, response) => {
try {
const currentLogoFilename = await SystemSettings.currentLogoFilename();
await removeCustomLogo(currentLogoFilename);
const { success, error } = await SystemSettings.updateSettings({
logo_filename: LOGO_FILENAME,
});
return response.status(success ? 200 : 500).json({
message: success
? "Logo removed successfully."
: error || "Failed to update with new logo.",
});
} catch (error) {
console.error("Error processing the logo removal:", error);
response.status(500).json({ message: "Error removing the logo." });
}
}
);
app.get(
"/system/can-delete-workspaces",
[validatedRequest],
async function (request, response) {
try {
if (!response.locals.multiUserMode) {
return response.status(200).json({ canDelete: true });
}
const user = await userFromSession(request, response);
if ([ROLES.admin, ROLES.manager].includes(user?.role)) {
return response.status(200).json({ canDelete: true });
}
const canDelete = await SystemSettings.canDeleteWorkspaces();
response.status(200).json({ canDelete });
} catch (error) {
console.error("Error fetching can delete workspaces:", error);
response.status(500).json({
success: false,
message: "Internal server error",
canDelete: false,
});
}
}
);
app.get(
"/system/welcome-messages",
[validatedRequest, flexUserRoleValid([ROLES.all])],
async function (_, response) {
try {
const welcomeMessages = await WelcomeMessages.getMessages();
response.status(200).json({ success: true, welcomeMessages });
} catch (error) {
console.error("Error fetching welcome messages:", error);
response
.status(500)
.json({ success: false, message: "Internal server error" });
}
}
);
app.post(
"/system/set-welcome-messages",
[validatedRequest, flexUserRoleValid([ROLES.admin, ROLES.manager])],
async (request, response) => {
try {
const { messages = [] } = reqBody(request);
if (!Array.isArray(messages)) {
return response.status(400).json({
success: false,
message: "Invalid message format. Expected an array of messages.",
});
}
await WelcomeMessages.saveAll(messages);
return response.status(200).json({
success: true,
message: "Welcome messages saved successfully.",
});
} catch (error) {
console.error("Error processing the welcome messages:", error);
response.status(500).json({
success: true,
message: "Error saving the welcome messages.",
});
}
}
);
app.get("/system/api-keys", [validatedRequest], async (_, response) => {
try {
if (response.locals.multiUserMode) {
return response.sendStatus(401).end();
}
const apiKeys = await ApiKey.where({});
return response.status(200).json({
apiKeys,
error: null,
});
} catch (error) {
console.error(error);
response.status(500).json({
apiKey: null,
error: "Could not find an API Key.",
});
}
});
app.post(
"/system/generate-api-key",
[validatedRequest],
async (_, response) => {
try {
if (response.locals.multiUserMode) {
return response.sendStatus(401).end();
}
const { apiKey, error } = await ApiKey.create();
await Telemetry.sendTelemetry("api_key_created");
await EventLogs.logEvent(
"api_key_created",
{},
response?.locals?.user?.id
);
return response.status(200).json({
apiKey,
error,
});
} catch (error) {
console.error(error);
response.status(500).json({
apiKey: null,
error: "Error generating api key.",
});
}
}
);
app.delete("/system/api-key", [validatedRequest], async (_, response) => {
try {
if (response.locals.multiUserMode) {
return response.sendStatus(401).end();
}
await ApiKey.delete();
await EventLogs.logEvent(
"api_key_deleted",
{ deletedBy: response.locals?.user?.username },
response?.locals?.user?.id
);
return response.status(200).end();
} catch (error) {
console.error(error);
response.status(500).end();
}
});
app.post(
"/system/custom-models",
[validatedRequest],
async (request, response) => {
try {
const { provider, apiKey = null, basePath = null } = reqBody(request);
const { models, error } = await getCustomModels(
provider,
apiKey,
basePath
);
return response.status(200).json({
models,
error,
});
} catch (error) {
console.error(error);
response.status(500).end();
}
}
);
app.post(
"/system/event-logs",
[validatedRequest, flexUserRoleValid([ROLES.admin])],
async (request, response) => {
try {
const { offset = 0, limit = 20 } = reqBody(request);
const logs = await EventLogs.whereWithData({}, limit, offset * limit, {
id: "desc",
});
const totalLogs = await EventLogs.count();
const hasPages = totalLogs > (offset + 1) * limit;
response.status(200).json({ logs: logs, hasPages, totalLogs });
} catch (e) {
console.error(e);
response.sendStatus(500).end();
}
}
);
app.delete(
"/system/event-logs",
[validatedRequest, flexUserRoleValid([ROLES.admin])],
async (_, response) => {
try {
await EventLogs.delete();
await EventLogs.logEvent(
"event_logs_cleared",
{},
response?.locals?.user?.id
);
response.json({ success: true });
} catch (e) {
console.error(e);
response.sendStatus(500).end();
}
}
);
app.post(
"/system/workspace-chats",
[validatedRequest, flexUserRoleValid([ROLES.admin, ROLES.manager])],
async (request, response) => {
try {
const { offset = 0, limit = 20 } = reqBody(request);
const chats = await WorkspaceChats.whereWithData(
{},
limit,
offset * limit,
{ id: "desc" }
);
const totalChats = await WorkspaceChats.count();
const hasPages = totalChats > (offset + 1) * limit;
response.status(200).json({ chats: chats, hasPages, totalChats });
} catch (e) {
console.error(e);
response.sendStatus(500).end();
}
}
);
app.delete(
"/system/workspace-chats/:id",
[validatedRequest, flexUserRoleValid([ROLES.admin, ROLES.manager])],
async (request, response) => {
try {
const { id } = request.params;
await WorkspaceChats.delete({ id: Number(id) });
response.json({ success: true, error: null });
} catch (e) {
console.error(e);
response.sendStatus(500).end();
}
}
);
app.get(
"/system/export-chats",
[validatedRequest, flexUserRoleValid([ROLES.manager, ROLES.admin])],
async (request, response) => {
try {
const { type = "jsonl" } = request.query;
const chats = await prepareWorkspaceChatsForExport(type);
const { contentType, data } = await exportChatsAsType(chats, type);
await EventLogs.logEvent(
"exported_chats",
{
type,
},
response.locals.user?.id
);
response.setHeader("Content-Type", contentType);
response.status(200).send(data);
} catch (e) {
console.error(e);
response.sendStatus(500).end();
}
}
);
// Used for when a user in multi-user updates their own profile
// from the UI.
app.post("/system/user", [validatedRequest], async (request, response) => {
try {
const sessionUser = await userFromSession(request, response);
const { username, password } = reqBody(request);
const id = Number(sessionUser.id);
if (!id) {
response.status(400).json({ success: false, error: "Invalid user ID" });
return;
}
const updates = {};
if (username) {
updates.username = username;
}
if (password) {
updates.password = password;
}
if (Object.keys(updates).length === 0) {
response
.status(400)
.json({ success: false, error: "No updates provided" });
return;
}
const { success, error } = await User.update(id, updates);
response.status(200).json({ success, error });
} catch (e) {
console.error(e);
response.sendStatus(500).end();
}
});
}
module.exports = { systemEndpoints };