anything-llm/server/endpoints/admin.js

548 lines
17 KiB
JavaScript
Raw Normal View History

const { ApiKey } = require("../models/apiKeys");
const { Document } = require("../models/documents");
const { EventLogs } = require("../models/eventLogs");
const { Invite } = require("../models/invite");
const { SystemSettings } = require("../models/systemSettings");
const { Telemetry } = require("../models/telemetry");
const { User } = require("../models/user");
const { DocumentVectors } = require("../models/vectors");
const { Workspace } = require("../models/workspace");
const { WorkspaceChats } = require("../models/workspaceChats");
const {
getVectorDbClass,
getEmbeddingEngineSelection,
} = require("../utils/helpers");
const {
validRoleSelection,
canModifyAdmin,
validCanModify,
} = require("../utils/helpers/admin");
const { reqBody, userFromSession, safeJsonParse } = require("../utils/http");
const {
strictMultiUserRoleValid,
flexUserRoleValid,
ROLES,
} = require("../utils/middleware/multiUserProtected");
const { validatedRequest } = require("../utils/middleware/validatedRequest");
const ImportedPlugin = require("../utils/agents/imported");
function adminEndpoints(app) {
if (!app) return;
app.get(
"/admin/users",
[validatedRequest, strictMultiUserRoleValid([ROLES.admin, ROLES.manager])],
async (_request, response) => {
try {
const users = await User.where();
response.status(200).json({ users });
} catch (e) {
console.error(e);
response.sendStatus(500).end();
}
}
);
app.post(
"/admin/users/new",
[validatedRequest, strictMultiUserRoleValid([ROLES.admin, ROLES.manager])],
async (request, response) => {
try {
const currUser = await userFromSession(request, response);
const newUserParams = reqBody(request);
const roleValidation = validRoleSelection(currUser, newUserParams);
if (!roleValidation.valid) {
response
.status(200)
.json({ user: null, error: roleValidation.error });
return;
}
const { user: newUser, error } = await User.create(newUserParams);
if (!!newUser) {
await EventLogs.logEvent(
"user_created",
{
userName: newUser.username,
createdBy: currUser.username,
},
currUser.id
);
}
response.status(200).json({ user: newUser, error });
} catch (e) {
console.error(e);
response.sendStatus(500).end();
}
}
);
app.post(
"/admin/user/:id",
[validatedRequest, strictMultiUserRoleValid([ROLES.admin, ROLES.manager])],
async (request, response) => {
try {
const currUser = await userFromSession(request, response);
const { id } = request.params;
const updates = reqBody(request);
const user = await User.get({ id: Number(id) });
const canModify = validCanModify(currUser, user);
if (!canModify.valid) {
response.status(200).json({ success: false, error: canModify.error });
return;
}
const roleValidation = validRoleSelection(currUser, updates);
if (!roleValidation.valid) {
response
.status(200)
.json({ success: false, error: roleValidation.error });
return;
}
const validAdminRoleModification = await canModifyAdmin(user, updates);
if (!validAdminRoleModification.valid) {
response
.status(200)
.json({ success: false, error: validAdminRoleModification.error });
return;
}
const { success, error } = await User.update(id, updates);
response.status(200).json({ success, error });
} catch (e) {
console.error(e);
response.sendStatus(500).end();
}
}
);
app.delete(
"/admin/user/:id",
[validatedRequest, strictMultiUserRoleValid([ROLES.admin, ROLES.manager])],
async (request, response) => {
try {
const currUser = await userFromSession(request, response);
const { id } = request.params;
const user = await User.get({ id: Number(id) });
const canModify = validCanModify(currUser, user);
if (!canModify.valid) {
response.status(200).json({ success: false, error: canModify.error });
return;
}
Replace custom sqlite dbms with prisma (#239) * WIP converted all sqlite models into prisma calls * modify db setup and fix ApiKey model calls in admin.js * renaming function params to be consistent * converted adminEndpoints to utilize prisma orm * converted chatEndpoints to utilize prisma orm * converted inviteEndpoints to utilize prisma orm * converted systemEndpoints to utilize prisma orm * converted workspaceEndpoints to utilize prisma orm * converting sql queries to prisma calls * fixed default param bug for orderBy and limit * fixed typo for workspace chats * fixed order of deletion to account for sql relations * fix invite CRUD and workspace management CRUD * fixed CRUD for api keys * created prisma setup scripts/docs for understanding how to use prisma * prisma dependency change * removing unneeded console.logs * removing unneeded sql escape function * linting and creating migration script * migration from depreciated sqlite script update * removing unneeded migrations in prisma folder * create backup of old sqlite db and use transactions to ensure all operations complete successfully * adding migrations to gitignore * updated PRISMA.md docs for info on how to use sqlite migration script * comment changes * adding back migrations folder to repo * Reviewing SQL and prisma integraiton on fresh repo * update inline key replacement * ensure migration script executes and maps foreign_keys regardless of db ordering * run migration endpoint * support new prisma backend * bump version * change migration call --------- Co-authored-by: timothycarambat <rambat1010@gmail.com>
2023-09-28 23:00:03 +02:00
await User.delete({ id: Number(id) });
await EventLogs.logEvent(
"user_deleted",
{
userName: user.username,
deletedBy: currUser.username,
},
currUser.id
);
response.status(200).json({ success: true, error: null });
} catch (e) {
console.error(e);
response.sendStatus(500).end();
}
}
);
app.get(
"/admin/invites",
[validatedRequest, strictMultiUserRoleValid([ROLES.admin, ROLES.manager])],
async (_request, response) => {
try {
const invites = await Invite.whereWithUsers();
response.status(200).json({ invites });
} catch (e) {
console.error(e);
response.sendStatus(500).end();
}
}
);
app.post(
"/admin/invite/new",
[validatedRequest, strictMultiUserRoleValid([ROLES.admin, ROLES.manager])],
async (request, response) => {
try {
const user = await userFromSession(request, response);
const body = reqBody(request);
const { invite, error } = await Invite.create({
createdByUserId: user.id,
workspaceIds: body?.workspaceIds || [],
});
await EventLogs.logEvent(
"invite_created",
{
inviteCode: invite.code,
createdBy: response.locals?.user?.username,
},
response.locals?.user?.id
);
response.status(200).json({ invite, error });
} catch (e) {
console.error(e);
response.sendStatus(500).end();
}
}
);
app.delete(
"/admin/invite/:id",
[validatedRequest, strictMultiUserRoleValid([ROLES.admin, ROLES.manager])],
async (request, response) => {
try {
const { id } = request.params;
const { success, error } = await Invite.deactivate(id);
await EventLogs.logEvent(
"invite_deleted",
{ deletedBy: response.locals?.user?.username },
response.locals?.user?.id
);
response.status(200).json({ success, error });
} catch (e) {
console.error(e);
response.sendStatus(500).end();
}
}
);
app.get(
"/admin/workspaces",
[validatedRequest, strictMultiUserRoleValid([ROLES.admin, ROLES.manager])],
async (_request, response) => {
try {
const workspaces = await Workspace.whereWithUsers();
response.status(200).json({ workspaces });
} catch (e) {
console.error(e);
response.sendStatus(500).end();
}
}
);
app.get(
"/admin/workspaces/:workspaceId/users",
[validatedRequest, strictMultiUserRoleValid([ROLES.admin, ROLES.manager])],
async (request, response) => {
try {
const { workspaceId } = request.params;
const users = await Workspace.workspaceUsers(workspaceId);
response.status(200).json({ users });
} catch (e) {
console.error(e);
response.sendStatus(500).end();
}
}
);
app.post(
"/admin/workspaces/new",
[validatedRequest, strictMultiUserRoleValid([ROLES.admin, ROLES.manager])],
async (request, response) => {
try {
const user = await userFromSession(request, response);
const { name } = reqBody(request);
const { workspace, message: error } = await Workspace.new(
name,
user.id
);
response.status(200).json({ workspace, error });
} catch (e) {
console.error(e);
response.sendStatus(500).end();
}
}
);
app.post(
"/admin/workspaces/:workspaceId/update-users",
[validatedRequest, strictMultiUserRoleValid([ROLES.admin, ROLES.manager])],
async (request, response) => {
try {
const { workspaceId } = request.params;
const { userIds } = reqBody(request);
const { success, error } = await Workspace.updateUsers(
Replace custom sqlite dbms with prisma (#239) * WIP converted all sqlite models into prisma calls * modify db setup and fix ApiKey model calls in admin.js * renaming function params to be consistent * converted adminEndpoints to utilize prisma orm * converted chatEndpoints to utilize prisma orm * converted inviteEndpoints to utilize prisma orm * converted systemEndpoints to utilize prisma orm * converted workspaceEndpoints to utilize prisma orm * converting sql queries to prisma calls * fixed default param bug for orderBy and limit * fixed typo for workspace chats * fixed order of deletion to account for sql relations * fix invite CRUD and workspace management CRUD * fixed CRUD for api keys * created prisma setup scripts/docs for understanding how to use prisma * prisma dependency change * removing unneeded console.logs * removing unneeded sql escape function * linting and creating migration script * migration from depreciated sqlite script update * removing unneeded migrations in prisma folder * create backup of old sqlite db and use transactions to ensure all operations complete successfully * adding migrations to gitignore * updated PRISMA.md docs for info on how to use sqlite migration script * comment changes * adding back migrations folder to repo * Reviewing SQL and prisma integraiton on fresh repo * update inline key replacement * ensure migration script executes and maps foreign_keys regardless of db ordering * run migration endpoint * support new prisma backend * bump version * change migration call --------- Co-authored-by: timothycarambat <rambat1010@gmail.com>
2023-09-28 23:00:03 +02:00
workspaceId,
userIds
);
response.status(200).json({ success, error });
} catch (e) {
console.error(e);
response.sendStatus(500).end();
}
}
);
app.delete(
"/admin/workspaces/:id",
[validatedRequest, strictMultiUserRoleValid([ROLES.admin, ROLES.manager])],
async (request, response) => {
try {
const { id } = request.params;
const VectorDb = getVectorDbClass();
Replace custom sqlite dbms with prisma (#239) * WIP converted all sqlite models into prisma calls * modify db setup and fix ApiKey model calls in admin.js * renaming function params to be consistent * converted adminEndpoints to utilize prisma orm * converted chatEndpoints to utilize prisma orm * converted inviteEndpoints to utilize prisma orm * converted systemEndpoints to utilize prisma orm * converted workspaceEndpoints to utilize prisma orm * converting sql queries to prisma calls * fixed default param bug for orderBy and limit * fixed typo for workspace chats * fixed order of deletion to account for sql relations * fix invite CRUD and workspace management CRUD * fixed CRUD for api keys * created prisma setup scripts/docs for understanding how to use prisma * prisma dependency change * removing unneeded console.logs * removing unneeded sql escape function * linting and creating migration script * migration from depreciated sqlite script update * removing unneeded migrations in prisma folder * create backup of old sqlite db and use transactions to ensure all operations complete successfully * adding migrations to gitignore * updated PRISMA.md docs for info on how to use sqlite migration script * comment changes * adding back migrations folder to repo * Reviewing SQL and prisma integraiton on fresh repo * update inline key replacement * ensure migration script executes and maps foreign_keys regardless of db ordering * run migration endpoint * support new prisma backend * bump version * change migration call --------- Co-authored-by: timothycarambat <rambat1010@gmail.com>
2023-09-28 23:00:03 +02:00
const workspace = await Workspace.get({ id: Number(id) });
if (!workspace) {
response.sendStatus(404).end();
return;
}
Replace custom sqlite dbms with prisma (#239) * WIP converted all sqlite models into prisma calls * modify db setup and fix ApiKey model calls in admin.js * renaming function params to be consistent * converted adminEndpoints to utilize prisma orm * converted chatEndpoints to utilize prisma orm * converted inviteEndpoints to utilize prisma orm * converted systemEndpoints to utilize prisma orm * converted workspaceEndpoints to utilize prisma orm * converting sql queries to prisma calls * fixed default param bug for orderBy and limit * fixed typo for workspace chats * fixed order of deletion to account for sql relations * fix invite CRUD and workspace management CRUD * fixed CRUD for api keys * created prisma setup scripts/docs for understanding how to use prisma * prisma dependency change * removing unneeded console.logs * removing unneeded sql escape function * linting and creating migration script * migration from depreciated sqlite script update * removing unneeded migrations in prisma folder * create backup of old sqlite db and use transactions to ensure all operations complete successfully * adding migrations to gitignore * updated PRISMA.md docs for info on how to use sqlite migration script * comment changes * adding back migrations folder to repo * Reviewing SQL and prisma integraiton on fresh repo * update inline key replacement * ensure migration script executes and maps foreign_keys regardless of db ordering * run migration endpoint * support new prisma backend * bump version * change migration call --------- Co-authored-by: timothycarambat <rambat1010@gmail.com>
2023-09-28 23:00:03 +02:00
await WorkspaceChats.delete({ workspaceId: Number(workspace.id) });
await DocumentVectors.deleteForWorkspace(Number(workspace.id));
await Document.delete({ workspaceId: Number(workspace.id) });
await Workspace.delete({ id: Number(workspace.id) });
try {
await VectorDb["delete-namespace"]({ namespace: workspace.slug });
} catch (e) {
console.error(e.message);
}
Replace custom sqlite dbms with prisma (#239) * WIP converted all sqlite models into prisma calls * modify db setup and fix ApiKey model calls in admin.js * renaming function params to be consistent * converted adminEndpoints to utilize prisma orm * converted chatEndpoints to utilize prisma orm * converted inviteEndpoints to utilize prisma orm * converted systemEndpoints to utilize prisma orm * converted workspaceEndpoints to utilize prisma orm * converting sql queries to prisma calls * fixed default param bug for orderBy and limit * fixed typo for workspace chats * fixed order of deletion to account for sql relations * fix invite CRUD and workspace management CRUD * fixed CRUD for api keys * created prisma setup scripts/docs for understanding how to use prisma * prisma dependency change * removing unneeded console.logs * removing unneeded sql escape function * linting and creating migration script * migration from depreciated sqlite script update * removing unneeded migrations in prisma folder * create backup of old sqlite db and use transactions to ensure all operations complete successfully * adding migrations to gitignore * updated PRISMA.md docs for info on how to use sqlite migration script * comment changes * adding back migrations folder to repo * Reviewing SQL and prisma integraiton on fresh repo * update inline key replacement * ensure migration script executes and maps foreign_keys regardless of db ordering * run migration endpoint * support new prisma backend * bump version * change migration call --------- Co-authored-by: timothycarambat <rambat1010@gmail.com>
2023-09-28 23:00:03 +02:00
response.status(200).json({ success: true, error: null });
} catch (e) {
console.error(e);
response.sendStatus(500).end();
}
}
);
// System preferences but only by array of labels
app.get(
"/admin/system-preferences-for",
[validatedRequest, flexUserRoleValid([ROLES.admin, ROLES.manager])],
async (request, response) => {
try {
const requestedSettings = {};
const labels = request.query.labels?.split(",") || [];
const needEmbedder = [
"text_splitter_chunk_size",
"max_embed_chunk_size",
];
const noRecord = [
"max_embed_chunk_size",
"agent_sql_connections",
"imported_agent_skills",
"feature_flags",
"meta_page_title",
"meta_page_favicon",
];
for (const label of labels) {
// Skip any settings that are not explicitly defined as public
if (!SystemSettings.publicFields.includes(label)) continue;
// Only get the embedder if the setting actually needs it
let embedder = needEmbedder.includes(label)
? getEmbeddingEngineSelection()
: null;
// Only get the record from db if the setting actually needs it
let setting = noRecord.includes(label)
? null
: await SystemSettings.get({ label });
switch (label) {
case "footer_data":
requestedSettings[label] = setting?.value ?? JSON.stringify([]);
break;
case "support_email":
requestedSettings[label] = setting?.value || null;
break;
case "text_splitter_chunk_size":
requestedSettings[label] =
setting?.value || embedder?.embeddingMaxChunkLength || null;
break;
case "text_splitter_chunk_overlap":
requestedSettings[label] = setting?.value || null;
break;
case "max_embed_chunk_size":
requestedSettings[label] =
embedder?.embeddingMaxChunkLength || 1000;
break;
case "agent_search_provider":
requestedSettings[label] = setting?.value || null;
break;
case "agent_sql_connections":
requestedSettings[label] =
await SystemSettings.brief.agent_sql_connections();
break;
case "default_agent_skills":
requestedSettings[label] = safeJsonParse(setting?.value, []);
break;
case "imported_agent_skills":
requestedSettings[label] = ImportedPlugin.listImportedPlugins();
break;
case "custom_app_name":
requestedSettings[label] = setting?.value || null;
break;
case "feature_flags":
requestedSettings[label] =
(await SystemSettings.getFeatureFlags()) || {};
break;
case "meta_page_title":
requestedSettings[label] =
await SystemSettings.getValueOrFallback({ label }, null);
break;
case "meta_page_favicon":
requestedSettings[label] =
await SystemSettings.getValueOrFallback({ label }, null);
break;
default:
break;
}
}
response.status(200).json({ settings: requestedSettings });
} catch (e) {
console.error(e);
response.sendStatus(500).end();
}
}
);
// TODO: Delete this endpoint
// DEPRECATED - use /admin/system-preferences-for instead with ?labels=... comma separated string of labels
app.get(
"/admin/system-preferences",
[validatedRequest, flexUserRoleValid([ROLES.admin, ROLES.manager])],
async (_, response) => {
try {
const embedder = getEmbeddingEngineSelection();
const settings = {
footer_data:
(await SystemSettings.get({ label: "footer_data" }))?.value ||
JSON.stringify([]),
support_email:
(await SystemSettings.get({ label: "support_email" }))?.value ||
null,
text_splitter_chunk_size:
(await SystemSettings.get({ label: "text_splitter_chunk_size" }))
?.value ||
embedder?.embeddingMaxChunkLength ||
null,
text_splitter_chunk_overlap:
(await SystemSettings.get({ label: "text_splitter_chunk_overlap" }))
?.value || null,
max_embed_chunk_size: embedder?.embeddingMaxChunkLength || 1000,
agent_search_provider:
(await SystemSettings.get({ label: "agent_search_provider" }))
?.value || null,
agent_sql_connections:
await SystemSettings.brief.agent_sql_connections(),
default_agent_skills:
safeJsonParse(
(await SystemSettings.get({ label: "default_agent_skills" }))
?.value,
[]
) || [],
imported_agent_skills: ImportedPlugin.listImportedPlugins(),
custom_app_name:
(await SystemSettings.get({ label: "custom_app_name" }))?.value ||
null,
[BETA] Live document sync (#1719) * wip bg workers for live document sync * Add ability to re-embed specific documents across many workspaces via background queue bgworkser is gated behind expieremental system setting flag that needs to be explictly enabled UI for watching/unwatching docments that are embedded. TODO: UI to easily manage all bg tasks and see run results TODO: UI to enable this feature and background endpoints to manage it * create frontend views and paths Move elements to correct experimental scope * update migration to delete runs on removal of watched document * Add watch support to YouTube transcripts (#1716) * Add watch support to YouTube transcripts refactor how sync is done for supported types * Watch specific files in Confluence space (#1718) Add failure-prune check for runs * create tmp workflow modifications for beta image * create tmp workflow modifications for beta image * create tmp workflow modifications for beta image * dual build update copy of alert modals * update job interval * Add support for live-sync of Github files * update copy for document sync feature * hide Experimental features from UI * update docs links * [FEAT] Implement new settings menu for experimental features (#1735) * implement new settings menu for experimental features * remove unused context save bar --------- Co-authored-by: timothycarambat <rambat1010@gmail.com> * dont run job on boot * unset workflow changes * Add persistent encryption service Relay key to collector so persistent encryption can be used Encrypt any private data in chunkSources used for replay during resync jobs * update jsDOC * Linting and organization * update modal copy for feature --------- Co-authored-by: Sean Hatfield <seanhatfield5@gmail.com>
2024-06-21 22:38:50 +02:00
feature_flags: (await SystemSettings.getFeatureFlags()) || {},
meta_page_title: await SystemSettings.getValueOrFallback(
{ label: "meta_page_title" },
null
),
meta_page_favicon: await SystemSettings.getValueOrFallback(
{ label: "meta_page_favicon" },
null
),
};
response.status(200).json({ settings });
} catch (e) {
console.error(e);
response.sendStatus(500).end();
}
}
);
app.post(
"/admin/system-preferences",
[validatedRequest, flexUserRoleValid([ROLES.admin, ROLES.manager])],
async (request, response) => {
try {
const updates = reqBody(request);
await SystemSettings.updateSettings(updates);
response.status(200).json({ success: true, error: null });
} catch (e) {
console.error(e);
response.sendStatus(500).end();
}
}
);
app.get(
"/admin/api-keys",
[validatedRequest, strictMultiUserRoleValid([ROLES.admin])],
async (_request, response) => {
try {
const apiKeys = await ApiKey.whereWithUser({});
return response.status(200).json({
apiKeys,
error: null,
});
} catch (error) {
console.error(error);
response.status(500).json({
apiKey: null,
error: "Could not find an API Keys.",
});
}
}
);
app.post(
"/admin/generate-api-key",
[validatedRequest, strictMultiUserRoleValid([ROLES.admin])],
async (request, response) => {
try {
const user = await userFromSession(request, response);
const { apiKey, error } = await ApiKey.create(user.id);
await Telemetry.sendTelemetry("api_key_created");
await EventLogs.logEvent(
"api_key_created",
{ createdBy: user?.username },
user?.id
);
return response.status(200).json({
apiKey,
error,
});
} catch (e) {
console.error(e);
response.sendStatus(500).end();
}
}
);
app.delete(
"/admin/delete-api-key/:id",
[validatedRequest, strictMultiUserRoleValid([ROLES.admin])],
async (request, response) => {
try {
const { id } = request.params;
Replace custom sqlite dbms with prisma (#239) * WIP converted all sqlite models into prisma calls * modify db setup and fix ApiKey model calls in admin.js * renaming function params to be consistent * converted adminEndpoints to utilize prisma orm * converted chatEndpoints to utilize prisma orm * converted inviteEndpoints to utilize prisma orm * converted systemEndpoints to utilize prisma orm * converted workspaceEndpoints to utilize prisma orm * converting sql queries to prisma calls * fixed default param bug for orderBy and limit * fixed typo for workspace chats * fixed order of deletion to account for sql relations * fix invite CRUD and workspace management CRUD * fixed CRUD for api keys * created prisma setup scripts/docs for understanding how to use prisma * prisma dependency change * removing unneeded console.logs * removing unneeded sql escape function * linting and creating migration script * migration from depreciated sqlite script update * removing unneeded migrations in prisma folder * create backup of old sqlite db and use transactions to ensure all operations complete successfully * adding migrations to gitignore * updated PRISMA.md docs for info on how to use sqlite migration script * comment changes * adding back migrations folder to repo * Reviewing SQL and prisma integraiton on fresh repo * update inline key replacement * ensure migration script executes and maps foreign_keys regardless of db ordering * run migration endpoint * support new prisma backend * bump version * change migration call --------- Co-authored-by: timothycarambat <rambat1010@gmail.com>
2023-09-28 23:00:03 +02:00
await ApiKey.delete({ id: Number(id) });
await EventLogs.logEvent(
"api_key_deleted",
{ deletedBy: response.locals?.user?.username },
response?.locals?.user?.id
);
return response.status(200).end();
} catch (e) {
console.error(e);
response.sendStatus(500).end();
}
}
);
}
module.exports = { adminEndpoints };