2023-08-24 04:15:07 +02:00
|
|
|
|
const { Telemetry } = require("../../../models/telemetry");
|
|
|
|
|
const { validApiKey } = require("../../../utils/middleware/validApiKey");
|
2024-04-01 21:06:47 +02:00
|
|
|
|
const { handleFileUpload } = require("../../../utils/files/multer");
|
2024-01-16 23:58:49 +01:00
|
|
|
|
const {
|
|
|
|
|
viewLocalFiles,
|
|
|
|
|
findDocumentInDocuments,
|
2024-03-21 00:10:30 +01:00
|
|
|
|
normalizePath,
|
2024-05-23 21:52:04 +02:00
|
|
|
|
isWithin,
|
2024-01-16 23:58:49 +01:00
|
|
|
|
} = require("../../../utils/files");
|
2024-01-17 01:04:22 +01:00
|
|
|
|
const { reqBody } = require("../../../utils/http");
|
[FEAT] Automated audit logging (#667)
* WIP event logging - new table for events and new settings view for viewing
* WIP add logging
* UI for log rows
* rename files to Logging to prevent getting gitignore
* add metadata for all logging events and colored badges in logs page
* remove unneeded comment
* cleanup namespace for logging
* clean up backend calls
* update logging to show to => from settings changes
* add logging for invitations, created, deleted, and accepted
* add logging for user created, updated, suspended, or removed
* add logging for workspace deleted
* add logging for chat logs exported
* add logging for API keys, LLM, embedder, vector db, embed chat, and reset button
* modify event logs
* update to event log types
* simplify rendering of event badges
---------
Co-authored-by: timothycarambat <rambat1010@gmail.com>
2024-02-07 00:21:40 +01:00
|
|
|
|
const { EventLogs } = require("../../../models/eventLogs");
|
2024-02-17 01:32:25 +01:00
|
|
|
|
const { CollectorApi } = require("../../../utils/collectorApi");
|
2024-03-21 00:10:30 +01:00
|
|
|
|
const fs = require("fs");
|
|
|
|
|
const path = require("path");
|
|
|
|
|
const { Document } = require("../../../models/documents");
|
2024-03-21 23:10:44 +01:00
|
|
|
|
const documentsPath =
|
|
|
|
|
process.env.NODE_ENV === "development"
|
|
|
|
|
? path.resolve(__dirname, "../../../storage/documents")
|
|
|
|
|
: path.resolve(process.env.STORAGE_DIR, `documents`);
|
2023-08-24 04:15:07 +02:00
|
|
|
|
|
|
|
|
|
function apiDocumentEndpoints(app) {
|
|
|
|
|
if (!app) return;
|
|
|
|
|
|
|
|
|
|
app.post(
|
|
|
|
|
"/v1/document/upload",
|
2024-04-01 21:06:47 +02:00
|
|
|
|
[validApiKey, handleFileUpload],
|
2023-08-24 04:15:07 +02:00
|
|
|
|
async (request, response) => {
|
[FEAT] Automated audit logging (#667)
* WIP event logging - new table for events and new settings view for viewing
* WIP add logging
* UI for log rows
* rename files to Logging to prevent getting gitignore
* add metadata for all logging events and colored badges in logs page
* remove unneeded comment
* cleanup namespace for logging
* clean up backend calls
* update logging to show to => from settings changes
* add logging for invitations, created, deleted, and accepted
* add logging for user created, updated, suspended, or removed
* add logging for workspace deleted
* add logging for chat logs exported
* add logging for API keys, LLM, embedder, vector db, embed chat, and reset button
* modify event logs
* update to event log types
* simplify rendering of event badges
---------
Co-authored-by: timothycarambat <rambat1010@gmail.com>
2024-02-07 00:21:40 +01:00
|
|
|
|
/*
|
2023-08-24 04:15:07 +02:00
|
|
|
|
#swagger.tags = ['Documents']
|
|
|
|
|
#swagger.description = 'Upload a new file to AnythingLLM to be parsed and prepared for embedding.'
|
|
|
|
|
#swagger.requestBody = {
|
|
|
|
|
description: 'File to be uploaded.',
|
|
|
|
|
required: true,
|
|
|
|
|
type: 'file',
|
|
|
|
|
content: {
|
|
|
|
|
"multipart/form-data": {
|
|
|
|
|
schema: {
|
|
|
|
|
type: 'object',
|
|
|
|
|
properties: {
|
|
|
|
|
file: {
|
|
|
|
|
type: 'string',
|
|
|
|
|
format: 'binary',
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
#swagger.responses[200] = {
|
|
|
|
|
content: {
|
|
|
|
|
"application/json": {
|
|
|
|
|
schema: {
|
|
|
|
|
type: 'object',
|
|
|
|
|
example: {
|
|
|
|
|
success: true,
|
|
|
|
|
error: null,
|
2024-01-17 01:04:22 +01:00
|
|
|
|
documents: [
|
|
|
|
|
{
|
|
|
|
|
"location": "custom-documents/anythingllm.txt-6e8be64c-c162-4b43-9997-b068c0071e8b.json",
|
|
|
|
|
"name": "anythingllm.txt-6e8be64c-c162-4b43-9997-b068c0071e8b.json",
|
|
|
|
|
"url": "file:///Users/tim/Documents/anything-llm/collector/hotdir/anythingllm.txt",
|
|
|
|
|
"title": "anythingllm.txt",
|
|
|
|
|
"docAuthor": "Unknown",
|
|
|
|
|
"description": "Unknown",
|
|
|
|
|
"docSource": "a text file uploaded by the user.",
|
|
|
|
|
"chunkSource": "anythingllm.txt",
|
|
|
|
|
"published": "1/16/2024, 3:07:00 PM",
|
|
|
|
|
"wordCount": 93,
|
|
|
|
|
"token_count_estimate": 115,
|
|
|
|
|
}
|
|
|
|
|
]
|
2023-08-24 04:15:07 +02:00
|
|
|
|
}
|
|
|
|
|
}
|
[FEAT] Automated audit logging (#667)
* WIP event logging - new table for events and new settings view for viewing
* WIP add logging
* UI for log rows
* rename files to Logging to prevent getting gitignore
* add metadata for all logging events and colored badges in logs page
* remove unneeded comment
* cleanup namespace for logging
* clean up backend calls
* update logging to show to => from settings changes
* add logging for invitations, created, deleted, and accepted
* add logging for user created, updated, suspended, or removed
* add logging for workspace deleted
* add logging for chat logs exported
* add logging for API keys, LLM, embedder, vector db, embed chat, and reset button
* modify event logs
* update to event log types
* simplify rendering of event badges
---------
Co-authored-by: timothycarambat <rambat1010@gmail.com>
2024-02-07 00:21:40 +01:00
|
|
|
|
}
|
2023-08-24 04:15:07 +02:00
|
|
|
|
}
|
[FEAT] Automated audit logging (#667)
* WIP event logging - new table for events and new settings view for viewing
* WIP add logging
* UI for log rows
* rename files to Logging to prevent getting gitignore
* add metadata for all logging events and colored badges in logs page
* remove unneeded comment
* cleanup namespace for logging
* clean up backend calls
* update logging to show to => from settings changes
* add logging for invitations, created, deleted, and accepted
* add logging for user created, updated, suspended, or removed
* add logging for workspace deleted
* add logging for chat logs exported
* add logging for API keys, LLM, embedder, vector db, embed chat, and reset button
* modify event logs
* update to event log types
* simplify rendering of event badges
---------
Co-authored-by: timothycarambat <rambat1010@gmail.com>
2024-02-07 00:21:40 +01:00
|
|
|
|
}
|
2023-08-24 04:15:07 +02:00
|
|
|
|
#swagger.responses[403] = {
|
|
|
|
|
schema: {
|
|
|
|
|
"$ref": "#/definitions/InvalidAPIKey"
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
*/
|
|
|
|
|
try {
|
2024-02-17 01:32:25 +01:00
|
|
|
|
const Collector = new CollectorApi();
|
2023-08-24 04:15:07 +02:00
|
|
|
|
const { originalname } = request.file;
|
2024-02-17 01:32:25 +01:00
|
|
|
|
const processingOnline = await Collector.online();
|
2023-08-24 04:15:07 +02:00
|
|
|
|
|
|
|
|
|
if (!processingOnline) {
|
|
|
|
|
response
|
|
|
|
|
.status(500)
|
|
|
|
|
.json({
|
|
|
|
|
success: false,
|
2023-12-15 00:14:56 +01:00
|
|
|
|
error: `Document processing API is not online. Document ${originalname} will not be processed automatically.`,
|
2023-08-24 04:15:07 +02:00
|
|
|
|
})
|
|
|
|
|
.end();
|
2024-02-08 00:17:32 +01:00
|
|
|
|
return;
|
2023-08-24 04:15:07 +02:00
|
|
|
|
}
|
|
|
|
|
|
2024-01-17 01:04:22 +01:00
|
|
|
|
const { success, reason, documents } =
|
2024-02-17 01:32:25 +01:00
|
|
|
|
await Collector.processDocument(originalname);
|
2023-08-24 04:15:07 +02:00
|
|
|
|
if (!success) {
|
2024-01-17 01:04:22 +01:00
|
|
|
|
response
|
|
|
|
|
.status(500)
|
|
|
|
|
.json({ success: false, error: reason, documents })
|
|
|
|
|
.end();
|
|
|
|
|
return;
|
2023-08-24 04:15:07 +02:00
|
|
|
|
}
|
|
|
|
|
|
2024-02-17 01:32:25 +01:00
|
|
|
|
Collector.log(
|
2023-08-24 04:15:07 +02:00
|
|
|
|
`Document ${originalname} uploaded processed and successfully. It is now available in documents.`
|
|
|
|
|
);
|
|
|
|
|
await Telemetry.sendTelemetry("document_uploaded");
|
[FEAT] Automated audit logging (#667)
* WIP event logging - new table for events and new settings view for viewing
* WIP add logging
* UI for log rows
* rename files to Logging to prevent getting gitignore
* add metadata for all logging events and colored badges in logs page
* remove unneeded comment
* cleanup namespace for logging
* clean up backend calls
* update logging to show to => from settings changes
* add logging for invitations, created, deleted, and accepted
* add logging for user created, updated, suspended, or removed
* add logging for workspace deleted
* add logging for chat logs exported
* add logging for API keys, LLM, embedder, vector db, embed chat, and reset button
* modify event logs
* update to event log types
* simplify rendering of event badges
---------
Co-authored-by: timothycarambat <rambat1010@gmail.com>
2024-02-07 00:21:40 +01:00
|
|
|
|
await EventLogs.logEvent("api_document_uploaded", {
|
|
|
|
|
documentName: originalname,
|
|
|
|
|
});
|
2024-01-17 01:04:22 +01:00
|
|
|
|
response.status(200).json({ success: true, error: null, documents });
|
|
|
|
|
} catch (e) {
|
2024-07-04 01:39:33 +02:00
|
|
|
|
console.error(e.message, e);
|
2024-01-17 01:04:22 +01:00
|
|
|
|
response.sendStatus(500).end();
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
app.post(
|
|
|
|
|
"/v1/document/upload-link",
|
|
|
|
|
[validApiKey],
|
|
|
|
|
async (request, response) => {
|
[FEAT] Automated audit logging (#667)
* WIP event logging - new table for events and new settings view for viewing
* WIP add logging
* UI for log rows
* rename files to Logging to prevent getting gitignore
* add metadata for all logging events and colored badges in logs page
* remove unneeded comment
* cleanup namespace for logging
* clean up backend calls
* update logging to show to => from settings changes
* add logging for invitations, created, deleted, and accepted
* add logging for user created, updated, suspended, or removed
* add logging for workspace deleted
* add logging for chat logs exported
* add logging for API keys, LLM, embedder, vector db, embed chat, and reset button
* modify event logs
* update to event log types
* simplify rendering of event badges
---------
Co-authored-by: timothycarambat <rambat1010@gmail.com>
2024-02-07 00:21:40 +01:00
|
|
|
|
/*
|
2024-01-17 01:04:22 +01:00
|
|
|
|
#swagger.tags = ['Documents']
|
|
|
|
|
#swagger.description = 'Upload a valid URL for AnythingLLM to scrape and prepare for embedding.'
|
|
|
|
|
#swagger.requestBody = {
|
|
|
|
|
description: 'Link of web address to be scraped.',
|
|
|
|
|
required: true,
|
2024-02-08 00:17:32 +01:00
|
|
|
|
type: 'object',
|
2024-01-17 01:04:22 +01:00
|
|
|
|
content: {
|
|
|
|
|
"application/json": {
|
|
|
|
|
schema: {
|
|
|
|
|
type: 'object',
|
|
|
|
|
example: {
|
|
|
|
|
"link": "https://useanything.com"
|
|
|
|
|
}
|
|
|
|
|
}
|
[FEAT] Automated audit logging (#667)
* WIP event logging - new table for events and new settings view for viewing
* WIP add logging
* UI for log rows
* rename files to Logging to prevent getting gitignore
* add metadata for all logging events and colored badges in logs page
* remove unneeded comment
* cleanup namespace for logging
* clean up backend calls
* update logging to show to => from settings changes
* add logging for invitations, created, deleted, and accepted
* add logging for user created, updated, suspended, or removed
* add logging for workspace deleted
* add logging for chat logs exported
* add logging for API keys, LLM, embedder, vector db, embed chat, and reset button
* modify event logs
* update to event log types
* simplify rendering of event badges
---------
Co-authored-by: timothycarambat <rambat1010@gmail.com>
2024-02-07 00:21:40 +01:00
|
|
|
|
}
|
2024-01-17 01:04:22 +01:00
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
#swagger.responses[200] = {
|
|
|
|
|
content: {
|
|
|
|
|
"application/json": {
|
|
|
|
|
schema: {
|
|
|
|
|
type: 'object',
|
|
|
|
|
example: {
|
|
|
|
|
success: true,
|
|
|
|
|
error: null,
|
|
|
|
|
documents: [
|
|
|
|
|
{
|
|
|
|
|
"id": "c530dbe6-bff1-4b9e-b87f-710d539d20bc",
|
|
|
|
|
"url": "file://useanything_com.html",
|
|
|
|
|
"title": "useanything_com.html",
|
|
|
|
|
"docAuthor": "no author found",
|
|
|
|
|
"description": "No description found.",
|
|
|
|
|
"docSource": "URL link uploaded by the user.",
|
|
|
|
|
"chunkSource": "https:useanything.com.html",
|
|
|
|
|
"published": "1/16/2024, 3:46:33 PM",
|
|
|
|
|
"wordCount": 252,
|
|
|
|
|
"pageContent": "AnythingLLM is the best....",
|
|
|
|
|
"token_count_estimate": 447,
|
|
|
|
|
"location": "custom-documents/url-useanything_com-c530dbe6-bff1-4b9e-b87f-710d539d20bc.json"
|
|
|
|
|
}
|
|
|
|
|
]
|
|
|
|
|
}
|
|
|
|
|
}
|
[FEAT] Automated audit logging (#667)
* WIP event logging - new table for events and new settings view for viewing
* WIP add logging
* UI for log rows
* rename files to Logging to prevent getting gitignore
* add metadata for all logging events and colored badges in logs page
* remove unneeded comment
* cleanup namespace for logging
* clean up backend calls
* update logging to show to => from settings changes
* add logging for invitations, created, deleted, and accepted
* add logging for user created, updated, suspended, or removed
* add logging for workspace deleted
* add logging for chat logs exported
* add logging for API keys, LLM, embedder, vector db, embed chat, and reset button
* modify event logs
* update to event log types
* simplify rendering of event badges
---------
Co-authored-by: timothycarambat <rambat1010@gmail.com>
2024-02-07 00:21:40 +01:00
|
|
|
|
}
|
2024-01-17 01:04:22 +01:00
|
|
|
|
}
|
[FEAT] Automated audit logging (#667)
* WIP event logging - new table for events and new settings view for viewing
* WIP add logging
* UI for log rows
* rename files to Logging to prevent getting gitignore
* add metadata for all logging events and colored badges in logs page
* remove unneeded comment
* cleanup namespace for logging
* clean up backend calls
* update logging to show to => from settings changes
* add logging for invitations, created, deleted, and accepted
* add logging for user created, updated, suspended, or removed
* add logging for workspace deleted
* add logging for chat logs exported
* add logging for API keys, LLM, embedder, vector db, embed chat, and reset button
* modify event logs
* update to event log types
* simplify rendering of event badges
---------
Co-authored-by: timothycarambat <rambat1010@gmail.com>
2024-02-07 00:21:40 +01:00
|
|
|
|
}
|
2024-01-17 01:04:22 +01:00
|
|
|
|
#swagger.responses[403] = {
|
|
|
|
|
schema: {
|
|
|
|
|
"$ref": "#/definitions/InvalidAPIKey"
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
*/
|
|
|
|
|
try {
|
2024-02-17 01:32:25 +01:00
|
|
|
|
const Collector = new CollectorApi();
|
2024-01-17 01:04:22 +01:00
|
|
|
|
const { link } = reqBody(request);
|
2024-02-17 01:32:25 +01:00
|
|
|
|
const processingOnline = await Collector.online();
|
2024-01-17 01:04:22 +01:00
|
|
|
|
|
|
|
|
|
if (!processingOnline) {
|
|
|
|
|
response
|
|
|
|
|
.status(500)
|
|
|
|
|
.json({
|
|
|
|
|
success: false,
|
|
|
|
|
error: `Document processing API is not online. Link ${link} will not be processed automatically.`,
|
|
|
|
|
})
|
|
|
|
|
.end();
|
2024-02-08 00:17:32 +01:00
|
|
|
|
return;
|
2024-01-17 01:04:22 +01:00
|
|
|
|
}
|
|
|
|
|
|
2024-02-17 01:32:25 +01:00
|
|
|
|
const { success, reason, documents } =
|
|
|
|
|
await Collector.processLink(link);
|
2024-01-17 01:04:22 +01:00
|
|
|
|
if (!success) {
|
|
|
|
|
response
|
|
|
|
|
.status(500)
|
|
|
|
|
.json({ success: false, error: reason, documents })
|
|
|
|
|
.end();
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
|
2024-02-17 01:32:25 +01:00
|
|
|
|
Collector.log(
|
2024-01-17 01:04:22 +01:00
|
|
|
|
`Link ${link} uploaded processed and successfully. It is now available in documents.`
|
|
|
|
|
);
|
[FEAT] Automated audit logging (#667)
* WIP event logging - new table for events and new settings view for viewing
* WIP add logging
* UI for log rows
* rename files to Logging to prevent getting gitignore
* add metadata for all logging events and colored badges in logs page
* remove unneeded comment
* cleanup namespace for logging
* clean up backend calls
* update logging to show to => from settings changes
* add logging for invitations, created, deleted, and accepted
* add logging for user created, updated, suspended, or removed
* add logging for workspace deleted
* add logging for chat logs exported
* add logging for API keys, LLM, embedder, vector db, embed chat, and reset button
* modify event logs
* update to event log types
* simplify rendering of event badges
---------
Co-authored-by: timothycarambat <rambat1010@gmail.com>
2024-02-07 00:21:40 +01:00
|
|
|
|
await Telemetry.sendTelemetry("link_uploaded");
|
|
|
|
|
await EventLogs.logEvent("api_link_uploaded", {
|
|
|
|
|
link,
|
|
|
|
|
});
|
2024-01-17 01:04:22 +01:00
|
|
|
|
response.status(200).json({ success: true, error: null, documents });
|
2023-08-24 04:15:07 +02:00
|
|
|
|
} catch (e) {
|
2024-07-04 01:39:33 +02:00
|
|
|
|
console.error(e.message, e);
|
2023-08-24 04:15:07 +02:00
|
|
|
|
response.sendStatus(500).end();
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
);
|
|
|
|
|
|
2024-02-08 00:17:32 +01:00
|
|
|
|
app.post(
|
|
|
|
|
"/v1/document/raw-text",
|
|
|
|
|
[validApiKey],
|
|
|
|
|
async (request, response) => {
|
|
|
|
|
/*
|
|
|
|
|
#swagger.tags = ['Documents']
|
|
|
|
|
#swagger.description = 'Upload a file by specifying its raw text content and metadata values without having to upload a file.'
|
|
|
|
|
#swagger.requestBody = {
|
|
|
|
|
description: 'Text content and metadata of the file to be saved to the system. Use metadata-schema endpoint to get the possible metadata keys',
|
|
|
|
|
required: true,
|
|
|
|
|
type: 'object',
|
|
|
|
|
content: {
|
|
|
|
|
"application/json": {
|
|
|
|
|
schema: {
|
|
|
|
|
type: 'object',
|
|
|
|
|
example: {
|
|
|
|
|
"textContent": "This is the raw text that will be saved as a document in AnythingLLM.",
|
|
|
|
|
"metadata": {
|
|
|
|
|
keyOne: "valueOne",
|
|
|
|
|
keyTwo: "valueTwo",
|
|
|
|
|
etc: "etc"
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
#swagger.responses[200] = {
|
|
|
|
|
content: {
|
|
|
|
|
"application/json": {
|
|
|
|
|
schema: {
|
|
|
|
|
type: 'object',
|
|
|
|
|
example: {
|
|
|
|
|
success: true,
|
|
|
|
|
error: null,
|
|
|
|
|
documents: [
|
|
|
|
|
{
|
|
|
|
|
"id": "c530dbe6-bff1-4b9e-b87f-710d539d20bc",
|
|
|
|
|
"url": "file://my-document.txt",
|
|
|
|
|
"title": "hello-world.txt",
|
|
|
|
|
"docAuthor": "no author found",
|
|
|
|
|
"description": "No description found.",
|
|
|
|
|
"docSource": "My custom description set during upload",
|
|
|
|
|
"chunkSource": "no chunk source specified",
|
|
|
|
|
"published": "1/16/2024, 3:46:33 PM",
|
|
|
|
|
"wordCount": 252,
|
|
|
|
|
"pageContent": "AnythingLLM is the best....",
|
|
|
|
|
"token_count_estimate": 447,
|
|
|
|
|
"location": "custom-documents/raw-my-doc-text-c530dbe6-bff1-4b9e-b87f-710d539d20bc.json"
|
|
|
|
|
}
|
|
|
|
|
]
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
#swagger.responses[403] = {
|
|
|
|
|
schema: {
|
|
|
|
|
"$ref": "#/definitions/InvalidAPIKey"
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
*/
|
|
|
|
|
try {
|
2024-02-17 01:32:25 +01:00
|
|
|
|
const Collector = new CollectorApi();
|
2024-02-08 00:17:32 +01:00
|
|
|
|
const requiredMetadata = ["title"];
|
|
|
|
|
const { textContent, metadata = {} } = reqBody(request);
|
2024-02-17 01:32:25 +01:00
|
|
|
|
const processingOnline = await Collector.online();
|
2024-02-08 00:17:32 +01:00
|
|
|
|
|
|
|
|
|
if (!processingOnline) {
|
|
|
|
|
response
|
|
|
|
|
.status(500)
|
|
|
|
|
.json({
|
|
|
|
|
success: false,
|
|
|
|
|
error: `Document processing API is not online. Request will not be processed.`,
|
|
|
|
|
})
|
|
|
|
|
.end();
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (
|
|
|
|
|
!requiredMetadata.every(
|
|
|
|
|
(reqKey) =>
|
|
|
|
|
Object.keys(metadata).includes(reqKey) && !!metadata[reqKey]
|
|
|
|
|
)
|
|
|
|
|
) {
|
|
|
|
|
response
|
|
|
|
|
.status(422)
|
|
|
|
|
.json({
|
|
|
|
|
success: false,
|
|
|
|
|
error: `You are missing required metadata key:value pairs in your request. Required metadata key:values are ${requiredMetadata
|
|
|
|
|
.map((v) => `'${v}'`)
|
|
|
|
|
.join(", ")}`,
|
|
|
|
|
})
|
|
|
|
|
.end();
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (!textContent || textContent?.length === 0) {
|
|
|
|
|
response
|
|
|
|
|
.status(422)
|
|
|
|
|
.json({
|
|
|
|
|
success: false,
|
|
|
|
|
error: `The 'textContent' key cannot have an empty value.`,
|
|
|
|
|
})
|
|
|
|
|
.end();
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
|
2024-02-17 01:32:25 +01:00
|
|
|
|
const { success, reason, documents } = await Collector.processRawText(
|
2024-02-08 00:17:32 +01:00
|
|
|
|
textContent,
|
|
|
|
|
metadata
|
|
|
|
|
);
|
|
|
|
|
if (!success) {
|
|
|
|
|
response
|
|
|
|
|
.status(500)
|
|
|
|
|
.json({ success: false, error: reason, documents })
|
|
|
|
|
.end();
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
|
2024-02-17 01:32:25 +01:00
|
|
|
|
Collector.log(
|
2024-02-08 00:17:32 +01:00
|
|
|
|
`Document created successfully. It is now available in documents.`
|
|
|
|
|
);
|
|
|
|
|
await Telemetry.sendTelemetry("raw_document_uploaded");
|
|
|
|
|
await EventLogs.logEvent("api_raw_document_uploaded");
|
|
|
|
|
response.status(200).json({ success: true, error: null, documents });
|
|
|
|
|
} catch (e) {
|
2024-07-04 01:39:33 +02:00
|
|
|
|
console.error(e.message, e);
|
2024-02-08 00:17:32 +01:00
|
|
|
|
response.sendStatus(500).end();
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
);
|
|
|
|
|
|
2023-08-24 04:15:07 +02:00
|
|
|
|
app.get("/v1/documents", [validApiKey], async (_, response) => {
|
[FEAT] Automated audit logging (#667)
* WIP event logging - new table for events and new settings view for viewing
* WIP add logging
* UI for log rows
* rename files to Logging to prevent getting gitignore
* add metadata for all logging events and colored badges in logs page
* remove unneeded comment
* cleanup namespace for logging
* clean up backend calls
* update logging to show to => from settings changes
* add logging for invitations, created, deleted, and accepted
* add logging for user created, updated, suspended, or removed
* add logging for workspace deleted
* add logging for chat logs exported
* add logging for API keys, LLM, embedder, vector db, embed chat, and reset button
* modify event logs
* update to event log types
* simplify rendering of event badges
---------
Co-authored-by: timothycarambat <rambat1010@gmail.com>
2024-02-07 00:21:40 +01:00
|
|
|
|
/*
|
2023-08-24 04:15:07 +02:00
|
|
|
|
#swagger.tags = ['Documents']
|
|
|
|
|
#swagger.description = 'List of all locally-stored documents in instance'
|
|
|
|
|
#swagger.responses[200] = {
|
|
|
|
|
content: {
|
|
|
|
|
"application/json": {
|
|
|
|
|
schema: {
|
|
|
|
|
type: 'object',
|
|
|
|
|
example: {
|
|
|
|
|
"localFiles": {
|
|
|
|
|
"name": "documents",
|
|
|
|
|
"type": "folder",
|
|
|
|
|
items: [
|
|
|
|
|
{
|
|
|
|
|
"name": "my-stored-document.json",
|
|
|
|
|
"type": "file",
|
|
|
|
|
"id": "bb07c334-4dab-4419-9462-9d00065a49a1",
|
|
|
|
|
"url": "file://my-stored-document.txt",
|
|
|
|
|
"title": "my-stored-document.txt",
|
|
|
|
|
"cached": false
|
|
|
|
|
},
|
|
|
|
|
]
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
[FEAT] Automated audit logging (#667)
* WIP event logging - new table for events and new settings view for viewing
* WIP add logging
* UI for log rows
* rename files to Logging to prevent getting gitignore
* add metadata for all logging events and colored badges in logs page
* remove unneeded comment
* cleanup namespace for logging
* clean up backend calls
* update logging to show to => from settings changes
* add logging for invitations, created, deleted, and accepted
* add logging for user created, updated, suspended, or removed
* add logging for workspace deleted
* add logging for chat logs exported
* add logging for API keys, LLM, embedder, vector db, embed chat, and reset button
* modify event logs
* update to event log types
* simplify rendering of event badges
---------
Co-authored-by: timothycarambat <rambat1010@gmail.com>
2024-02-07 00:21:40 +01:00
|
|
|
|
}
|
2023-08-24 04:15:07 +02:00
|
|
|
|
}
|
[FEAT] Automated audit logging (#667)
* WIP event logging - new table for events and new settings view for viewing
* WIP add logging
* UI for log rows
* rename files to Logging to prevent getting gitignore
* add metadata for all logging events and colored badges in logs page
* remove unneeded comment
* cleanup namespace for logging
* clean up backend calls
* update logging to show to => from settings changes
* add logging for invitations, created, deleted, and accepted
* add logging for user created, updated, suspended, or removed
* add logging for workspace deleted
* add logging for chat logs exported
* add logging for API keys, LLM, embedder, vector db, embed chat, and reset button
* modify event logs
* update to event log types
* simplify rendering of event badges
---------
Co-authored-by: timothycarambat <rambat1010@gmail.com>
2024-02-07 00:21:40 +01:00
|
|
|
|
}
|
2023-08-24 04:15:07 +02:00
|
|
|
|
#swagger.responses[403] = {
|
|
|
|
|
schema: {
|
|
|
|
|
"$ref": "#/definitions/InvalidAPIKey"
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
*/
|
|
|
|
|
try {
|
|
|
|
|
const localFiles = await viewLocalFiles();
|
|
|
|
|
response.status(200).json({ localFiles });
|
|
|
|
|
} catch (e) {
|
2024-07-04 01:39:33 +02:00
|
|
|
|
console.error(e.message, e);
|
2023-08-24 04:15:07 +02:00
|
|
|
|
response.sendStatus(500).end();
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
app.get(
|
|
|
|
|
"/v1/document/accepted-file-types",
|
|
|
|
|
[validApiKey],
|
|
|
|
|
async (_, response) => {
|
[FEAT] Automated audit logging (#667)
* WIP event logging - new table for events and new settings view for viewing
* WIP add logging
* UI for log rows
* rename files to Logging to prevent getting gitignore
* add metadata for all logging events and colored badges in logs page
* remove unneeded comment
* cleanup namespace for logging
* clean up backend calls
* update logging to show to => from settings changes
* add logging for invitations, created, deleted, and accepted
* add logging for user created, updated, suspended, or removed
* add logging for workspace deleted
* add logging for chat logs exported
* add logging for API keys, LLM, embedder, vector db, embed chat, and reset button
* modify event logs
* update to event log types
* simplify rendering of event badges
---------
Co-authored-by: timothycarambat <rambat1010@gmail.com>
2024-02-07 00:21:40 +01:00
|
|
|
|
/*
|
2023-08-24 04:15:07 +02:00
|
|
|
|
#swagger.tags = ['Documents']
|
|
|
|
|
#swagger.description = 'Check available filetypes and MIMEs that can be uploaded.'
|
|
|
|
|
#swagger.responses[200] = {
|
|
|
|
|
content: {
|
|
|
|
|
"application/json": {
|
|
|
|
|
schema: {
|
|
|
|
|
type: 'object',
|
|
|
|
|
example: {
|
|
|
|
|
"types": {
|
|
|
|
|
"application/mbox": [
|
|
|
|
|
".mbox"
|
|
|
|
|
],
|
|
|
|
|
"application/pdf": [
|
|
|
|
|
".pdf"
|
|
|
|
|
],
|
|
|
|
|
"application/vnd.oasis.opendocument.text": [
|
|
|
|
|
".odt"
|
|
|
|
|
],
|
|
|
|
|
"application/vnd.openxmlformats-officedocument.wordprocessingml.document": [
|
|
|
|
|
".docx"
|
|
|
|
|
],
|
|
|
|
|
"text/plain": [
|
|
|
|
|
".txt",
|
|
|
|
|
".md"
|
|
|
|
|
]
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
[FEAT] Automated audit logging (#667)
* WIP event logging - new table for events and new settings view for viewing
* WIP add logging
* UI for log rows
* rename files to Logging to prevent getting gitignore
* add metadata for all logging events and colored badges in logs page
* remove unneeded comment
* cleanup namespace for logging
* clean up backend calls
* update logging to show to => from settings changes
* add logging for invitations, created, deleted, and accepted
* add logging for user created, updated, suspended, or removed
* add logging for workspace deleted
* add logging for chat logs exported
* add logging for API keys, LLM, embedder, vector db, embed chat, and reset button
* modify event logs
* update to event log types
* simplify rendering of event badges
---------
Co-authored-by: timothycarambat <rambat1010@gmail.com>
2024-02-07 00:21:40 +01:00
|
|
|
|
}
|
2023-08-24 04:15:07 +02:00
|
|
|
|
}
|
[FEAT] Automated audit logging (#667)
* WIP event logging - new table for events and new settings view for viewing
* WIP add logging
* UI for log rows
* rename files to Logging to prevent getting gitignore
* add metadata for all logging events and colored badges in logs page
* remove unneeded comment
* cleanup namespace for logging
* clean up backend calls
* update logging to show to => from settings changes
* add logging for invitations, created, deleted, and accepted
* add logging for user created, updated, suspended, or removed
* add logging for workspace deleted
* add logging for chat logs exported
* add logging for API keys, LLM, embedder, vector db, embed chat, and reset button
* modify event logs
* update to event log types
* simplify rendering of event badges
---------
Co-authored-by: timothycarambat <rambat1010@gmail.com>
2024-02-07 00:21:40 +01:00
|
|
|
|
}
|
2023-08-24 04:15:07 +02:00
|
|
|
|
#swagger.responses[403] = {
|
|
|
|
|
schema: {
|
|
|
|
|
"$ref": "#/definitions/InvalidAPIKey"
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
*/
|
|
|
|
|
try {
|
2024-02-17 01:32:25 +01:00
|
|
|
|
const types = await new CollectorApi().acceptedFileTypes();
|
2023-08-24 04:15:07 +02:00
|
|
|
|
if (!types) {
|
|
|
|
|
response.sendStatus(404).end();
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
response.status(200).json({ types });
|
|
|
|
|
} catch (e) {
|
2024-07-04 01:39:33 +02:00
|
|
|
|
console.error(e.message, e);
|
2023-08-24 04:15:07 +02:00
|
|
|
|
response.sendStatus(500).end();
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
);
|
2024-02-08 00:17:32 +01:00
|
|
|
|
|
|
|
|
|
app.get(
|
|
|
|
|
"/v1/document/metadata-schema",
|
|
|
|
|
[validApiKey],
|
|
|
|
|
async (_, response) => {
|
|
|
|
|
/*
|
|
|
|
|
#swagger.tags = ['Documents']
|
|
|
|
|
#swagger.description = 'Get the known available metadata schema for when doing a raw-text upload and the acceptable type of value for each key.'
|
|
|
|
|
#swagger.responses[200] = {
|
|
|
|
|
content: {
|
|
|
|
|
"application/json": {
|
|
|
|
|
schema: {
|
|
|
|
|
type: 'object',
|
|
|
|
|
example: {
|
|
|
|
|
"schema": {
|
|
|
|
|
"keyOne": "string | number | nullable",
|
|
|
|
|
"keyTwo": "string | number | nullable",
|
|
|
|
|
"specialKey": "number",
|
|
|
|
|
"title": "string",
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
#swagger.responses[403] = {
|
|
|
|
|
schema: {
|
|
|
|
|
"$ref": "#/definitions/InvalidAPIKey"
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
*/
|
|
|
|
|
try {
|
|
|
|
|
response.status(200).json({
|
|
|
|
|
schema: {
|
|
|
|
|
// If you are updating this be sure to update the collector METADATA_KEYS constant in /processRawText.
|
|
|
|
|
url: "string | nullable",
|
|
|
|
|
title: "string",
|
|
|
|
|
docAuthor: "string | nullable",
|
|
|
|
|
description: "string | nullable",
|
|
|
|
|
docSource: "string | nullable",
|
|
|
|
|
chunkSource: "string | nullable",
|
|
|
|
|
published: "epoch timestamp in ms | nullable",
|
|
|
|
|
},
|
|
|
|
|
});
|
|
|
|
|
} catch (e) {
|
2024-07-04 01:39:33 +02:00
|
|
|
|
console.error(e.message, e);
|
2024-02-08 00:17:32 +01:00
|
|
|
|
response.sendStatus(500).end();
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
);
|
2024-02-17 01:32:25 +01:00
|
|
|
|
|
|
|
|
|
// Be careful and place as last route to prevent override of the other /document/ GET
|
|
|
|
|
// endpoints!
|
|
|
|
|
app.get("/v1/document/:docName", [validApiKey], async (request, response) => {
|
|
|
|
|
/*
|
|
|
|
|
#swagger.tags = ['Documents']
|
|
|
|
|
#swagger.description = 'Get a single document by its unique AnythingLLM document name'
|
|
|
|
|
#swagger.parameters['docName'] = {
|
|
|
|
|
in: 'path',
|
|
|
|
|
description: 'Unique document name to find (name in /documents)',
|
|
|
|
|
required: true,
|
|
|
|
|
type: 'string'
|
|
|
|
|
}
|
|
|
|
|
#swagger.responses[200] = {
|
|
|
|
|
content: {
|
|
|
|
|
"application/json": {
|
|
|
|
|
schema: {
|
|
|
|
|
type: 'object',
|
|
|
|
|
example: {
|
|
|
|
|
"localFiles": {
|
|
|
|
|
"name": "documents",
|
|
|
|
|
"type": "folder",
|
|
|
|
|
items: [
|
|
|
|
|
{
|
|
|
|
|
"name": "my-stored-document.txt-uuid1234.json",
|
|
|
|
|
"type": "file",
|
|
|
|
|
"id": "bb07c334-4dab-4419-9462-9d00065a49a1",
|
|
|
|
|
"url": "file://my-stored-document.txt",
|
|
|
|
|
"title": "my-stored-document.txt",
|
|
|
|
|
"cached": false
|
|
|
|
|
},
|
|
|
|
|
]
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
#swagger.responses[403] = {
|
|
|
|
|
schema: {
|
|
|
|
|
"$ref": "#/definitions/InvalidAPIKey"
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
*/
|
|
|
|
|
try {
|
|
|
|
|
const { docName } = request.params;
|
|
|
|
|
const document = await findDocumentInDocuments(docName);
|
|
|
|
|
if (!document) {
|
|
|
|
|
response.sendStatus(404).end();
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
response.status(200).json({ document });
|
|
|
|
|
} catch (e) {
|
2024-07-04 01:39:33 +02:00
|
|
|
|
console.error(e.message, e);
|
2024-02-17 01:32:25 +01:00
|
|
|
|
response.sendStatus(500).end();
|
|
|
|
|
}
|
|
|
|
|
});
|
2024-03-21 00:10:30 +01:00
|
|
|
|
|
|
|
|
|
app.post(
|
|
|
|
|
"/v1/document/create-folder",
|
|
|
|
|
[validApiKey],
|
|
|
|
|
async (request, response) => {
|
|
|
|
|
/*
|
|
|
|
|
#swagger.tags = ['Documents']
|
|
|
|
|
#swagger.description = 'Create a new folder inside the documents storage directory.'
|
|
|
|
|
#swagger.requestBody = {
|
|
|
|
|
description: 'Name of the folder to create.',
|
|
|
|
|
required: true,
|
|
|
|
|
type: 'object',
|
|
|
|
|
content: {
|
|
|
|
|
"application/json": {
|
|
|
|
|
schema: {
|
|
|
|
|
type: 'object',
|
|
|
|
|
example: {
|
|
|
|
|
"name": "new-folder"
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
#swagger.responses[200] = {
|
|
|
|
|
content: {
|
|
|
|
|
"application/json": {
|
|
|
|
|
schema: {
|
|
|
|
|
type: 'object',
|
|
|
|
|
example: {
|
|
|
|
|
success: true,
|
|
|
|
|
message: null
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
#swagger.responses[403] = {
|
|
|
|
|
schema: {
|
|
|
|
|
"$ref": "#/definitions/InvalidAPIKey"
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
*/
|
|
|
|
|
try {
|
|
|
|
|
const { name } = reqBody(request);
|
2024-03-21 23:10:44 +01:00
|
|
|
|
const storagePath = path.join(documentsPath, normalizePath(name));
|
2024-05-23 21:52:04 +02:00
|
|
|
|
if (!isWithin(path.resolve(documentsPath), path.resolve(storagePath)))
|
|
|
|
|
throw new Error("Invalid path name");
|
2024-03-21 00:10:30 +01:00
|
|
|
|
|
|
|
|
|
if (fs.existsSync(storagePath)) {
|
|
|
|
|
response.status(500).json({
|
|
|
|
|
success: false,
|
|
|
|
|
message: "Folder by that name already exists",
|
|
|
|
|
});
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
fs.mkdirSync(storagePath, { recursive: true });
|
|
|
|
|
response.status(200).json({ success: true, message: null });
|
|
|
|
|
} catch (e) {
|
|
|
|
|
console.error(e);
|
|
|
|
|
response.status(500).json({
|
|
|
|
|
success: false,
|
|
|
|
|
message: `Failed to create folder: ${e.message}`,
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
app.post(
|
|
|
|
|
"/v1/document/move-files",
|
|
|
|
|
[validApiKey],
|
|
|
|
|
async (request, response) => {
|
|
|
|
|
/*
|
|
|
|
|
#swagger.tags = ['Documents']
|
|
|
|
|
#swagger.description = 'Move files within the documents storage directory.'
|
|
|
|
|
#swagger.requestBody = {
|
|
|
|
|
description: 'Array of objects containing source and destination paths of files to move.',
|
|
|
|
|
required: true,
|
|
|
|
|
type: 'object',
|
|
|
|
|
content: {
|
|
|
|
|
"application/json": {
|
|
|
|
|
schema: {
|
|
|
|
|
type: 'object',
|
|
|
|
|
example: {
|
|
|
|
|
"files": [
|
|
|
|
|
{
|
|
|
|
|
"from": "custom-documents/file.txt-fc4beeeb-e436-454d-8bb4-e5b8979cb48f.json",
|
|
|
|
|
"to": "folder/file.txt-fc4beeeb-e436-454d-8bb4-e5b8979cb48f.json"
|
|
|
|
|
}
|
|
|
|
|
]
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
#swagger.responses[200] = {
|
|
|
|
|
content: {
|
|
|
|
|
"application/json": {
|
|
|
|
|
schema: {
|
|
|
|
|
type: 'object',
|
|
|
|
|
example: {
|
|
|
|
|
success: true,
|
|
|
|
|
message: null
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
#swagger.responses[403] = {
|
|
|
|
|
schema: {
|
|
|
|
|
"$ref": "#/definitions/InvalidAPIKey"
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
*/
|
|
|
|
|
try {
|
|
|
|
|
const { files } = reqBody(request);
|
|
|
|
|
const docpaths = files.map(({ from }) => from);
|
|
|
|
|
const documents = await Document.where({ docpath: { in: docpaths } });
|
|
|
|
|
const embeddedFiles = documents.map((doc) => doc.docpath);
|
|
|
|
|
const moveableFiles = files.filter(
|
|
|
|
|
({ from }) => !embeddedFiles.includes(from)
|
|
|
|
|
);
|
|
|
|
|
const movePromises = moveableFiles.map(({ from, to }) => {
|
2024-03-21 23:10:44 +01:00
|
|
|
|
const sourcePath = path.join(documentsPath, normalizePath(from));
|
|
|
|
|
const destinationPath = path.join(documentsPath, normalizePath(to));
|
2024-03-21 00:10:30 +01:00
|
|
|
|
return new Promise((resolve, reject) => {
|
|
|
|
|
fs.rename(sourcePath, destinationPath, (err) => {
|
|
|
|
|
if (err) {
|
|
|
|
|
console.error(`Error moving file ${from} to ${to}:`, err);
|
|
|
|
|
reject(err);
|
|
|
|
|
} else {
|
|
|
|
|
resolve();
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
});
|
|
|
|
|
});
|
|
|
|
|
Promise.all(movePromises)
|
|
|
|
|
.then(() => {
|
|
|
|
|
const unmovableCount = files.length - moveableFiles.length;
|
|
|
|
|
if (unmovableCount > 0) {
|
|
|
|
|
response.status(200).json({
|
|
|
|
|
success: true,
|
|
|
|
|
message: `${unmovableCount}/${files.length} files not moved. Unembed them from all workspaces.`,
|
|
|
|
|
});
|
|
|
|
|
} else {
|
|
|
|
|
response.status(200).json({
|
|
|
|
|
success: true,
|
|
|
|
|
message: null,
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
})
|
|
|
|
|
.catch((err) => {
|
|
|
|
|
console.error("Error moving files:", err);
|
|
|
|
|
response
|
|
|
|
|
.status(500)
|
|
|
|
|
.json({ success: false, message: "Failed to move some files." });
|
|
|
|
|
});
|
|
|
|
|
} catch (e) {
|
|
|
|
|
console.error(e);
|
|
|
|
|
response
|
|
|
|
|
.status(500)
|
|
|
|
|
.json({ success: false, message: "Failed to move files." });
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
);
|
2023-08-24 04:15:07 +02:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
module.exports = { apiDocumentEndpoints };
|