mirror of
https://github.com/Mintplex-Labs/anything-llm.git
synced 2024-09-12 07:10:08 +02:00
* wip: init refactor of document processor to JS * add NodeJs PDF support * wip: partity with python processor feat: add pptx support * fix: forgot files * Remove python scripts totally * wip:update docker to boot new collector * add package.json support * update dockerfile for new build * update gitignore and linting * add more protections on file lookup * update package.json * test build * update docker commands to use cap-add=SYS_ADMIN so web scraper can run update all scripts to reflect this remove docker build for branch
195 lines
5.1 KiB
JavaScript
195 lines
5.1 KiB
JavaScript
const { Telemetry } = require("../../../models/telemetry");
|
|
const { validApiKey } = require("../../../utils/middleware/validApiKey");
|
|
const { setupMulter } = require("../../../utils/files/multer");
|
|
const {
|
|
checkProcessorAlive,
|
|
acceptedFileTypes,
|
|
processDocument,
|
|
} = require("../../../utils/files/documentProcessor");
|
|
const { viewLocalFiles } = require("../../../utils/files");
|
|
const { handleUploads } = setupMulter();
|
|
|
|
function apiDocumentEndpoints(app) {
|
|
if (!app) return;
|
|
|
|
app.post(
|
|
"/v1/document/upload",
|
|
[validApiKey],
|
|
handleUploads.single("file"),
|
|
async (request, response) => {
|
|
/*
|
|
#swagger.tags = ['Documents']
|
|
#swagger.description = 'Upload a new file to AnythingLLM to be parsed and prepared for embedding.'
|
|
|
|
#swagger.requestBody = {
|
|
description: 'File to be uploaded.',
|
|
required: true,
|
|
type: 'file',
|
|
content: {
|
|
"multipart/form-data": {
|
|
schema: {
|
|
type: 'object',
|
|
properties: {
|
|
file: {
|
|
type: 'string',
|
|
format: 'binary',
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
#swagger.responses[200] = {
|
|
content: {
|
|
"application/json": {
|
|
schema: {
|
|
type: 'object',
|
|
example: {
|
|
success: true,
|
|
error: null,
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
#swagger.responses[403] = {
|
|
schema: {
|
|
"$ref": "#/definitions/InvalidAPIKey"
|
|
}
|
|
}
|
|
*/
|
|
try {
|
|
const { originalname } = request.file;
|
|
const processingOnline = await checkProcessorAlive();
|
|
|
|
if (!processingOnline) {
|
|
response
|
|
.status(500)
|
|
.json({
|
|
success: false,
|
|
error: `Document processing API is not online. Document ${originalname} will not be processed automatically.`,
|
|
})
|
|
.end();
|
|
}
|
|
|
|
const { success, reason } = await processDocument(originalname);
|
|
if (!success) {
|
|
response.status(500).json({ success: false, error: reason }).end();
|
|
}
|
|
|
|
console.log(
|
|
`Document ${originalname} uploaded processed and successfully. It is now available in documents.`
|
|
);
|
|
await Telemetry.sendTelemetry("document_uploaded");
|
|
response.status(200).json({ success: true, error: null });
|
|
} catch (e) {
|
|
console.log(e.message, e);
|
|
response.sendStatus(500).end();
|
|
}
|
|
}
|
|
);
|
|
|
|
app.get("/v1/documents", [validApiKey], async (_, response) => {
|
|
/*
|
|
#swagger.tags = ['Documents']
|
|
#swagger.description = 'List of all locally-stored documents in instance'
|
|
#swagger.responses[200] = {
|
|
content: {
|
|
"application/json": {
|
|
schema: {
|
|
type: 'object',
|
|
example: {
|
|
"localFiles": {
|
|
"name": "documents",
|
|
"type": "folder",
|
|
items: [
|
|
{
|
|
"name": "my-stored-document.json",
|
|
"type": "file",
|
|
"id": "bb07c334-4dab-4419-9462-9d00065a49a1",
|
|
"url": "file://my-stored-document.txt",
|
|
"title": "my-stored-document.txt",
|
|
"cached": false
|
|
},
|
|
]
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
#swagger.responses[403] = {
|
|
schema: {
|
|
"$ref": "#/definitions/InvalidAPIKey"
|
|
}
|
|
}
|
|
*/
|
|
try {
|
|
const localFiles = await viewLocalFiles();
|
|
response.status(200).json({ localFiles });
|
|
} catch (e) {
|
|
console.log(e.message, e);
|
|
response.sendStatus(500).end();
|
|
}
|
|
});
|
|
|
|
app.get(
|
|
"/v1/document/accepted-file-types",
|
|
[validApiKey],
|
|
async (_, response) => {
|
|
/*
|
|
#swagger.tags = ['Documents']
|
|
#swagger.description = 'Check available filetypes and MIMEs that can be uploaded.'
|
|
#swagger.responses[200] = {
|
|
content: {
|
|
"application/json": {
|
|
schema: {
|
|
type: 'object',
|
|
example: {
|
|
"types": {
|
|
"application/mbox": [
|
|
".mbox"
|
|
],
|
|
"application/pdf": [
|
|
".pdf"
|
|
],
|
|
"application/vnd.oasis.opendocument.text": [
|
|
".odt"
|
|
],
|
|
"application/vnd.openxmlformats-officedocument.wordprocessingml.document": [
|
|
".docx"
|
|
],
|
|
"text/plain": [
|
|
".txt",
|
|
".md"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
#swagger.responses[403] = {
|
|
schema: {
|
|
"$ref": "#/definitions/InvalidAPIKey"
|
|
}
|
|
}
|
|
*/
|
|
try {
|
|
const types = await acceptedFileTypes();
|
|
if (!types) {
|
|
response.sendStatus(404).end();
|
|
return;
|
|
}
|
|
|
|
response.status(200).json({ types });
|
|
} catch (e) {
|
|
console.log(e.message, e);
|
|
response.sendStatus(500).end();
|
|
}
|
|
}
|
|
);
|
|
}
|
|
|
|
module.exports = { apiDocumentEndpoints };
|