anything-llm/server/utils/files/documentProcessor.js
Timothy Carambat 452582489e
GitHub loader extension + extension support v1 (#469)
* feat: implement github repo loading
fix: purge of folders
fix: rendering of sub-files

* noshow delete on custom-documents

* Add API key support because of rate limits

* WIP for frontend of data connectors

* wip

* Add frontend form for GitHub repo data connector

* remove console.logs
block custom-documents from being deleted

* remove _meta unused arg

* Add support for ignore pathing in request
Ignore path input via tagging

* Update hint
2023-12-18 15:48:02 -08:00

91 lines
2.6 KiB
JavaScript

// When running locally will occupy the 0.0.0.0 hostname space but when deployed inside
// of docker this endpoint is not exposed so it is only on the Docker instances internal network
// so no additional security is needed on the endpoint directly. Auth is done however by the express
// middleware prior to leaving the node-side of the application so that is good enough >:)
const PROCESSOR_API = "http://0.0.0.0:8888";
async function checkProcessorAlive() {
return await fetch(`${PROCESSOR_API}`)
.then((res) => res.ok)
.catch((e) => false);
}
async function acceptedFileTypes() {
return await fetch(`${PROCESSOR_API}/accepts`)
.then((res) => {
if (!res.ok) throw new Error("Could not reach");
return res.json();
})
.then((res) => res)
.catch(() => null);
}
async function processDocument(filename = "") {
if (!filename) return false;
return await fetch(`${PROCESSOR_API}/process`, {
method: "POST",
headers: {
"Content-Type": "application/json",
},
body: JSON.stringify({ filename }),
})
.then((res) => {
if (!res.ok) throw new Error("Response could not be completed");
return res.json();
})
.then((res) => res)
.catch((e) => {
console.log(e.message);
return { success: false, reason: e.message };
});
}
async function processLink(link = "") {
if (!link) return false;
return await fetch(`${PROCESSOR_API}/process-link`, {
method: "POST",
headers: {
"Content-Type": "application/json",
},
body: JSON.stringify({ link }),
})
.then((res) => {
if (!res.ok) throw new Error("Response could not be completed");
return res.json();
})
.then((res) => res)
.catch((e) => {
console.log(e.message);
return { success: false, reason: e.message };
});
}
// We will not ever expose the document processor to the frontend API so instead we relay
// all requests through the server. You can use this function to directly expose a specific endpoint
// on the document processor.
async function forwardExtensionRequest({ endpoint, method, body }) {
return await fetch(`${PROCESSOR_API}${endpoint}`, {
method,
body, // Stringified JSON!
headers: {
"Content-Type": "application/json",
},
})
.then((res) => {
if (!res.ok) throw new Error("Response could not be completed");
return res.json();
})
.then((res) => res)
.catch((e) => {
console.log(e.message);
return { success: false, data: {}, reason: e.message };
});
}
module.exports = {
checkProcessorAlive,
processDocument,
processLink,
acceptedFileTypes,
forwardExtensionRequest,
};