anything-llm/collector/processSingleFile/index.js

79 lines
2.0 KiB
JavaScript
Raw Normal View History

const path = require("path");
const fs = require("fs");
const {
WATCH_DIRECTORY,
SUPPORTED_FILETYPE_CONVERTERS,
} = require("../utils/constants");
const {
trashFile,
isTextType,
normalizePath,
isWithin,
} = require("../utils/files");
2023-12-21 04:41:16 +01:00
const RESERVED_FILES = ["__HOTDIR__.md"];
async function processSingleFile(targetFilename, options = {}) {
const fullFilePath = path.resolve(
WATCH_DIRECTORY,
normalizePath(targetFilename)
);
if (!isWithin(path.resolve(WATCH_DIRECTORY), fullFilePath))
return {
success: false,
reason: "Filename is a not a valid path to process.",
documents: [],
};
if (RESERVED_FILES.includes(targetFilename))
return {
success: false,
reason: "Filename is a reserved filename and cannot be processed.",
documents: [],
};
if (!fs.existsSync(fullFilePath))
return {
success: false,
reason: "File does not exist in upload directory.",
documents: [],
};
const fileExtension = path.extname(fullFilePath).toLowerCase();
if (!fileExtension) {
return {
success: false,
reason: `No file extension found. This file cannot be processed.`,
documents: [],
};
}
let processFileAs = fileExtension;
if (!SUPPORTED_FILETYPE_CONVERTERS.hasOwnProperty(fileExtension)) {
if (isTextType(fullFilePath)) {
console.log(
`\x1b[33m[Collector]\x1b[0m The provided filetype of ${fileExtension} does not have a preset and will be processed as .txt.`
);
processFileAs = ".txt";
} else {
trashFile(fullFilePath);
return {
success: false,
reason: `File extension ${fileExtension} not supported for parsing and cannot be assumed as text file type.`,
documents: [],
};
}
}
const FileTypeProcessor = require(SUPPORTED_FILETYPE_CONVERTERS[
processFileAs
]);
return await FileTypeProcessor({
fullFilePath,
filename: targetFilename,
options,
});
}
module.exports = {
processSingleFile,
};