kamp/anything-llm
1
0
Fork 0
mirror of https://github.com/Mintplex-Labs/anything-llm.git synced 2024-11-19 12:40:09 +01:00
Code Issues Packages Projects Releases Wiki Activity

Add exception handling for special case files like Dockerfile and Jenkinsfile (#2410)

Browse Source
This commit is contained in:
Timothy Carambat 2024-10-02 15:13:31 -07:00 committed by GitHub
parent 5ac6020480
commit 93d64642f3
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 21 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
collector/processSingleFile/index.js
View File

@ -38,7 +38,7 @@ async function processSingleFile(targetFilename, options = {}) {
};
const fileExtension = path.extname(fullFilePath).toLowerCase();
if (!fileExtension) {
if (fullFilePath.includes(".") && !fileExtension) {
return {
success: false,
reason: `No file extension found. This file cannot be processed.`,

22
collector/utils/files/mime.js
View File

@ -1,5 +1,5 @@
const MimeLib = require("mime");
const path = require("path");
class MimeDetector {
nonTextTypes = ["multipart", "image", "model", "audio", "video"];
badMimes = [
@ -44,8 +44,26 @@ class MimeDetector {
);
}
// These are file types that are not detected by the mime library and need to be processed as text files.
// You should only add file types that are not detected by the mime library, are parsable as text, and are files
// with no extension. Otherwise, their extension should be added to the overrides array.
#specialTextFileTypes = ["dockerfile", "jenkinsfile"];
/**
* Returns the MIME type of the file. If the file has no extension found, it will be processed as a text file.
* @param {string} filepath
* @returns {string}
*/
getType(filepath) {
return this.lib.getType(filepath);
const parsedMime = this.lib.getType(filepath);
if (!!parsedMime) return parsedMime;
// If the mime could not be parsed, it could be a special file type like Dockerfile or Jenkinsfile
// which we can reliably process as text files.
const baseName = path.basename(filepath)?.toLowerCase();
if (this.#specialTextFileTypes.includes(baseName)) return "text/plain";
return null;
}
}