mirror of
https://github.com/Mintplex-Labs/anything-llm.git
synced 2024-11-11 01:10:11 +01:00
forgot epub file!
This commit is contained in:
parent
752e3e22ed
commit
117c3b2bfb
55
collector/processSingleFile/convert/asEPub.js
Normal file
55
collector/processSingleFile/convert/asEPub.js
Normal file
@ -0,0 +1,55 @@
|
||||
const { v4 } = require("uuid");
|
||||
const { EPubLoader } = require("langchain/document_loaders/fs/epub");
|
||||
const { tokenizeString } = require("../../utils/tokenizer");
|
||||
const {
|
||||
createdDate,
|
||||
trashFile,
|
||||
writeToServerDocuments,
|
||||
} = require("../../utils/files");
|
||||
const { default: slugify } = require("slugify");
|
||||
|
||||
async function asEPub({ fullFilePath = "", filename = "" }) {
|
||||
let content = "";
|
||||
try {
|
||||
const loader = new EPubLoader(fullFilePath, { splitChapters: false });
|
||||
const docs = await loader.load();
|
||||
docs.forEach((doc) => (content += doc.pageContent));
|
||||
} catch (err) {
|
||||
console.error("Could not read epub file!", err);
|
||||
}
|
||||
|
||||
if (!content?.length) {
|
||||
console.error(`Resulting text content was empty for ${filename}.`);
|
||||
trashFile(fullFilePath);
|
||||
return {
|
||||
success: false,
|
||||
reason: `No text content found in ${filename}.`,
|
||||
documents: [],
|
||||
};
|
||||
}
|
||||
|
||||
console.log(`-- Working ${filename} --`);
|
||||
const data = {
|
||||
id: v4(),
|
||||
url: "file://" + fullFilePath,
|
||||
title: filename,
|
||||
docAuthor: "Unknown", // TODO: Find a better author
|
||||
description: "Unknown", // TODO: Find a better description
|
||||
docSource: "a epub file uploaded by the user.",
|
||||
chunkSource: "",
|
||||
published: createdDate(fullFilePath),
|
||||
wordCount: content.split(" ").length,
|
||||
pageContent: content,
|
||||
token_count_estimate: tokenizeString(content).length,
|
||||
};
|
||||
|
||||
const document = writeToServerDocuments(
|
||||
data,
|
||||
`${slugify(filename)}-${data.id}`
|
||||
);
|
||||
trashFile(fullFilePath);
|
||||
console.log(`[SUCCESS]: ${filename} converted & ready for embedding.\n`);
|
||||
return { success: true, reason: null, documents: [document] };
|
||||
}
|
||||
|
||||
module.exports = asEPub;
|
Loading…
Reference in New Issue
Block a user