anything-llm/collector/processSingleFile/convert/asPDF.js
Timothy Carambat 719521c307
Document Processor v2 (#442)
* wip: init refactor of document processor to JS

* add NodeJs PDF support

* wip: partity with python processor
feat: add pptx support

* fix: forgot files

* Remove python scripts totally

* wip:update docker to boot new collector

* add package.json support

* update dockerfile for new build

* update gitignore and linting

* add more protections on file lookup

* update package.json

* test build

* update docker commands to use cap-add=SYS_ADMIN so web scraper can run
update all scripts to reflect this
remove docker build for branch
2023-12-14 15:14:56 -08:00

57 lines
1.7 KiB
JavaScript

const { v4 } = require("uuid");
const { PDFLoader } = require("langchain/document_loaders/fs/pdf");
const {
createdDate,
trashFile,
writeToServerDocuments,
} = require("../../utils/files");
const { tokenizeString } = require("../../utils/tokenizer");
const { default: slugify } = require("slugify");
async function asPDF({ fullFilePath = "", filename = "" }) {
const pdfLoader = new PDFLoader(fullFilePath, {
splitPages: true,
});
console.log(`-- Working ${filename} --`);
const pageContent = [];
const docs = await pdfLoader.load();
for (const doc of docs) {
console.log(
`-- Parsing content from pg ${
doc.metadata?.loc?.pageNumber || "unknown"
} --`
);
if (!doc.pageContent.length) continue;
pageContent.push(doc.pageContent);
}
if (!pageContent.length) {
console.error(`Resulting text content was empty for ${filename}.`);
trashFile(fullFilePath);
return { success: false, reason: `No text content found in ${filename}.` };
}
const content = pageContent.join("");
data = {
id: v4(),
url: "file://" + fullFilePath,
title: docs[0]?.metadata?.pdf?.info?.Title || filename,
docAuthor: docs[0]?.metadata?.pdf?.info?.Creator || "no author found",
description: "No description found.",
docSource: "pdf file uploaded by the user.",
chunkSource: filename,
published: createdDate(fullFilePath),
wordCount: content.split(" ").length,
pageContent: content,
token_count_estimate: tokenizeString(content).length,
};
writeToServerDocuments(data, `${slugify(filename)}-${data.id}`);
trashFile(fullFilePath);
console.log(`[SUCCESS]: ${filename} converted & ready for embedding.\n`);
return { success: true, reason: null };
}
module.exports = asPDF;