[FIX] PDFLoader module bug fix (#1879)

use pdf.js by importing it from pdf-parse and fix custom PDFLoader module

collector/processSingleFile/convert/asPDF/PDFLoader/index.js

@@ -1,5 +1,4 @@
 const fs = require("fs").promises;
-const pdf = require("pdf-parse");
 
 class PDFLoader {
   constructor(filePath, { splitPages = true } = {}) {
@@ -9,54 +8,90 @@ class PDFLoader {
 
   async load() {
     const buffer = await fs.readFile(this.filePath);
+    const { getDocument, version } = await this.getPdfJS();
 
-    const options = {
+    const pdf = await getDocument({
-      pagerender: this.splitPages ? this.renderPage : null,
+      data: new Uint8Array(buffer),
-    };
+      useWorkerFetch: false,
+      isEvalSupported: false,
+      useSystemFonts: true,
+    }).promise;
 
-    const { text, numpages, info, metadata, version } = await pdf(
+    const meta = await pdf.getMetadata().catch(() => null);
-      buffer,
+    const documents = [];
-      options
-    );
 
-    if (!this.splitPages) {
+    for (let i = 1; i <= pdf.numPages; i += 1) {
-      return [
+      const page = await pdf.getPage(i);
-        {
+      const content = await page.getTextContent();
-          pageContent: text.trim(),
+
-          metadata: {
+      if (content.items.length === 0) {
-            source: this.filePath,
+        continue;
-            pdf: { version, info, metadata, totalPages: numpages },
+      }
+
+      let lastY;
+      const textItems = [];
+      for (const item of content.items) {
+        if ("str" in item) {
+          if (lastY === item.transform[5] || !lastY) {
+            textItems.push(item.str);
+          } else {
+            textItems.push(`\n${item.str}`);
+          }
+          lastY = item.transform[5];
+        }
+      }
+
+      const text = textItems.join("");
+      documents.push({
+        pageContent: text.trim(),
+        metadata: {
+          source: this.filePath,
+          pdf: {
+            version,
+            info: meta?.info,
+            metadata: meta?.metadata,
+            totalPages: pdf.numPages,
+          },
+          loc: { pageNumber: i },
+        },
+      });
+    }
+
+    if (this.splitPages) {
+      return documents;
+    }
+
+    if (documents.length === 0) {
+      return [];
+    }
+
+    return [
+      {
+        pageContent: documents.map((doc) => doc.pageContent).join("\n\n"),
+        metadata: {
+          source: this.filePath,
+          pdf: {
+            version,
+            info: meta?.info,
+            metadata: meta?.metadata,
+            totalPages: pdf.numPages,
           },
         },
-      ];
-    }
-
-    return this.pages.map((pageContent, index) => ({
-      pageContent: pageContent.trim(),
-      metadata: {
-        source: this.filePath,
-        pdf: { version, info, metadata, totalPages: numpages },
-        loc: { pageNumber: index + 1 },
       },
-    }));
+    ];
   }
 
-  pages = [];
+  async getPdfJS() {
-
+    try {
-  renderPage = async (pageData) => {
+      const pdfjs = await import("pdf-parse/lib/pdf.js/v1.10.100/build/pdf.js");
-    const textContent = await pageData.getTextContent();
+      return { getDocument: pdfjs.getDocument, version: pdfjs.version };
-    let lastY,
+    } catch (e) {
-      text = "";
+      console.error(e);
-    for (const item of textContent.items) {
+      throw new Error(
-      if (lastY !== item.transform[5] && lastY !== undefined) {
+        "Failed to load pdf-parse. Please install it with eg. `npm install pdf-parse`."
-        text += "\n";
+      );
-      }
-      text += item.str;
-      lastY = item.transform[5];
     }
-    this.pages.push(text);
+  }
-    return text;
-  };
 }
 
 module.exports = PDFLoader;