kamp/anything-llm
1
0
Fork 0
mirror of https://github.com/Mintplex-Labs/anything-llm.git synced 2024-11-19 20:50:09 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of github.com:Mintplex-Labs/anything-llm into render

Browse Source
This commit is contained in:
timothycarambat 2024-05-14 12:57:44 -07:00
commit c8dac6177a
1 changed files with 15 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
collector/utils/extensions/WebsiteDepth/index.js
View File

@ -76,7 +76,7 @@ function extractLinks(html, baseUrl) {
return Array.from(extractedLinks);
}
async function bulkScrapePages(links, outputFolder) {
async function bulkScrapePages(links, outFolderPath) {
const scrapedData = [];
for (let i = 0; i < links.length; i++) {
@ -118,7 +118,7 @@ async function bulkScrapePages(links, outputFolder) {
token_count_estimate: tokenizeString(content).length,
};
writeToServerDocuments(data, data.title, outputFolder);
writeToServerDocuments(data, data.title, outFolderPath);
scrapedData.push(data);
console.log(`Successfully scraped ${link}.`);
@ -132,19 +132,25 @@ async function bulkScrapePages(links, outputFolder) {
async function websiteScraper(startUrl, depth = 1, maxLinks = 20) {
const websiteName = new URL(startUrl).hostname;
const outputFolder = path.resolve(
__dirname,
`../../../../server/storage/documents/${slugify(websiteName)}`
);
fs.mkdirSync(outputFolder, { recursive: true });
const outFolder = slugify(
`${slugify(websiteName)}-${v4().slice(0, 4)}`
).toLowerCase();
const outFolderPath =
process.env.NODE_ENV === "development"
? path.resolve(
__dirname,
`../../../../server/storage/documents/${outFolder}`
)
: path.resolve(process.env.STORAGE_DIR, `documents/${outFolder}`);
console.log("Discovering links...");
const linksToScrape = await discoverLinks(startUrl, depth, maxLinks);
console.log(`Found ${linksToScrape.length} links to scrape.`);
if (!fs.existsSync(outFolderPath))
fs.mkdirSync(outFolderPath, { recursive: true });
console.log("Starting bulk scraping...");
const scrapedData = await bulkScrapePages(linksToScrape, outputFolder);
const scrapedData = await bulkScrapePages(linksToScrape, outFolderPath);
console.log(`Scraped ${scrapedData.length} pages.`);
return scrapedData;