kamp/anything-llm
1
0
Fork 0
mirror of https://github.com/Mintplex-Labs/anything-llm.git synced 2024-11-14 18:40:11 +01:00
Code Issues Packages Projects Releases Wiki Activity

Ignore SSL errors for web scraper

Browse Source 
resolves #2114
This commit is contained in:
timothycarambat 2024-08-14 09:11:22 -07:00
parent b541623c9e
commit 619f6b3884
2 changed files with 4 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
collector/processLink/convert/generic.js
View File

@ -57,6 +57,7 @@ async function getPageContent(link) {
const loader = new PuppeteerWebBaseLoader(link, { const loader = new PuppeteerWebBaseLoader(link, {
launchOptions: { launchOptions: {
headless: "new", headless: "new",
ignoreHTTPSErrors: true,
}, },
gotoOptions: { gotoOptions: {
waitUntil: "domcontentloaded", waitUntil: "domcontentloaded",

6
collector/utils/url/index.js
View File

@ -5,11 +5,11 @@
* and is simply to prevent the user from accidentally putting in non-valid websites, which is all this protects * and is simply to prevent the user from accidentally putting in non-valid websites, which is all this protects
* since _all urls must be submitted by the user anyway_ and cannot be done with authentication and manager or admin roles. * since _all urls must be submitted by the user anyway_ and cannot be done with authentication and manager or admin roles.
* If an attacker has those roles then the system is already vulnerable and this is not a primary concern. * If an attacker has those roles then the system is already vulnerable and this is not a primary concern.
* *
* We have gotten this report may times, marked them as duplicate or information and continue to get them. We communicate * We have gotten this report may times, marked them as duplicate or information and continue to get them. We communicate
* already that deployment (and security) of an instance is on the deployer and system admin deploying it. This would include * already that deployment (and security) of an instance is on the deployer and system admin deploying it. This would include
* isolation, firewalls, and the general security of the instance. * isolation, firewalls, and the general security of the instance.
*/ */
const VALID_PROTOCOLS = ["https:", "http:"]; const VALID_PROTOCOLS = ["https:", "http:"];
const INVALID_OCTETS = [192, 172, 10, 127]; const INVALID_OCTETS = [192, 172, 10, 127];
@ -32,7 +32,7 @@ function validURL(url) {
if (!VALID_PROTOCOLS.includes(destination.protocol)) return false; if (!VALID_PROTOCOLS.includes(destination.protocol)) return false;
if (isInvalidIp(destination)) return false; if (isInvalidIp(destination)) return false;
return true; return true;
} catch { } } catch {}
return false; return false;
} }