kamp/anything-llm
1
0
Fork 0
mirror of https://github.com/Mintplex-Labs/anything-llm.git synced 2024-11-14 18:40:11 +01:00
Code Issues Packages Projects Releases Wiki Activity

Ignore SSL errors for web scraper

Browse Source 
resolves #2114
This commit is contained in:
timothycarambat 2024-08-14 09:11:22 -07:00
parent b541623c9e
commit 619f6b3884
2 changed files with 4 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
collector/processLink/convert/generic.js
View File

@ -57,6 +57,7 @@ async function getPageContent(link) {
const loader = new PuppeteerWebBaseLoader(link, {
launchOptions: {
headless: "new",
ignoreHTTPSErrors: true,
},
gotoOptions: {
waitUntil: "domcontentloaded",

4
collector/utils/url/index.js
View File

@ -9,7 +9,7 @@
* We have gotten this report may times, marked them as duplicate or information and continue to get them. We communicate
* already that deployment (and security) of an instance is on the deployer and system admin deploying it. This would include
* isolation, firewalls, and the general security of the instance.
*/
*/
const VALID_PROTOCOLS = ["https:", "http:"];
const INVALID_OCTETS = [192, 172, 10, 127];
@ -32,7 +32,7 @@ function validURL(url) {
if (!VALID_PROTOCOLS.includes(destination.protocol)) return false;
if (isInvalidIp(destination)) return false;
return true;
} catch { }
} catch {}
return false;
}