kamp/anything-llm
1
0
Fork 0
mirror of https://github.com/Mintplex-Labs/anything-llm.git synced 2024-11-10 17:00:11 +01:00
Code Issues Packages Projects Releases Wiki Activity

Add support for searXNG search for agents (#1733)

Browse Source 
resolves #1367
This commit is contained in:
Timothy Carambat 2024-06-20 14:08:00 -07:00 committed by GitHub
parent 524edd6e69
commit 8b1ceb30c1
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
9 changed files with 132 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.vscode/settings.json vendored
View File

@ -32,7 +32,9 @@
"opendocument",
"openrouter",
"Qdrant",
"searxng",
"Serper",
"Serply",
"textgenwebui",
"togetherai",
"vectordbs",

3
docker/.env.example
View File

@ -245,3 +245,6 @@ GID='1000'
#------ Serply.io ----------- https://serply.io/
# AGENT_SERPLY_API_KEY=
#------ SearXNG ----------- https://github.com/searxng/searxng
# AGENT_SEARXNG_API_URL=

22
frontend/src/pages/Admin/Agents/WebSearchSelection/SearchProviderOptions/index.jsx
View File

@ -182,3 +182,25 @@ export function SerplySearchOptions({ settings }) {
</>
);
}
export function SearXNGOptions({ settings }) {
return (
<div className="flex gap-x-4">
<div className="flex flex-col w-60">
<label className="text-white text-sm font-semibold block mb-4">
SearXNG API base URL
</label>
<input
type="url"
name="env::AgentSearXNGApiUrl"
className="border-none bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:border-white block w-full p-2.5"
placeholder="SearXNG API Key"
defaultValue={settings?.AgentSearXNGApiUrl}
required={true}
autoComplete="off"
spellCheck={false}
/>
</div>
</div>
);
}

BIN
frontend/src/pages/Admin/Agents/WebSearchSelection/icons/searxng.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 7.5 KiB

10
frontend/src/pages/Admin/Agents/WebSearchSelection/index.jsx
View File

@ -4,6 +4,7 @@ import GoogleSearchIcon from "./icons/google.png";
import SerperDotDevIcon from "./icons/serper.png";
import BingSearchIcon from "./icons/bing.png";
import SerplySearchIcon from "./icons/serply.png";
import SearXNGSearchIcon from "./icons/searxng.png";
import {
CaretUpDown,
MagnifyingGlass,
@ -17,6 +18,7 @@ import {
GoogleSearchOptions,
BingSearchOptions,
SerplySearchOptions,
SearXNGOptions,
} from "./SearchProviderOptions";
const SEARCH_PROVIDERS = [
@ -60,6 +62,14 @@ const SEARCH_PROVIDERS = [
description:
"Serply.io web-search. Free account with a 100 calls/month forever.",
},
{
name: "SearXNG",
value: "searxng-engine",
logo: SearXNGSearchIcon,
options: (settings) => <SearXNGOptions settings={settings} />,
description:
"Free, open-source, internet meta-search engine with no tracking.",
},
];
export default function AgentWebSearchSelection({

3
server/.env.example
View File

@ -241,3 +241,6 @@ TTS_PROVIDER="native"
#------ Serply.io ----------- https://serply.io/
# AGENT_SERPLY_API_KEY=
#------ SearXNG ----------- https://github.com/searxng/searxng
# AGENT_SEARXNG_API_URL=

10
server/models/systemSettings.js
View File

@ -76,6 +76,7 @@ const SystemSettings = {
"serper-dot-dev",
"bing-search",
"serply-engine",
"searxng-engine",
].includes(update)
)
throw new Error("Invalid SERP provider.");
@ -176,10 +177,11 @@ const SystemSettings = {
// Agent Settings & Configs
// --------------------------------------------------------
AgentGoogleSearchEngineId: process.env.AGENT_GSE_CTX || null,
AgentGoogleSearchEngineKey: process.env.AGENT_GSE_KEY || null,
AgentSerperApiKey: process.env.AGENT_SERPER_DEV_KEY || null,
AgentBingSearchApiKey: process.env.AGENT_BING_SEARCH_API_KEY || null,
AgentSerplyApiKey: process.env.AGENT_SERPLY_API_KEY || null,
AgentGoogleSearchEngineKey: !!process.env.AGENT_GSE_KEY || null,
AgentSerperApiKey: !!process.env.AGENT_SERPER_DEV_KEY || null,
AgentBingSearchApiKey: !!process.env.AGENT_BING_SEARCH_API_KEY || null,
AgentSerplyApiKey: !!process.env.AGENT_SERPLY_API_KEY || null,
AgentSearXNGApiUrl: process.env.AGENT_SEARXNG_API_URL || null,
};
},

85
server/utils/agents/aibitat/plugins/web-browsing.js
View File

@ -71,6 +71,9 @@ const webBrowsing = {
case "serply-engine":
engine = "_serplyEngine";
break;
case "searxng-engine":
engine = "_searXNGEngine";
break;
default:
engine = "_googleSearchEngine";
}
@ -102,7 +105,7 @@ const webBrowsing = {
query.length > 100 ? `${query.slice(0, 100)}...` : query
}"`
);
const searchResponse = await fetch(searchURL)
const data = await fetch(searchURL)
.then((res) => res.json())
.then((searchResult) => searchResult?.items || [])
.then((items) => {
@ -116,10 +119,15 @@ const webBrowsing = {
})
.catch((e) => {
console.log(e);
return {};
return [];
});
return JSON.stringify(searchResponse);
if (data.length === 0)
return `No information was found online for the search query.`;
this.super.introspect(
`${this.caller}: I found ${data.length} results - looking over them now.`
);
return JSON.stringify(data);
},
/**
@ -176,6 +184,9 @@ const webBrowsing = {
if (data.length === 0)
return `No information was found online for the search query.`;
this.super.introspect(
`${this.caller}: I found ${data.length} results - looking over them now.`
);
return JSON.stringify(data);
},
_bingWebSearch: async function (query) {
@ -219,6 +230,9 @@ const webBrowsing = {
if (searchResponse.length === 0)
return `No information was found online for the search query.`;
this.super.introspect(
`${this.caller}: I found ${data.length} results - looking over them now.`
);
return JSON.stringify(searchResponse);
},
_serplyEngine: async function (
@ -293,6 +307,71 @@ const webBrowsing = {
if (data.length === 0)
return `No information was found online for the search query.`;
this.super.introspect(
`${this.caller}: I found ${data.length} results - looking over them now.`
);
return JSON.stringify(data);
},
_searXNGEngine: async function (query) {
let searchURL;
if (!process.env.AGENT_SEARXNG_API_URL) {
this.super.introspect(
`${this.caller}: I can't use SearXNG searching because the user has not defined the required base URL.\nPlease set this value in the agent skill settings.`
);
return `Search is disabled and no content was found. This functionality is disabled because the user has not set it up yet.`;
}
try {
searchURL = new URL(process.env.AGENT_SEARXNG_API_URL);
searchURL.searchParams.append("q", encodeURIComponent(query));
searchURL.searchParams.append("format", "json");
} catch (e) {
this.super.handlerProps.log(`SearXNG Search: ${e.message}`);
this.super.introspect(
`${this.caller}: I can't use SearXNG searching because the url provided is not a valid URL.`
);
return `Search is disabled and no content was found. This functionality is disabled because the user has not set it up yet.`;
}
this.super.introspect(
`${this.caller}: Using SearXNG to search for "${
query.length > 100 ? `${query.slice(0, 100)}...` : query
}"`
);
const { response, error } = await fetch(searchURL.toString(), {
method: "GET",
headers: {
"Content-Type": "application/json",
"User-Agent": "anything-llm",
},
})
.then((res) => res.json())
.then((data) => {
return { response: data, error: null };
})
.catch((e) => {
return { response: null, error: e.message };
});
if (error)
return `There was an error searching for content. ${error}`;
const data = [];
response.results?.forEach((searchResult) => {
const { url, title, content, publishedDate } = searchResult;
data.push({
title,
link: url,
snippet: content,
publishedDate,
});
});
if (data.length === 0)
return `No information was found online for the search query.`;
this.super.introspect(
`${this.caller}: I found ${data.length} results - looking over them now.`
);
return JSON.stringify(data);
},
});

4
server/utils/helpers/updateENV.js
View File

@ -407,6 +407,10 @@ const KEY_MAPPING = {
envKey: "AGENT_SERPLY_API_KEY",
checks: [],
},
AgentSearXNGApiUrl: {
envKey: "AGENT_SEARXNG_API_URL",
checks: [],
},
// TTS/STT Integration ENVS
TextToSpeechProvider: {