diff --git a/.vscode/settings.json b/.vscode/settings.json index 4930aa2d..8d924b71 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -32,7 +32,9 @@ "opendocument", "openrouter", "Qdrant", + "searxng", "Serper", + "Serply", "textgenwebui", "togetherai", "vectordbs", diff --git a/docker/.env.example b/docker/.env.example index a38b4c5a..71572cc8 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -245,3 +245,6 @@ GID='1000' #------ Serply.io ----------- https://serply.io/ # AGENT_SERPLY_API_KEY= + +#------ SearXNG ----------- https://github.com/searxng/searxng +# AGENT_SEARXNG_API_URL= \ No newline at end of file diff --git a/frontend/src/pages/Admin/Agents/WebSearchSelection/SearchProviderOptions/index.jsx b/frontend/src/pages/Admin/Agents/WebSearchSelection/SearchProviderOptions/index.jsx index 58ceb844..c5ccd260 100644 --- a/frontend/src/pages/Admin/Agents/WebSearchSelection/SearchProviderOptions/index.jsx +++ b/frontend/src/pages/Admin/Agents/WebSearchSelection/SearchProviderOptions/index.jsx @@ -182,3 +182,25 @@ export function SerplySearchOptions({ settings }) { ); } + +export function SearXNGOptions({ settings }) { + return ( +
+
+ + +
+
+ ); +} diff --git a/frontend/src/pages/Admin/Agents/WebSearchSelection/icons/searxng.png b/frontend/src/pages/Admin/Agents/WebSearchSelection/icons/searxng.png new file mode 100644 index 00000000..434e570f Binary files /dev/null and b/frontend/src/pages/Admin/Agents/WebSearchSelection/icons/searxng.png differ diff --git a/frontend/src/pages/Admin/Agents/WebSearchSelection/index.jsx b/frontend/src/pages/Admin/Agents/WebSearchSelection/index.jsx index 9650c38f..438be111 100644 --- a/frontend/src/pages/Admin/Agents/WebSearchSelection/index.jsx +++ b/frontend/src/pages/Admin/Agents/WebSearchSelection/index.jsx @@ -4,6 +4,7 @@ import GoogleSearchIcon from "./icons/google.png"; import SerperDotDevIcon from "./icons/serper.png"; import BingSearchIcon from "./icons/bing.png"; import SerplySearchIcon from "./icons/serply.png"; +import SearXNGSearchIcon from "./icons/searxng.png"; import { CaretUpDown, MagnifyingGlass, @@ -17,6 +18,7 @@ import { GoogleSearchOptions, BingSearchOptions, SerplySearchOptions, + SearXNGOptions, } from "./SearchProviderOptions"; const SEARCH_PROVIDERS = [ @@ -60,6 +62,14 @@ const SEARCH_PROVIDERS = [ description: "Serply.io web-search. Free account with a 100 calls/month forever.", }, + { + name: "SearXNG", + value: "searxng-engine", + logo: SearXNGSearchIcon, + options: (settings) => , + description: + "Free, open-source, internet meta-search engine with no tracking.", + }, ]; export default function AgentWebSearchSelection({ diff --git a/server/.env.example b/server/.env.example index a88a8a03..3a4fb072 100644 --- a/server/.env.example +++ b/server/.env.example @@ -241,3 +241,6 @@ TTS_PROVIDER="native" #------ Serply.io ----------- https://serply.io/ # AGENT_SERPLY_API_KEY= + +#------ SearXNG ----------- https://github.com/searxng/searxng +# AGENT_SEARXNG_API_URL= \ No newline at end of file diff --git a/server/models/systemSettings.js b/server/models/systemSettings.js index 8d548c7b..4d998e81 100644 --- a/server/models/systemSettings.js +++ b/server/models/systemSettings.js @@ -76,6 +76,7 @@ const SystemSettings = { "serper-dot-dev", "bing-search", "serply-engine", + "searxng-engine", ].includes(update) ) throw new Error("Invalid SERP provider."); @@ -176,10 +177,11 @@ const SystemSettings = { // Agent Settings & Configs // -------------------------------------------------------- AgentGoogleSearchEngineId: process.env.AGENT_GSE_CTX || null, - AgentGoogleSearchEngineKey: process.env.AGENT_GSE_KEY || null, - AgentSerperApiKey: process.env.AGENT_SERPER_DEV_KEY || null, - AgentBingSearchApiKey: process.env.AGENT_BING_SEARCH_API_KEY || null, - AgentSerplyApiKey: process.env.AGENT_SERPLY_API_KEY || null, + AgentGoogleSearchEngineKey: !!process.env.AGENT_GSE_KEY || null, + AgentSerperApiKey: !!process.env.AGENT_SERPER_DEV_KEY || null, + AgentBingSearchApiKey: !!process.env.AGENT_BING_SEARCH_API_KEY || null, + AgentSerplyApiKey: !!process.env.AGENT_SERPLY_API_KEY || null, + AgentSearXNGApiUrl: process.env.AGENT_SEARXNG_API_URL || null, }; }, diff --git a/server/utils/agents/aibitat/plugins/web-browsing.js b/server/utils/agents/aibitat/plugins/web-browsing.js index 81314f17..f4269fe1 100644 --- a/server/utils/agents/aibitat/plugins/web-browsing.js +++ b/server/utils/agents/aibitat/plugins/web-browsing.js @@ -71,6 +71,9 @@ const webBrowsing = { case "serply-engine": engine = "_serplyEngine"; break; + case "searxng-engine": + engine = "_searXNGEngine"; + break; default: engine = "_googleSearchEngine"; } @@ -102,7 +105,7 @@ const webBrowsing = { query.length > 100 ? `${query.slice(0, 100)}...` : query }"` ); - const searchResponse = await fetch(searchURL) + const data = await fetch(searchURL) .then((res) => res.json()) .then((searchResult) => searchResult?.items || []) .then((items) => { @@ -116,10 +119,15 @@ const webBrowsing = { }) .catch((e) => { console.log(e); - return {}; + return []; }); - return JSON.stringify(searchResponse); + if (data.length === 0) + return `No information was found online for the search query.`; + this.super.introspect( + `${this.caller}: I found ${data.length} results - looking over them now.` + ); + return JSON.stringify(data); }, /** @@ -176,6 +184,9 @@ const webBrowsing = { if (data.length === 0) return `No information was found online for the search query.`; + this.super.introspect( + `${this.caller}: I found ${data.length} results - looking over them now.` + ); return JSON.stringify(data); }, _bingWebSearch: async function (query) { @@ -219,6 +230,9 @@ const webBrowsing = { if (searchResponse.length === 0) return `No information was found online for the search query.`; + this.super.introspect( + `${this.caller}: I found ${data.length} results - looking over them now.` + ); return JSON.stringify(searchResponse); }, _serplyEngine: async function ( @@ -293,6 +307,71 @@ const webBrowsing = { if (data.length === 0) return `No information was found online for the search query.`; + this.super.introspect( + `${this.caller}: I found ${data.length} results - looking over them now.` + ); + return JSON.stringify(data); + }, + _searXNGEngine: async function (query) { + let searchURL; + if (!process.env.AGENT_SEARXNG_API_URL) { + this.super.introspect( + `${this.caller}: I can't use SearXNG searching because the user has not defined the required base URL.\nPlease set this value in the agent skill settings.` + ); + return `Search is disabled and no content was found. This functionality is disabled because the user has not set it up yet.`; + } + + try { + searchURL = new URL(process.env.AGENT_SEARXNG_API_URL); + searchURL.searchParams.append("q", encodeURIComponent(query)); + searchURL.searchParams.append("format", "json"); + } catch (e) { + this.super.handlerProps.log(`SearXNG Search: ${e.message}`); + this.super.introspect( + `${this.caller}: I can't use SearXNG searching because the url provided is not a valid URL.` + ); + return `Search is disabled and no content was found. This functionality is disabled because the user has not set it up yet.`; + } + + this.super.introspect( + `${this.caller}: Using SearXNG to search for "${ + query.length > 100 ? `${query.slice(0, 100)}...` : query + }"` + ); + + const { response, error } = await fetch(searchURL.toString(), { + method: "GET", + headers: { + "Content-Type": "application/json", + "User-Agent": "anything-llm", + }, + }) + .then((res) => res.json()) + .then((data) => { + return { response: data, error: null }; + }) + .catch((e) => { + return { response: null, error: e.message }; + }); + if (error) + return `There was an error searching for content. ${error}`; + + const data = []; + response.results?.forEach((searchResult) => { + const { url, title, content, publishedDate } = searchResult; + data.push({ + title, + link: url, + snippet: content, + publishedDate, + }); + }); + + if (data.length === 0) + return `No information was found online for the search query.`; + this.super.introspect( + `${this.caller}: I found ${data.length} results - looking over them now.` + ); return JSON.stringify(data); }, }); diff --git a/server/utils/helpers/updateENV.js b/server/utils/helpers/updateENV.js index 51364191..6b170da3 100644 --- a/server/utils/helpers/updateENV.js +++ b/server/utils/helpers/updateENV.js @@ -407,6 +407,10 @@ const KEY_MAPPING = { envKey: "AGENT_SERPLY_API_KEY", checks: [], }, + AgentSearXNGApiUrl: { + envKey: "AGENT_SEARXNG_API_URL", + checks: [], + }, // TTS/STT Integration ENVS TextToSpeechProvider: {