From 8b1ceb30c159cf3a10efa16275bc6849d84e4ea8 Mon Sep 17 00:00:00 2001 From: Timothy Carambat Date: Thu, 20 Jun 2024 14:08:00 -0700 Subject: [PATCH] Add support for searXNG search for agents (#1733) resolves #1367 --- .vscode/settings.json | 2 + docker/.env.example | 3 + .../SearchProviderOptions/index.jsx | 22 +++++ .../WebSearchSelection/icons/searxng.png | Bin 0 -> 7722 bytes .../Admin/Agents/WebSearchSelection/index.jsx | 10 +++ server/.env.example | 3 + server/models/systemSettings.js | 10 ++- .../agents/aibitat/plugins/web-browsing.js | 85 +++++++++++++++++- server/utils/helpers/updateENV.js | 4 + 9 files changed, 132 insertions(+), 7 deletions(-) create mode 100644 frontend/src/pages/Admin/Agents/WebSearchSelection/icons/searxng.png diff --git a/.vscode/settings.json b/.vscode/settings.json index 4930aa2d..8d924b71 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -32,7 +32,9 @@ "opendocument", "openrouter", "Qdrant", + "searxng", "Serper", + "Serply", "textgenwebui", "togetherai", "vectordbs", diff --git a/docker/.env.example b/docker/.env.example index a38b4c5a..71572cc8 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -245,3 +245,6 @@ GID='1000' #------ Serply.io ----------- https://serply.io/ # AGENT_SERPLY_API_KEY= + +#------ SearXNG ----------- https://github.com/searxng/searxng +# AGENT_SEARXNG_API_URL= \ No newline at end of file diff --git a/frontend/src/pages/Admin/Agents/WebSearchSelection/SearchProviderOptions/index.jsx b/frontend/src/pages/Admin/Agents/WebSearchSelection/SearchProviderOptions/index.jsx index 58ceb844..c5ccd260 100644 --- a/frontend/src/pages/Admin/Agents/WebSearchSelection/SearchProviderOptions/index.jsx +++ b/frontend/src/pages/Admin/Agents/WebSearchSelection/SearchProviderOptions/index.jsx @@ -182,3 +182,25 @@ export function SerplySearchOptions({ settings }) { ); } + +export function SearXNGOptions({ settings }) { + return ( +
+
+ + +
+
+ ); +} diff --git a/frontend/src/pages/Admin/Agents/WebSearchSelection/icons/searxng.png b/frontend/src/pages/Admin/Agents/WebSearchSelection/icons/searxng.png new file mode 100644 index 0000000000000000000000000000000000000000..434e570f8bfe0f8752b186a8fac772e73c40fe18 GIT binary patch literal 7722 zcmd5>YkCtF1yx#7KmNg+;2S3f9NMdc^SGM}Up75bTAlU}3SK z)xaP_zuY|pp&#krd;FtA7>vsy1NzIJls9%+{H(?JwP01v`EJdR_NUFMbOKMET&tTc z?B2uAee{-#1N7$1Z&sI;C_!|TgiD^sq1)TWX;Plr+ma_!6Ia2ki6aInYq!}OIZLbl z))(vRcEZD{?$ih@d&gdoT^tsEj0iR}$1s5#l!5*Ki{X!8`y;W@@u(UGTwHGcuntlbZhp)pZ1*g?p{=XX{j6i+`B zkb(WUtBjOus`P(lW|_u9AW9d7D)fRIKizZX69>m<4m0$*A+Hg-?9ATpi$es!^C~6n zxqJf~Y6GNQEHv%>0MPv&C!d(jO|ZalrwvNgcLJn!0XpR!MZaG@?L|rfkH5 z)U`%#%S_M2h=`P0)<0ocYqETn&{DeS+w-Vl%j=1lHny8w2wZ(P`y7H>Q}XkHvdLBO zE^7x`jpO{F7u3d><1_H{p@Y&0y|-4HB1azbp-60-e8%;w@QucvJbyd6G_uH#gE3Jy zf)QE#+BQ_!6FSAUJ}~slUDpe+TW1b$WVhN${K*Ed5pKleVW&Rn9&YfiTLQNfivnC_ z7xAah*HzlXvIJBma=d7+ZK)eR#(q>%G~(N+0XZ#ddI1C{WI+u}bzY=&!H}l+#jhz# zRe-C0te-=QZ`{CiYfH)LB_yQR)K~V~jKz8AdO3al!H9Fhp zWy$XB!Z_69SW5SpHFLuYLMRbY_h7@Jn-blQc9gtDT4%5kCY|$M zg@A#AsNKrx3ea)`{)Xcp#{`tTmyj%t*hmo>AluM!l#SC$d5yuICzITX^LHdiqX2fPSgT#C@>o@0V8Uoo7?|JbKTjD-68tD77liwKB8g@D^&qyZW0 z!Hke3aqI{E?M6>`IgRVIAli&%cM&t8x5jCT!lBQqqGApPyhtcY!(PP&={Us7{?@DI z%9a491k3=cj0DGTIm{%JQsWH%y6usW@vK;dbR+$kFB*3{_%nRh)ooS3sy>3hWRlRCJg zC@hh#BPA_6VcNuQNqTd;vb*+RU*wI{@3;$~p05{Usp5!;lPr-@$nNeTdEd!UG%9Nx}5ynM!@gByRVd=%nIt!&u^Bm!b0d}nam zse$MM#t-(Zx&6(Gq4~K13hvYQHkf0>{wLp;6MmXsoa{TQ5JWvKTYg|g#xk<(vU|{f z>g5YIZ=|*VT5a^(u5ZE}_0yu}dRMI@@}7PE?&^5Jg7s9_=bA`gn>C*`#&cJ5m<1K} z>!}}fUHn6imh?ksa^!fW>h#Nx-Cmq2IZ^9&eu3&H7F(TqJPc`y6m-~Y$2fuf{(1NN zPo%uU%9jz+&3#SqceIjn!MM(2vi#Ox)wd+HMEn`TZpW>jin9bJ{F!@^d&@FHmCJBU zP4lUJ{-K3UA2M8FR<#=dDuSAj?MmCID*Al#XQjBga+nPs;K+YaE=MeQ{H?;vC2dF)SphYo{a#|MTq~h`nxRw#2VAHUd^bNM*PaHTI! z_+~^4i1U$F;%x9lqsCj))s}Rg04t*i6?0pCFlP$;+E?b?-R;O)1&;&2gLRz550}>} z>oi~RY_BFn%PTg2%%3D0oEu1}Xd`!$E`!Rve(Xe}(r3PqBF#f2nB>hhO{Q?8NoFRQ z`L7ciY)*zRLlS1{C+^yJ*5g|`v4fg$YIWRCn|oNv*PGvRGyU>r2*l0XVoVxk3%+9) zMQImTMIlQh%M@c@S>^lH1GF^8T-FlbDkCTSqa;Gm?&}>cXls3AG0CJw)x@qah?frS z*LdZx62LMK4@$Y-qSv-Yui#BZsmS9kia*5FM%QpR0qPMdH)2RTm!zl3W|kxoEQ+KZSGF^f4jkb zSPC1jd{g;F-7U)S?LEF30+Z*i_J{A(Tm#Mr9#SZ%XD*quU94+L5~ z)YQ}*{d%ii(ocN|v?^svtyRpWcNe@)mns?)^7_6_*WW~YTGQTPm3Q%k8d~%`E>X~( zK@?O)*v(tyFM=XJx$5|Q_X<#aCWSRnP{QNg!WO#2t6D~rrkO76`fNMS_v80rfk2TL zRw0doE-*pH6#cplw*R71lzOj>rdKkfbCD+ScYzA&lY`XcGH5C9t?B42{!k6KQ(^+c zqx<>&?zNW&BZD|NJ1YfFyT47y$rV%$)cFCfasGs2Yd-6i%Pu$QDa|a%#MslzJ-JK0 z+rp&=`sbm~E9~ve@ptx=nB^1=-fh05o$hE_EP5dy3V;i+AVN-9gWAj;bE?urP=s4G zMHK`kU(wly=T0o8XW&M^miLhrc)+sL5@jjOF5jb2soi_~z!a15=0_1D+6$RHd!b|z zy8tq8$t$0Ff9Jf5(yi6@b1hYLE&97{zs{WXxdyXvKRYoT1QbtCr2l|FMJZNIfB`rmco%Q6{iqNtOd7+)HTrE=9BT(IA>oWHhg5y<&R_E8n`Ke z)|`!?M5i#vj}9BDg=Xnq+8u037GHfnZt(u~Zn;byK!J{PT$b2RGe4qPy) zm)SxSsA+b7rx!*eW#^~_7{@=@^Q8kQ7r(5h57?6FI|GUI1MmdPxCN-J2=&gKuSIe* zA04lJTjD)&OW&-iVSi2-o0+H#lHU4t%5qK`O1B`|mG=(&VqWn?S#-rhEv0RXLT|F@ zX6M{eTZs;CLE~7g2c6>Xenm?5@_B}AZO5CJI#(_dd(fqq=QMCiLK+1+tzlu6(h9s( zYBol_D7xbT9beY;rQW5sAr0^Ba4-7!FfS{#SCtm`fz*@i9Hw?(=+bsj@ zGmfv054=W6q>{{5a|k6T%2JY{e@{$&Z05=#Zky%fv>hdT12p(**P2STL*iTLqOc3~JDj zm}9E14I$nlDoc8BZoqo-@?3%6D0>^J@3c>vI5hd`_TB%LbIkRrbVaq3e!O$ z|7f~r1@?W*HdN_da--e}-6PlX;!FK|p;qDNkd;@xRnL=!>sJZT#3w8p5x%SXS0!S_7S zyJnI`PiX07bm{FI?-MwRpeD#Ye}fR1$B?~8)M+dUeDCBMp7K}sn_n~qOkabmL=T!e zZ86~4@^-bi93i>_7KgTxzHCVSe5^AY1SCp6hBbxAXq~98wi>&pf2Z9~$>S>EP?Z&5 zMvzdKb_NAq%V~)z8=6Lkjtg)aH(-?qp)VOO}u!>X1cUCtGNBSc6>%qmGob4fn90IUh;rov64N%KfTzn z>RNIS-d=OC6; z6Q2JWGz&w(tRma^weZnD%1n)JlqHOhK|2!0l2`udhr7p<-BPL-#m>-W$r7{lTgW{3 zdJ8YJ4=1exjC95meCC$**YLaMEF13C?6<1C)VSsqg5B0=9g{*`v+TSJtzd4vTH1iAK#T z|JZOZi%NdSo8wPjVViY+%(di`u$$)!lfI`FwiO1kP*Wf61|JFny=Q9ek&?hNP3LJ! z&0H<9#i0@wy&~EF8c>v`YdMr-d;gvhm-XG06Sz+QHWw~1s-BhhiOmf0Gauy+4tcah zO_lP(BiS+f9n9Y{>Croq_nh{=ieYrfN zc>@|~%Zg8WD9v3ceyd_wt!X^@8?24|T^WB1o=+GbANglce__%PHmq+NEh!J3@~HG{ zwf{-&Ft4d|WPuWy);I3(VG)Xdp1HE@77Gz^C-Y$`{tOgiW z%Kr15xO$Z~wr$U2EdRCL?(3B8Pnprw*Y>}7=!l0qBSH{7?|0)TBVnFK1%q}kb#|gy z8mAeVnE2Z9aq!tj0XY{G8HgVMTwJFmFj3mo=CX&k18xtE{lFkg*g;<1kx`X)G)#QD!An3( zhg|W_>@lqIwEV&dW(&c_z5`zrkq40I!x%ZH|FwrFo$buas7U`K%t|6DXIpoW!|jzURxhs@ph@8(%;!; z((@@?weIXls&f82>5`zt{ld8aJ%Ay=x6hu1lDQDh>IxI;-Ikk)`MtV!>MgJtJw{v4 zHZ{LyG2;!z+97%S+v^u_4~}4Q$tivxxN}??me_v7%iG^wa_KB1anOIp?|o5L1${Y@ z7ri_WzId9j8jfQ#gr}MPc)!D|r8q#Uc(=sT^EV)dEwo{bKE?eH_xWus3%*wXdl3ot zov<(V?rc}3Gd~^y)h9TBfOXrxzA)HR_Da}XB`x?(_MXLuCgj1--iZgb|iVF~GAbQPLl2sfq~&t;6p zFj97t#+_mi8}E4u9CvOdDwp{+@8AtdfQ>3Y}9UBC<&U99S6!!ajIpXy^QH z(!KD^N5wh1@lV1GMV(yWzL$PaCW%%xIqv{{74#D?WU|%54A(CpmmfH54~yT2{)=<3 z&9tS2&L1GP!sOi8`8vfw7<45)nz<@{j@ z!h-M4587mEeG1YK$yP4*e67^@+H$)8kJ_+II4SwiZA`_x3Fj^;)zBm#QW-Mj5h3%7 z&ku2^)qmo8|Loz5}+oecS93t=Z$2Y zNO?Y5{LWY7T(A0IylbqImp$_ZRI*&bsy(}p@HnnRy$<@(Z?i!jep}|**)EQuemTgJ zsT(|@sbLxzDnG@}2Sb{Avok44(9k`reJLGghQMqd9UVXad)I^_Eb9oIXHpcjk16Sj z&v-36$m|C@^eadPd$70TFw{MUdd9*qox>Kc78pav)ZbEh|K>TGbjORPy9zq?)oD$a3RP4BvNXhU%^4}d&q$rYmS6JtO2 z`)hQMY$GsUtb*mX^d)PC?YFlA#(Z2o2?s3T3tM`8*j7*{iDx$rA|mDY@1yf-6vE64 za4`(>P{J@?&JeI9k^_QUVm)Xe2flD8XXc=94q_>vl3b0&5qT_Jmx$t0$Aq&~_&9tS z+UH1(h)The`@DD!`M55~%#rcK_Bq$YE1&Nu`CMS(b7c}5#Gti1)WTAkYgl_^N)eOu z%mzYun-g9AbFCj_H!9SNa{kqz;swwS_{PpfB6y=-+9ai_sc#z5^41370}qy}^n5)y zgt9XiBBQz~c*eB8Uhh$YL8ti><*QQIj7pPup}MD|Bq1w8@|IfZ0$EBVZz@NA-~BA` z=j0IM?4xR?^yya)5ttrc>`W;;UpQw^>F}XcYc{Zl1u0WH}wL9&X);rk-_1YY#2~7 z*!W2DVg04d4tc+l!YAhvZ^;e{{FoQ-)@q}(VwsB+hhD4L4x7pe>I{SIw55Wm6gQyW z1{13}%FjYNwduJw*hKy^>-4@)fhJX+g)q$z-I}H@{!4aX=Gbm(jPdiAodH@nH(F`& zXL9%f^MV7~(S0P(iW%*q-nGa;a1A2=Nn-|Wth}F%O#2F4UaSk%74$2iBBSCf$tVC% zV9j3~zYCRJ(Kzek9s$|Ck*p(MWyFEJJ1h7XpOLRyBhAyj)SGJFMwzl6^Yx7AbYP`+ z{lVN|!H^a|hSIcy)O8xHN2w*=b8N~G6(G$j6eua*Y>(6K+uH;`DrER!(9P`FV_!$( z4rR!SGPfHxy)@A^99*Fm8$rrjCP=m<>g+R=?OAAMej$|^KW@%v)u&kjmK~};4g%Yy z26duWRZpXRu89#fmw%a*wJ7mp;Em>#^x@r>d6!g@l7n4>dCZLT1B zKJ%c(1m{ghvdCfNANRR$VXkj?-ir!sVcN>P@HhQIH!WM0@#dTWlRLZ3_YFS#u>Vg= k=KoPtj~*UjVLdz`FUFcg*kXb(8Zefck~aAJE1R(Y0TNBj@Bjb+ literal 0 HcmV?d00001 diff --git a/frontend/src/pages/Admin/Agents/WebSearchSelection/index.jsx b/frontend/src/pages/Admin/Agents/WebSearchSelection/index.jsx index 9650c38f..438be111 100644 --- a/frontend/src/pages/Admin/Agents/WebSearchSelection/index.jsx +++ b/frontend/src/pages/Admin/Agents/WebSearchSelection/index.jsx @@ -4,6 +4,7 @@ import GoogleSearchIcon from "./icons/google.png"; import SerperDotDevIcon from "./icons/serper.png"; import BingSearchIcon from "./icons/bing.png"; import SerplySearchIcon from "./icons/serply.png"; +import SearXNGSearchIcon from "./icons/searxng.png"; import { CaretUpDown, MagnifyingGlass, @@ -17,6 +18,7 @@ import { GoogleSearchOptions, BingSearchOptions, SerplySearchOptions, + SearXNGOptions, } from "./SearchProviderOptions"; const SEARCH_PROVIDERS = [ @@ -60,6 +62,14 @@ const SEARCH_PROVIDERS = [ description: "Serply.io web-search. Free account with a 100 calls/month forever.", }, + { + name: "SearXNG", + value: "searxng-engine", + logo: SearXNGSearchIcon, + options: (settings) => , + description: + "Free, open-source, internet meta-search engine with no tracking.", + }, ]; export default function AgentWebSearchSelection({ diff --git a/server/.env.example b/server/.env.example index a88a8a03..3a4fb072 100644 --- a/server/.env.example +++ b/server/.env.example @@ -241,3 +241,6 @@ TTS_PROVIDER="native" #------ Serply.io ----------- https://serply.io/ # AGENT_SERPLY_API_KEY= + +#------ SearXNG ----------- https://github.com/searxng/searxng +# AGENT_SEARXNG_API_URL= \ No newline at end of file diff --git a/server/models/systemSettings.js b/server/models/systemSettings.js index 8d548c7b..4d998e81 100644 --- a/server/models/systemSettings.js +++ b/server/models/systemSettings.js @@ -76,6 +76,7 @@ const SystemSettings = { "serper-dot-dev", "bing-search", "serply-engine", + "searxng-engine", ].includes(update) ) throw new Error("Invalid SERP provider."); @@ -176,10 +177,11 @@ const SystemSettings = { // Agent Settings & Configs // -------------------------------------------------------- AgentGoogleSearchEngineId: process.env.AGENT_GSE_CTX || null, - AgentGoogleSearchEngineKey: process.env.AGENT_GSE_KEY || null, - AgentSerperApiKey: process.env.AGENT_SERPER_DEV_KEY || null, - AgentBingSearchApiKey: process.env.AGENT_BING_SEARCH_API_KEY || null, - AgentSerplyApiKey: process.env.AGENT_SERPLY_API_KEY || null, + AgentGoogleSearchEngineKey: !!process.env.AGENT_GSE_KEY || null, + AgentSerperApiKey: !!process.env.AGENT_SERPER_DEV_KEY || null, + AgentBingSearchApiKey: !!process.env.AGENT_BING_SEARCH_API_KEY || null, + AgentSerplyApiKey: !!process.env.AGENT_SERPLY_API_KEY || null, + AgentSearXNGApiUrl: process.env.AGENT_SEARXNG_API_URL || null, }; }, diff --git a/server/utils/agents/aibitat/plugins/web-browsing.js b/server/utils/agents/aibitat/plugins/web-browsing.js index 81314f17..f4269fe1 100644 --- a/server/utils/agents/aibitat/plugins/web-browsing.js +++ b/server/utils/agents/aibitat/plugins/web-browsing.js @@ -71,6 +71,9 @@ const webBrowsing = { case "serply-engine": engine = "_serplyEngine"; break; + case "searxng-engine": + engine = "_searXNGEngine"; + break; default: engine = "_googleSearchEngine"; } @@ -102,7 +105,7 @@ const webBrowsing = { query.length > 100 ? `${query.slice(0, 100)}...` : query }"` ); - const searchResponse = await fetch(searchURL) + const data = await fetch(searchURL) .then((res) => res.json()) .then((searchResult) => searchResult?.items || []) .then((items) => { @@ -116,10 +119,15 @@ const webBrowsing = { }) .catch((e) => { console.log(e); - return {}; + return []; }); - return JSON.stringify(searchResponse); + if (data.length === 0) + return `No information was found online for the search query.`; + this.super.introspect( + `${this.caller}: I found ${data.length} results - looking over them now.` + ); + return JSON.stringify(data); }, /** @@ -176,6 +184,9 @@ const webBrowsing = { if (data.length === 0) return `No information was found online for the search query.`; + this.super.introspect( + `${this.caller}: I found ${data.length} results - looking over them now.` + ); return JSON.stringify(data); }, _bingWebSearch: async function (query) { @@ -219,6 +230,9 @@ const webBrowsing = { if (searchResponse.length === 0) return `No information was found online for the search query.`; + this.super.introspect( + `${this.caller}: I found ${data.length} results - looking over them now.` + ); return JSON.stringify(searchResponse); }, _serplyEngine: async function ( @@ -293,6 +307,71 @@ const webBrowsing = { if (data.length === 0) return `No information was found online for the search query.`; + this.super.introspect( + `${this.caller}: I found ${data.length} results - looking over them now.` + ); + return JSON.stringify(data); + }, + _searXNGEngine: async function (query) { + let searchURL; + if (!process.env.AGENT_SEARXNG_API_URL) { + this.super.introspect( + `${this.caller}: I can't use SearXNG searching because the user has not defined the required base URL.\nPlease set this value in the agent skill settings.` + ); + return `Search is disabled and no content was found. This functionality is disabled because the user has not set it up yet.`; + } + + try { + searchURL = new URL(process.env.AGENT_SEARXNG_API_URL); + searchURL.searchParams.append("q", encodeURIComponent(query)); + searchURL.searchParams.append("format", "json"); + } catch (e) { + this.super.handlerProps.log(`SearXNG Search: ${e.message}`); + this.super.introspect( + `${this.caller}: I can't use SearXNG searching because the url provided is not a valid URL.` + ); + return `Search is disabled and no content was found. This functionality is disabled because the user has not set it up yet.`; + } + + this.super.introspect( + `${this.caller}: Using SearXNG to search for "${ + query.length > 100 ? `${query.slice(0, 100)}...` : query + }"` + ); + + const { response, error } = await fetch(searchURL.toString(), { + method: "GET", + headers: { + "Content-Type": "application/json", + "User-Agent": "anything-llm", + }, + }) + .then((res) => res.json()) + .then((data) => { + return { response: data, error: null }; + }) + .catch((e) => { + return { response: null, error: e.message }; + }); + if (error) + return `There was an error searching for content. ${error}`; + + const data = []; + response.results?.forEach((searchResult) => { + const { url, title, content, publishedDate } = searchResult; + data.push({ + title, + link: url, + snippet: content, + publishedDate, + }); + }); + + if (data.length === 0) + return `No information was found online for the search query.`; + this.super.introspect( + `${this.caller}: I found ${data.length} results - looking over them now.` + ); return JSON.stringify(data); }, }); diff --git a/server/utils/helpers/updateENV.js b/server/utils/helpers/updateENV.js index 51364191..6b170da3 100644 --- a/server/utils/helpers/updateENV.js +++ b/server/utils/helpers/updateENV.js @@ -407,6 +407,10 @@ const KEY_MAPPING = { envKey: "AGENT_SERPLY_API_KEY", checks: [], }, + AgentSearXNGApiUrl: { + envKey: "AGENT_SEARXNG_API_URL", + checks: [], + }, // TTS/STT Integration ENVS TextToSpeechProvider: {