mirror of
https://github.com/Mintplex-Labs/anything-llm.git
synced 2024-11-19 04:30:10 +01:00
Feature/add searchapi web browsing (#2224)
* Add SearchApi to web browsing * UI modifications for SearchAPI --------- Co-authored-by: Sebastjan Prachovskij <sebastjan.prachovskij@gmail.com>
This commit is contained in:
Timothy Carambat
GitHub
parent
aa4c9533aa
commit
b8b55b5899
1
.vscode/settings.json
vendored
1
.vscode/settings.json
vendored
@ -41,6 +41,7 @@
|
||||
"Qdrant",
|
||||
"royalblue",
|
||||
"searxng",
|
||||
"SearchApi",
|
||||
"Serper",
|
||||
"Serply",
|
||||
"streamable",
|
||||
|
||||
@ -252,6 +252,10 @@ GID='1000'
|
||||
# AGENT_GSE_KEY=
|
||||
# AGENT_GSE_CTX=
|
||||
|
||||
#------ SearchApi.io ----------- https://www.searchapi.io/
|
||||
# AGENT_SEARCHAPI_API_KEY=
|
||||
# AGENT_SEARCHAPI_ENGINE=google
|
||||
|
||||
#------ Serper.dev ----------- https://serper.dev/
|
||||
# AGENT_SERPER_DEV_KEY=
|
||||
|
||||
|
||||
@ -117,8 +117,8 @@ services:
|
||||
- WHISPER_PROVIDER=local
|
||||
- TTS_PROVIDER=native
|
||||
- PASSWORDMINCHAR=8
|
||||
- AGENT_SERPER_DEV_KEY="SERPER DEV API KEY"
|
||||
- AGENT_SERPLY_API_KEY="Serply.io API KEY"
|
||||
# Add any other keys here for services or settings
|
||||
# you can find in the docker/.env.example file
|
||||
volumes:
|
||||
- anythingllm_storage:/app/server/storage
|
||||
restart: always
|
||||
|
||||
@ -50,6 +50,83 @@ export function GoogleSearchOptions({ settings }) {
|
||||
);
|
||||
}
|
||||
|
||||
const SearchApiEngines = [
|
||||
{ name: "Google Search", value: "google" },
|
||||
{ name: "Google Maps", value: "google_maps" },
|
||||
{ name: "Google Shopping", value: "google_shopping" },
|
||||
{ name: "Google News", value: "google_news" },
|
||||
{ name: "Google Jobs", value: "google_jobs" },
|
||||
{ name: "Google Scholar", value: "google_scholar" },
|
||||
{ name: "Google Finance", value: "google_finance" },
|
||||
{ name: "Google Patents", value: "google_patents" },
|
||||
{ name: "YouTube", value: "youtube" },
|
||||
{ name: "Bing", value: "bing" },
|
||||
{ name: "Bing News", value: "bing_news" },
|
||||
{ name: "Amazon Product Search", value: "amazon_search" },
|
||||
{ name: "Baidu", value: "baidu" },
|
||||
];
|
||||
export function SearchApiOptions({ settings }) {
|
||||
return (
|
||||
<>
|
||||
<p className="text-sm text-white/60 my-2">
|
||||
You can get a free API key{" "}
|
||||
<a
|
||||
href="https://www.searchapi.io/"
|
||||
target="_blank"
|
||||
rel="noreferrer"
|
||||
className="text-blue-300 underline"
|
||||
>
|
||||
from SearchApi.
|
||||
</a>
|
||||
</p>
|
||||
<div className="flex gap-x-4">
|
||||
<div className="flex flex-col w-60">
|
||||
<label className="text-white text-sm font-semibold block mb-3">
|
||||
API Key
|
||||
</label>
|
||||
<input
|
||||
type="password"
|
||||
name="env::AgentSearchApiKey"
|
||||
className="border-none bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:outline-primary-button active:outline-primary-button outline-none block w-full p-2.5"
|
||||
placeholder="SearchApi API Key"
|
||||
defaultValue={settings?.AgentSearchApiKey ? "*".repeat(20) : ""}
|
||||
required={true}
|
||||
autoComplete="off"
|
||||
spellCheck={false}
|
||||
/>
|
||||
</div>
|
||||
<div className="flex flex-col w-60">
|
||||
<label className="text-white text-sm font-semibold block mb-3">
|
||||
Engine
|
||||
</label>
|
||||
<select
|
||||
name="env::AgentSearchApiEngine"
|
||||
required={true}
|
||||
className="border-none bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:outline-primary-button active:outline-primary-button outline-none block w-full p-2.5"
|
||||
defaultValue={settings?.AgentSearchApiEngine || "google"}
|
||||
>
|
||||
{SearchApiEngines.map(({ name, value }) => (
|
||||
<option key={name} value={value}>
|
||||
{name}
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
{/* <input
|
||||
type="text"
|
||||
name="env::AgentSearchApiEngine"
|
||||
className="border-none bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:outline-primary-button active:outline-primary-button outline-none block w-full p-2.5"
|
||||
placeholder="SearchApi engine (Google, Bing...)"
|
||||
defaultValue={settings?.AgentSearchApiEngine || "google"}
|
||||
required={true}
|
||||
autoComplete="off"
|
||||
spellCheck={false}
|
||||
/> */}
|
||||
</div>
|
||||
</div>
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
export function SerperDotDevOptions({ settings }) {
|
||||
return (
|
||||
<>
|
||||
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 4.4 KiB |
@ -1,6 +1,7 @@
|
||||
import React, { useEffect, useRef, useState } from "react";
|
||||
import AnythingLLMIcon from "@/media/logo/anything-llm-icon.png";
|
||||
import GoogleSearchIcon from "./icons/google.png";
|
||||
import SearchApiIcon from "./icons/searchapi.png";
|
||||
import SerperDotDevIcon from "./icons/serper.png";
|
||||
import BingSearchIcon from "./icons/bing.png";
|
||||
import SerplySearchIcon from "./icons/serply.png";
|
||||
@ -14,6 +15,7 @@ import {
|
||||
import SearchProviderItem from "./SearchProviderItem";
|
||||
import WebSearchImage from "@/media/agents/scrape-websites.png";
|
||||
import {
|
||||
SearchApiOptions,
|
||||
SerperDotDevOptions,
|
||||
GoogleSearchOptions,
|
||||
BingSearchOptions,
|
||||
@ -38,6 +40,14 @@ const SEARCH_PROVIDERS = [
|
||||
description:
|
||||
"Web search powered by a custom Google Search Engine. Free for 100 queries per day.",
|
||||
},
|
||||
{
|
||||
name: "SearchApi",
|
||||
value: "searchapi",
|
||||
logo: SearchApiIcon,
|
||||
options: (settings) => <SearchApiOptions settings={settings} />,
|
||||
description:
|
||||
"SearchApi delivers structured data from multiple search engines. Free for 100 queries, but then paid. ",
|
||||
},
|
||||
{
|
||||
name: "Serper.dev",
|
||||
value: "serper-dot-dev",
|
||||
|
||||
@ -241,6 +241,10 @@ TTS_PROVIDER="native"
|
||||
# AGENT_GSE_KEY=
|
||||
# AGENT_GSE_CTX=
|
||||
|
||||
#------ SearchApi.io ----------- https://www.searchapi.io/
|
||||
# AGENT_SEARCHAPI_API_KEY=
|
||||
# AGENT_SEARCHAPI_ENGINE=google
|
||||
|
||||
#------ Serper.dev ----------- https://serper.dev/
|
||||
# AGENT_SERPER_DEV_KEY=
|
||||
|
||||
|
||||
@ -81,6 +81,7 @@ const SystemSettings = {
|
||||
if (
|
||||
![
|
||||
"google-search-engine",
|
||||
"searchapi",
|
||||
"serper-dot-dev",
|
||||
"bing-search",
|
||||
"serply-engine",
|
||||
@ -218,6 +219,8 @@ const SystemSettings = {
|
||||
// --------------------------------------------------------
|
||||
AgentGoogleSearchEngineId: process.env.AGENT_GSE_CTX || null,
|
||||
AgentGoogleSearchEngineKey: !!process.env.AGENT_GSE_KEY || null,
|
||||
AgentSearchApiKey: !!process.env.AGENT_SEARCHAPI_API_KEY || null,
|
||||
AgentSearchApiEngine: process.env.AGENT_SEARCHAPI_ENGINE || "google",
|
||||
AgentSerperApiKey: !!process.env.AGENT_SERPER_DEV_KEY || null,
|
||||
AgentBingSearchApiKey: !!process.env.AGENT_BING_SEARCH_API_KEY || null,
|
||||
AgentSerplyApiKey: !!process.env.AGENT_SERPLY_API_KEY || null,
|
||||
|
||||
@ -62,6 +62,9 @@ const webBrowsing = {
|
||||
case "google-search-engine":
|
||||
engine = "_googleSearchEngine";
|
||||
break;
|
||||
case "searchapi":
|
||||
engine = "_searchApi";
|
||||
break;
|
||||
case "serper-dot-dev":
|
||||
engine = "_serperDotDev";
|
||||
break;
|
||||
@ -130,6 +133,72 @@ const webBrowsing = {
|
||||
return JSON.stringify(data);
|
||||
},
|
||||
|
||||
/**
|
||||
* Use SearchApi
|
||||
* SearchApi supports multiple search engines like Google Search, Bing Search, Baidu Search, Google News, YouTube, and many more.
|
||||
* https://www.searchapi.io/
|
||||
*/
|
||||
_searchApi: async function (query) {
|
||||
if (!process.env.AGENT_SEARCHAPI_API_KEY) {
|
||||
this.super.introspect(
|
||||
`${this.caller}: I can't use SearchApi searching because the user has not defined the required API key.\nVisit: https://www.searchapi.io/ to create the API key for free.`
|
||||
);
|
||||
return `Search is disabled and no content was found. This functionality is disabled because the user has not set it up yet.`;
|
||||
}
|
||||
|
||||
this.super.introspect(
|
||||
`${this.caller}: Using SearchApi to search for "${
|
||||
query.length > 100 ? `${query.slice(0, 100)}...` : query
|
||||
}"`
|
||||
);
|
||||
|
||||
const engine = process.env.AGENT_SEARCHAPI_ENGINE;
|
||||
const params = new URLSearchParams({
|
||||
engine: engine,
|
||||
q: query,
|
||||
});
|
||||
|
||||
const url = `https://www.searchapi.io/api/v1/search?${params.toString()}`;
|
||||
const { response, error } = await fetch(url, {
|
||||
method: "GET",
|
||||
headers: {
|
||||
Authorization: `Bearer ${process.env.AGENT_SEARCHAPI_API_KEY}`,
|
||||
"Content-Type": "application/json",
|
||||
"X-SearchApi-Source": "AnythingLLM",
|
||||
},
|
||||
})
|
||||
.then((res) => res.json())
|
||||
.then((data) => {
|
||||
return { response: data, error: null };
|
||||
})
|
||||
.catch((e) => {
|
||||
return { response: null, error: e.message };
|
||||
});
|
||||
if (error)
|
||||
return `There was an error searching for content. ${error}`;
|
||||
|
||||
const data = [];
|
||||
if (response.hasOwnProperty("knowledge_graph"))
|
||||
data.push(response.knowledge_graph?.description);
|
||||
if (response.hasOwnProperty("answer_box"))
|
||||
data.push(response.answer_box?.answer);
|
||||
response.organic_results?.forEach((searchResult) => {
|
||||
const { title, link, snippet } = searchResult;
|
||||
data.push({
|
||||
title,
|
||||
link,
|
||||
snippet,
|
||||
});
|
||||
});
|
||||
|
||||
if (data.length === 0)
|
||||
return `No information was found online for the search query.`;
|
||||
this.super.introspect(
|
||||
`${this.caller}: I found ${data.length} results - looking over them now.`
|
||||
);
|
||||
return JSON.stringify(data);
|
||||
},
|
||||
|
||||
/**
|
||||
* Use Serper.dev
|
||||
* Free to set up, easy to use, 2,500 calls for free one-time
|
||||
|
||||
@ -435,6 +435,14 @@ const KEY_MAPPING = {
|
||||
envKey: "AGENT_GSE_KEY",
|
||||
checks: [],
|
||||
},
|
||||
AgentSearchApiKey: {
|
||||
envKey: "AGENT_SEARCHAPI_API_KEY",
|
||||
checks: [],
|
||||
},
|
||||
AgentSearchApiEngine: {
|
||||
envKey: "AGENT_SEARCHAPI_ENGINE",
|
||||
checks: [],
|
||||
},
|
||||
AgentSerperApiKey: {
|
||||
envKey: "AGENT_SERPER_DEV_KEY",
|
||||
checks: [],
|
||||
|
||||
Loading…
Reference in New Issue
Block a user