mirror of
https://github.com/Mintplex-Labs/anything-llm.git
synced 2024-11-19 12:40:09 +01:00
merge with master
This commit is contained in:
timothycarambat
commit
a99deaa89e
1
.vscode/settings.json
vendored
1
.vscode/settings.json
vendored
@ -31,7 +31,6 @@
|
||||
"Mintplex",
|
||||
"moderations",
|
||||
"numpages",
|
||||
"odbc",
|
||||
"Ollama",
|
||||
"Oobabooga",
|
||||
"openai",
|
||||
|
||||
20
README.md
20
README.md
@ -75,6 +75,7 @@ Some cool features of AnythingLLM
|
||||
- [OpenAI](https://openai.com)
|
||||
- [OpenAI (Generic)](https://openai.com)
|
||||
- [Azure OpenAI](https://azure.microsoft.com/en-us/products/ai-services/openai-service)
|
||||
- [AWS Bedrock](https://aws.amazon.com/bedrock/)
|
||||
- [Anthropic](https://www.anthropic.com/)
|
||||
- [Google Gemini Pro](https://ai.google.dev/)
|
||||
- [Hugging Face (chat models)](https://huggingface.co/)
|
||||
@ -162,12 +163,6 @@ Mintplex Labs & the community maintain a number of deployment methods, scripts,
|
||||
|
||||
[Learn about vector caching](./server/storage/vector-cache/VECTOR_CACHE.md)
|
||||
|
||||
## Contributing
|
||||
|
||||
- create issue
|
||||
- create PR with branch name format of `<issue number>-<short name>`
|
||||
- yee haw let's merge
|
||||
|
||||
## Telemetry & Privacy
|
||||
|
||||
AnythingLLM by Mintplex Labs Inc contains a telemetry feature that collects anonymous usage information.
|
||||
@ -199,6 +194,19 @@ You can verify these claims by finding all locations `Telemetry.sendTelemetry` i
|
||||
|
||||
</details>
|
||||
|
||||
|
||||
## 👋 Contributing
|
||||
|
||||
- create issue
|
||||
- create PR with branch name format of `<issue number>-<short name>`
|
||||
- LGTM from core-team
|
||||
|
||||
## 🌟 Contributors
|
||||
|
||||
[](https://github.com/mintplex-labs/anything-llm/graphs/contributors)
|
||||
|
||||
[](https://star-history.com/#mintplex-labs/anything-llm&Date)
|
||||
|
||||
## 🔗 More Products
|
||||
|
||||
- **[VectorAdmin][vector-admin]:** An all-in-one GUI & tool-suite for managing vector databases.
|
||||
|
||||
@ -94,6 +94,13 @@ GID='1000'
|
||||
# COHERE_API_KEY=
|
||||
# COHERE_MODEL_PREF='command-r'
|
||||
|
||||
# LLM_PROVIDER='bedrock'
|
||||
# AWS_BEDROCK_LLM_ACCESS_KEY_ID=
|
||||
# AWS_BEDROCK_LLM_ACCESS_KEY=
|
||||
# AWS_BEDROCK_LLM_REGION=us-west-2
|
||||
# AWS_BEDROCK_LLM_MODEL_PREFERENCE=meta.llama3-1-8b-instruct-v1:0
|
||||
# AWS_BEDROCK_LLM_MODEL_TOKEN_LIMIT=8191
|
||||
|
||||
###########################################
|
||||
######## Embedding API SElECTION ##########
|
||||
###########################################
|
||||
|
||||
@ -0,0 +1,122 @@
|
||||
import { ArrowSquareOut, Info } from "@phosphor-icons/react";
|
||||
import { AWS_REGIONS } from "./regions";
|
||||
|
||||
export default function AwsBedrockLLMOptions({ settings }) {
|
||||
return (
|
||||
<div className="w-full flex flex-col">
|
||||
{!settings?.credentialsOnly && (
|
||||
<div className="flex flex-col md:flex-row md:items-center gap-x-2 text-white mb-4 bg-blue-800/30 w-fit rounded-lg px-4 py-2">
|
||||
<div className="gap-x-2 flex items-center">
|
||||
<Info size={40} />
|
||||
<p className="text-base">
|
||||
You should use a properly defined IAM user for inferencing.
|
||||
<br />
|
||||
<a
|
||||
href="https://docs.useanything.com/setup/llm-configuration/cloud/aws-bedrock"
|
||||
target="_blank"
|
||||
className="underline flex gap-x-1 items-center"
|
||||
>
|
||||
Read more on how to use AWS Bedrock in AnythingLLM
|
||||
<ArrowSquareOut size={14} />
|
||||
</a>
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className="w-full flex items-center gap-[36px] my-1.5">
|
||||
<div className="flex flex-col w-60">
|
||||
<label className="text-white text-sm font-semibold block mb-3">
|
||||
AWS Bedrock IAM Access ID
|
||||
</label>
|
||||
<input
|
||||
type="password"
|
||||
name="AwsBedrockLLMAccessKeyId"
|
||||
className="border-none bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:outline-primary-button active:outline-primary-button outline-none block w-full p-2.5"
|
||||
placeholder="AWS Bedrock IAM User Access ID"
|
||||
defaultValue={
|
||||
settings?.AwsBedrockLLMAccessKeyId ? "*".repeat(20) : ""
|
||||
}
|
||||
required={true}
|
||||
autoComplete="off"
|
||||
spellCheck={false}
|
||||
/>
|
||||
</div>
|
||||
<div className="flex flex-col w-60">
|
||||
<label className="text-white text-sm font-semibold block mb-3">
|
||||
AWS Bedrock IAM Access Key
|
||||
</label>
|
||||
<input
|
||||
type="password"
|
||||
name="AwsBedrockLLMAccessKey"
|
||||
className="border-none bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:outline-primary-button active:outline-primary-button outline-none block w-full p-2.5"
|
||||
placeholder="AWS Bedrock IAM User Access Key"
|
||||
defaultValue={
|
||||
settings?.AwsBedrockLLMAccessKey ? "*".repeat(20) : ""
|
||||
}
|
||||
required={true}
|
||||
autoComplete="off"
|
||||
spellCheck={false}
|
||||
/>
|
||||
</div>
|
||||
<div className="flex flex-col w-60">
|
||||
<label className="text-white text-sm font-semibold block mb-3">
|
||||
AWS region
|
||||
</label>
|
||||
<select
|
||||
name="AwsBedrockLLMRegion"
|
||||
defaultValue={settings?.AwsBedrockLLMRegion || "us-west-2"}
|
||||
required={true}
|
||||
className="border-none bg-zinc-900 border-gray-500 text-white text-sm rounded-lg block w-full p-2.5"
|
||||
>
|
||||
{AWS_REGIONS.map((region) => {
|
||||
return (
|
||||
<option key={region.code} value={region.code}>
|
||||
{region.name} ({region.code})
|
||||
</option>
|
||||
);
|
||||
})}
|
||||
</select>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="w-full flex items-center gap-[36px] my-1.5">
|
||||
{!settings?.credentialsOnly && (
|
||||
<>
|
||||
<div className="flex flex-col w-60">
|
||||
<label className="text-white text-sm font-semibold block mb-3">
|
||||
Model ID
|
||||
</label>
|
||||
<input
|
||||
type="text"
|
||||
name="AwsBedrockLLMModel"
|
||||
className="border-none bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:outline-primary-button active:outline-primary-button outline-none block w-full p-2.5"
|
||||
placeholder="Model id from AWS eg: meta.llama3.1-v0.1"
|
||||
defaultValue={settings?.AwsBedrockLLMModel}
|
||||
required={true}
|
||||
autoComplete="off"
|
||||
spellCheck={false}
|
||||
/>
|
||||
</div>
|
||||
<div className="flex flex-col w-60">
|
||||
<label className="text-white text-sm font-semibold block mb-3">
|
||||
Model context window
|
||||
</label>
|
||||
<input
|
||||
type="number"
|
||||
name="AwsBedrockLLMTokenLimit"
|
||||
className="border-none bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:outline-primary-button active:outline-primary-button outline-none block w-full p-2.5"
|
||||
placeholder="Content window limit (eg: 4096)"
|
||||
min={1}
|
||||
onScroll={(e) => e.target.blur()}
|
||||
defaultValue={settings?.AwsBedrockLLMTokenLimit}
|
||||
required={true}
|
||||
autoComplete="off"
|
||||
/>
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@ -0,0 +1,204 @@
|
||||
export const AWS_REGIONS = [
|
||||
{
|
||||
name: "N. Virginia",
|
||||
full_name: "US East (N. Virginia)",
|
||||
code: "us-east-1",
|
||||
public: true,
|
||||
zones: [
|
||||
"us-east-1a",
|
||||
"us-east-1b",
|
||||
"us-east-1c",
|
||||
"us-east-1d",
|
||||
"us-east-1e",
|
||||
"us-east-1f",
|
||||
],
|
||||
},
|
||||
{
|
||||
name: "Ohio",
|
||||
full_name: "US East (Ohio)",
|
||||
code: "us-east-2",
|
||||
public: true,
|
||||
zones: ["us-east-2a", "us-east-2b", "us-east-2c"],
|
||||
},
|
||||
{
|
||||
name: "N. California",
|
||||
full_name: "US West (N. California)",
|
||||
code: "us-west-1",
|
||||
public: true,
|
||||
zone_limit: 2,
|
||||
zones: ["us-west-1a", "us-west-1b", "us-west-1c"],
|
||||
},
|
||||
{
|
||||
name: "Oregon",
|
||||
full_name: "US West (Oregon)",
|
||||
code: "us-west-2",
|
||||
public: true,
|
||||
zones: ["us-west-2a", "us-west-2b", "us-west-2c", "us-west-2d"],
|
||||
},
|
||||
{
|
||||
name: "GovCloud West",
|
||||
full_name: "AWS GovCloud (US)",
|
||||
code: "us-gov-west-1",
|
||||
public: false,
|
||||
zones: ["us-gov-west-1a", "us-gov-west-1b", "us-gov-west-1c"],
|
||||
},
|
||||
{
|
||||
name: "GovCloud East",
|
||||
full_name: "AWS GovCloud (US-East)",
|
||||
code: "us-gov-east-1",
|
||||
public: false,
|
||||
zones: ["us-gov-east-1a", "us-gov-east-1b", "us-gov-east-1c"],
|
||||
},
|
||||
{
|
||||
name: "Canada",
|
||||
full_name: "Canada (Central)",
|
||||
code: "ca-central-1",
|
||||
public: true,
|
||||
zones: ["ca-central-1a", "ca-central-1b", "ca-central-1c", "ca-central-1d"],
|
||||
},
|
||||
{
|
||||
name: "Stockholm",
|
||||
full_name: "EU (Stockholm)",
|
||||
code: "eu-north-1",
|
||||
public: true,
|
||||
zones: ["eu-north-1a", "eu-north-1b", "eu-north-1c"],
|
||||
},
|
||||
{
|
||||
name: "Ireland",
|
||||
full_name: "EU (Ireland)",
|
||||
code: "eu-west-1",
|
||||
public: true,
|
||||
zones: ["eu-west-1a", "eu-west-1b", "eu-west-1c"],
|
||||
},
|
||||
{
|
||||
name: "London",
|
||||
full_name: "EU (London)",
|
||||
code: "eu-west-2",
|
||||
public: true,
|
||||
zones: ["eu-west-2a", "eu-west-2b", "eu-west-2c"],
|
||||
},
|
||||
{
|
||||
name: "Paris",
|
||||
full_name: "EU (Paris)",
|
||||
code: "eu-west-3",
|
||||
public: true,
|
||||
zones: ["eu-west-3a", "eu-west-3b", "eu-west-3c"],
|
||||
},
|
||||
{
|
||||
name: "Frankfurt",
|
||||
full_name: "EU (Frankfurt)",
|
||||
code: "eu-central-1",
|
||||
public: true,
|
||||
zones: ["eu-central-1a", "eu-central-1b", "eu-central-1c"],
|
||||
},
|
||||
{
|
||||
name: "Milan",
|
||||
full_name: "EU (Milan)",
|
||||
code: "eu-south-1",
|
||||
public: true,
|
||||
zones: ["eu-south-1a", "eu-south-1b", "eu-south-1c"],
|
||||
},
|
||||
{
|
||||
name: "Cape Town",
|
||||
full_name: "Africa (Cape Town)",
|
||||
code: "af-south-1",
|
||||
public: true,
|
||||
zones: ["af-south-1a", "af-south-1b", "af-south-1c"],
|
||||
},
|
||||
{
|
||||
name: "Tokyo",
|
||||
full_name: "Asia Pacific (Tokyo)",
|
||||
code: "ap-northeast-1",
|
||||
public: true,
|
||||
zone_limit: 3,
|
||||
zones: [
|
||||
"ap-northeast-1a",
|
||||
"ap-northeast-1b",
|
||||
"ap-northeast-1c",
|
||||
"ap-northeast-1d",
|
||||
],
|
||||
},
|
||||
{
|
||||
name: "Seoul",
|
||||
full_name: "Asia Pacific (Seoul)",
|
||||
code: "ap-northeast-2",
|
||||
public: true,
|
||||
zones: [
|
||||
"ap-northeast-2a",
|
||||
"ap-northeast-2b",
|
||||
"ap-northeast-2c",
|
||||
"ap-northeast-2d",
|
||||
],
|
||||
},
|
||||
{
|
||||
name: "Osaka",
|
||||
full_name: "Asia Pacific (Osaka-Local)",
|
||||
code: "ap-northeast-3",
|
||||
public: true,
|
||||
zones: ["ap-northeast-3a", "ap-northeast-3b", "ap-northeast-3c"],
|
||||
},
|
||||
{
|
||||
name: "Singapore",
|
||||
full_name: "Asia Pacific (Singapore)",
|
||||
code: "ap-southeast-1",
|
||||
public: true,
|
||||
zones: ["ap-southeast-1a", "ap-southeast-1b", "ap-southeast-1c"],
|
||||
},
|
||||
{
|
||||
name: "Sydney",
|
||||
full_name: "Asia Pacific (Sydney)",
|
||||
code: "ap-southeast-2",
|
||||
public: true,
|
||||
zones: ["ap-southeast-2a", "ap-southeast-2b", "ap-southeast-2c"],
|
||||
},
|
||||
{
|
||||
name: "Jakarta",
|
||||
full_name: "Asia Pacific (Jakarta)",
|
||||
code: "ap-southeast-3",
|
||||
public: true,
|
||||
zones: ["ap-southeast-3a", "ap-southeast-3b", "ap-southeast-3c"],
|
||||
},
|
||||
{
|
||||
name: "Hong Kong",
|
||||
full_name: "Asia Pacific (Hong Kong)",
|
||||
code: "ap-east-1",
|
||||
public: true,
|
||||
zones: ["ap-east-1a", "ap-east-1b", "ap-east-1c"],
|
||||
},
|
||||
{
|
||||
name: "Mumbai",
|
||||
full_name: "Asia Pacific (Mumbai)",
|
||||
code: "ap-south-1",
|
||||
public: true,
|
||||
zones: ["ap-south-1a", "ap-south-1b", "ap-south-1c"],
|
||||
},
|
||||
{
|
||||
name: "São Paulo",
|
||||
full_name: "South America (São Paulo)",
|
||||
code: "sa-east-1",
|
||||
public: true,
|
||||
zone_limit: 2,
|
||||
zones: ["sa-east-1a", "sa-east-1b", "sa-east-1c"],
|
||||
},
|
||||
{
|
||||
name: "Bahrain",
|
||||
full_name: "Middle East (Bahrain)",
|
||||
code: "me-south-1",
|
||||
public: true,
|
||||
zones: ["me-south-1a", "me-south-1b", "me-south-1c"],
|
||||
},
|
||||
{
|
||||
name: "Beijing",
|
||||
full_name: "China (Beijing)",
|
||||
code: "cn-north-1",
|
||||
public: false,
|
||||
zones: ["cn-north-1a", "cn-north-1b", "cn-north-1c"],
|
||||
},
|
||||
{
|
||||
name: "Ningxia",
|
||||
full_name: "China (Ningxia)",
|
||||
code: "cn-northwest-1",
|
||||
public: false,
|
||||
zones: ["cn-northwest-1a", "cn-northwest-1b", "cn-northwest-1c"],
|
||||
},
|
||||
];
|
||||
@ -8,6 +8,7 @@ export const DISABLED_PROVIDERS = [
|
||||
"native",
|
||||
"textgenwebui",
|
||||
"generic-openai",
|
||||
"bedrock",
|
||||
];
|
||||
const PROVIDER_DEFAULT_MODELS = {
|
||||
openai: [],
|
||||
@ -48,6 +49,7 @@ const PROVIDER_DEFAULT_MODELS = {
|
||||
],
|
||||
textgenwebui: [],
|
||||
"generic-openai": [],
|
||||
bedrock: [],
|
||||
};
|
||||
|
||||
// For togetherAi, which has a large model list - we subgroup the options
|
||||
|
||||
BIN
frontend/src/media/llmprovider/bedrock.png
Normal file
BIN
frontend/src/media/llmprovider/bedrock.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 56 KiB |
@ -1,14 +1,12 @@
|
||||
import PostgreSQLLogo from "./icons/postgresql.png";
|
||||
import MySQLLogo from "./icons/mysql.png";
|
||||
import MSSQLLogo from "./icons/mssql.png";
|
||||
import ODBCLogo from "./icons/odbc.png";
|
||||
import { X } from "@phosphor-icons/react";
|
||||
|
||||
export const DB_LOGOS = {
|
||||
postgresql: PostgreSQLLogo,
|
||||
mysql: MySQLLogo,
|
||||
"sql-server": MSSQLLogo,
|
||||
odbc: ODBCLogo,
|
||||
};
|
||||
|
||||
export default function DBConnection({ connection, onRemove, setHasChanges }) {
|
||||
|
||||
@ -11,7 +11,6 @@ function assembleConnectionString({
|
||||
host = "",
|
||||
port = "",
|
||||
database = "",
|
||||
driver = "",
|
||||
}) {
|
||||
if ([username, password, host, database].every((i) => !!i) === false)
|
||||
return `Please fill out all the fields above.`;
|
||||
@ -22,9 +21,6 @@ function assembleConnectionString({
|
||||
return `mysql://${username}:${password}@${host}:${port}/${database}`;
|
||||
case "sql-server":
|
||||
return `mssql://${username}:${password}@${host}:${port}/${database}`;
|
||||
case "odbc":
|
||||
if (!driver) return `Please fill out the driver field.`;
|
||||
return `Driver={${driver}};Server=${host};Port=${port};Database=${database};UID=${username};PWD=${password}`;
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
@ -37,7 +33,6 @@ const DEFAULT_CONFIG = {
|
||||
host: null,
|
||||
port: null,
|
||||
database: null,
|
||||
driver: null,
|
||||
};
|
||||
|
||||
export default function NewSQLConnection({ isOpen, closeModal, onSubmit }) {
|
||||
@ -53,14 +48,12 @@ export default function NewSQLConnection({ isOpen, closeModal, onSubmit }) {
|
||||
|
||||
function onFormChange() {
|
||||
const form = new FormData(document.getElementById("sql-connection-form"));
|
||||
|
||||
setConfig({
|
||||
username: form.get("username").trim(),
|
||||
password: form.get("password"),
|
||||
host: form.get("host").trim(),
|
||||
port: form.get("port").trim(),
|
||||
database: form.get("database").trim(),
|
||||
driver: form.get("driver")?.trim(),
|
||||
});
|
||||
}
|
||||
|
||||
@ -81,7 +74,7 @@ export default function NewSQLConnection({ isOpen, closeModal, onSubmit }) {
|
||||
// to the parent container form so we don't have nested forms.
|
||||
return createPortal(
|
||||
<ModalWrapper isOpen={isOpen}>
|
||||
<div className="relative w-full md:w-fit max-w-2xl max-h-full md:mt-8">
|
||||
<div className="relative w-full md:w-1/3 max-w-2xl max-h-full md:mt-8">
|
||||
<div className="relative bg-main-gradient rounded-xl shadow-[0_4px_14px_rgba(0,0,0,0.25)] max-h-[85vh] overflow-y-scroll no-scroll">
|
||||
<div className="flex items-start justify-between p-4 border-b rounded-t border-gray-500/50">
|
||||
<h3 className="text-xl font-semibold text-white">
|
||||
@ -121,7 +114,7 @@ export default function NewSQLConnection({ isOpen, closeModal, onSubmit }) {
|
||||
<label className="text-white text-sm font-semibold block my-4">
|
||||
Select your SQL engine
|
||||
</label>
|
||||
<div className="flex flex-wrap gap-x-4 gap-y-4">
|
||||
<div className="grid md:grid-cols-4 gap-4 grid-cols-2">
|
||||
<DBEngine
|
||||
provider="postgresql"
|
||||
active={engine === "postgresql"}
|
||||
@ -137,11 +130,6 @@ export default function NewSQLConnection({ isOpen, closeModal, onSubmit }) {
|
||||
active={engine === "sql-server"}
|
||||
onClick={() => setEngine("sql-server")}
|
||||
/>
|
||||
<DBEngine
|
||||
provider="odbc"
|
||||
active={engine === "odbc"}
|
||||
onClick={() => setEngine("odbc")}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@ -236,23 +224,6 @@ export default function NewSQLConnection({ isOpen, closeModal, onSubmit }) {
|
||||
spellCheck={false}
|
||||
/>
|
||||
</div>
|
||||
|
||||
{engine === "odbc" && (
|
||||
<div className="flex flex-col">
|
||||
<label className="text-white text-sm font-semibold block mb-3">
|
||||
Driver
|
||||
</label>
|
||||
<input
|
||||
type="text"
|
||||
name="driver"
|
||||
className="border-none bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:outline-primary-button active:outline-primary-button outline-none block w-full p-2.5"
|
||||
placeholder="the driver to use eg: MongoDB ODBC 1.2.0 ANSI Driver"
|
||||
required={true}
|
||||
autoComplete="off"
|
||||
spellCheck={false}
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
<p className="text-white/40 text-sm">
|
||||
{assembleConnectionString({ engine, ...config })}
|
||||
</p>
|
||||
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 19 KiB |
@ -22,6 +22,8 @@ import KoboldCPPLogo from "@/media/llmprovider/koboldcpp.png";
|
||||
import TextGenWebUILogo from "@/media/llmprovider/text-generation-webui.png";
|
||||
import CohereLogo from "@/media/llmprovider/cohere.png";
|
||||
import LiteLLMLogo from "@/media/llmprovider/litellm.png";
|
||||
import AWSBedrockLogo from "@/media/llmprovider/bedrock.png";
|
||||
|
||||
import PreLoader from "@/components/Preloader";
|
||||
import OpenAiOptions from "@/components/LLMSelection/OpenAiOptions";
|
||||
import GenericOpenAiOptions from "@/components/LLMSelection/GenericOpenAiOptions";
|
||||
@ -41,6 +43,7 @@ import CohereAiOptions from "@/components/LLMSelection/CohereAiOptions";
|
||||
import KoboldCPPOptions from "@/components/LLMSelection/KoboldCPPOptions";
|
||||
import TextGenWebUIOptions from "@/components/LLMSelection/TextGenWebUIOptions";
|
||||
import LiteLLMOptions from "@/components/LLMSelection/LiteLLMOptions";
|
||||
import AWSBedrockLLMOptions from "@/components/LLMSelection/AwsBedrockLLMOptions";
|
||||
|
||||
import LLMItem from "@/components/LLMSelection/LLMItem";
|
||||
import { CaretUpDown, MagnifyingGlass, X } from "@phosphor-icons/react";
|
||||
@ -218,6 +221,19 @@ export const AVAILABLE_LLM_PROVIDERS = [
|
||||
// "Use a downloaded custom Llama model for chatting on this AnythingLLM instance.",
|
||||
// requiredConfig: [],
|
||||
// },
|
||||
{
|
||||
name: "AWS Bedrock",
|
||||
value: "bedrock",
|
||||
logo: AWSBedrockLogo,
|
||||
options: (settings) => <AWSBedrockLLMOptions settings={settings} />,
|
||||
description: "Run powerful foundation models privately with AWS Bedrock.",
|
||||
requiredConfig: [
|
||||
"AwsBedrockLLMAccessKeyId",
|
||||
"AwsBedrockLLMAccessKey",
|
||||
"AwsBedrockLLMRegion",
|
||||
"AwsBedrockLLMModel",
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
export default function GeneralLLMPreference() {
|
||||
|
||||
@ -18,6 +18,7 @@ import GroqLogo from "@/media/llmprovider/groq.png";
|
||||
import KoboldCPPLogo from "@/media/llmprovider/koboldcpp.png";
|
||||
import TextGenWebUILogo from "@/media/llmprovider/text-generation-webui.png";
|
||||
import LiteLLMLogo from "@/media/llmprovider/litellm.png";
|
||||
import AWSBedrockLogo from "@/media/llmprovider/bedrock.png";
|
||||
|
||||
import CohereLogo from "@/media/llmprovider/cohere.png";
|
||||
import ZillizLogo from "@/media/vectordbs/zilliz.png";
|
||||
@ -179,6 +180,13 @@ export const LLM_SELECTION_PRIVACY = {
|
||||
],
|
||||
logo: LiteLLMLogo,
|
||||
},
|
||||
bedrock: {
|
||||
name: "AWS Bedrock",
|
||||
description: [
|
||||
"You model and chat contents are subject to the agreed EULA for AWS and the model provider on aws.amazon.com",
|
||||
],
|
||||
logo: AWSBedrockLogo,
|
||||
},
|
||||
};
|
||||
|
||||
export const VECTOR_DB_PRIVACY = {
|
||||
|
||||
@ -17,6 +17,7 @@ import GroqLogo from "@/media/llmprovider/groq.png";
|
||||
import KoboldCPPLogo from "@/media/llmprovider/koboldcpp.png";
|
||||
import TextGenWebUILogo from "@/media/llmprovider/text-generation-webui.png";
|
||||
import LiteLLMLogo from "@/media/llmprovider/litellm.png";
|
||||
import AWSBedrockLogo from "@/media/llmprovider/bedrock.png";
|
||||
|
||||
import CohereLogo from "@/media/llmprovider/cohere.png";
|
||||
import OpenAiOptions from "@/components/LLMSelection/OpenAiOptions";
|
||||
@ -37,6 +38,7 @@ import CohereAiOptions from "@/components/LLMSelection/CohereAiOptions";
|
||||
import KoboldCPPOptions from "@/components/LLMSelection/KoboldCPPOptions";
|
||||
import TextGenWebUIOptions from "@/components/LLMSelection/TextGenWebUIOptions";
|
||||
import LiteLLMOptions from "@/components/LLMSelection/LiteLLMOptions";
|
||||
import AWSBedrockLLMOptions from "@/components/LLMSelection/AwsBedrockLLMOptions";
|
||||
|
||||
import LLMItem from "@/components/LLMSelection/LLMItem";
|
||||
import System from "@/models/system";
|
||||
@ -188,6 +190,13 @@ const LLMS = [
|
||||
// description:
|
||||
// "Use a downloaded custom Llama model for chatting on this AnythingLLM instance.",
|
||||
// },
|
||||
{
|
||||
name: "AWS Bedrock",
|
||||
value: "bedrock",
|
||||
logo: AWSBedrockLogo,
|
||||
options: (settings) => <AWSBedrockLLMOptions settings={settings} />,
|
||||
description: "Run powerful foundation models privately with AWS Bedrock.",
|
||||
},
|
||||
];
|
||||
|
||||
export default function LLMPreference({
|
||||
|
||||
@ -21,6 +21,7 @@ const ENABLED_PROVIDERS = [
|
||||
"perplexity",
|
||||
"textgenwebui",
|
||||
"generic-openai",
|
||||
"bedrock",
|
||||
// TODO: More agent support.
|
||||
// "cohere", // Has tool calling and will need to build explicit support
|
||||
// "huggingface" // Can be done but already has issues with no-chat templated. Needs to be tested.
|
||||
|
||||
@ -10,7 +10,12 @@ import paths from "@/utils/paths";
|
||||
|
||||
// Some providers can only be associated with a single model.
|
||||
// In that case there is no selection to be made so we can just move on.
|
||||
const NO_MODEL_SELECTION = ["default", "huggingface", "generic-openai"];
|
||||
const NO_MODEL_SELECTION = [
|
||||
"default",
|
||||
"huggingface",
|
||||
"generic-openai",
|
||||
"bedrock",
|
||||
];
|
||||
const DISABLED_PROVIDERS = ["azure", "lmstudio", "native"];
|
||||
const LLM_DEFAULT = {
|
||||
name: "System default",
|
||||
|
||||
@ -470,6 +470,12 @@ const SystemSettings = {
|
||||
GenericOpenAiKey: !!process.env.GENERIC_OPEN_AI_API_KEY,
|
||||
GenericOpenAiMaxTokens: process.env.GENERIC_OPEN_AI_MAX_TOKENS,
|
||||
|
||||
AwsBedrockLLMAccessKeyId: !!process.env.AWS_BEDROCK_LLM_ACCESS_KEY_ID,
|
||||
AwsBedrockLLMAccessKey: !!process.env.AWS_BEDROCK_LLM_ACCESS_KEY,
|
||||
AwsBedrockLLMRegion: process.env.AWS_BEDROCK_LLM_REGION,
|
||||
AwsBedrockLLMModel: process.env.AWS_BEDROCK_LLM_MODEL_PREFERENCE,
|
||||
AwsBedrockLLMTokenLimit: process.env.AWS_BEDROCK_LLM_MODEL_TOKEN_LIMIT,
|
||||
|
||||
// Cohere API Keys
|
||||
CohereApiKey: !!process.env.COHERE_API_KEY,
|
||||
CohereModelPref: process.env.COHERE_MODEL_PREF,
|
||||
|
||||
@ -28,6 +28,7 @@
|
||||
"@ladjs/graceful": "^3.2.2",
|
||||
"@lancedb/lancedb": "0.5.2",
|
||||
"@langchain/anthropic": "0.1.16",
|
||||
"@langchain/aws": "^0.0.5",
|
||||
"@langchain/community": "0.0.53",
|
||||
"@langchain/core": "0.1.61",
|
||||
"@langchain/openai": "0.0.28",
|
||||
@ -66,7 +67,6 @@
|
||||
"mysql2": "^3.9.8",
|
||||
"node-html-markdown": "^1.3.0",
|
||||
"node-llama-cpp": "^2.8.0",
|
||||
"odbc": "^2.4.8",
|
||||
"ollama": "^0.5.0",
|
||||
"openai": "4.38.5",
|
||||
"pg": "^8.11.5",
|
||||
|
||||
217
server/utils/AiProviders/bedrock/index.js
Normal file
217
server/utils/AiProviders/bedrock/index.js
Normal file
@ -0,0 +1,217 @@
|
||||
const { StringOutputParser } = require("@langchain/core/output_parsers");
|
||||
const {
|
||||
writeResponseChunk,
|
||||
clientAbortedHandler,
|
||||
} = require("../../helpers/chat/responses");
|
||||
const { NativeEmbedder } = require("../../EmbeddingEngines/native");
|
||||
|
||||
// Docs: https://js.langchain.com/v0.2/docs/integrations/chat/bedrock_converse
|
||||
class AWSBedrockLLM {
|
||||
constructor(embedder = null, modelPreference = null) {
|
||||
if (!process.env.AWS_BEDROCK_LLM_ACCESS_KEY_ID)
|
||||
throw new Error("No AWS Bedrock LLM profile id was set.");
|
||||
|
||||
if (!process.env.AWS_BEDROCK_LLM_ACCESS_KEY)
|
||||
throw new Error("No AWS Bedrock LLM access key was set.");
|
||||
|
||||
if (!process.env.AWS_BEDROCK_LLM_REGION)
|
||||
throw new Error("No AWS Bedrock LLM region was set.");
|
||||
|
||||
this.model =
|
||||
modelPreference || process.env.AWS_BEDROCK_LLM_MODEL_PREFERENCE;
|
||||
this.limits = {
|
||||
history: this.promptWindowLimit() * 0.15,
|
||||
system: this.promptWindowLimit() * 0.15,
|
||||
user: this.promptWindowLimit() * 0.7,
|
||||
};
|
||||
|
||||
this.embedder = embedder ?? new NativeEmbedder();
|
||||
this.defaultTemp = 0.7;
|
||||
}
|
||||
|
||||
#bedrockClient({ temperature = 0.7 }) {
|
||||
const { ChatBedrockConverse } = require("@langchain/aws");
|
||||
return new ChatBedrockConverse({
|
||||
model: process.env.AWS_BEDROCK_LLM_MODEL_PREFERENCE,
|
||||
region: process.env.AWS_BEDROCK_LLM_REGION,
|
||||
credentials: {
|
||||
accessKeyId: process.env.AWS_BEDROCK_LLM_ACCESS_KEY_ID,
|
||||
secretAccessKey: process.env.AWS_BEDROCK_LLM_ACCESS_KEY,
|
||||
},
|
||||
temperature,
|
||||
});
|
||||
}
|
||||
|
||||
// For streaming we use Langchain's wrapper to handle weird chunks
|
||||
// or otherwise absorb headaches that can arise from Ollama models
|
||||
#convertToLangchainPrototypes(chats = []) {
|
||||
const {
|
||||
HumanMessage,
|
||||
SystemMessage,
|
||||
AIMessage,
|
||||
} = require("@langchain/core/messages");
|
||||
const langchainChats = [];
|
||||
const roleToMessageMap = {
|
||||
system: SystemMessage,
|
||||
user: HumanMessage,
|
||||
assistant: AIMessage,
|
||||
};
|
||||
|
||||
for (const chat of chats) {
|
||||
if (!roleToMessageMap.hasOwnProperty(chat.role)) continue;
|
||||
const MessageClass = roleToMessageMap[chat.role];
|
||||
langchainChats.push(new MessageClass({ content: chat.content }));
|
||||
}
|
||||
|
||||
return langchainChats;
|
||||
}
|
||||
|
||||
#appendContext(contextTexts = []) {
|
||||
if (!contextTexts || !contextTexts.length) return "";
|
||||
return (
|
||||
"\nContext:\n" +
|
||||
contextTexts
|
||||
.map((text, i) => {
|
||||
return `[CONTEXT ${i}]:\n${text}\n[END CONTEXT ${i}]\n\n`;
|
||||
})
|
||||
.join("")
|
||||
);
|
||||
}
|
||||
|
||||
streamingEnabled() {
|
||||
return "streamGetChatCompletion" in this;
|
||||
}
|
||||
|
||||
// Ensure the user set a value for the token limit
|
||||
// and if undefined - assume 4096 window.
|
||||
promptWindowLimit() {
|
||||
const limit = process.env.AWS_BEDROCK_LLM_MODEL_TOKEN_LIMIT || 8191;
|
||||
if (!limit || isNaN(Number(limit)))
|
||||
throw new Error("No valid token context limit was set.");
|
||||
return Number(limit);
|
||||
}
|
||||
|
||||
async isValidChatCompletionModel(_ = "") {
|
||||
return true;
|
||||
}
|
||||
|
||||
constructPrompt({
|
||||
systemPrompt = "",
|
||||
contextTexts = [],
|
||||
chatHistory = [],
|
||||
userPrompt = "",
|
||||
}) {
|
||||
// AWS Mistral models do not support system prompts
|
||||
if (this.model.startsWith("mistral"))
|
||||
return [...chatHistory, { role: "user", content: userPrompt }];
|
||||
|
||||
const prompt = {
|
||||
role: "system",
|
||||
content: `${systemPrompt}${this.#appendContext(contextTexts)}`,
|
||||
};
|
||||
return [prompt, ...chatHistory, { role: "user", content: userPrompt }];
|
||||
}
|
||||
|
||||
async getChatCompletion(messages = null, { temperature = 0.7 }) {
|
||||
const model = this.#bedrockClient({ temperature });
|
||||
const textResponse = await model
|
||||
.pipe(new StringOutputParser())
|
||||
.invoke(this.#convertToLangchainPrototypes(messages))
|
||||
.catch((e) => {
|
||||
throw new Error(
|
||||
`AWSBedrock::getChatCompletion failed to communicate with Ollama. ${e.message}`
|
||||
);
|
||||
});
|
||||
|
||||
if (!textResponse || !textResponse.length)
|
||||
throw new Error(`AWSBedrock::getChatCompletion text response was empty.`);
|
||||
|
||||
return textResponse;
|
||||
}
|
||||
|
||||
async streamGetChatCompletion(messages = null, { temperature = 0.7 }) {
|
||||
const model = this.#bedrockClient({ temperature });
|
||||
const stream = await model
|
||||
.pipe(new StringOutputParser())
|
||||
.stream(this.#convertToLangchainPrototypes(messages));
|
||||
return stream;
|
||||
}
|
||||
|
||||
handleStream(response, stream, responseProps) {
|
||||
const { uuid = uuidv4(), sources = [] } = responseProps;
|
||||
|
||||
return new Promise(async (resolve) => {
|
||||
let fullText = "";
|
||||
|
||||
// Establish listener to early-abort a streaming response
|
||||
// in case things go sideways or the user does not like the response.
|
||||
// We preserve the generated text but continue as if chat was completed
|
||||
// to preserve previously generated content.
|
||||
const handleAbort = () => clientAbortedHandler(resolve, fullText);
|
||||
response.on("close", handleAbort);
|
||||
|
||||
try {
|
||||
for await (const chunk of stream) {
|
||||
if (chunk === undefined)
|
||||
throw new Error(
|
||||
"Stream returned undefined chunk. Aborting reply - check model provider logs."
|
||||
);
|
||||
|
||||
const content = chunk.hasOwnProperty("content")
|
||||
? chunk.content
|
||||
: chunk;
|
||||
fullText += content;
|
||||
writeResponseChunk(response, {
|
||||
uuid,
|
||||
sources: [],
|
||||
type: "textResponseChunk",
|
||||
textResponse: content,
|
||||
close: false,
|
||||
error: false,
|
||||
});
|
||||
}
|
||||
|
||||
writeResponseChunk(response, {
|
||||
uuid,
|
||||
sources,
|
||||
type: "textResponseChunk",
|
||||
textResponse: "",
|
||||
close: true,
|
||||
error: false,
|
||||
});
|
||||
response.removeListener("close", handleAbort);
|
||||
resolve(fullText);
|
||||
} catch (error) {
|
||||
writeResponseChunk(response, {
|
||||
uuid,
|
||||
sources: [],
|
||||
type: "textResponseChunk",
|
||||
textResponse: "",
|
||||
close: true,
|
||||
error: `AWSBedrock:streaming - could not stream chat. ${
|
||||
error?.cause ?? error.message
|
||||
}`,
|
||||
});
|
||||
response.removeListener("close", handleAbort);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Simple wrapper for dynamic embedder & normalize interface for all LLM implementations
|
||||
async embedTextInput(textInput) {
|
||||
return await this.embedder.embedTextInput(textInput);
|
||||
}
|
||||
async embedChunks(textChunks = []) {
|
||||
return await this.embedder.embedChunks(textChunks);
|
||||
}
|
||||
|
||||
async compressMessages(promptArgs = {}, rawHistory = []) {
|
||||
const { messageArrayCompressor } = require("../../helpers/chat");
|
||||
const messageArray = this.constructPrompt(promptArgs);
|
||||
return await messageArrayCompressor(this, messageArray, rawHistory);
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
AWSBedrockLLM,
|
||||
};
|
||||
@ -775,6 +775,8 @@ ${this.getHistory({ to: route.to })
|
||||
return new Providers.PerplexityProvider({ model: config.model });
|
||||
case "textgenwebui":
|
||||
return new Providers.TextWebGenUiProvider({});
|
||||
case "bedrock":
|
||||
return new Providers.AWSBedrockProvider({});
|
||||
|
||||
default:
|
||||
throw new Error(
|
||||
|
||||
@ -1,60 +0,0 @@
|
||||
const odbc = require("odbc");
|
||||
const UrlPattern = require("url-pattern");
|
||||
|
||||
class ODBCConnector {
|
||||
#connected = false;
|
||||
database_id = "";
|
||||
constructor(
|
||||
config = {
|
||||
connectionString: null,
|
||||
}
|
||||
) {
|
||||
this.connectionString = config.connectionString;
|
||||
this._client = null;
|
||||
this.database_id = this.#parseDatabase();
|
||||
}
|
||||
|
||||
#parseDatabase() {
|
||||
const regex = /Database=([^;]+)/;
|
||||
const match = this.connectionString.match(regex);
|
||||
return match ? match[1] : null;
|
||||
}
|
||||
|
||||
async connect() {
|
||||
this._client = await odbc.connect(this.connectionString);
|
||||
this.#connected = true;
|
||||
return this._client;
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param {string} queryString the SQL query to be run
|
||||
* @returns {import(".").QueryResult}
|
||||
*/
|
||||
async runQuery(queryString = "") {
|
||||
const result = { rows: [], count: 0, error: null };
|
||||
try {
|
||||
if (!this.#connected) await this.connect();
|
||||
const query = await this._client.query(queryString);
|
||||
result.rows = query;
|
||||
result.count = query.length;
|
||||
} catch (err) {
|
||||
console.log(this.constructor.name, err);
|
||||
result.error = err.message;
|
||||
} finally {
|
||||
await this._client.close();
|
||||
this.#connected = false;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
getTablesSql() {
|
||||
return `SELECT table_name FROM information_schema.tables WHERE table_schema = '${this.database_id}'`;
|
||||
}
|
||||
|
||||
getTableSchemaSql(table_name) {
|
||||
return `SHOW COLUMNS FROM ${this.database_id}.${table_name};`;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports.ODBCConnector = ODBCConnector;
|
||||
@ -2,7 +2,7 @@ const { SystemSettings } = require("../../../../../../models/systemSettings");
|
||||
const { safeJsonParse } = require("../../../../../http");
|
||||
|
||||
/**
|
||||
* @typedef {('postgresql'|'mysql'|'sql-server'|'odbc')} SQLEngine
|
||||
* @typedef {('postgresql'|'mysql'|'sql-server')} SQLEngine
|
||||
*/
|
||||
|
||||
/**
|
||||
@ -36,9 +36,6 @@ function getDBClient(identifier = "", connectionConfig = {}) {
|
||||
case "sql-server":
|
||||
const { MSSQLConnector } = require("./MSSQL");
|
||||
return new MSSQLConnector(connectionConfig);
|
||||
case "odbc":
|
||||
const { ODBCConnector } = require("./ODBC");
|
||||
return new ODBCConnector(connectionConfig);
|
||||
default:
|
||||
throw new Error(
|
||||
`There is no supported database connector for ${identifier}`
|
||||
|
||||
@ -12,6 +12,7 @@
|
||||
|
||||
const { ChatOpenAI } = require("@langchain/openai");
|
||||
const { ChatAnthropic } = require("@langchain/anthropic");
|
||||
const { ChatBedrockConverse } = require("@langchain/aws");
|
||||
const { ChatOllama } = require("@langchain/community/chat_models/ollama");
|
||||
const { toValidNumber } = require("../../../http");
|
||||
|
||||
@ -113,6 +114,16 @@ class Provider {
|
||||
),
|
||||
...config,
|
||||
});
|
||||
case "bedrock":
|
||||
return new ChatBedrockConverse({
|
||||
model: process.env.AWS_BEDROCK_LLM_MODEL_PREFERENCE,
|
||||
region: process.env.AWS_BEDROCK_LLM_REGION,
|
||||
credentials: {
|
||||
accessKeyId: process.env.AWS_BEDROCK_LLM_ACCESS_KEY_ID,
|
||||
secretAccessKey: process.env.AWS_BEDROCK_LLM_ACCESS_KEY,
|
||||
},
|
||||
...config,
|
||||
});
|
||||
|
||||
// OSS Model Runners
|
||||
// case "anythingllm_ollama":
|
||||
|
||||
136
server/utils/agents/aibitat/providers/bedrock.js
Normal file
136
server/utils/agents/aibitat/providers/bedrock.js
Normal file
@ -0,0 +1,136 @@
|
||||
const Provider = require("./ai-provider.js");
|
||||
const InheritMultiple = require("./helpers/classes.js");
|
||||
const UnTooled = require("./helpers/untooled.js");
|
||||
const { ChatBedrockConverse } = require("@langchain/aws");
|
||||
const {
|
||||
HumanMessage,
|
||||
SystemMessage,
|
||||
AIMessage,
|
||||
} = require("@langchain/core/messages");
|
||||
|
||||
/**
|
||||
* The agent provider for the AWS Bedrock provider.
|
||||
*/
|
||||
class AWSBedrockProvider extends InheritMultiple([Provider, UnTooled]) {
|
||||
model;
|
||||
|
||||
constructor(_config = {}) {
|
||||
super();
|
||||
const model = process.env.AWS_BEDROCK_LLM_MODEL_PREFERENCE ?? null;
|
||||
const client = new ChatBedrockConverse({
|
||||
region: process.env.AWS_BEDROCK_LLM_REGION,
|
||||
credentials: {
|
||||
accessKeyId: process.env.AWS_BEDROCK_LLM_ACCESS_KEY_ID,
|
||||
secretAccessKey: process.env.AWS_BEDROCK_LLM_ACCESS_KEY,
|
||||
},
|
||||
model,
|
||||
});
|
||||
|
||||
this._client = client;
|
||||
this.model = model;
|
||||
this.verbose = true;
|
||||
}
|
||||
|
||||
get client() {
|
||||
return this._client;
|
||||
}
|
||||
|
||||
// For streaming we use Langchain's wrapper to handle weird chunks
|
||||
// or otherwise absorb headaches that can arise from Ollama models
|
||||
#convertToLangchainPrototypes(chats = []) {
|
||||
const langchainChats = [];
|
||||
const roleToMessageMap = {
|
||||
system: SystemMessage,
|
||||
user: HumanMessage,
|
||||
assistant: AIMessage,
|
||||
};
|
||||
|
||||
for (const chat of chats) {
|
||||
if (!roleToMessageMap.hasOwnProperty(chat.role)) continue;
|
||||
const MessageClass = roleToMessageMap[chat.role];
|
||||
langchainChats.push(new MessageClass({ content: chat.content }));
|
||||
}
|
||||
|
||||
return langchainChats;
|
||||
}
|
||||
|
||||
async #handleFunctionCallChat({ messages = [] }) {
|
||||
const response = await this.client
|
||||
.invoke(this.#convertToLangchainPrototypes(messages))
|
||||
.then((res) => res)
|
||||
.catch((e) => {
|
||||
console.error(e);
|
||||
return null;
|
||||
});
|
||||
|
||||
return response?.content;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a completion based on the received messages.
|
||||
*
|
||||
* @param messages A list of messages to send to the API.
|
||||
* @param functions
|
||||
* @returns The completion.
|
||||
*/
|
||||
async complete(messages, functions = null) {
|
||||
try {
|
||||
let completion;
|
||||
if (functions.length > 0) {
|
||||
const { toolCall, text } = await this.functionCall(
|
||||
messages,
|
||||
functions,
|
||||
this.#handleFunctionCallChat.bind(this)
|
||||
);
|
||||
|
||||
if (toolCall !== null) {
|
||||
this.providerLog(`Valid tool call found - running ${toolCall.name}.`);
|
||||
this.deduplicator.trackRun(toolCall.name, toolCall.arguments);
|
||||
return {
|
||||
result: null,
|
||||
functionCall: {
|
||||
name: toolCall.name,
|
||||
arguments: toolCall.arguments,
|
||||
},
|
||||
cost: 0,
|
||||
};
|
||||
}
|
||||
completion = { content: text };
|
||||
}
|
||||
|
||||
if (!completion?.content) {
|
||||
this.providerLog(
|
||||
"Will assume chat completion without tool call inputs."
|
||||
);
|
||||
const response = await this.client.invoke(
|
||||
this.#convertToLangchainPrototypes(this.cleanMsgs(messages))
|
||||
);
|
||||
completion = response;
|
||||
}
|
||||
|
||||
// The UnTooled class inherited Deduplicator is mostly useful to prevent the agent
|
||||
// from calling the exact same function over and over in a loop within a single chat exchange
|
||||
// _but_ we should enable it to call previously used tools in a new chat interaction.
|
||||
this.deduplicator.reset("runs");
|
||||
return {
|
||||
result: completion.content,
|
||||
cost: 0,
|
||||
};
|
||||
} catch (error) {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the cost of the completion.
|
||||
*
|
||||
* @param _usage The completion to get the cost for.
|
||||
* @returns The cost of the completion.
|
||||
* Stubbed since KoboldCPP has no cost basis.
|
||||
*/
|
||||
getCost(_usage) {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = AWSBedrockProvider;
|
||||
@ -12,6 +12,7 @@ const MistralProvider = require("./mistral.js");
|
||||
const GenericOpenAiProvider = require("./genericOpenAi.js");
|
||||
const PerplexityProvider = require("./perplexity.js");
|
||||
const TextWebGenUiProvider = require("./textgenwebui.js");
|
||||
const AWSBedrockProvider = require("./bedrock.js");
|
||||
|
||||
module.exports = {
|
||||
OpenAIProvider,
|
||||
@ -28,4 +29,5 @@ module.exports = {
|
||||
GenericOpenAiProvider,
|
||||
PerplexityProvider,
|
||||
TextWebGenUiProvider,
|
||||
AWSBedrockProvider,
|
||||
};
|
||||
|
||||
@ -143,6 +143,17 @@ class AgentHandler {
|
||||
"TextWebGenUI API base path must be provided to use agents."
|
||||
);
|
||||
break;
|
||||
case "bedrock":
|
||||
if (
|
||||
!process.env.AWS_BEDROCK_LLM_ACCESS_KEY_ID ||
|
||||
!process.env.AWS_BEDROCK_LLM_ACCESS_KEY ||
|
||||
!process.env.AWS_BEDROCK_LLM_REGION ||
|
||||
!process.env.AWS_BEDROCK_LLM_MODEL_PREFERENCE
|
||||
)
|
||||
throw new Error(
|
||||
"AWS Bedrock Access Keys, model and region must be provided to use agents."
|
||||
);
|
||||
break;
|
||||
|
||||
default:
|
||||
throw new Error(
|
||||
@ -183,6 +194,8 @@ class AgentHandler {
|
||||
return "sonar-small-online";
|
||||
case "textgenwebui":
|
||||
return null;
|
||||
case "bedrock":
|
||||
return null;
|
||||
default:
|
||||
return "unknown";
|
||||
}
|
||||
@ -210,7 +223,7 @@ class AgentHandler {
|
||||
}
|
||||
|
||||
#providerSetupAndCheck() {
|
||||
this.provider = this.invocation.workspace.agentProvider || "openai";
|
||||
this.provider = this.invocation.workspace.agentProvider;
|
||||
this.model = this.#fetchModel();
|
||||
this.log(`Start ${this.#invocationUUID}::${this.provider}:${this.model}`);
|
||||
this.#checkSetup();
|
||||
|
||||
@ -140,6 +140,9 @@ function getLLMProvider({ provider = null, model = null } = {}) {
|
||||
case "generic-openai":
|
||||
const { GenericOpenAiLLM } = require("../AiProviders/genericOpenAi");
|
||||
return new GenericOpenAiLLM(embedder, model);
|
||||
case "bedrock":
|
||||
const { AWSBedrockLLM } = require("../AiProviders/bedrock");
|
||||
return new AWSBedrockLLM(embedder, model);
|
||||
default:
|
||||
throw new Error(
|
||||
`ENV: No valid LLM_PROVIDER value found in environment! Using ${process.env.LLM_PROVIDER}`
|
||||
|
||||
@ -208,6 +208,28 @@ const KEY_MAPPING = {
|
||||
checks: [nonZero],
|
||||
},
|
||||
|
||||
// AWS Bedrock LLM InferenceSettings
|
||||
AwsBedrockLLMAccessKeyId: {
|
||||
envKey: "AWS_BEDROCK_LLM_ACCESS_KEY_ID",
|
||||
checks: [isNotEmpty],
|
||||
},
|
||||
AwsBedrockLLMAccessKey: {
|
||||
envKey: "AWS_BEDROCK_LLM_ACCESS_KEY",
|
||||
checks: [isNotEmpty],
|
||||
},
|
||||
AwsBedrockLLMRegion: {
|
||||
envKey: "AWS_BEDROCK_LLM_REGION",
|
||||
checks: [isNotEmpty],
|
||||
},
|
||||
AwsBedrockLLMModel: {
|
||||
envKey: "AWS_BEDROCK_LLM_MODEL_PREFERENCE",
|
||||
checks: [isNotEmpty],
|
||||
},
|
||||
AwsBedrockLLMTokenLimit: {
|
||||
envKey: "AWS_BEDROCK_LLM_MODEL_TOKEN_LIMIT",
|
||||
checks: [nonZero],
|
||||
},
|
||||
|
||||
EmbeddingEngine: {
|
||||
envKey: "EMBEDDING_ENGINE",
|
||||
checks: [supportedEmbeddingModel],
|
||||
@ -541,6 +563,7 @@ function supportedLLM(input = "") {
|
||||
"cohere",
|
||||
"litellm",
|
||||
"generic-openai",
|
||||
"bedrock",
|
||||
].includes(input);
|
||||
return validSelection ? null : `${input} is not a valid LLM provider.`;
|
||||
}
|
||||
|
||||
1064
server/yarn.lock
1064
server/yarn.lock
File diff suppressed because it is too large
Load Diff
Loading…
Reference in New Issue
Block a user