Add AWS bedrock support for LLM + agents (#1935)

add AWS bedrock support for LLM + agents

README.md

@@ -75,6 +75,7 @@ Some cool features of AnythingLLM
 - [OpenAI](https://openai.com)
 - [OpenAI (Generic)](https://openai.com)
 - [Azure OpenAI](https://azure.microsoft.com/en-us/products/ai-services/openai-service)
+- [AWS Bedrock](https://aws.amazon.com/bedrock/)
 - [Anthropic](https://www.anthropic.com/)
 - [Google Gemini Pro](https://ai.google.dev/)
 - [Hugging Face (chat models)](https://huggingface.co/)

docker/.env.example

@@ -94,6 +94,13 @@ GID='1000'
 # COHERE_API_KEY=
 # COHERE_MODEL_PREF='command-r'
 
+# LLM_PROVIDER='bedrock'
+# AWS_BEDROCK_LLM_ACCESS_KEY_ID=
+# AWS_BEDROCK_LLM_ACCESS_KEY=
+# AWS_BEDROCK_LLM_REGION=us-west-2
+# AWS_BEDROCK_LLM_MODEL_PREFERENCE=meta.llama3-1-8b-instruct-v1:0
+# AWS_BEDROCK_LLM_MODEL_TOKEN_LIMIT=8191
+
 ###########################################
 ######## Embedding API SElECTION ##########
 ###########################################

frontend/src/components/LLMSelection/AwsBedrockLLMOptions/index.jsx

@@ -0,0 +1,122 @@
+import { ArrowSquareOut, Info } from "@phosphor-icons/react";
+import { AWS_REGIONS } from "./regions";
+
+export default function AwsBedrockLLMOptions({ settings }) {
+  return (
+    <div className="w-full flex flex-col">
+      {!settings?.credentialsOnly && (
+        <div className="flex flex-col md:flex-row md:items-center gap-x-2 text-white mb-4 bg-blue-800/30 w-fit rounded-lg px-4 py-2">
+          <div className="gap-x-2 flex items-center">
+            <Info size={40} />
+            <p className="text-base">
+              You should use a properly defined IAM user for inferencing.
+              <br />
+              <a
+                href="https://docs.useanything.com/setup/llm-configuration/cloud/aws-bedrock"
+                target="_blank"
+                className="underline flex gap-x-1 items-center"
+              >
+                Read more on how to use AWS Bedrock in AnythingLLM
+                <ArrowSquareOut size={14} />
+              </a>
+            </p>
+          </div>
+        </div>
+      )}
+
+      <div className="w-full flex items-center gap-[36px] my-1.5">
+        <div className="flex flex-col w-60">
+          <label className="text-white text-sm font-semibold block mb-3">
+            AWS Bedrock IAM Access ID
+          </label>
+          <input
+            type="password"
+            name="AwsBedrockLLMAccessKeyId"
+            className="border-none bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:outline-primary-button active:outline-primary-button outline-none block w-full p-2.5"
+            placeholder="AWS Bedrock IAM User Access ID"
+            defaultValue={
+              settings?.AwsBedrockLLMAccessKeyId ? "*".repeat(20) : ""
+            }
+            required={true}
+            autoComplete="off"
+            spellCheck={false}
+          />
+        </div>
+        <div className="flex flex-col w-60">
+          <label className="text-white text-sm font-semibold block mb-3">
+            AWS Bedrock IAM Access Key
+          </label>
+          <input
+            type="password"
+            name="AwsBedrockLLMAccessKey"
+            className="border-none bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:outline-primary-button active:outline-primary-button outline-none block w-full p-2.5"
+            placeholder="AWS Bedrock IAM User Access Key"
+            defaultValue={
+              settings?.AwsBedrockLLMAccessKey ? "*".repeat(20) : ""
+            }
+            required={true}
+            autoComplete="off"
+            spellCheck={false}
+          />
+        </div>
+        <div className="flex flex-col w-60">
+          <label className="text-white text-sm font-semibold block mb-3">
+            AWS region
+          </label>
+          <select
+            name="AwsBedrockLLMRegion"
+            defaultValue={settings?.AwsBedrockLLMRegion || "us-west-2"}
+            required={true}
+            className="border-none bg-zinc-900 border-gray-500 text-white text-sm rounded-lg block w-full p-2.5"
+          >
+            {AWS_REGIONS.map((region) => {
+              return (
+                <option key={region.code} value={region.code}>
+                  {region.name} ({region.code})
+                </option>
+              );
+            })}
+          </select>
+        </div>
+      </div>
+
+      <div className="w-full flex items-center gap-[36px] my-1.5">
+        {!settings?.credentialsOnly && (
+          <>
+            <div className="flex flex-col w-60">
+              <label className="text-white text-sm font-semibold block mb-3">
+                Model ID
+              </label>
+              <input
+                type="text"
+                name="AwsBedrockLLMModel"
+                className="border-none bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:outline-primary-button active:outline-primary-button outline-none block w-full p-2.5"
+                placeholder="Model id from AWS eg: meta.llama3.1-v0.1"
+                defaultValue={settings?.AwsBedrockLLMModel}
+                required={true}
+                autoComplete="off"
+                spellCheck={false}
+              />
+            </div>
+            <div className="flex flex-col w-60">
+              <label className="text-white text-sm font-semibold block mb-3">
+                Model context window
+              </label>
+              <input
+                type="number"
+                name="AwsBedrockLLMTokenLimit"
+                className="border-none bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:outline-primary-button active:outline-primary-button outline-none block w-full p-2.5"
+                placeholder="Content window limit (eg: 4096)"
+                min={1}
+                onScroll={(e) => e.target.blur()}
+                defaultValue={settings?.AwsBedrockLLMTokenLimit}
+                required={true}
+                autoComplete="off"
+              />
+            </div>
+          </>
+        )}
+      </div>
+    </div>
+  );
+}

frontend/src/components/LLMSelection/AwsBedrockLLMOptions/regions.js

@@ -0,0 +1,204 @@
+export const AWS_REGIONS = [
+  {
+    name: "N. Virginia",
+    full_name: "US East (N. Virginia)",
+    code: "us-east-1",
+    public: true,
+    zones: [
+      "us-east-1a",
+      "us-east-1b",
+      "us-east-1c",
+      "us-east-1d",
+      "us-east-1e",
+      "us-east-1f",
+    ],
+  },
+  {
+    name: "Ohio",
+    full_name: "US East (Ohio)",
+    code: "us-east-2",
+    public: true,
+    zones: ["us-east-2a", "us-east-2b", "us-east-2c"],
+  },
+  {
+    name: "N. California",
+    full_name: "US West (N. California)",
+    code: "us-west-1",
+    public: true,
+    zone_limit: 2,
+    zones: ["us-west-1a", "us-west-1b", "us-west-1c"],
+  },
+  {
+    name: "Oregon",
+    full_name: "US West (Oregon)",
+    code: "us-west-2",
+    public: true,
+    zones: ["us-west-2a", "us-west-2b", "us-west-2c", "us-west-2d"],
+  },
+  {
+    name: "GovCloud West",
+    full_name: "AWS GovCloud (US)",
+    code: "us-gov-west-1",
+    public: false,
+    zones: ["us-gov-west-1a", "us-gov-west-1b", "us-gov-west-1c"],
+  },
+  {
+    name: "GovCloud East",
+    full_name: "AWS GovCloud (US-East)",
+    code: "us-gov-east-1",
+    public: false,
+    zones: ["us-gov-east-1a", "us-gov-east-1b", "us-gov-east-1c"],
+  },
+  {
+    name: "Canada",
+    full_name: "Canada (Central)",
+    code: "ca-central-1",
+    public: true,
+    zones: ["ca-central-1a", "ca-central-1b", "ca-central-1c", "ca-central-1d"],
+  },
+  {
+    name: "Stockholm",
+    full_name: "EU (Stockholm)",
+    code: "eu-north-1",
+    public: true,
+    zones: ["eu-north-1a", "eu-north-1b", "eu-north-1c"],
+  },
+  {
+    name: "Ireland",
+    full_name: "EU (Ireland)",
+    code: "eu-west-1",
+    public: true,
+    zones: ["eu-west-1a", "eu-west-1b", "eu-west-1c"],
+  },
+  {
+    name: "London",
+    full_name: "EU (London)",
+    code: "eu-west-2",
+    public: true,
+    zones: ["eu-west-2a", "eu-west-2b", "eu-west-2c"],
+  },
+  {
+    name: "Paris",
+    full_name: "EU (Paris)",
+    code: "eu-west-3",
+    public: true,
+    zones: ["eu-west-3a", "eu-west-3b", "eu-west-3c"],
+  },
+  {
+    name: "Frankfurt",
+    full_name: "EU (Frankfurt)",
+    code: "eu-central-1",
+    public: true,
+    zones: ["eu-central-1a", "eu-central-1b", "eu-central-1c"],
+  },
+  {
+    name: "Milan",
+    full_name: "EU (Milan)",
+    code: "eu-south-1",
+    public: true,
+    zones: ["eu-south-1a", "eu-south-1b", "eu-south-1c"],
+  },
+  {
+    name: "Cape Town",
+    full_name: "Africa (Cape Town)",
+    code: "af-south-1",
+    public: true,
+    zones: ["af-south-1a", "af-south-1b", "af-south-1c"],
+  },
+  {
+    name: "Tokyo",
+    full_name: "Asia Pacific (Tokyo)",
+    code: "ap-northeast-1",
+    public: true,
+    zone_limit: 3,
+    zones: [
+      "ap-northeast-1a",
+      "ap-northeast-1b",
+      "ap-northeast-1c",
+      "ap-northeast-1d",
+    ],
+  },
+  {
+    name: "Seoul",
+    full_name: "Asia Pacific (Seoul)",
+    code: "ap-northeast-2",
+    public: true,
+    zones: [
+      "ap-northeast-2a",
+      "ap-northeast-2b",
+      "ap-northeast-2c",
+      "ap-northeast-2d",
+    ],
+  },
+  {
+    name: "Osaka",
+    full_name: "Asia Pacific (Osaka-Local)",
+    code: "ap-northeast-3",
+    public: true,
+    zones: ["ap-northeast-3a", "ap-northeast-3b", "ap-northeast-3c"],
+  },
+  {
+    name: "Singapore",
+    full_name: "Asia Pacific (Singapore)",
+    code: "ap-southeast-1",
+    public: true,
+    zones: ["ap-southeast-1a", "ap-southeast-1b", "ap-southeast-1c"],
+  },
+  {
+    name: "Sydney",
+    full_name: "Asia Pacific (Sydney)",
+    code: "ap-southeast-2",
+    public: true,
+    zones: ["ap-southeast-2a", "ap-southeast-2b", "ap-southeast-2c"],
+  },
+  {
+    name: "Jakarta",
+    full_name: "Asia Pacific (Jakarta)",
+    code: "ap-southeast-3",
+    public: true,
+    zones: ["ap-southeast-3a", "ap-southeast-3b", "ap-southeast-3c"],
+  },
+  {
+    name: "Hong Kong",
+    full_name: "Asia Pacific (Hong Kong)",
+    code: "ap-east-1",
+    public: true,
+    zones: ["ap-east-1a", "ap-east-1b", "ap-east-1c"],
+  },
+  {
+    name: "Mumbai",
+    full_name: "Asia Pacific (Mumbai)",
+    code: "ap-south-1",
+    public: true,
+    zones: ["ap-south-1a", "ap-south-1b", "ap-south-1c"],
+  },
+  {
+    name: "São Paulo",
+    full_name: "South America (São Paulo)",
+    code: "sa-east-1",
+    public: true,
+    zone_limit: 2,
+    zones: ["sa-east-1a", "sa-east-1b", "sa-east-1c"],
+  },
+  {
+    name: "Bahrain",
+    full_name: "Middle East (Bahrain)",
+    code: "me-south-1",
+    public: true,
+    zones: ["me-south-1a", "me-south-1b", "me-south-1c"],
+  },
+  {
+    name: "Beijing",
+    full_name: "China (Beijing)",
+    code: "cn-north-1",
+    public: false,
+    zones: ["cn-north-1a", "cn-north-1b", "cn-north-1c"],
+  },
+  {
+    name: "Ningxia",
+    full_name: "China (Ningxia)",
+    code: "cn-northwest-1",
+    public: false,
+    zones: ["cn-northwest-1a", "cn-northwest-1b", "cn-northwest-1c"],
+  },
+];

frontend/src/hooks/useGetProvidersModels.js

@@ -8,6 +8,7 @@ export const DISABLED_PROVIDERS = [
   "native",
   "textgenwebui",
   "generic-openai",
+  "bedrock",
 ];
 const PROVIDER_DEFAULT_MODELS = {
   openai: [],
@@ -48,6 +49,7 @@ const PROVIDER_DEFAULT_MODELS = {
   ],
   textgenwebui: [],
   "generic-openai": [],
+  bedrock: [],
 };
 
 // For togetherAi, which has a large model list - we subgroup the options

frontend/src/pages/GeneralSettings/LLMPreference/index.jsx

@@ -23,6 +23,8 @@ import KoboldCPPLogo from "@/media/llmprovider/koboldcpp.png";
 import TextGenWebUILogo from "@/media/llmprovider/text-generation-webui.png";
 import CohereLogo from "@/media/llmprovider/cohere.png";
 import LiteLLMLogo from "@/media/llmprovider/litellm.png";
+import AWSBedrockLogo from "@/media/llmprovider/bedrock.png";
+
 import PreLoader from "@/components/Preloader";
 import OpenAiOptions from "@/components/LLMSelection/OpenAiOptions";
 import GenericOpenAiOptions from "@/components/LLMSelection/GenericOpenAiOptions";
@@ -43,6 +45,7 @@ import CohereAiOptions from "@/components/LLMSelection/CohereAiOptions";
 import KoboldCPPOptions from "@/components/LLMSelection/KoboldCPPOptions";
 import TextGenWebUIOptions from "@/components/LLMSelection/TextGenWebUIOptions";
 import LiteLLMOptions from "@/components/LLMSelection/LiteLLMOptions";
+import AWSBedrockLLMOptions from "@/components/LLMSelection/AwsBedrockLLMOptions";
 
 import LLMItem from "@/components/LLMSelection/LLMItem";
 import { CaretUpDown, MagnifyingGlass, X } from "@phosphor-icons/react";
@@ -211,6 +214,19 @@ export const AVAILABLE_LLM_PROVIDERS = [
       "GenericOpenAiKey",
     ],
   },
+  {
+    name: "AWS Bedrock",
+    value: "bedrock",
+    logo: AWSBedrockLogo,
+    options: (settings) => <AWSBedrockLLMOptions settings={settings} />,
+    description: "Run powerful foundation models privately with AWS Bedrock.",
+    requiredConfig: [
+      "AwsBedrockLLMAccessKeyId",
+      "AwsBedrockLLMAccessKey",
+      "AwsBedrockLLMRegion",
+      "AwsBedrockLLMModel",
+    ],
+  },
   {
     name: "Native",
     value: "native",

frontend/src/pages/OnboardingFlow/Steps/DataHandling/index.jsx

@@ -18,6 +18,7 @@ import GroqLogo from "@/media/llmprovider/groq.png";
 import KoboldCPPLogo from "@/media/llmprovider/koboldcpp.png";
 import TextGenWebUILogo from "@/media/llmprovider/text-generation-webui.png";
 import LiteLLMLogo from "@/media/llmprovider/litellm.png";
+import AWSBedrockLogo from "@/media/llmprovider/bedrock.png";
 
 import CohereLogo from "@/media/llmprovider/cohere.png";
 import ZillizLogo from "@/media/vectordbs/zilliz.png";
@@ -179,6 +180,13 @@ export const LLM_SELECTION_PRIVACY = {
     ],
     logo: LiteLLMLogo,
   },
+  bedrock: {
+    name: "AWS Bedrock",
+    description: [
+      "You model and chat contents are subject to the agreed EULA for AWS and the model provider on aws.amazon.com",
+    ],
+    logo: AWSBedrockLogo,
+  },
 };
 
 export const VECTOR_DB_PRIVACY = {

frontend/src/pages/OnboardingFlow/Steps/LLMPreference/index.jsx

@@ -18,6 +18,7 @@ import GroqLogo from "@/media/llmprovider/groq.png";
 import KoboldCPPLogo from "@/media/llmprovider/koboldcpp.png";
 import TextGenWebUILogo from "@/media/llmprovider/text-generation-webui.png";
 import LiteLLMLogo from "@/media/llmprovider/litellm.png";
+import AWSBedrockLogo from "@/media/llmprovider/bedrock.png";
 
 import CohereLogo from "@/media/llmprovider/cohere.png";
 import OpenAiOptions from "@/components/LLMSelection/OpenAiOptions";
@@ -39,6 +40,7 @@ import CohereAiOptions from "@/components/LLMSelection/CohereAiOptions";
 import KoboldCPPOptions from "@/components/LLMSelection/KoboldCPPOptions";
 import TextGenWebUIOptions from "@/components/LLMSelection/TextGenWebUIOptions";
 import LiteLLMOptions from "@/components/LLMSelection/LiteLLMOptions";
+import AWSBedrockLLMOptions from "@/components/LLMSelection/AwsBedrockLLMOptions";
 
 import LLMItem from "@/components/LLMSelection/LLMItem";
 import System from "@/models/system";
@@ -182,6 +184,13 @@ const LLMS = [
     description:
       "Connect to any OpenAi-compatible service via a custom configuration",
   },
+  {
+    name: "AWS Bedrock",
+    value: "bedrock",
+    logo: AWSBedrockLogo,
+    options: (settings) => <AWSBedrockLLMOptions settings={settings} />,
+    description: "Run powerful foundation models privately with AWS Bedrock.",
+  },
   {
     name: "Native",
     value: "native",

frontend/src/pages/WorkspaceSettings/AgentConfig/AgentLLMSelection/index.jsx

@@ -21,6 +21,7 @@ const ENABLED_PROVIDERS = [
   "perplexity",
   "textgenwebui",
   "generic-openai",
+  "bedrock",
   // TODO: More agent support.
   // "cohere",         // Has tool calling and will need to build explicit support
   // "huggingface"     // Can be done but already has issues with no-chat templated. Needs to be tested.

frontend/src/pages/WorkspaceSettings/ChatSettings/WorkspaceLLMSelection/index.jsx

@@ -10,7 +10,12 @@ import paths from "@/utils/paths";
 
 // Some providers can only be associated with a single model.
 // In that case there is no selection to be made so we can just move on.
-const NO_MODEL_SELECTION = ["default", "huggingface", "generic-openai"];
+const NO_MODEL_SELECTION = [
+  "default",
+  "huggingface",
+  "generic-openai",
+  "bedrock",
+];
 const DISABLED_PROVIDERS = ["azure", "lmstudio", "native"];
 const LLM_DEFAULT = {
   name: "System default",

server/models/systemSettings.js

@@ -465,6 +465,12 @@ const SystemSettings = {
       GenericOpenAiKey: !!process.env.GENERIC_OPEN_AI_API_KEY,
       GenericOpenAiMaxTokens: process.env.GENERIC_OPEN_AI_MAX_TOKENS,
 
+      AwsBedrockLLMAccessKeyId: !!process.env.AWS_BEDROCK_LLM_ACCESS_KEY_ID,
+      AwsBedrockLLMAccessKey: !!process.env.AWS_BEDROCK_LLM_ACCESS_KEY,
+      AwsBedrockLLMRegion: process.env.AWS_BEDROCK_LLM_REGION,
+      AwsBedrockLLMModel: process.env.AWS_BEDROCK_LLM_MODEL_PREFERENCE,
+      AwsBedrockLLMTokenLimit: process.env.AWS_BEDROCK_LLM_MODEL_TOKEN_LIMIT,
+
       // Cohere API Keys
       CohereApiKey: !!process.env.COHERE_API_KEY,
       CohereModelPref: process.env.COHERE_MODEL_PREF,

server/package.json

@@ -28,6 +28,7 @@
     "@ladjs/graceful": "^3.2.2",
     "@lancedb/lancedb": "0.5.2",
     "@langchain/anthropic": "0.1.16",
+    "@langchain/aws": "^0.0.5",
     "@langchain/community": "0.0.53",
     "@langchain/core": "0.1.61",
     "@langchain/openai": "0.0.28",

server/utils/AiProviders/bedrock/index.js

@@ -0,0 +1,217 @@
+const { StringOutputParser } = require("@langchain/core/output_parsers");
+const {
+  writeResponseChunk,
+  clientAbortedHandler,
+} = require("../../helpers/chat/responses");
+const { NativeEmbedder } = require("../../EmbeddingEngines/native");
+
+// Docs: https://js.langchain.com/v0.2/docs/integrations/chat/bedrock_converse
+class AWSBedrockLLM {
+  constructor(embedder = null, modelPreference = null) {
+    if (!process.env.AWS_BEDROCK_LLM_ACCESS_KEY_ID)
+      throw new Error("No AWS Bedrock LLM profile id was set.");
+
+    if (!process.env.AWS_BEDROCK_LLM_ACCESS_KEY)
+      throw new Error("No AWS Bedrock LLM access key was set.");
+
+    if (!process.env.AWS_BEDROCK_LLM_REGION)
+      throw new Error("No AWS Bedrock LLM region was set.");
+
+    this.model =
+      modelPreference || process.env.AWS_BEDROCK_LLM_MODEL_PREFERENCE;
+    this.limits = {
+      history: this.promptWindowLimit() * 0.15,
+      system: this.promptWindowLimit() * 0.15,
+      user: this.promptWindowLimit() * 0.7,
+    };
+
+    this.embedder = embedder ?? new NativeEmbedder();
+    this.defaultTemp = 0.7;
+  }
+
+  #bedrockClient({ temperature = 0.7 }) {
+    const { ChatBedrockConverse } = require("@langchain/aws");
+    return new ChatBedrockConverse({
+      model: process.env.AWS_BEDROCK_LLM_MODEL_PREFERENCE,
+      region: process.env.AWS_BEDROCK_LLM_REGION,
+      credentials: {
+        accessKeyId: process.env.AWS_BEDROCK_LLM_ACCESS_KEY_ID,
+        secretAccessKey: process.env.AWS_BEDROCK_LLM_ACCESS_KEY,
+      },
+      temperature,
+    });
+  }
+
+  // For streaming we use Langchain's wrapper to handle weird chunks
+  // or otherwise absorb headaches that can arise from Ollama models
+  #convertToLangchainPrototypes(chats = []) {
+    const {
+      HumanMessage,
+      SystemMessage,
+      AIMessage,
+    } = require("@langchain/core/messages");
+    const langchainChats = [];
+    const roleToMessageMap = {
+      system: SystemMessage,
+      user: HumanMessage,
+      assistant: AIMessage,
+    };
+
+    for (const chat of chats) {
+      if (!roleToMessageMap.hasOwnProperty(chat.role)) continue;
+      const MessageClass = roleToMessageMap[chat.role];
+      langchainChats.push(new MessageClass({ content: chat.content }));
+    }
+
+    return langchainChats;
+  }
+
+  #appendContext(contextTexts = []) {
+    if (!contextTexts || !contextTexts.length) return "";
+    return (
+      "\nContext:\n" +
+      contextTexts
+        .map((text, i) => {
+          return `[CONTEXT ${i}]:\n${text}\n[END CONTEXT ${i}]\n\n`;
+        })
+        .join("")
+    );
+  }
+
+  streamingEnabled() {
+    return "streamGetChatCompletion" in this;
+  }
+
+  // Ensure the user set a value for the token limit
+  // and if undefined - assume 4096 window.
+  promptWindowLimit() {
+    const limit = process.env.AWS_BEDROCK_LLM_MODEL_TOKEN_LIMIT || 8191;
+    if (!limit || isNaN(Number(limit)))
+      throw new Error("No valid token context limit was set.");
+    return Number(limit);
+  }
+
+  async isValidChatCompletionModel(_ = "") {
+    return true;
+  }
+
+  constructPrompt({
+    systemPrompt = "",
+    contextTexts = [],
+    chatHistory = [],
+    userPrompt = "",
+  }) {
+    // AWS Mistral models do not support system prompts
+    if (this.model.startsWith("mistral"))
+      return [...chatHistory, { role: "user", content: userPrompt }];
+
+    const prompt = {
+      role: "system",
+      content: `${systemPrompt}${this.#appendContext(contextTexts)}`,
+    };
+    return [prompt, ...chatHistory, { role: "user", content: userPrompt }];
+  }
+
+  async getChatCompletion(messages = null, { temperature = 0.7 }) {
+    const model = this.#bedrockClient({ temperature });
+    const textResponse = await model
+      .pipe(new StringOutputParser())
+      .invoke(this.#convertToLangchainPrototypes(messages))
+      .catch((e) => {
+        throw new Error(
+          `AWSBedrock::getChatCompletion failed to communicate with Ollama. ${e.message}`
+        );
+      });
+
+    if (!textResponse || !textResponse.length)
+      throw new Error(`AWSBedrock::getChatCompletion text response was empty.`);
+
+    return textResponse;
+  }
+
+  async streamGetChatCompletion(messages = null, { temperature = 0.7 }) {
+    const model = this.#bedrockClient({ temperature });
+    const stream = await model
+      .pipe(new StringOutputParser())
+      .stream(this.#convertToLangchainPrototypes(messages));
+    return stream;
+  }
+
+  handleStream(response, stream, responseProps) {
+    const { uuid = uuidv4(), sources = [] } = responseProps;
+
+    return new Promise(async (resolve) => {
+      let fullText = "";
+
+      // Establish listener to early-abort a streaming response
+      // in case things go sideways or the user does not like the response.
+      // We preserve the generated text but continue as if chat was completed
+      // to preserve previously generated content.
+      const handleAbort = () => clientAbortedHandler(resolve, fullText);
+      response.on("close", handleAbort);
+
+      try {
+        for await (const chunk of stream) {
+          if (chunk === undefined)
+            throw new Error(
+              "Stream returned undefined chunk. Aborting reply - check model provider logs."
+            );
+
+          const content = chunk.hasOwnProperty("content")
+            ? chunk.content
+            : chunk;
+          fullText += content;
+          writeResponseChunk(response, {
+            uuid,
+            sources: [],
+            type: "textResponseChunk",
+            textResponse: content,
+            close: false,
+            error: false,
+          });
+        }
+
+        writeResponseChunk(response, {
+          uuid,
+          sources,
+          type: "textResponseChunk",
+          textResponse: "",
+          close: true,
+          error: false,
+        });
+        response.removeListener("close", handleAbort);
+        resolve(fullText);
+      } catch (error) {
+        writeResponseChunk(response, {
+          uuid,
+          sources: [],
+          type: "textResponseChunk",
+          textResponse: "",
+          close: true,
+          error: `AWSBedrock:streaming - could not stream chat. ${
+            error?.cause ?? error.message
+          }`,
+        });
+        response.removeListener("close", handleAbort);
+      }
+    });
+  }
+
+  // Simple wrapper for dynamic embedder & normalize interface for all LLM implementations
+  async embedTextInput(textInput) {
+    return await this.embedder.embedTextInput(textInput);
+  }
+  async embedChunks(textChunks = []) {
+    return await this.embedder.embedChunks(textChunks);
+  }
+
+  async compressMessages(promptArgs = {}, rawHistory = []) {
+    const { messageArrayCompressor } = require("../../helpers/chat");
+    const messageArray = this.constructPrompt(promptArgs);
+    return await messageArrayCompressor(this, messageArray, rawHistory);
+  }
+}
+
+module.exports = {
+  AWSBedrockLLM,
+};

server/utils/agents/aibitat/index.js

@@ -775,6 +775,8 @@ ${this.getHistory({ to: route.to })
         return new Providers.PerplexityProvider({ model: config.model });
       case "textgenwebui":
         return new Providers.TextWebGenUiProvider({});
+      case "bedrock":
+        return new Providers.AWSBedrockProvider({});
 
       default:
         throw new Error(

server/utils/agents/aibitat/providers/ai-provider.js

@@ -12,6 +12,7 @@
 
 const { ChatOpenAI } = require("@langchain/openai");
 const { ChatAnthropic } = require("@langchain/anthropic");
+const { ChatBedrockConverse } = require("@langchain/aws");
 const { ChatOllama } = require("@langchain/community/chat_models/ollama");
 const { toValidNumber } = require("../../../http");
 
@@ -113,6 +114,16 @@ class Provider {
           ),
           ...config,
         });
+      case "bedrock":
+        return new ChatBedrockConverse({
+          model: process.env.AWS_BEDROCK_LLM_MODEL_PREFERENCE,
+          region: process.env.AWS_BEDROCK_LLM_REGION,
+          credentials: {
+            accessKeyId: process.env.AWS_BEDROCK_LLM_ACCESS_KEY_ID,
+            secretAccessKey: process.env.AWS_BEDROCK_LLM_ACCESS_KEY,
+          },
+          ...config,
+        });
 
       // OSS Model Runners
       // case "anythingllm_ollama":

server/utils/agents/aibitat/providers/bedrock.js

@@ -0,0 +1,136 @@
+const Provider = require("./ai-provider.js");
+const InheritMultiple = require("./helpers/classes.js");
+const UnTooled = require("./helpers/untooled.js");
+const { ChatBedrockConverse } = require("@langchain/aws");
+const {
+  HumanMessage,
+  SystemMessage,
+  AIMessage,
+} = require("@langchain/core/messages");
+
+/**
+ * The agent provider for the AWS Bedrock provider.
+ */
+class AWSBedrockProvider extends InheritMultiple([Provider, UnTooled]) {
+  model;
+
+  constructor(_config = {}) {
+    super();
+    const model = process.env.AWS_BEDROCK_LLM_MODEL_PREFERENCE ?? null;
+    const client = new ChatBedrockConverse({
+      region: process.env.AWS_BEDROCK_LLM_REGION,
+      credentials: {
+        accessKeyId: process.env.AWS_BEDROCK_LLM_ACCESS_KEY_ID,
+        secretAccessKey: process.env.AWS_BEDROCK_LLM_ACCESS_KEY,
+      },
+      model,
+    });
+
+    this._client = client;
+    this.model = model;
+    this.verbose = true;
+  }
+
+  get client() {
+    return this._client;
+  }
+
+  // For streaming we use Langchain's wrapper to handle weird chunks
+  // or otherwise absorb headaches that can arise from Ollama models
+  #convertToLangchainPrototypes(chats = []) {
+    const langchainChats = [];
+    const roleToMessageMap = {
+      system: SystemMessage,
+      user: HumanMessage,
+      assistant: AIMessage,
+    };
+
+    for (const chat of chats) {
+      if (!roleToMessageMap.hasOwnProperty(chat.role)) continue;
+      const MessageClass = roleToMessageMap[chat.role];
+      langchainChats.push(new MessageClass({ content: chat.content }));
+    }
+
+    return langchainChats;
+  }
+
+  async #handleFunctionCallChat({ messages = [] }) {
+    const response = await this.client
+      .invoke(this.#convertToLangchainPrototypes(messages))
+      .then((res) => res)
+      .catch((e) => {
+        console.error(e);
+        return null;
+      });
+
+    return response?.content;
+  }
+
+  /**
+   * Create a completion based on the received messages.
+   *
+   * @param messages A list of messages to send to the API.
+   * @param functions
+   * @returns The completion.
+   */
+  async complete(messages, functions = null) {
+    try {
+      let completion;
+      if (functions.length > 0) {
+        const { toolCall, text } = await this.functionCall(
+          messages,
+          functions,
+          this.#handleFunctionCallChat.bind(this)
+        );
+
+        if (toolCall !== null) {
+          this.providerLog(`Valid tool call found - running ${toolCall.name}.`);
+          this.deduplicator.trackRun(toolCall.name, toolCall.arguments);
+          return {
+            result: null,
+            functionCall: {
+              name: toolCall.name,
+              arguments: toolCall.arguments,
+            },
+            cost: 0,
+          };
+        }
+        completion = { content: text };
+      }
+
+      if (!completion?.content) {
+        this.providerLog(
+          "Will assume chat completion without tool call inputs."
+        );
+        const response = await this.client.invoke(
+          this.#convertToLangchainPrototypes(this.cleanMsgs(messages))
+        );
+        completion = response;
+      }
+
+      // The UnTooled class inherited Deduplicator is mostly useful to prevent the agent
+      // from calling the exact same function over and over in a loop within a single chat exchange
+      // _but_ we should enable it to call previously used tools in a new chat interaction.
+      this.deduplicator.reset("runs");
+      return {
+        result: completion.content,
+        cost: 0,
+      };
+    } catch (error) {
+      throw error;
+    }
+  }
+
+  /**
+   * Get the cost of the completion.
+   *
+   * @param _usage The completion to get the cost for.
+   * @returns The cost of the completion.
+   * Stubbed since KoboldCPP has no cost basis.
+   */
+  getCost(_usage) {
+    return 0;
+  }
+}
+
+module.exports = AWSBedrockProvider;

server/utils/agents/aibitat/providers/index.js

@@ -12,6 +12,7 @@ const MistralProvider = require("./mistral.js");
 const GenericOpenAiProvider = require("./genericOpenAi.js");
 const PerplexityProvider = require("./perplexity.js");
 const TextWebGenUiProvider = require("./textgenwebui.js");
+const AWSBedrockProvider = require("./bedrock.js");
 
 module.exports = {
   OpenAIProvider,
@@ -28,4 +29,5 @@ module.exports = {
   GenericOpenAiProvider,
   PerplexityProvider,
   TextWebGenUiProvider,
+  AWSBedrockProvider,
 };

server/utils/agents/index.js

@@ -143,6 +143,17 @@ class AgentHandler {
             "TextWebGenUI API base path must be provided to use agents."
           );
         break;
+      case "bedrock":
+        if (
+          !process.env.AWS_BEDROCK_LLM_ACCESS_KEY_ID ||
+          !process.env.AWS_BEDROCK_LLM_ACCESS_KEY ||
+          !process.env.AWS_BEDROCK_LLM_REGION ||
+          !process.env.AWS_BEDROCK_LLM_MODEL_PREFERENCE
+        )
+          throw new Error(
+            "AWS Bedrock Access Keys, model and region must be provided to use agents."
+          );
+        break;
 
       default:
         throw new Error(
@@ -183,6 +194,8 @@ class AgentHandler {
         return "sonar-small-online";
       case "textgenwebui":
         return null;
+      case "bedrock":
+        return null;
       default:
         return "unknown";
     }
@@ -210,7 +223,7 @@ class AgentHandler {
   }
 
   #providerSetupAndCheck() {
-    this.provider = this.invocation.workspace.agentProvider || "openai";
+    this.provider = this.invocation.workspace.agentProvider;
     this.model = this.#fetchModel();
     this.log(`Start ${this.#invocationUUID}::${this.provider}:${this.model}`);
     this.#checkSetup();

server/utils/helpers/index.js

@@ -140,6 +140,9 @@ function getLLMProvider({ provider = null, model = null } = {}) {
     case "generic-openai":
       const { GenericOpenAiLLM } = require("../AiProviders/genericOpenAi");
       return new GenericOpenAiLLM(embedder, model);
+    case "bedrock":
+      const { AWSBedrockLLM } = require("../AiProviders/bedrock");
+      return new AWSBedrockLLM(embedder, model);
     default:
       throw new Error(
         `ENV: No valid LLM_PROVIDER value found in environment! Using ${process.env.LLM_PROVIDER}`

server/utils/helpers/updateENV.js

@@ -208,6 +208,28 @@ const KEY_MAPPING = {
     checks: [nonZero],
   },
 
+  // AWS Bedrock LLM InferenceSettings
+  AwsBedrockLLMAccessKeyId: {
+    envKey: "AWS_BEDROCK_LLM_ACCESS_KEY_ID",
+    checks: [isNotEmpty],
+  },
+  AwsBedrockLLMAccessKey: {
+    envKey: "AWS_BEDROCK_LLM_ACCESS_KEY",
+    checks: [isNotEmpty],
+  },
+  AwsBedrockLLMRegion: {
+    envKey: "AWS_BEDROCK_LLM_REGION",
+    checks: [isNotEmpty],
+  },
+  AwsBedrockLLMModel: {
+    envKey: "AWS_BEDROCK_LLM_MODEL_PREFERENCE",
+    checks: [isNotEmpty],
+  },
+  AwsBedrockLLMTokenLimit: {
+    envKey: "AWS_BEDROCK_LLM_MODEL_TOKEN_LIMIT",
+    checks: [nonZero],
+  },
+
   EmbeddingEngine: {
     envKey: "EMBEDDING_ENGINE",
     checks: [supportedEmbeddingModel],
@@ -541,6 +563,7 @@ function supportedLLM(input = "") {
     "cohere",
     "litellm",
     "generic-openai",
+    "bedrock",
   ].includes(input);
   return validSelection ? null : `${input} is not a valid LLM provider.`;
 }