merge with master

.vscode/settings.json

@@ -31,7 +31,6 @@
     "Mintplex",
     "moderations",
     "numpages",
-    "odbc",
     "Ollama",
     "Oobabooga",
     "openai",

README.md

@@ -75,6 +75,7 @@ Some cool features of AnythingLLM
 - [OpenAI](https://openai.com)
 - [OpenAI (Generic)](https://openai.com)
 - [Azure OpenAI](https://azure.microsoft.com/en-us/products/ai-services/openai-service)
+- [AWS Bedrock](https://aws.amazon.com/bedrock/)
 - [Anthropic](https://www.anthropic.com/)
 - [Google Gemini Pro](https://ai.google.dev/)
 - [Hugging Face (chat models)](https://huggingface.co/)
@@ -162,12 +163,6 @@ Mintplex Labs & the community maintain a number of deployment methods, scripts,
 
 [Learn about vector caching](./server/storage/vector-cache/VECTOR_CACHE.md)
 
-## Contributing
-
-- create issue
-- create PR with branch name format of `<issue number>-<short name>`
-- yee haw let's merge
-
 ## Telemetry & Privacy
 
 AnythingLLM by Mintplex Labs Inc contains a telemetry feature that collects anonymous usage information.
@@ -199,6 +194,19 @@ You can verify these claims by finding all locations `Telemetry.sendTelemetry` i
 
 </details>
 
+
+## 👋 Contributing
+
+- create issue
+- create PR with branch name format of `<issue number>-<short name>`
+- LGTM from core-team
+
+## 🌟 Contributors
+
+[![anythingllm contributors](https://contrib.rocks/image?repo=mintplex-labs/anything-llm)](https://github.com/mintplex-labs/anything-llm/graphs/contributors)
+
+[![Star History Chart](https://api.star-history.com/svg?repos=mintplex-labs/anything-llm&type=Timeline)](https://star-history.com/#mintplex-labs/anything-llm&Date)
+
 ## 🔗 More Products
 
 - **[VectorAdmin][vector-admin]:** An all-in-one GUI & tool-suite for managing vector databases.

docker/.env.example

@@ -94,6 +94,13 @@ GID='1000'
 # COHERE_API_KEY=
 # COHERE_MODEL_PREF='command-r'
 
+# LLM_PROVIDER='bedrock'
+# AWS_BEDROCK_LLM_ACCESS_KEY_ID=
+# AWS_BEDROCK_LLM_ACCESS_KEY=
+# AWS_BEDROCK_LLM_REGION=us-west-2
+# AWS_BEDROCK_LLM_MODEL_PREFERENCE=meta.llama3-1-8b-instruct-v1:0
+# AWS_BEDROCK_LLM_MODEL_TOKEN_LIMIT=8191
+
 ###########################################
 ######## Embedding API SElECTION ##########
 ###########################################

frontend/src/components/LLMSelection/AwsBedrockLLMOptions/index.jsx

@@ -0,0 +1,122 @@
+import { ArrowSquareOut, Info } from "@phosphor-icons/react";
+import { AWS_REGIONS } from "./regions";
+
+export default function AwsBedrockLLMOptions({ settings }) {
+  return (
+    <div className="w-full flex flex-col">
+      {!settings?.credentialsOnly && (
+        <div className="flex flex-col md:flex-row md:items-center gap-x-2 text-white mb-4 bg-blue-800/30 w-fit rounded-lg px-4 py-2">
+          <div className="gap-x-2 flex items-center">
+            <Info size={40} />
+            <p className="text-base">
+              You should use a properly defined IAM user for inferencing.
+              <br />
+              <a
+                href="https://docs.useanything.com/setup/llm-configuration/cloud/aws-bedrock"
+                target="_blank"
+                className="underline flex gap-x-1 items-center"
+              >
+                Read more on how to use AWS Bedrock in AnythingLLM
+                <ArrowSquareOut size={14} />
+              </a>
+            </p>
+          </div>
+        </div>
+      )}
+
+      <div className="w-full flex items-center gap-[36px] my-1.5">
+        <div className="flex flex-col w-60">
+          <label className="text-white text-sm font-semibold block mb-3">
+            AWS Bedrock IAM Access ID
+          </label>
+          <input
+            type="password"
+            name="AwsBedrockLLMAccessKeyId"
+            className="border-none bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:outline-primary-button active:outline-primary-button outline-none block w-full p-2.5"
+            placeholder="AWS Bedrock IAM User Access ID"
+            defaultValue={
+              settings?.AwsBedrockLLMAccessKeyId ? "*".repeat(20) : ""
+            }
+            required={true}
+            autoComplete="off"
+            spellCheck={false}
+          />
+        </div>
+        <div className="flex flex-col w-60">
+          <label className="text-white text-sm font-semibold block mb-3">
+            AWS Bedrock IAM Access Key
+          </label>
+          <input
+            type="password"
+            name="AwsBedrockLLMAccessKey"
+            className="border-none bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:outline-primary-button active:outline-primary-button outline-none block w-full p-2.5"
+            placeholder="AWS Bedrock IAM User Access Key"
+            defaultValue={
+              settings?.AwsBedrockLLMAccessKey ? "*".repeat(20) : ""
+            }
+            required={true}
+            autoComplete="off"
+            spellCheck={false}
+          />
+        </div>
+        <div className="flex flex-col w-60">
+          <label className="text-white text-sm font-semibold block mb-3">
+            AWS region
+          </label>
+          <select
+            name="AwsBedrockLLMRegion"
+            defaultValue={settings?.AwsBedrockLLMRegion || "us-west-2"}
+            required={true}
+            className="border-none bg-zinc-900 border-gray-500 text-white text-sm rounded-lg block w-full p-2.5"
+          >
+            {AWS_REGIONS.map((region) => {
+              return (
+                <option key={region.code} value={region.code}>
+                  {region.name} ({region.code})
+                </option>
+              );
+            })}
+          </select>
+        </div>
+      </div>
+
+      <div className="w-full flex items-center gap-[36px] my-1.5">
+        {!settings?.credentialsOnly && (
+          <>
+            <div className="flex flex-col w-60">
+              <label className="text-white text-sm font-semibold block mb-3">
+                Model ID
+              </label>
+              <input
+                type="text"
+                name="AwsBedrockLLMModel"
+                className="border-none bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:outline-primary-button active:outline-primary-button outline-none block w-full p-2.5"
+                placeholder="Model id from AWS eg: meta.llama3.1-v0.1"
+                defaultValue={settings?.AwsBedrockLLMModel}
+                required={true}
+                autoComplete="off"
+                spellCheck={false}
+              />
+            </div>
+            <div className="flex flex-col w-60">
+              <label className="text-white text-sm font-semibold block mb-3">
+                Model context window
+              </label>
+              <input
+                type="number"
+                name="AwsBedrockLLMTokenLimit"
+                className="border-none bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:outline-primary-button active:outline-primary-button outline-none block w-full p-2.5"
+                placeholder="Content window limit (eg: 4096)"
+                min={1}
+                onScroll={(e) => e.target.blur()}
+                defaultValue={settings?.AwsBedrockLLMTokenLimit}
+                required={true}
+                autoComplete="off"
+              />
+            </div>
+          </>
+        )}
+      </div>
+    </div>
+  );
+}

frontend/src/components/LLMSelection/AwsBedrockLLMOptions/regions.js

@@ -0,0 +1,204 @@
+export const AWS_REGIONS = [
+  {
+    name: "N. Virginia",
+    full_name: "US East (N. Virginia)",
+    code: "us-east-1",
+    public: true,
+    zones: [
+      "us-east-1a",
+      "us-east-1b",
+      "us-east-1c",
+      "us-east-1d",
+      "us-east-1e",
+      "us-east-1f",
+    ],
+  },
+  {
+    name: "Ohio",
+    full_name: "US East (Ohio)",
+    code: "us-east-2",
+    public: true,
+    zones: ["us-east-2a", "us-east-2b", "us-east-2c"],
+  },
+  {
+    name: "N. California",
+    full_name: "US West (N. California)",
+    code: "us-west-1",
+    public: true,
+    zone_limit: 2,
+    zones: ["us-west-1a", "us-west-1b", "us-west-1c"],
+  },
+  {
+    name: "Oregon",
+    full_name: "US West (Oregon)",
+    code: "us-west-2",
+    public: true,
+    zones: ["us-west-2a", "us-west-2b", "us-west-2c", "us-west-2d"],
+  },
+  {
+    name: "GovCloud West",
+    full_name: "AWS GovCloud (US)",
+    code: "us-gov-west-1",
+    public: false,
+    zones: ["us-gov-west-1a", "us-gov-west-1b", "us-gov-west-1c"],
+  },
+  {
+    name: "GovCloud East",
+    full_name: "AWS GovCloud (US-East)",
+    code: "us-gov-east-1",
+    public: false,
+    zones: ["us-gov-east-1a", "us-gov-east-1b", "us-gov-east-1c"],
+  },
+  {
+    name: "Canada",
+    full_name: "Canada (Central)",
+    code: "ca-central-1",
+    public: true,
+    zones: ["ca-central-1a", "ca-central-1b", "ca-central-1c", "ca-central-1d"],
+  },
+  {
+    name: "Stockholm",
+    full_name: "EU (Stockholm)",
+    code: "eu-north-1",
+    public: true,
+    zones: ["eu-north-1a", "eu-north-1b", "eu-north-1c"],
+  },
+  {
+    name: "Ireland",
+    full_name: "EU (Ireland)",
+    code: "eu-west-1",
+    public: true,
+    zones: ["eu-west-1a", "eu-west-1b", "eu-west-1c"],
+  },
+  {
+    name: "London",
+    full_name: "EU (London)",
+    code: "eu-west-2",
+    public: true,
+    zones: ["eu-west-2a", "eu-west-2b", "eu-west-2c"],
+  },
+  {
+    name: "Paris",
+    full_name: "EU (Paris)",
+    code: "eu-west-3",
+    public: true,
+    zones: ["eu-west-3a", "eu-west-3b", "eu-west-3c"],
+  },
+  {
+    name: "Frankfurt",
+    full_name: "EU (Frankfurt)",
+    code: "eu-central-1",
+    public: true,
+    zones: ["eu-central-1a", "eu-central-1b", "eu-central-1c"],
+  },
+  {
+    name: "Milan",
+    full_name: "EU (Milan)",
+    code: "eu-south-1",
+    public: true,
+    zones: ["eu-south-1a", "eu-south-1b", "eu-south-1c"],
+  },
+  {
+    name: "Cape Town",
+    full_name: "Africa (Cape Town)",
+    code: "af-south-1",
+    public: true,
+    zones: ["af-south-1a", "af-south-1b", "af-south-1c"],
+  },
+  {
+    name: "Tokyo",
+    full_name: "Asia Pacific (Tokyo)",
+    code: "ap-northeast-1",
+    public: true,
+    zone_limit: 3,
+    zones: [
+      "ap-northeast-1a",
+      "ap-northeast-1b",
+      "ap-northeast-1c",
+      "ap-northeast-1d",
+    ],
+  },
+  {
+    name: "Seoul",
+    full_name: "Asia Pacific (Seoul)",
+    code: "ap-northeast-2",
+    public: true,
+    zones: [
+      "ap-northeast-2a",
+      "ap-northeast-2b",
+      "ap-northeast-2c",
+      "ap-northeast-2d",
+    ],
+  },
+  {
+    name: "Osaka",
+    full_name: "Asia Pacific (Osaka-Local)",
+    code: "ap-northeast-3",
+    public: true,
+    zones: ["ap-northeast-3a", "ap-northeast-3b", "ap-northeast-3c"],
+  },
+  {
+    name: "Singapore",
+    full_name: "Asia Pacific (Singapore)",
+    code: "ap-southeast-1",
+    public: true,
+    zones: ["ap-southeast-1a", "ap-southeast-1b", "ap-southeast-1c"],
+  },
+  {
+    name: "Sydney",
+    full_name: "Asia Pacific (Sydney)",
+    code: "ap-southeast-2",
+    public: true,
+    zones: ["ap-southeast-2a", "ap-southeast-2b", "ap-southeast-2c"],
+  },
+  {
+    name: "Jakarta",
+    full_name: "Asia Pacific (Jakarta)",
+    code: "ap-southeast-3",
+    public: true,
+    zones: ["ap-southeast-3a", "ap-southeast-3b", "ap-southeast-3c"],
+  },
+  {
+    name: "Hong Kong",
+    full_name: "Asia Pacific (Hong Kong)",
+    code: "ap-east-1",
+    public: true,
+    zones: ["ap-east-1a", "ap-east-1b", "ap-east-1c"],
+  },
+  {
+    name: "Mumbai",
+    full_name: "Asia Pacific (Mumbai)",
+    code: "ap-south-1",
+    public: true,
+    zones: ["ap-south-1a", "ap-south-1b", "ap-south-1c"],
+  },
+  {
+    name: "São Paulo",
+    full_name: "South America (São Paulo)",
+    code: "sa-east-1",
+    public: true,
+    zone_limit: 2,
+    zones: ["sa-east-1a", "sa-east-1b", "sa-east-1c"],
+  },
+  {
+    name: "Bahrain",
+    full_name: "Middle East (Bahrain)",
+    code: "me-south-1",
+    public: true,
+    zones: ["me-south-1a", "me-south-1b", "me-south-1c"],
+  },
+  {
+    name: "Beijing",
+    full_name: "China (Beijing)",
+    code: "cn-north-1",
+    public: false,
+    zones: ["cn-north-1a", "cn-north-1b", "cn-north-1c"],
+  },
+  {
+    name: "Ningxia",
+    full_name: "China (Ningxia)",
+    code: "cn-northwest-1",
+    public: false,
+    zones: ["cn-northwest-1a", "cn-northwest-1b", "cn-northwest-1c"],
+  },
+];

frontend/src/hooks/useGetProvidersModels.js

@@ -8,6 +8,7 @@ export const DISABLED_PROVIDERS = [
   "native",
   "textgenwebui",
   "generic-openai",
+  "bedrock",
 ];
 const PROVIDER_DEFAULT_MODELS = {
   openai: [],
@@ -48,6 +49,7 @@ const PROVIDER_DEFAULT_MODELS = {
   ],
   textgenwebui: [],
   "generic-openai": [],
+  bedrock: [],
 };
 
 // For togetherAi, which has a large model list - we subgroup the options

frontend/src/pages/Admin/Agents/SQLConnectorSelection/DBConnection.jsx

@@ -1,14 +1,12 @@
 import PostgreSQLLogo from "./icons/postgresql.png";
 import MySQLLogo from "./icons/mysql.png";
 import MSSQLLogo from "./icons/mssql.png";
-import ODBCLogo from "./icons/odbc.png";
 import { X } from "@phosphor-icons/react";
 
 export const DB_LOGOS = {
   postgresql: PostgreSQLLogo,
   mysql: MySQLLogo,
   "sql-server": MSSQLLogo,
-  odbc: ODBCLogo,
 };
 
 export default function DBConnection({ connection, onRemove, setHasChanges }) {

frontend/src/pages/Admin/Agents/SQLConnectorSelection/NewConnectionModal.jsx

@@ -11,7 +11,6 @@ function assembleConnectionString({
   host = "",
   port = "",
   database = "",
-  driver = "",
 }) {
   if ([username, password, host, database].every((i) => !!i) === false)
     return `Please fill out all the fields above.`;
@@ -22,9 +21,6 @@ function assembleConnectionString({
       return `mysql://${username}:${password}@${host}:${port}/${database}`;
     case "sql-server":
       return `mssql://${username}:${password}@${host}:${port}/${database}`;
-    case "odbc":
-      if (!driver) return `Please fill out the driver field.`;
-      return `Driver={${driver}};Server=${host};Port=${port};Database=${database};UID=${username};PWD=${password}`;
     default:
       return null;
   }
@@ -37,7 +33,6 @@ const DEFAULT_CONFIG = {
   host: null,
   port: null,
   database: null,
-  driver: null,
 };
 
 export default function NewSQLConnection({ isOpen, closeModal, onSubmit }) {
@@ -53,14 +48,12 @@ export default function NewSQLConnection({ isOpen, closeModal, onSubmit }) {
 
   function onFormChange() {
     const form = new FormData(document.getElementById("sql-connection-form"));
-
     setConfig({
       username: form.get("username").trim(),
       password: form.get("password"),
       host: form.get("host").trim(),
       port: form.get("port").trim(),
       database: form.get("database").trim(),
-      driver: form.get("driver")?.trim(),
     });
   }
 
@@ -81,7 +74,7 @@ export default function NewSQLConnection({ isOpen, closeModal, onSubmit }) {
   // to the parent container form so we don't have nested forms.
   return createPortal(
     <ModalWrapper isOpen={isOpen}>
-      <div className="relative w-full md:w-fit max-w-2xl max-h-full md:mt-8">
+      <div className="relative w-full md:w-1/3 max-w-2xl max-h-full md:mt-8">
         <div className="relative bg-main-gradient rounded-xl shadow-[0_4px_14px_rgba(0,0,0,0.25)] max-h-[85vh] overflow-y-scroll no-scroll">
           <div className="flex items-start justify-between p-4 border-b rounded-t border-gray-500/50">
             <h3 className="text-xl font-semibold text-white">
@@ -121,7 +114,7 @@ export default function NewSQLConnection({ isOpen, closeModal, onSubmit }) {
                 <label className="text-white text-sm font-semibold block my-4">
                   Select your SQL engine
                 </label>
-                <div className="flex flex-wrap gap-x-4 gap-y-4">
+                <div className="grid md:grid-cols-4 gap-4 grid-cols-2">
                   <DBEngine
                     provider="postgresql"
                     active={engine === "postgresql"}
@@ -137,11 +130,6 @@ export default function NewSQLConnection({ isOpen, closeModal, onSubmit }) {
                     active={engine === "sql-server"}
                     onClick={() => setEngine("sql-server")}
                   />
-                  <DBEngine
-                    provider="odbc"
-                    active={engine === "odbc"}
-                    onClick={() => setEngine("odbc")}
-                  />
                 </div>
               </div>
 
@@ -236,23 +224,6 @@ export default function NewSQLConnection({ isOpen, closeModal, onSubmit }) {
                   spellCheck={false}
                 />
               </div>
-
-              {engine === "odbc" && (
-                <div className="flex flex-col">
-                  <label className="text-white text-sm font-semibold block mb-3">
-                    Driver
-                  </label>
-                  <input
-                    type="text"
-                    name="driver"
-                    className="border-none bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:outline-primary-button active:outline-primary-button outline-none block w-full p-2.5"
-                    placeholder="the driver to use eg: MongoDB ODBC 1.2.0 ANSI Driver"
-                    required={true}
-                    autoComplete="off"
-                    spellCheck={false}
-                  />
-                </div>
-              )}
               <p className="text-white/40 text-sm">
                 {assembleConnectionString({ engine, ...config })}
               </p>

frontend/src/pages/GeneralSettings/LLMPreference/index.jsx

@@ -22,6 +22,8 @@ import KoboldCPPLogo from "@/media/llmprovider/koboldcpp.png";
 import TextGenWebUILogo from "@/media/llmprovider/text-generation-webui.png";
 import CohereLogo from "@/media/llmprovider/cohere.png";
 import LiteLLMLogo from "@/media/llmprovider/litellm.png";
+import AWSBedrockLogo from "@/media/llmprovider/bedrock.png";
+
 import PreLoader from "@/components/Preloader";
 import OpenAiOptions from "@/components/LLMSelection/OpenAiOptions";
 import GenericOpenAiOptions from "@/components/LLMSelection/GenericOpenAiOptions";
@@ -41,6 +43,7 @@ import CohereAiOptions from "@/components/LLMSelection/CohereAiOptions";
 import KoboldCPPOptions from "@/components/LLMSelection/KoboldCPPOptions";
 import TextGenWebUIOptions from "@/components/LLMSelection/TextGenWebUIOptions";
 import LiteLLMOptions from "@/components/LLMSelection/LiteLLMOptions";
+import AWSBedrockLLMOptions from "@/components/LLMSelection/AwsBedrockLLMOptions";
 
 import LLMItem from "@/components/LLMSelection/LLMItem";
 import { CaretUpDown, MagnifyingGlass, X } from "@phosphor-icons/react";
@@ -218,6 +221,19 @@ export const AVAILABLE_LLM_PROVIDERS = [
   //     "Use a downloaded custom Llama model for chatting on this AnythingLLM instance.",
   //   requiredConfig: [],
   // },
+  {
+    name: "AWS Bedrock",
+    value: "bedrock",
+    logo: AWSBedrockLogo,
+    options: (settings) => <AWSBedrockLLMOptions settings={settings} />,
+    description: "Run powerful foundation models privately with AWS Bedrock.",
+    requiredConfig: [
+      "AwsBedrockLLMAccessKeyId",
+      "AwsBedrockLLMAccessKey",
+      "AwsBedrockLLMRegion",
+      "AwsBedrockLLMModel",
+    ],
+  },
 ];
 
 export default function GeneralLLMPreference() {

frontend/src/pages/OnboardingFlow/Steps/DataHandling/index.jsx

@@ -18,6 +18,7 @@ import GroqLogo from "@/media/llmprovider/groq.png";
 import KoboldCPPLogo from "@/media/llmprovider/koboldcpp.png";
 import TextGenWebUILogo from "@/media/llmprovider/text-generation-webui.png";
 import LiteLLMLogo from "@/media/llmprovider/litellm.png";
+import AWSBedrockLogo from "@/media/llmprovider/bedrock.png";
 
 import CohereLogo from "@/media/llmprovider/cohere.png";
 import ZillizLogo from "@/media/vectordbs/zilliz.png";
@@ -179,6 +180,13 @@ export const LLM_SELECTION_PRIVACY = {
     ],
     logo: LiteLLMLogo,
   },
+  bedrock: {
+    name: "AWS Bedrock",
+    description: [
+      "You model and chat contents are subject to the agreed EULA for AWS and the model provider on aws.amazon.com",
+    ],
+    logo: AWSBedrockLogo,
+  },
 };
 
 export const VECTOR_DB_PRIVACY = {

frontend/src/pages/OnboardingFlow/Steps/LLMPreference/index.jsx

@@ -17,6 +17,7 @@ import GroqLogo from "@/media/llmprovider/groq.png";
 import KoboldCPPLogo from "@/media/llmprovider/koboldcpp.png";
 import TextGenWebUILogo from "@/media/llmprovider/text-generation-webui.png";
 import LiteLLMLogo from "@/media/llmprovider/litellm.png";
+import AWSBedrockLogo from "@/media/llmprovider/bedrock.png";
 
 import CohereLogo from "@/media/llmprovider/cohere.png";
 import OpenAiOptions from "@/components/LLMSelection/OpenAiOptions";
@@ -37,6 +38,7 @@ import CohereAiOptions from "@/components/LLMSelection/CohereAiOptions";
 import KoboldCPPOptions from "@/components/LLMSelection/KoboldCPPOptions";
 import TextGenWebUIOptions from "@/components/LLMSelection/TextGenWebUIOptions";
 import LiteLLMOptions from "@/components/LLMSelection/LiteLLMOptions";
+import AWSBedrockLLMOptions from "@/components/LLMSelection/AwsBedrockLLMOptions";
 
 import LLMItem from "@/components/LLMSelection/LLMItem";
 import System from "@/models/system";
@@ -188,6 +190,13 @@ const LLMS = [
   //   description:
   //     "Use a downloaded custom Llama model for chatting on this AnythingLLM instance.",
   // },
+  {
+    name: "AWS Bedrock",
+    value: "bedrock",
+    logo: AWSBedrockLogo,
+    options: (settings) => <AWSBedrockLLMOptions settings={settings} />,
+    description: "Run powerful foundation models privately with AWS Bedrock.",
+  },
 ];
 
 export default function LLMPreference({

frontend/src/pages/WorkspaceSettings/AgentConfig/AgentLLMSelection/index.jsx

@@ -21,6 +21,7 @@ const ENABLED_PROVIDERS = [
   "perplexity",
   "textgenwebui",
   "generic-openai",
+  "bedrock",
   // TODO: More agent support.
   // "cohere",         // Has tool calling and will need to build explicit support
   // "huggingface"     // Can be done but already has issues with no-chat templated. Needs to be tested.

frontend/src/pages/WorkspaceSettings/ChatSettings/WorkspaceLLMSelection/index.jsx

@@ -10,7 +10,12 @@ import paths from "@/utils/paths";
 
 // Some providers can only be associated with a single model.
 // In that case there is no selection to be made so we can just move on.
-const NO_MODEL_SELECTION = ["default", "huggingface", "generic-openai"];
+const NO_MODEL_SELECTION = [
+  "default",
+  "huggingface",
+  "generic-openai",
+  "bedrock",
+];
 const DISABLED_PROVIDERS = ["azure", "lmstudio", "native"];
 const LLM_DEFAULT = {
   name: "System default",

server/models/systemSettings.js

@@ -470,6 +470,12 @@ const SystemSettings = {
       GenericOpenAiKey: !!process.env.GENERIC_OPEN_AI_API_KEY,
       GenericOpenAiMaxTokens: process.env.GENERIC_OPEN_AI_MAX_TOKENS,
 
+      AwsBedrockLLMAccessKeyId: !!process.env.AWS_BEDROCK_LLM_ACCESS_KEY_ID,
+      AwsBedrockLLMAccessKey: !!process.env.AWS_BEDROCK_LLM_ACCESS_KEY,
+      AwsBedrockLLMRegion: process.env.AWS_BEDROCK_LLM_REGION,
+      AwsBedrockLLMModel: process.env.AWS_BEDROCK_LLM_MODEL_PREFERENCE,
+      AwsBedrockLLMTokenLimit: process.env.AWS_BEDROCK_LLM_MODEL_TOKEN_LIMIT,
+
       // Cohere API Keys
       CohereApiKey: !!process.env.COHERE_API_KEY,
       CohereModelPref: process.env.COHERE_MODEL_PREF,

server/package.json

@@ -28,6 +28,7 @@
     "@ladjs/graceful": "^3.2.2",
     "@lancedb/lancedb": "0.5.2",
     "@langchain/anthropic": "0.1.16",
+    "@langchain/aws": "^0.0.5",
     "@langchain/community": "0.0.53",
     "@langchain/core": "0.1.61",
     "@langchain/openai": "0.0.28",
@@ -66,7 +67,6 @@
     "mysql2": "^3.9.8",
     "node-html-markdown": "^1.3.0",
     "node-llama-cpp": "^2.8.0",
-    "odbc": "^2.4.8",
     "ollama": "^0.5.0",
     "openai": "4.38.5",
     "pg": "^8.11.5",

server/utils/AiProviders/bedrock/index.js

@@ -0,0 +1,217 @@
+const { StringOutputParser } = require("@langchain/core/output_parsers");
+const {
+  writeResponseChunk,
+  clientAbortedHandler,
+} = require("../../helpers/chat/responses");
+const { NativeEmbedder } = require("../../EmbeddingEngines/native");
+
+// Docs: https://js.langchain.com/v0.2/docs/integrations/chat/bedrock_converse
+class AWSBedrockLLM {
+  constructor(embedder = null, modelPreference = null) {
+    if (!process.env.AWS_BEDROCK_LLM_ACCESS_KEY_ID)
+      throw new Error("No AWS Bedrock LLM profile id was set.");
+
+    if (!process.env.AWS_BEDROCK_LLM_ACCESS_KEY)
+      throw new Error("No AWS Bedrock LLM access key was set.");
+
+    if (!process.env.AWS_BEDROCK_LLM_REGION)
+      throw new Error("No AWS Bedrock LLM region was set.");
+
+    this.model =
+      modelPreference || process.env.AWS_BEDROCK_LLM_MODEL_PREFERENCE;
+    this.limits = {
+      history: this.promptWindowLimit() * 0.15,
+      system: this.promptWindowLimit() * 0.15,
+      user: this.promptWindowLimit() * 0.7,
+    };
+
+    this.embedder = embedder ?? new NativeEmbedder();
+    this.defaultTemp = 0.7;
+  }
+
+  #bedrockClient({ temperature = 0.7 }) {
+    const { ChatBedrockConverse } = require("@langchain/aws");
+    return new ChatBedrockConverse({
+      model: process.env.AWS_BEDROCK_LLM_MODEL_PREFERENCE,
+      region: process.env.AWS_BEDROCK_LLM_REGION,
+      credentials: {
+        accessKeyId: process.env.AWS_BEDROCK_LLM_ACCESS_KEY_ID,
+        secretAccessKey: process.env.AWS_BEDROCK_LLM_ACCESS_KEY,
+      },
+      temperature,
+    });
+  }
+
+  // For streaming we use Langchain's wrapper to handle weird chunks
+  // or otherwise absorb headaches that can arise from Ollama models
+  #convertToLangchainPrototypes(chats = []) {
+    const {
+      HumanMessage,
+      SystemMessage,
+      AIMessage,
+    } = require("@langchain/core/messages");
+    const langchainChats = [];
+    const roleToMessageMap = {
+      system: SystemMessage,
+      user: HumanMessage,
+      assistant: AIMessage,
+    };
+
+    for (const chat of chats) {
+      if (!roleToMessageMap.hasOwnProperty(chat.role)) continue;
+      const MessageClass = roleToMessageMap[chat.role];
+      langchainChats.push(new MessageClass({ content: chat.content }));
+    }
+
+    return langchainChats;
+  }
+
+  #appendContext(contextTexts = []) {
+    if (!contextTexts || !contextTexts.length) return "";
+    return (
+      "\nContext:\n" +
+      contextTexts
+        .map((text, i) => {
+          return `[CONTEXT ${i}]:\n${text}\n[END CONTEXT ${i}]\n\n`;
+        })
+        .join("")
+    );
+  }
+
+  streamingEnabled() {
+    return "streamGetChatCompletion" in this;
+  }
+
+  // Ensure the user set a value for the token limit
+  // and if undefined - assume 4096 window.
+  promptWindowLimit() {
+    const limit = process.env.AWS_BEDROCK_LLM_MODEL_TOKEN_LIMIT || 8191;
+    if (!limit || isNaN(Number(limit)))
+      throw new Error("No valid token context limit was set.");
+    return Number(limit);
+  }
+
+  async isValidChatCompletionModel(_ = "") {
+    return true;
+  }
+
+  constructPrompt({
+    systemPrompt = "",
+    contextTexts = [],
+    chatHistory = [],
+    userPrompt = "",
+  }) {
+    // AWS Mistral models do not support system prompts
+    if (this.model.startsWith("mistral"))
+      return [...chatHistory, { role: "user", content: userPrompt }];
+
+    const prompt = {
+      role: "system",
+      content: `${systemPrompt}${this.#appendContext(contextTexts)}`,
+    };
+    return [prompt, ...chatHistory, { role: "user", content: userPrompt }];
+  }
+
+  async getChatCompletion(messages = null, { temperature = 0.7 }) {
+    const model = this.#bedrockClient({ temperature });
+    const textResponse = await model
+      .pipe(new StringOutputParser())
+      .invoke(this.#convertToLangchainPrototypes(messages))
+      .catch((e) => {
+        throw new Error(
+          `AWSBedrock::getChatCompletion failed to communicate with Ollama. ${e.message}`
+        );
+      });
+
+    if (!textResponse || !textResponse.length)
+      throw new Error(`AWSBedrock::getChatCompletion text response was empty.`);
+
+    return textResponse;
+  }
+
+  async streamGetChatCompletion(messages = null, { temperature = 0.7 }) {
+    const model = this.#bedrockClient({ temperature });
+    const stream = await model
+      .pipe(new StringOutputParser())
+      .stream(this.#convertToLangchainPrototypes(messages));
+    return stream;
+  }
+
+  handleStream(response, stream, responseProps) {
+    const { uuid = uuidv4(), sources = [] } = responseProps;
+
+    return new Promise(async (resolve) => {
+      let fullText = "";
+
+      // Establish listener to early-abort a streaming response
+      // in case things go sideways or the user does not like the response.
+      // We preserve the generated text but continue as if chat was completed
+      // to preserve previously generated content.
+      const handleAbort = () => clientAbortedHandler(resolve, fullText);
+      response.on("close", handleAbort);
+
+      try {
+        for await (const chunk of stream) {
+          if (chunk === undefined)
+            throw new Error(
+              "Stream returned undefined chunk. Aborting reply - check model provider logs."
+            );
+
+          const content = chunk.hasOwnProperty("content")
+            ? chunk.content
+            : chunk;
+          fullText += content;
+          writeResponseChunk(response, {
+            uuid,
+            sources: [],
+            type: "textResponseChunk",
+            textResponse: content,
+            close: false,
+            error: false,
+          });
+        }
+
+        writeResponseChunk(response, {
+          uuid,
+          sources,
+          type: "textResponseChunk",
+          textResponse: "",
+          close: true,
+          error: false,
+        });
+        response.removeListener("close", handleAbort);
+        resolve(fullText);
+      } catch (error) {
+        writeResponseChunk(response, {
+          uuid,
+          sources: [],
+          type: "textResponseChunk",
+          textResponse: "",
+          close: true,
+          error: `AWSBedrock:streaming - could not stream chat. ${
+            error?.cause ?? error.message
+          }`,
+        });
+        response.removeListener("close", handleAbort);
+      }
+    });
+  }
+
+  // Simple wrapper for dynamic embedder & normalize interface for all LLM implementations
+  async embedTextInput(textInput) {
+    return await this.embedder.embedTextInput(textInput);
+  }
+  async embedChunks(textChunks = []) {
+    return await this.embedder.embedChunks(textChunks);
+  }
+
+  async compressMessages(promptArgs = {}, rawHistory = []) {
+    const { messageArrayCompressor } = require("../../helpers/chat");
+    const messageArray = this.constructPrompt(promptArgs);
+    return await messageArrayCompressor(this, messageArray, rawHistory);
+  }
+}
+
+module.exports = {
+  AWSBedrockLLM,
+};

server/utils/agents/aibitat/index.js

@@ -775,6 +775,8 @@ ${this.getHistory({ to: route.to })
         return new Providers.PerplexityProvider({ model: config.model });
       case "textgenwebui":
         return new Providers.TextWebGenUiProvider({});
+      case "bedrock":
+        return new Providers.AWSBedrockProvider({});
 
       default:
         throw new Error(

server/utils/agents/aibitat/plugins/sql-agent/SQLConnectors/ODBC.js

@@ -1,60 +0,0 @@
-const odbc = require("odbc");
-const UrlPattern = require("url-pattern");
-
-class ODBCConnector {
-  #connected = false;
-  database_id = "";
-  constructor(
-    config = {
-      connectionString: null,
-    }
-  ) {
-    this.connectionString = config.connectionString;
-    this._client = null;
-    this.database_id = this.#parseDatabase();
-  }
-
-  #parseDatabase() {
-    const regex = /Database=([^;]+)/;
-    const match = this.connectionString.match(regex);
-    return match ? match[1] : null;
-  }
-
-  async connect() {
-    this._client = await odbc.connect(this.connectionString);
-    this.#connected = true;
-    return this._client;
-  }
-
-  /**
-   *
-   * @param {string} queryString the SQL query to be run
-   * @returns {import(".").QueryResult}
-   */
-  async runQuery(queryString = "") {
-    const result = { rows: [], count: 0, error: null };
-    try {
-      if (!this.#connected) await this.connect();
-      const query = await this._client.query(queryString);
-      result.rows = query;
-      result.count = query.length;
-    } catch (err) {
-      console.log(this.constructor.name, err);
-      result.error = err.message;
-    } finally {
-      await this._client.close();
-      this.#connected = false;
-    }
-    return result;
-  }
-
-  getTablesSql() {
-    return `SELECT table_name FROM information_schema.tables WHERE table_schema = '${this.database_id}'`;
-  }
-
-  getTableSchemaSql(table_name) {
-    return `SHOW COLUMNS FROM ${this.database_id}.${table_name};`;
-  }
-}
-
-module.exports.ODBCConnector = ODBCConnector;

server/utils/agents/aibitat/plugins/sql-agent/SQLConnectors/index.js

@@ -2,7 +2,7 @@ const { SystemSettings } = require("../../../../../../models/systemSettings");
 const { safeJsonParse } = require("../../../../../http");
 
 /**
- * @typedef {('postgresql'|'mysql'|'sql-server'|'odbc')} SQLEngine
+ * @typedef {('postgresql'|'mysql'|'sql-server')} SQLEngine
  */
 
 /**
@@ -36,9 +36,6 @@ function getDBClient(identifier = "", connectionConfig = {}) {
     case "sql-server":
       const { MSSQLConnector } = require("./MSSQL");
       return new MSSQLConnector(connectionConfig);
-    case "odbc":
-      const { ODBCConnector } = require("./ODBC");
-      return new ODBCConnector(connectionConfig);
     default:
       throw new Error(
         `There is no supported database connector for ${identifier}`

server/utils/agents/aibitat/providers/ai-provider.js

@@ -12,6 +12,7 @@
 
 const { ChatOpenAI } = require("@langchain/openai");
 const { ChatAnthropic } = require("@langchain/anthropic");
+const { ChatBedrockConverse } = require("@langchain/aws");
 const { ChatOllama } = require("@langchain/community/chat_models/ollama");
 const { toValidNumber } = require("../../../http");
 
@@ -113,6 +114,16 @@ class Provider {
           ),
           ...config,
         });
+      case "bedrock":
+        return new ChatBedrockConverse({
+          model: process.env.AWS_BEDROCK_LLM_MODEL_PREFERENCE,
+          region: process.env.AWS_BEDROCK_LLM_REGION,
+          credentials: {
+            accessKeyId: process.env.AWS_BEDROCK_LLM_ACCESS_KEY_ID,
+            secretAccessKey: process.env.AWS_BEDROCK_LLM_ACCESS_KEY,
+          },
+          ...config,
+        });
 
       // OSS Model Runners
       // case "anythingllm_ollama":

server/utils/agents/aibitat/providers/bedrock.js

@@ -0,0 +1,136 @@
+const Provider = require("./ai-provider.js");
+const InheritMultiple = require("./helpers/classes.js");
+const UnTooled = require("./helpers/untooled.js");
+const { ChatBedrockConverse } = require("@langchain/aws");
+const {
+  HumanMessage,
+  SystemMessage,
+  AIMessage,
+} = require("@langchain/core/messages");
+
+/**
+ * The agent provider for the AWS Bedrock provider.
+ */
+class AWSBedrockProvider extends InheritMultiple([Provider, UnTooled]) {
+  model;
+
+  constructor(_config = {}) {
+    super();
+    const model = process.env.AWS_BEDROCK_LLM_MODEL_PREFERENCE ?? null;
+    const client = new ChatBedrockConverse({
+      region: process.env.AWS_BEDROCK_LLM_REGION,
+      credentials: {
+        accessKeyId: process.env.AWS_BEDROCK_LLM_ACCESS_KEY_ID,
+        secretAccessKey: process.env.AWS_BEDROCK_LLM_ACCESS_KEY,
+      },
+      model,
+    });
+
+    this._client = client;
+    this.model = model;
+    this.verbose = true;
+  }
+
+  get client() {
+    return this._client;
+  }
+
+  // For streaming we use Langchain's wrapper to handle weird chunks
+  // or otherwise absorb headaches that can arise from Ollama models
+  #convertToLangchainPrototypes(chats = []) {
+    const langchainChats = [];
+    const roleToMessageMap = {
+      system: SystemMessage,
+      user: HumanMessage,
+      assistant: AIMessage,
+    };
+
+    for (const chat of chats) {
+      if (!roleToMessageMap.hasOwnProperty(chat.role)) continue;
+      const MessageClass = roleToMessageMap[chat.role];
+      langchainChats.push(new MessageClass({ content: chat.content }));
+    }
+
+    return langchainChats;
+  }
+
+  async #handleFunctionCallChat({ messages = [] }) {
+    const response = await this.client
+      .invoke(this.#convertToLangchainPrototypes(messages))
+      .then((res) => res)
+      .catch((e) => {
+        console.error(e);
+        return null;
+      });
+
+    return response?.content;
+  }
+
+  /**
+   * Create a completion based on the received messages.
+   *
+   * @param messages A list of messages to send to the API.
+   * @param functions
+   * @returns The completion.
+   */
+  async complete(messages, functions = null) {
+    try {
+      let completion;
+      if (functions.length > 0) {
+        const { toolCall, text } = await this.functionCall(
+          messages,
+          functions,
+          this.#handleFunctionCallChat.bind(this)
+        );
+
+        if (toolCall !== null) {
+          this.providerLog(`Valid tool call found - running ${toolCall.name}.`);
+          this.deduplicator.trackRun(toolCall.name, toolCall.arguments);
+          return {
+            result: null,
+            functionCall: {
+              name: toolCall.name,
+              arguments: toolCall.arguments,
+            },
+            cost: 0,
+          };
+        }
+        completion = { content: text };
+      }
+
+      if (!completion?.content) {
+        this.providerLog(
+          "Will assume chat completion without tool call inputs."
+        );
+        const response = await this.client.invoke(
+          this.#convertToLangchainPrototypes(this.cleanMsgs(messages))
+        );
+        completion = response;
+      }
+
+      // The UnTooled class inherited Deduplicator is mostly useful to prevent the agent
+      // from calling the exact same function over and over in a loop within a single chat exchange
+      // _but_ we should enable it to call previously used tools in a new chat interaction.
+      this.deduplicator.reset("runs");
+      return {
+        result: completion.content,
+        cost: 0,
+      };
+    } catch (error) {
+      throw error;
+    }
+  }
+
+  /**
+   * Get the cost of the completion.
+   *
+   * @param _usage The completion to get the cost for.
+   * @returns The cost of the completion.
+   * Stubbed since KoboldCPP has no cost basis.
+   */
+  getCost(_usage) {
+    return 0;
+  }
+}
+
+module.exports = AWSBedrockProvider;

server/utils/agents/aibitat/providers/index.js

@@ -12,6 +12,7 @@ const MistralProvider = require("./mistral.js");
 const GenericOpenAiProvider = require("./genericOpenAi.js");
 const PerplexityProvider = require("./perplexity.js");
 const TextWebGenUiProvider = require("./textgenwebui.js");
+const AWSBedrockProvider = require("./bedrock.js");
 
 module.exports = {
   OpenAIProvider,
@@ -28,4 +29,5 @@ module.exports = {
   GenericOpenAiProvider,
   PerplexityProvider,
   TextWebGenUiProvider,
+  AWSBedrockProvider,
 };

server/utils/agents/index.js

@@ -143,6 +143,17 @@ class AgentHandler {
             "TextWebGenUI API base path must be provided to use agents."
           );
         break;
+      case "bedrock":
+        if (
+          !process.env.AWS_BEDROCK_LLM_ACCESS_KEY_ID ||
+          !process.env.AWS_BEDROCK_LLM_ACCESS_KEY ||
+          !process.env.AWS_BEDROCK_LLM_REGION ||
+          !process.env.AWS_BEDROCK_LLM_MODEL_PREFERENCE
+        )
+          throw new Error(
+            "AWS Bedrock Access Keys, model and region must be provided to use agents."
+          );
+        break;
 
       default:
         throw new Error(
@@ -183,6 +194,8 @@ class AgentHandler {
         return "sonar-small-online";
       case "textgenwebui":
         return null;
+      case "bedrock":
+        return null;
       default:
         return "unknown";
     }
@@ -210,7 +223,7 @@ class AgentHandler {
   }
 
   #providerSetupAndCheck() {
-    this.provider = this.invocation.workspace.agentProvider || "openai";
+    this.provider = this.invocation.workspace.agentProvider;
     this.model = this.#fetchModel();
     this.log(`Start ${this.#invocationUUID}::${this.provider}:${this.model}`);
     this.#checkSetup();

server/utils/helpers/index.js

@@ -140,6 +140,9 @@ function getLLMProvider({ provider = null, model = null } = {}) {
     case "generic-openai":
       const { GenericOpenAiLLM } = require("../AiProviders/genericOpenAi");
       return new GenericOpenAiLLM(embedder, model);
+    case "bedrock":
+      const { AWSBedrockLLM } = require("../AiProviders/bedrock");
+      return new AWSBedrockLLM(embedder, model);
     default:
       throw new Error(
         `ENV: No valid LLM_PROVIDER value found in environment! Using ${process.env.LLM_PROVIDER}`

server/utils/helpers/updateENV.js

@@ -208,6 +208,28 @@ const KEY_MAPPING = {
     checks: [nonZero],
   },
 
+  // AWS Bedrock LLM InferenceSettings
+  AwsBedrockLLMAccessKeyId: {
+    envKey: "AWS_BEDROCK_LLM_ACCESS_KEY_ID",
+    checks: [isNotEmpty],
+  },
+  AwsBedrockLLMAccessKey: {
+    envKey: "AWS_BEDROCK_LLM_ACCESS_KEY",
+    checks: [isNotEmpty],
+  },
+  AwsBedrockLLMRegion: {
+    envKey: "AWS_BEDROCK_LLM_REGION",
+    checks: [isNotEmpty],
+  },
+  AwsBedrockLLMModel: {
+    envKey: "AWS_BEDROCK_LLM_MODEL_PREFERENCE",
+    checks: [isNotEmpty],
+  },
+  AwsBedrockLLMTokenLimit: {
+    envKey: "AWS_BEDROCK_LLM_MODEL_TOKEN_LIMIT",
+    checks: [nonZero],
+  },
+
   EmbeddingEngine: {
     envKey: "EMBEDDING_ENGINE",
     checks: [supportedEmbeddingModel],
@@ -541,6 +563,7 @@ function supportedLLM(input = "") {
     "cohere",
     "litellm",
     "generic-openai",
+    "bedrock",
   ].includes(input);
   return validSelection ? null : `${input} is not a valid LLM provider.`;
 }