diff --git a/README.md b/README.md
index 032b5892..00301a36 100644
--- a/README.md
+++ b/README.md
@@ -52,9 +52,10 @@ Some cool features of AnythingLLM
### Supported LLMs and Vector Databases
**Supported LLMs:**
-- OpenAI
-- Azure OpenAI
-- Anthropic ClaudeV2
+- [OpenAI](https://openai.com)
+- [Azure OpenAI](https://azure.microsoft.com/en-us/products/ai-services/openai-service)
+- [Anthropic ClaudeV2](https://www.anthropic.com/)
+- [LM Studio (all models)](https://lmstudio.ai)
**Supported Vector Databases:**
- [LanceDB](https://github.com/lancedb/lancedb) (default)
@@ -73,7 +74,7 @@ This monorepo consists of three main sections:
### Requirements
- `yarn` and `node` on your machine
- `python` 3.9+ for running scripts in `collector/`.
-- access to an LLM like `GPT-3.5`, `GPT-4`, etc.
+- access to an LLM service like `GPT-3.5`, `GPT-4`, `Mistral`, `LLama`, etc.
- (optional) a vector database like Pinecone, qDrant, Weaviate, or Chroma*.
*AnythingLLM by default uses a built-in vector db called LanceDB.
diff --git a/docker/.env.example b/docker/.env.example
index 4ab09a1e..1bd2b708 100644
--- a/docker/.env.example
+++ b/docker/.env.example
@@ -19,6 +19,10 @@ CACHE_VECTORS="true"
# ANTHROPIC_API_KEY=sk-ant-xxxx
# ANTHROPIC_MODEL_PREF='claude-2'
+# LLM_PROVIDER='lmstudio'
+# LMSTUDIO_BASE_PATH='http://your-server:1234/v1'
+# LMSTUDIO_MODEL_TOKEN_LIMIT=4096
+
###########################################
######## Embedding API SElECTION ##########
###########################################
diff --git a/frontend/src/components/LLMSelection/LMStudioOptions/index.jsx b/frontend/src/components/LLMSelection/LMStudioOptions/index.jsx
new file mode 100644
index 00000000..1f00c070
--- /dev/null
+++ b/frontend/src/components/LLMSelection/LMStudioOptions/index.jsx
@@ -0,0 +1,59 @@
+import { Info } from "@phosphor-icons/react";
+import paths from "../../../utils/paths";
+
+export default function LMStudioOptions({ settings, showAlert = false }) {
+ return (
+
+ {showAlert && (
+
+
+
+
+ LMStudio as your LLM requires you to set an embedding service to
+ use.
+
+
+
+ Manage embedding →
+
+
+ )}
+
+
{llmChoice === "openai" && (
@@ -141,6 +152,9 @@ export default function GeneralLLMPreference() {
{llmChoice === "anthropic" && (
)}
+ {llmChoice === "lmstudio" && (
+
+ )}
{llmChoice === "openai" &&
}
@@ -101,6 +114,9 @@ function LLMSelection({ nextStep, prevStep, currentStep }) {
{llmChoice === "anthropic" && (
)}
+ {llmChoice === "lmstudio" && (
+
+ )}
diff --git a/server/.env.example b/server/.env.example
index d7a9cbe7..327aa6ee 100644
--- a/server/.env.example
+++ b/server/.env.example
@@ -19,6 +19,10 @@ JWT_SECRET="my-random-string-for-seeding" # Please generate random string at lea
# ANTHROPIC_API_KEY=sk-ant-xxxx
# ANTHROPIC_MODEL_PREF='claude-2'
+# LLM_PROVIDER='lmstudio'
+# LMSTUDIO_BASE_PATH='http://your-server:1234/v1'
+# LMSTUDIO_MODEL_TOKEN_LIMIT=4096
+
###########################################
######## Embedding API SElECTION ##########
###########################################
@@ -58,4 +62,4 @@ VECTOR_DB="lancedb"
# CLOUD DEPLOYMENT VARIRABLES ONLY
# AUTH_TOKEN="hunter2" # This is the password to your application if remote hosting.
# STORAGE_DIR= # absolute filesystem path with no trailing slash
-# NO_DEBUG="true"
\ No newline at end of file
+# NO_DEBUG="true"
diff --git a/server/models/systemSettings.js b/server/models/systemSettings.js
index d15f7306..b28c5e86 100644
--- a/server/models/systemSettings.js
+++ b/server/models/systemSettings.js
@@ -81,6 +81,19 @@ const SystemSettings = {
AzureOpenAiEmbeddingModelPref: process.env.EMBEDDING_MODEL_PREF,
}
: {}),
+
+ ...(llmProvider === "lmstudio"
+ ? {
+ LMStudioBasePath: process.env.LMSTUDIO_BASE_PATH,
+ LMStudioTokenLimit: process.env.LMSTUDIO_MODEL_TOKEN_LIMIT,
+
+ // For embedding credentials when lmstudio is selected.
+ OpenAiKey: !!process.env.OPEN_AI_KEY,
+ AzureOpenAiEndpoint: process.env.AZURE_OPENAI_ENDPOINT,
+ AzureOpenAiKey: !!process.env.AZURE_OPENAI_KEY,
+ AzureOpenAiEmbeddingModelPref: process.env.EMBEDDING_MODEL_PREF,
+ }
+ : {}),
};
},
diff --git a/server/utils/AiProviders/lmStudio/index.js b/server/utils/AiProviders/lmStudio/index.js
new file mode 100644
index 00000000..bb025b3b
--- /dev/null
+++ b/server/utils/AiProviders/lmStudio/index.js
@@ -0,0 +1,139 @@
+const { chatPrompt } = require("../../chats");
+
+// hybrid of openAi LLM chat completion for LMStudio
+class LMStudioLLM {
+ constructor(embedder = null) {
+ if (!process.env.LMSTUDIO_BASE_PATH)
+ throw new Error("No LMStudio API Base Path was set.");
+
+ const { Configuration, OpenAIApi } = require("openai");
+ const config = new Configuration({
+ basePath: process.env.LMSTUDIO_BASE_PATH?.replace(/\/+$/, ""), // here is the URL to your LMStudio instance
+ });
+ this.lmstudio = new OpenAIApi(config);
+ // When using LMStudios inference server - the model param is not required so
+ // we can stub it here.
+ this.model = "model-placeholder";
+ this.limits = {
+ history: this.promptWindowLimit() * 0.15,
+ system: this.promptWindowLimit() * 0.15,
+ user: this.promptWindowLimit() * 0.7,
+ };
+
+ if (!embedder)
+ throw new Error(
+ "INVALID LM STUDIO SETUP. No embedding engine has been set. Go to instance settings and set up an embedding interface to use LMStudio as your LLM."
+ );
+ this.embedder = embedder;
+ }
+
+ // Ensure the user set a value for the token limit
+ // and if undefined - assume 4096 window.
+ promptWindowLimit() {
+ const limit = process.env.LMSTUDIO_MODEL_TOKEN_LIMIT || 4096;
+ if (!limit || isNaN(Number(limit)))
+ throw new Error("No LMStudio token context limit was set.");
+ return Number(limit);
+ }
+
+ async isValidChatCompletionModel(_ = "") {
+ // LMStudio may be anything. The user must do it correctly.
+ // See comment about this.model declaration in constructor
+ return true;
+ }
+
+ constructPrompt({
+ systemPrompt = "",
+ contextTexts = [],
+ chatHistory = [],
+ userPrompt = "",
+ }) {
+ const prompt = {
+ role: "system",
+ content: `${systemPrompt}
+Context:
+ ${contextTexts
+ .map((text, i) => {
+ return `[CONTEXT ${i}]:\n${text}\n[END CONTEXT ${i}]\n\n`;
+ })
+ .join("")}`,
+ };
+ return [prompt, ...chatHistory, { role: "user", content: userPrompt }];
+ }
+
+ async isSafe(_input = "") {
+ // Not implemented so must be stubbed
+ return { safe: true, reasons: [] };
+ }
+
+ async sendChat(chatHistory = [], prompt, workspace = {}, rawHistory = []) {
+ if (!this.model)
+ throw new Error(
+ `LMStudio chat: ${model} is not valid or defined for chat completion!`
+ );
+
+ const textResponse = await this.lmstudio
+ .createChatCompletion({
+ model: this.model,
+ temperature: Number(workspace?.openAiTemp ?? 0.7),
+ n: 1,
+ messages: await this.compressMessages(
+ {
+ systemPrompt: chatPrompt(workspace),
+ userPrompt: prompt,
+ chatHistory,
+ },
+ rawHistory
+ ),
+ })
+ .then((json) => {
+ const res = json.data;
+ if (!res.hasOwnProperty("choices"))
+ throw new Error("LMStudio chat: No results!");
+ if (res.choices.length === 0)
+ throw new Error("LMStudio chat: No results length!");
+ return res.choices[0].message.content;
+ })
+ .catch((error) => {
+ throw new Error(
+ `LMStudio::createChatCompletion failed with: ${error.message}`
+ );
+ });
+
+ return textResponse;
+ }
+
+ async getChatCompletion(messages = null, { temperature = 0.7 }) {
+ if (!this.model)
+ throw new Error(
+ `LMStudio chat: ${this.model} is not valid or defined model for chat completion!`
+ );
+
+ const { data } = await this.lmstudio.createChatCompletion({
+ model: this.model,
+ messages,
+ temperature,
+ });
+
+ if (!data.hasOwnProperty("choices")) return null;
+ return data.choices[0].message.content;
+ }
+
+ // Simple wrapper for dynamic embedder & normalize interface for all LLM implementations
+ async embedTextInput(textInput) {
+ return await this.embedder.embedTextInput(textInput);
+ }
+ async embedChunks(textChunks = []) {
+ return await this.embedder.embedChunks(textChunks);
+ }
+
+ async compressMessages(promptArgs = {}, rawHistory = []) {
+ const { messageArrayCompressor } = require("../../helpers/chat");
+ const messageArray = this.constructPrompt(promptArgs);
+ return await messageArrayCompressor(this, messageArray, rawHistory);
+ }
+}
+
+module.exports = {
+ LMStudioLLM,
+};
diff --git a/server/utils/helpers/index.js b/server/utils/helpers/index.js
index 9df2e8f1..cf48937a 100644
--- a/server/utils/helpers/index.js
+++ b/server/utils/helpers/index.js
@@ -23,6 +23,7 @@ function getVectorDbClass() {
function getLLMProvider() {
const vectorSelection = process.env.LLM_PROVIDER || "openai";
+ let embedder = null;
switch (vectorSelection) {
case "openai":
const { OpenAiLLM } = require("../AiProviders/openAi");
@@ -32,8 +33,12 @@ function getLLMProvider() {
return new AzureOpenAiLLM();
case "anthropic":
const { AnthropicLLM } = require("../AiProviders/anthropic");
- const embedder = getEmbeddingEngineSelection();
+ embedder = getEmbeddingEngineSelection();
return new AnthropicLLM(embedder);
+ case "lmstudio":
+ const { LMStudioLLM } = require("../AiProviders/lmStudio");
+ embedder = getEmbeddingEngineSelection();
+ return new LMStudioLLM(embedder);
default:
throw new Error("ENV: No LLM_PROVIDER value found in environment!");
}
diff --git a/server/utils/helpers/updateENV.js b/server/utils/helpers/updateENV.js
index 976849d9..e97f9791 100644
--- a/server/utils/helpers/updateENV.js
+++ b/server/utils/helpers/updateENV.js
@@ -44,6 +44,16 @@ const KEY_MAPPING = {
checks: [isNotEmpty, validAnthropicModel],
},
+ // LMStudio Settings
+ LMStudioBasePath: {
+ envKey: "LMSTUDIO_BASE_PATH",
+ checks: [isNotEmpty, validLMStudioBasePath],
+ },
+ LMStudioTokenLimit: {
+ envKey: "LMSTUDIO_MODEL_TOKEN_LIMIT",
+ checks: [nonZero],
+ },
+
EmbeddingEngine: {
envKey: "EMBEDDING_ENGINE",
checks: [supportedEmbeddingModel],
@@ -117,6 +127,11 @@ function isNotEmpty(input = "") {
return !input || input.length === 0 ? "Value cannot be empty" : null;
}
+function nonZero(input = "") {
+ if (isNaN(Number(input))) return "Value must be a number";
+ return Number(input) <= 0 ? "Value must be greater than zero" : null;
+}
+
function isValidURL(input = "") {
try {
new URL(input);
@@ -136,8 +151,20 @@ function validAnthropicApiKey(input = "") {
: "Anthropic Key must start with sk-ant-";
}
+function validLMStudioBasePath(input = "") {
+ try {
+ new URL(input);
+ if (!input.includes("v1")) return "URL must include /v1";
+ if (input.split("").slice(-1)?.[0] === "/")
+ return "URL cannot end with a slash";
+ return null;
+ } catch {
+ return "Not a valid URL";
+ }
+}
+
function supportedLLM(input = "") {
- return ["openai", "azure", "anthropic"].includes(input);
+ return ["openai", "azure", "anthropic", "lmstudio"].includes(input);
}
function validAnthropicModel(input = "") {