mirror of
https://github.com/Mintplex-Labs/anything-llm.git
synced 2024-10-03 17:30:11 +02:00
Add Ollama as LLM provider option (#494)
* Add support for Ollama as LLM provider resolves #493
This commit is contained in:
parent
24227e48a7
commit
e0a0a8976d
1
.vscode/settings.json
vendored
1
.vscode/settings.json
vendored
@ -1,5 +1,6 @@
|
|||||||
{
|
{
|
||||||
"cSpell.words": [
|
"cSpell.words": [
|
||||||
|
"Ollama",
|
||||||
"openai",
|
"openai",
|
||||||
"Qdrant",
|
"Qdrant",
|
||||||
"Weaviate"
|
"Weaviate"
|
||||||
|
@ -59,6 +59,7 @@ Some cool features of AnythingLLM
|
|||||||
- [Azure OpenAI](https://azure.microsoft.com/en-us/products/ai-services/openai-service)
|
- [Azure OpenAI](https://azure.microsoft.com/en-us/products/ai-services/openai-service)
|
||||||
- [Anthropic ClaudeV2](https://www.anthropic.com/)
|
- [Anthropic ClaudeV2](https://www.anthropic.com/)
|
||||||
- [Google Gemini Pro](https://ai.google.dev/)
|
- [Google Gemini Pro](https://ai.google.dev/)
|
||||||
|
- [Ollama (chat models)](https://ollama.ai/)
|
||||||
- [LM Studio (all models)](https://lmstudio.ai)
|
- [LM Studio (all models)](https://lmstudio.ai)
|
||||||
- [LocalAi (all models)](https://localai.io/)
|
- [LocalAi (all models)](https://localai.io/)
|
||||||
|
|
||||||
|
@ -35,6 +35,11 @@ GID='1000'
|
|||||||
# LOCAL_AI_MODEL_TOKEN_LIMIT=4096
|
# LOCAL_AI_MODEL_TOKEN_LIMIT=4096
|
||||||
# LOCAL_AI_API_KEY="sk-123abc"
|
# LOCAL_AI_API_KEY="sk-123abc"
|
||||||
|
|
||||||
|
# LLM_PROVIDER='ollama'
|
||||||
|
# OLLAMA_BASE_PATH='http://host.docker.internal:11434'
|
||||||
|
# OLLAMA_MODEL_PREF='llama2'
|
||||||
|
# OLLAMA_MODEL_TOKEN_LIMIT=4096
|
||||||
|
|
||||||
###########################################
|
###########################################
|
||||||
######## Embedding API SElECTION ##########
|
######## Embedding API SElECTION ##########
|
||||||
###########################################
|
###########################################
|
||||||
|
120
frontend/src/components/LLMSelection/OllamaLLMOptions/index.jsx
Normal file
120
frontend/src/components/LLMSelection/OllamaLLMOptions/index.jsx
Normal file
@ -0,0 +1,120 @@
|
|||||||
|
import { useEffect, useState } from "react";
|
||||||
|
import System from "@/models/system";
|
||||||
|
|
||||||
|
export default function OllamaLLMOptions({ settings }) {
|
||||||
|
const [basePathValue, setBasePathValue] = useState(
|
||||||
|
settings?.OllamaLLMBasePath
|
||||||
|
);
|
||||||
|
const [basePath, setBasePath] = useState(settings?.OllamaLLMBasePath);
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="w-full flex flex-col gap-y-4">
|
||||||
|
<div className="w-full flex items-center gap-4">
|
||||||
|
<div className="flex flex-col w-60">
|
||||||
|
<label className="text-white text-sm font-semibold block mb-4">
|
||||||
|
Ollama Base URL
|
||||||
|
</label>
|
||||||
|
<input
|
||||||
|
type="url"
|
||||||
|
name="OllamaLLMBasePath"
|
||||||
|
className="bg-zinc-900 text-white placeholder-white placeholder-opacity-60 text-sm rounded-lg focus:border-white block w-full p-2.5"
|
||||||
|
placeholder="http://127.0.0.1:11434"
|
||||||
|
defaultValue={settings?.OllamaLLMBasePath}
|
||||||
|
required={true}
|
||||||
|
autoComplete="off"
|
||||||
|
spellCheck={false}
|
||||||
|
onChange={(e) => setBasePathValue(e.target.value)}
|
||||||
|
onBlur={() => setBasePath(basePathValue)}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
<OllamaLLMModelSelection settings={settings} basePath={basePath} />
|
||||||
|
<div className="flex flex-col w-60">
|
||||||
|
<label className="text-white text-sm font-semibold block mb-4">
|
||||||
|
Token context window
|
||||||
|
</label>
|
||||||
|
<input
|
||||||
|
type="number"
|
||||||
|
name="OllamaLLMTokenLimit"
|
||||||
|
className="bg-zinc-900 text-white placeholder-white placeholder-opacity-60 text-sm rounded-lg focus:border-white block w-full p-2.5"
|
||||||
|
placeholder="4096"
|
||||||
|
min={1}
|
||||||
|
onScroll={(e) => e.target.blur()}
|
||||||
|
defaultValue={settings?.OllamaLLMTokenLimit}
|
||||||
|
required={true}
|
||||||
|
autoComplete="off"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function OllamaLLMModelSelection({ settings, basePath = null }) {
|
||||||
|
const [customModels, setCustomModels] = useState([]);
|
||||||
|
const [loading, setLoading] = useState(true);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
async function findCustomModels() {
|
||||||
|
if (!basePath) {
|
||||||
|
setCustomModels([]);
|
||||||
|
setLoading(false);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
setLoading(true);
|
||||||
|
const { models } = await System.customModels("ollama", null, basePath);
|
||||||
|
setCustomModels(models || []);
|
||||||
|
setLoading(false);
|
||||||
|
}
|
||||||
|
findCustomModels();
|
||||||
|
}, [basePath]);
|
||||||
|
|
||||||
|
if (loading || customModels.length == 0) {
|
||||||
|
return (
|
||||||
|
<div className="flex flex-col w-60">
|
||||||
|
<label className="text-white text-sm font-semibold block mb-4">
|
||||||
|
Chat Model Selection
|
||||||
|
</label>
|
||||||
|
<select
|
||||||
|
name="OllamaLLMModelPref"
|
||||||
|
disabled={true}
|
||||||
|
className="bg-zinc-900 border border-gray-500 text-white text-sm rounded-lg block w-full p-2.5"
|
||||||
|
>
|
||||||
|
<option disabled={true} selected={true}>
|
||||||
|
{!!basePath
|
||||||
|
? "-- loading available models --"
|
||||||
|
: "-- waiting for URL --"}
|
||||||
|
</option>
|
||||||
|
</select>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="flex flex-col w-60">
|
||||||
|
<label className="text-white text-sm font-semibold block mb-4">
|
||||||
|
Chat Model Selection
|
||||||
|
</label>
|
||||||
|
<select
|
||||||
|
name="OllamaLLMModelPref"
|
||||||
|
required={true}
|
||||||
|
className="bg-zinc-900 border border-gray-500 text-white text-sm rounded-lg block w-full p-2.5"
|
||||||
|
>
|
||||||
|
{customModels.length > 0 && (
|
||||||
|
<optgroup label="Your loaded models">
|
||||||
|
{customModels.map((model) => {
|
||||||
|
return (
|
||||||
|
<option
|
||||||
|
key={model.id}
|
||||||
|
value={model.id}
|
||||||
|
selected={settings.OllamaLLMModelPref === model.id}
|
||||||
|
>
|
||||||
|
{model.id}
|
||||||
|
</option>
|
||||||
|
);
|
||||||
|
})}
|
||||||
|
</optgroup>
|
||||||
|
)}
|
||||||
|
</select>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
BIN
frontend/src/media/llmprovider/ollama.png
Normal file
BIN
frontend/src/media/llmprovider/ollama.png
Normal file
Binary file not shown.
After Width: | Height: | Size: 23 KiB |
@ -8,6 +8,7 @@ import OpenAiLogo from "@/media/llmprovider/openai.png";
|
|||||||
import AzureOpenAiLogo from "@/media/llmprovider/azure.png";
|
import AzureOpenAiLogo from "@/media/llmprovider/azure.png";
|
||||||
import AnthropicLogo from "@/media/llmprovider/anthropic.png";
|
import AnthropicLogo from "@/media/llmprovider/anthropic.png";
|
||||||
import GeminiLogo from "@/media/llmprovider/gemini.png";
|
import GeminiLogo from "@/media/llmprovider/gemini.png";
|
||||||
|
import OllamaLogo from "@/media/llmprovider/ollama.png";
|
||||||
import LMStudioLogo from "@/media/llmprovider/lmstudio.png";
|
import LMStudioLogo from "@/media/llmprovider/lmstudio.png";
|
||||||
import LocalAiLogo from "@/media/llmprovider/localai.png";
|
import LocalAiLogo from "@/media/llmprovider/localai.png";
|
||||||
import PreLoader from "@/components/Preloader";
|
import PreLoader from "@/components/Preloader";
|
||||||
@ -19,6 +20,7 @@ import LMStudioOptions from "@/components/LLMSelection/LMStudioOptions";
|
|||||||
import LocalAiOptions from "@/components/LLMSelection/LocalAiOptions";
|
import LocalAiOptions from "@/components/LLMSelection/LocalAiOptions";
|
||||||
import NativeLLMOptions from "@/components/LLMSelection/NativeLLMOptions";
|
import NativeLLMOptions from "@/components/LLMSelection/NativeLLMOptions";
|
||||||
import GeminiLLMOptions from "@/components/LLMSelection/GeminiLLMOptions";
|
import GeminiLLMOptions from "@/components/LLMSelection/GeminiLLMOptions";
|
||||||
|
import OllamaLLMOptions from "@/components/LLMSelection/OllamaLLMOptions";
|
||||||
|
|
||||||
export default function GeneralLLMPreference() {
|
export default function GeneralLLMPreference() {
|
||||||
const [saving, setSaving] = useState(false);
|
const [saving, setSaving] = useState(false);
|
||||||
@ -163,6 +165,15 @@ export default function GeneralLLMPreference() {
|
|||||||
image={LocalAiLogo}
|
image={LocalAiLogo}
|
||||||
onClick={updateLLMChoice}
|
onClick={updateLLMChoice}
|
||||||
/>
|
/>
|
||||||
|
<LLMProviderOption
|
||||||
|
name="Ollama"
|
||||||
|
value="ollama"
|
||||||
|
link="ollama.ai"
|
||||||
|
description="Run LLMs locally on your own machine."
|
||||||
|
checked={llmChoice === "ollama"}
|
||||||
|
image={OllamaLogo}
|
||||||
|
onClick={updateLLMChoice}
|
||||||
|
/>
|
||||||
{!window.location.hostname.includes("useanything.com") && (
|
{!window.location.hostname.includes("useanything.com") && (
|
||||||
<LLMProviderOption
|
<LLMProviderOption
|
||||||
name="Custom Llama Model"
|
name="Custom Llama Model"
|
||||||
@ -193,6 +204,9 @@ export default function GeneralLLMPreference() {
|
|||||||
{llmChoice === "localai" && (
|
{llmChoice === "localai" && (
|
||||||
<LocalAiOptions settings={settings} showAlert={true} />
|
<LocalAiOptions settings={settings} showAlert={true} />
|
||||||
)}
|
)}
|
||||||
|
{llmChoice === "ollama" && (
|
||||||
|
<OllamaLLMOptions settings={settings} />
|
||||||
|
)}
|
||||||
{llmChoice === "native" && (
|
{llmChoice === "native" && (
|
||||||
<NativeLLMOptions settings={settings} />
|
<NativeLLMOptions settings={settings} />
|
||||||
)}
|
)}
|
||||||
|
@ -5,6 +5,7 @@ import OpenAiLogo from "@/media/llmprovider/openai.png";
|
|||||||
import AzureOpenAiLogo from "@/media/llmprovider/azure.png";
|
import AzureOpenAiLogo from "@/media/llmprovider/azure.png";
|
||||||
import AnthropicLogo from "@/media/llmprovider/anthropic.png";
|
import AnthropicLogo from "@/media/llmprovider/anthropic.png";
|
||||||
import GeminiLogo from "@/media/llmprovider/gemini.png";
|
import GeminiLogo from "@/media/llmprovider/gemini.png";
|
||||||
|
import OllamaLogo from "@/media/llmprovider/ollama.png";
|
||||||
import LMStudioLogo from "@/media/llmprovider/lmstudio.png";
|
import LMStudioLogo from "@/media/llmprovider/lmstudio.png";
|
||||||
import LocalAiLogo from "@/media/llmprovider/localai.png";
|
import LocalAiLogo from "@/media/llmprovider/localai.png";
|
||||||
import ChromaLogo from "@/media/vectordbs/chroma.png";
|
import ChromaLogo from "@/media/vectordbs/chroma.png";
|
||||||
@ -61,6 +62,13 @@ const LLM_SELECTION_PRIVACY = {
|
|||||||
],
|
],
|
||||||
logo: LocalAiLogo,
|
logo: LocalAiLogo,
|
||||||
},
|
},
|
||||||
|
ollama: {
|
||||||
|
name: "Ollama",
|
||||||
|
description: [
|
||||||
|
"Your model and chats are only accessible on the machine running Ollama models",
|
||||||
|
],
|
||||||
|
logo: OllamaLogo,
|
||||||
|
},
|
||||||
native: {
|
native: {
|
||||||
name: "Custom Llama Model",
|
name: "Custom Llama Model",
|
||||||
description: [
|
description: [
|
||||||
|
@ -4,6 +4,7 @@ import OpenAiLogo from "@/media/llmprovider/openai.png";
|
|||||||
import AzureOpenAiLogo from "@/media/llmprovider/azure.png";
|
import AzureOpenAiLogo from "@/media/llmprovider/azure.png";
|
||||||
import AnthropicLogo from "@/media/llmprovider/anthropic.png";
|
import AnthropicLogo from "@/media/llmprovider/anthropic.png";
|
||||||
import GeminiLogo from "@/media/llmprovider/gemini.png";
|
import GeminiLogo from "@/media/llmprovider/gemini.png";
|
||||||
|
import OllamaLogo from "@/media/llmprovider/ollama.png";
|
||||||
import LMStudioLogo from "@/media/llmprovider/lmstudio.png";
|
import LMStudioLogo from "@/media/llmprovider/lmstudio.png";
|
||||||
import LocalAiLogo from "@/media/llmprovider/localai.png";
|
import LocalAiLogo from "@/media/llmprovider/localai.png";
|
||||||
import System from "@/models/system";
|
import System from "@/models/system";
|
||||||
@ -16,6 +17,7 @@ import LMStudioOptions from "@/components/LLMSelection/LMStudioOptions";
|
|||||||
import LocalAiOptions from "@/components/LLMSelection/LocalAiOptions";
|
import LocalAiOptions from "@/components/LLMSelection/LocalAiOptions";
|
||||||
import NativeLLMOptions from "@/components/LLMSelection/NativeLLMOptions";
|
import NativeLLMOptions from "@/components/LLMSelection/NativeLLMOptions";
|
||||||
import GeminiLLMOptions from "@/components/LLMSelection/GeminiLLMOptions";
|
import GeminiLLMOptions from "@/components/LLMSelection/GeminiLLMOptions";
|
||||||
|
import OllamaLLMOptions from "@/components/LLMSelection/OllamaLLMOptions";
|
||||||
|
|
||||||
function LLMSelection({ nextStep, prevStep, currentStep }) {
|
function LLMSelection({ nextStep, prevStep, currentStep }) {
|
||||||
const [llmChoice, setLLMChoice] = useState("openai");
|
const [llmChoice, setLLMChoice] = useState("openai");
|
||||||
@ -124,13 +126,24 @@ function LLMSelection({ nextStep, prevStep, currentStep }) {
|
|||||||
onClick={updateLLMChoice}
|
onClick={updateLLMChoice}
|
||||||
/>
|
/>
|
||||||
<LLMProviderOption
|
<LLMProviderOption
|
||||||
name="Custom Llama Model"
|
name="Ollama"
|
||||||
value="native"
|
value="ollama"
|
||||||
description="Use a downloaded custom Llama model for chatting on this AnythingLLM instance."
|
link="ollama.ai"
|
||||||
checked={llmChoice === "native"}
|
description="Run LLMs locally on your own machine."
|
||||||
image={AnythingLLMIcon}
|
checked={llmChoice === "ollama"}
|
||||||
|
image={OllamaLogo}
|
||||||
onClick={updateLLMChoice}
|
onClick={updateLLMChoice}
|
||||||
/>
|
/>
|
||||||
|
{!window.location.hostname.includes("useanything.com") && (
|
||||||
|
<LLMProviderOption
|
||||||
|
name="Custom Llama Model"
|
||||||
|
value="native"
|
||||||
|
description="Use a downloaded custom Llama model for chatting on this AnythingLLM instance."
|
||||||
|
checked={llmChoice === "native"}
|
||||||
|
image={AnythingLLMIcon}
|
||||||
|
onClick={updateLLMChoice}
|
||||||
|
/>
|
||||||
|
)}
|
||||||
</div>
|
</div>
|
||||||
<div className="mt-4 flex flex-wrap gap-4 max-w-[752px]">
|
<div className="mt-4 flex flex-wrap gap-4 max-w-[752px]">
|
||||||
{llmChoice === "openai" && <OpenAiOptions settings={settings} />}
|
{llmChoice === "openai" && <OpenAiOptions settings={settings} />}
|
||||||
@ -143,6 +156,7 @@ function LLMSelection({ nextStep, prevStep, currentStep }) {
|
|||||||
<LMStudioOptions settings={settings} />
|
<LMStudioOptions settings={settings} />
|
||||||
)}
|
)}
|
||||||
{llmChoice === "localai" && <LocalAiOptions settings={settings} />}
|
{llmChoice === "localai" && <LocalAiOptions settings={settings} />}
|
||||||
|
{llmChoice === "ollama" && <OllamaLLMOptions settings={settings} />}
|
||||||
{llmChoice === "native" && <NativeLLMOptions settings={settings} />}
|
{llmChoice === "native" && <NativeLLMOptions settings={settings} />}
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
@ -32,6 +32,11 @@ JWT_SECRET="my-random-string-for-seeding" # Please generate random string at lea
|
|||||||
# LOCAL_AI_MODEL_TOKEN_LIMIT=4096
|
# LOCAL_AI_MODEL_TOKEN_LIMIT=4096
|
||||||
# LOCAL_AI_API_KEY="sk-123abc"
|
# LOCAL_AI_API_KEY="sk-123abc"
|
||||||
|
|
||||||
|
# LLM_PROVIDER='ollama'
|
||||||
|
# OLLAMA_BASE_PATH='http://host.docker.internal:11434'
|
||||||
|
# OLLAMA_MODEL_PREF='llama2'
|
||||||
|
# OLLAMA_MODEL_TOKEN_LIMIT=4096
|
||||||
|
|
||||||
###########################################
|
###########################################
|
||||||
######## Embedding API SElECTION ##########
|
######## Embedding API SElECTION ##########
|
||||||
###########################################
|
###########################################
|
||||||
|
@ -126,6 +126,20 @@ const SystemSettings = {
|
|||||||
AzureOpenAiEmbeddingModelPref: process.env.EMBEDDING_MODEL_PREF,
|
AzureOpenAiEmbeddingModelPref: process.env.EMBEDDING_MODEL_PREF,
|
||||||
}
|
}
|
||||||
: {}),
|
: {}),
|
||||||
|
|
||||||
|
...(llmProvider === "ollama"
|
||||||
|
? {
|
||||||
|
OllamaLLMBasePath: process.env.OLLAMA_BASE_PATH,
|
||||||
|
OllamaLLMModelPref: process.env.OLLAMA_MODEL_PREF,
|
||||||
|
OllamaLLMTokenLimit: process.env.OLLAMA_MODEL_TOKEN_LIMIT,
|
||||||
|
|
||||||
|
// For embedding credentials when ollama is selected.
|
||||||
|
OpenAiKey: !!process.env.OPEN_AI_KEY,
|
||||||
|
AzureOpenAiEndpoint: process.env.AZURE_OPENAI_ENDPOINT,
|
||||||
|
AzureOpenAiKey: !!process.env.AZURE_OPENAI_KEY,
|
||||||
|
AzureOpenAiEmbeddingModelPref: process.env.EMBEDDING_MODEL_PREF,
|
||||||
|
}
|
||||||
|
: {}),
|
||||||
...(llmProvider === "native"
|
...(llmProvider === "native"
|
||||||
? {
|
? {
|
||||||
NativeLLMModelPref: process.env.NATIVE_LLM_MODEL_PREF,
|
NativeLLMModelPref: process.env.NATIVE_LLM_MODEL_PREF,
|
||||||
|
208
server/utils/AiProviders/ollama/index.js
Normal file
208
server/utils/AiProviders/ollama/index.js
Normal file
@ -0,0 +1,208 @@
|
|||||||
|
const { chatPrompt } = require("../../chats");
|
||||||
|
|
||||||
|
// Docs: https://github.com/jmorganca/ollama/blob/main/docs/api.md
|
||||||
|
class OllamaAILLM {
|
||||||
|
constructor(embedder = null) {
|
||||||
|
if (!process.env.OLLAMA_BASE_PATH)
|
||||||
|
throw new Error("No Ollama Base Path was set.");
|
||||||
|
|
||||||
|
this.basePath = process.env.OLLAMA_BASE_PATH;
|
||||||
|
this.model = process.env.OLLAMA_MODEL_PREF;
|
||||||
|
this.limits = {
|
||||||
|
history: this.promptWindowLimit() * 0.15,
|
||||||
|
system: this.promptWindowLimit() * 0.15,
|
||||||
|
user: this.promptWindowLimit() * 0.7,
|
||||||
|
};
|
||||||
|
|
||||||
|
if (!embedder)
|
||||||
|
throw new Error(
|
||||||
|
"INVALID OLLAMA SETUP. No embedding engine has been set. Go to instance settings and set up an embedding interface to use Ollama as your LLM."
|
||||||
|
);
|
||||||
|
this.embedder = embedder;
|
||||||
|
}
|
||||||
|
|
||||||
|
streamingEnabled() {
|
||||||
|
return "streamChat" in this && "streamGetChatCompletion" in this;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Ensure the user set a value for the token limit
|
||||||
|
// and if undefined - assume 4096 window.
|
||||||
|
promptWindowLimit() {
|
||||||
|
const limit = process.env.OLLAMA_MODEL_TOKEN_LIMIT || 4096;
|
||||||
|
if (!limit || isNaN(Number(limit)))
|
||||||
|
throw new Error("No Ollama token context limit was set.");
|
||||||
|
return Number(limit);
|
||||||
|
}
|
||||||
|
|
||||||
|
async isValidChatCompletionModel(_ = "") {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
constructPrompt({
|
||||||
|
systemPrompt = "",
|
||||||
|
contextTexts = [],
|
||||||
|
chatHistory = [],
|
||||||
|
userPrompt = "",
|
||||||
|
}) {
|
||||||
|
const prompt = {
|
||||||
|
role: "system",
|
||||||
|
content: `${systemPrompt}
|
||||||
|
Context:
|
||||||
|
${contextTexts
|
||||||
|
.map((text, i) => {
|
||||||
|
return `[CONTEXT ${i}]:\n${text}\n[END CONTEXT ${i}]\n\n`;
|
||||||
|
})
|
||||||
|
.join("")}`,
|
||||||
|
};
|
||||||
|
return [prompt, ...chatHistory, { role: "user", content: userPrompt }];
|
||||||
|
}
|
||||||
|
|
||||||
|
async isSafe(_input = "") {
|
||||||
|
// Not implemented so must be stubbed
|
||||||
|
return { safe: true, reasons: [] };
|
||||||
|
}
|
||||||
|
|
||||||
|
async sendChat(chatHistory = [], prompt, workspace = {}, rawHistory = []) {
|
||||||
|
const textResponse = await fetch(`${this.basePath}/api/chat`, {
|
||||||
|
method: "POST",
|
||||||
|
headers: {
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
},
|
||||||
|
body: JSON.stringify({
|
||||||
|
model: this.model,
|
||||||
|
stream: false,
|
||||||
|
options: {
|
||||||
|
temperature: Number(workspace?.openAiTemp ?? 0.7),
|
||||||
|
},
|
||||||
|
messages: await this.compressMessages(
|
||||||
|
{
|
||||||
|
systemPrompt: chatPrompt(workspace),
|
||||||
|
userPrompt: prompt,
|
||||||
|
chatHistory,
|
||||||
|
},
|
||||||
|
rawHistory
|
||||||
|
),
|
||||||
|
}),
|
||||||
|
})
|
||||||
|
.then((res) => {
|
||||||
|
if (!res.ok)
|
||||||
|
throw new Error(`Ollama:sendChat ${res.status} ${res.statusText}`);
|
||||||
|
return res.json();
|
||||||
|
})
|
||||||
|
.then((data) => data?.message?.content)
|
||||||
|
.catch((e) => {
|
||||||
|
console.error(e);
|
||||||
|
throw new Error(`Ollama::sendChat failed with: ${error.message}`);
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!textResponse.length)
|
||||||
|
throw new Error(`Ollama::sendChat text response was empty.`);
|
||||||
|
|
||||||
|
return textResponse;
|
||||||
|
}
|
||||||
|
|
||||||
|
async streamChat(chatHistory = [], prompt, workspace = {}, rawHistory = []) {
|
||||||
|
const response = await fetch(`${this.basePath}/api/chat`, {
|
||||||
|
method: "POST",
|
||||||
|
headers: {
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
},
|
||||||
|
body: JSON.stringify({
|
||||||
|
model: this.model,
|
||||||
|
stream: true,
|
||||||
|
options: {
|
||||||
|
temperature: Number(workspace?.openAiTemp ?? 0.7),
|
||||||
|
},
|
||||||
|
messages: await this.compressMessages(
|
||||||
|
{
|
||||||
|
systemPrompt: chatPrompt(workspace),
|
||||||
|
userPrompt: prompt,
|
||||||
|
chatHistory,
|
||||||
|
},
|
||||||
|
rawHistory
|
||||||
|
),
|
||||||
|
}),
|
||||||
|
}).catch((e) => {
|
||||||
|
console.error(e);
|
||||||
|
throw new Error(`Ollama:streamChat ${error.message}`);
|
||||||
|
});
|
||||||
|
|
||||||
|
return { type: "ollamaStream", response };
|
||||||
|
}
|
||||||
|
|
||||||
|
async getChatCompletion(messages = null, { temperature = 0.7 }) {
|
||||||
|
const textResponse = await fetch(`${this.basePath}/api/chat`, {
|
||||||
|
method: "POST",
|
||||||
|
headers: {
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
},
|
||||||
|
body: JSON.stringify({
|
||||||
|
model: this.model,
|
||||||
|
messages,
|
||||||
|
stream: false,
|
||||||
|
options: {
|
||||||
|
temperature,
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
})
|
||||||
|
.then((res) => {
|
||||||
|
if (!res.ok)
|
||||||
|
throw new Error(
|
||||||
|
`Ollama:getChatCompletion ${res.status} ${res.statusText}`
|
||||||
|
);
|
||||||
|
return res.json();
|
||||||
|
})
|
||||||
|
.then((data) => data?.message?.content)
|
||||||
|
.catch((e) => {
|
||||||
|
console.error(e);
|
||||||
|
throw new Error(
|
||||||
|
`Ollama::getChatCompletion failed with: ${error.message}`
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!textResponse.length)
|
||||||
|
throw new Error(`Ollama::getChatCompletion text response was empty.`);
|
||||||
|
|
||||||
|
return textResponse;
|
||||||
|
}
|
||||||
|
|
||||||
|
async streamGetChatCompletion(messages = null, { temperature = 0.7 }) {
|
||||||
|
const response = await fetch(`${this.basePath}/api/chat`, {
|
||||||
|
method: "POST",
|
||||||
|
headers: {
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
},
|
||||||
|
body: JSON.stringify({
|
||||||
|
model: this.model,
|
||||||
|
stream: true,
|
||||||
|
messages,
|
||||||
|
options: {
|
||||||
|
temperature,
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
}).catch((e) => {
|
||||||
|
console.error(e);
|
||||||
|
throw new Error(`Ollama:streamGetChatCompletion ${error.message}`);
|
||||||
|
});
|
||||||
|
|
||||||
|
return { type: "ollamaStream", response };
|
||||||
|
}
|
||||||
|
|
||||||
|
// Simple wrapper for dynamic embedder & normalize interface for all LLM implementations
|
||||||
|
async embedTextInput(textInput) {
|
||||||
|
return await this.embedder.embedTextInput(textInput);
|
||||||
|
}
|
||||||
|
async embedChunks(textChunks = []) {
|
||||||
|
return await this.embedder.embedChunks(textChunks);
|
||||||
|
}
|
||||||
|
|
||||||
|
async compressMessages(promptArgs = {}, rawHistory = []) {
|
||||||
|
const { messageArrayCompressor } = require("../../helpers/chat");
|
||||||
|
const messageArray = this.constructPrompt(promptArgs);
|
||||||
|
return await messageArrayCompressor(this, messageArray, rawHistory);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = {
|
||||||
|
OllamaAILLM,
|
||||||
|
};
|
@ -199,6 +199,7 @@ async function streamEmptyEmbeddingChat({
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TODO: Refactor this implementation
|
||||||
function handleStreamResponses(response, stream, responseProps) {
|
function handleStreamResponses(response, stream, responseProps) {
|
||||||
const { uuid = uuidv4(), sources = [] } = responseProps;
|
const { uuid = uuidv4(), sources = [] } = responseProps;
|
||||||
|
|
||||||
@ -231,6 +232,34 @@ function handleStreamResponses(response, stream, responseProps) {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (stream?.type === "ollamaStream") {
|
||||||
|
return new Promise(async (resolve) => {
|
||||||
|
let fullText = "";
|
||||||
|
for await (const dataChunk of stream.response.body) {
|
||||||
|
const chunk = JSON.parse(Buffer.from(dataChunk).toString());
|
||||||
|
fullText += chunk.message.content;
|
||||||
|
writeResponseChunk(response, {
|
||||||
|
uuid,
|
||||||
|
sources: [],
|
||||||
|
type: "textResponseChunk",
|
||||||
|
textResponse: chunk.message.content,
|
||||||
|
close: false,
|
||||||
|
error: false,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
writeResponseChunk(response, {
|
||||||
|
uuid,
|
||||||
|
sources,
|
||||||
|
type: "textResponseChunk",
|
||||||
|
textResponse: "",
|
||||||
|
close: true,
|
||||||
|
error: false,
|
||||||
|
});
|
||||||
|
resolve(fullText);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
// If stream is not a regular OpenAI Stream (like if using native model)
|
// If stream is not a regular OpenAI Stream (like if using native model)
|
||||||
// we can just iterate the stream content instead.
|
// we can just iterate the stream content instead.
|
||||||
if (!stream.hasOwnProperty("data")) {
|
if (!stream.hasOwnProperty("data")) {
|
||||||
|
@ -1,4 +1,4 @@
|
|||||||
const SUPPORT_CUSTOM_MODELS = ["openai", "localai", "native-llm"];
|
const SUPPORT_CUSTOM_MODELS = ["openai", "localai", "ollama", "native-llm"];
|
||||||
|
|
||||||
async function getCustomModels(provider = "", apiKey = null, basePath = null) {
|
async function getCustomModels(provider = "", apiKey = null, basePath = null) {
|
||||||
if (!SUPPORT_CUSTOM_MODELS.includes(provider))
|
if (!SUPPORT_CUSTOM_MODELS.includes(provider))
|
||||||
@ -9,6 +9,8 @@ async function getCustomModels(provider = "", apiKey = null, basePath = null) {
|
|||||||
return await openAiModels(apiKey);
|
return await openAiModels(apiKey);
|
||||||
case "localai":
|
case "localai":
|
||||||
return await localAIModels(basePath, apiKey);
|
return await localAIModels(basePath, apiKey);
|
||||||
|
case "ollama":
|
||||||
|
return await ollamaAIModels(basePath, apiKey);
|
||||||
case "native-llm":
|
case "native-llm":
|
||||||
return nativeLLMModels();
|
return nativeLLMModels();
|
||||||
default:
|
default:
|
||||||
@ -59,6 +61,37 @@ async function localAIModels(basePath = null, apiKey = null) {
|
|||||||
return { models, error: null };
|
return { models, error: null };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function ollamaAIModels(basePath = null, _apiKey = null) {
|
||||||
|
let url;
|
||||||
|
try {
|
||||||
|
new URL(basePath);
|
||||||
|
if (basePath.split("").slice(-1)?.[0] === "/")
|
||||||
|
throw new Error("BasePath Cannot end in /!");
|
||||||
|
url = basePath;
|
||||||
|
} catch {
|
||||||
|
return { models: [], error: "Not a valid URL." };
|
||||||
|
}
|
||||||
|
|
||||||
|
const models = await fetch(`${url}/api/tags`)
|
||||||
|
.then((res) => {
|
||||||
|
if (!res.ok)
|
||||||
|
throw new Error(`Could not reach Ollama server! ${res.status}`);
|
||||||
|
return res.json();
|
||||||
|
})
|
||||||
|
.then((data) => data?.models || [])
|
||||||
|
.then((models) =>
|
||||||
|
models.map((model) => {
|
||||||
|
return { id: model.name };
|
||||||
|
})
|
||||||
|
)
|
||||||
|
.catch((e) => {
|
||||||
|
console.error(e);
|
||||||
|
return [];
|
||||||
|
});
|
||||||
|
|
||||||
|
return { models, error: null };
|
||||||
|
}
|
||||||
|
|
||||||
function nativeLLMModels() {
|
function nativeLLMModels() {
|
||||||
const fs = require("fs");
|
const fs = require("fs");
|
||||||
const path = require("path");
|
const path = require("path");
|
||||||
|
@ -43,6 +43,9 @@ function getLLMProvider() {
|
|||||||
case "localai":
|
case "localai":
|
||||||
const { LocalAiLLM } = require("../AiProviders/localAi");
|
const { LocalAiLLM } = require("../AiProviders/localAi");
|
||||||
return new LocalAiLLM(embedder);
|
return new LocalAiLLM(embedder);
|
||||||
|
case "ollama":
|
||||||
|
const { OllamaAILLM } = require("../AiProviders/ollama");
|
||||||
|
return new OllamaAILLM(embedder);
|
||||||
case "native":
|
case "native":
|
||||||
const { NativeLLM } = require("../AiProviders/native");
|
const { NativeLLM } = require("../AiProviders/native");
|
||||||
return new NativeLLM(embedder);
|
return new NativeLLM(embedder);
|
||||||
|
@ -81,6 +81,19 @@ const KEY_MAPPING = {
|
|||||||
checks: [],
|
checks: [],
|
||||||
},
|
},
|
||||||
|
|
||||||
|
OllamaLLMBasePath: {
|
||||||
|
envKey: "OLLAMA_BASE_PATH",
|
||||||
|
checks: [isNotEmpty, validOllamaLLMBasePath],
|
||||||
|
},
|
||||||
|
OllamaLLMModelPref: {
|
||||||
|
envKey: "OLLAMA_MODEL_PREF",
|
||||||
|
checks: [],
|
||||||
|
},
|
||||||
|
OllamaLLMTokenLimit: {
|
||||||
|
envKey: "OLLAMA_MODEL_TOKEN_LIMIT",
|
||||||
|
checks: [nonZero],
|
||||||
|
},
|
||||||
|
|
||||||
// Native LLM Settings
|
// Native LLM Settings
|
||||||
NativeLLMModelPref: {
|
NativeLLMModelPref: {
|
||||||
envKey: "NATIVE_LLM_MODEL_PREF",
|
envKey: "NATIVE_LLM_MODEL_PREF",
|
||||||
@ -208,6 +221,17 @@ function validLLMExternalBasePath(input = "") {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function validOllamaLLMBasePath(input = "") {
|
||||||
|
try {
|
||||||
|
new URL(input);
|
||||||
|
if (input.split("").slice(-1)?.[0] === "/")
|
||||||
|
return "URL cannot end with a slash";
|
||||||
|
return null;
|
||||||
|
} catch {
|
||||||
|
return "Not a valid URL";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
function supportedLLM(input = "") {
|
function supportedLLM(input = "") {
|
||||||
return [
|
return [
|
||||||
"openai",
|
"openai",
|
||||||
@ -216,6 +240,7 @@ function supportedLLM(input = "") {
|
|||||||
"gemini",
|
"gemini",
|
||||||
"lmstudio",
|
"lmstudio",
|
||||||
"localai",
|
"localai",
|
||||||
|
"ollama",
|
||||||
"native",
|
"native",
|
||||||
].includes(input);
|
].includes(input);
|
||||||
}
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user