From ec90060d3698c8f471df0c8877c7d425f1cf6036 Mon Sep 17 00:00:00 2001 From: Timothy Carambat Date: Thu, 29 Feb 2024 10:05:03 -0800 Subject: [PATCH 1/8] Re-map some file mimes to support text (#842) re-map some file mimes to support text --- collector/utils/files/index.js | 24 ++++++---------------- collector/utils/files/mime.js | 37 ++++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 18 deletions(-) create mode 100644 collector/utils/files/mime.js diff --git a/collector/utils/files/index.js b/collector/utils/files/index.js index 3e6ce344..4bca62f9 100644 --- a/collector/utils/files/index.js +++ b/collector/utils/files/index.js @@ -1,28 +1,16 @@ const fs = require("fs"); const path = require("path"); -const { getType } = require("mime"); +const { MimeDetector } = require("./mime"); function isTextType(filepath) { - if (!fs.existsSync(filepath)) return false; - // These are types of mime primary classes that for sure - // cannot also for forced into a text type. - const nonTextTypes = ["multipart", "image", "model", "audio", "video"]; - // These are full-mimes we for sure cannot parse or interpret as text - // documents - const BAD_MIMES = [ - "application/octet-stream", - "application/zip", - "application/pkcs8", - "application/vnd.microsoft.portable-executable", - "application/x-msdownload", - ]; - try { - const mime = getType(filepath); - if (BAD_MIMES.includes(mime)) return false; + if (!fs.existsSync(filepath)) return false; + const mimeLib = new MimeDetector(); + const mime = mimeLib.getType(filepath); + if (mimeLib.badMimes.includes(mime)) return false; const type = mime.split("/")[0]; - if (nonTextTypes.includes(type)) return false; + if (mimeLib.nonTextTypes.includes(type)) return false; return true; } catch { return false; diff --git a/collector/utils/files/mime.js b/collector/utils/files/mime.js new file mode 100644 index 00000000..feabd620 --- /dev/null +++ b/collector/utils/files/mime.js @@ -0,0 +1,37 @@ +const MimeLib = require("mime"); + +class MimeDetector { + nonTextTypes = ["multipart", "image", "model", "audio", "video"]; + badMimes = [ + "application/octet-stream", + "application/zip", + "application/pkcs8", + "application/vnd.microsoft.portable-executable", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", // XLSX are binaries and need to be handled explicitly. + "application/x-msdownload", + ]; + + constructor() { + this.lib = MimeLib; + this.setOverrides(); + } + + setOverrides() { + // the .ts extension maps to video/mp2t because of https://en.wikipedia.org/wiki/MPEG_transport_stream + // which has had this extension far before TS was invented. So need to force re-map this MIME map. + this.lib.define( + { + "text/plain": ["ts", "py", "opts", "lock", "jsonl"], + }, + true + ); + } + + getType(filepath) { + return this.lib.getType(filepath); + } +} + +module.exports = { + MimeDetector, +}; From 147426704c6b4b4dae9963312ec6cc9ff3d9be0c Mon Sep 17 00:00:00 2001 From: Timothy Carambat Date: Thu, 29 Feb 2024 10:09:49 -0800 Subject: [PATCH 2/8] adjust max upload window height (#844) --- .../Modals/MangeWorkspace/Documents/UploadFile/index.jsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/components/Modals/MangeWorkspace/Documents/UploadFile/index.jsx b/frontend/src/components/Modals/MangeWorkspace/Documents/UploadFile/index.jsx index 2ed91102..406fda86 100644 --- a/frontend/src/components/Modals/MangeWorkspace/Documents/UploadFile/index.jsx +++ b/frontend/src/components/Modals/MangeWorkspace/Documents/UploadFile/index.jsx @@ -105,7 +105,7 @@ export default function UploadFile({ workspace, fetchKeys, setLoading }) { ) : ( -
+
{files.map((file) => ( Date: Thu, 29 Feb 2024 17:04:59 -0800 Subject: [PATCH 3/8] [FEAT] JSON export append all metadata fields to workspace chats (#845) have JSON export append all metadata fields --- server/utils/helpers/chat/convertTo.js | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/server/utils/helpers/chat/convertTo.js b/server/utils/helpers/chat/convertTo.js index 119c155a..7d2b5f69 100644 --- a/server/utils/helpers/chat/convertTo.js +++ b/server/utils/helpers/chat/convertTo.js @@ -21,12 +21,8 @@ async function convertToCSV(preparedData) { return rows.join("\n"); } -async function convertToJSON(workspaceChatsMap) { - const allMessages = [].concat.apply( - [], - Object.values(workspaceChatsMap).map((workspace) => workspace.messages) - ); - return JSON.stringify(allMessages, null, 4); +async function convertToJSON(preparedData) { + return JSON.stringify(preparedData, null, 4); } // ref: https://raw.githubusercontent.com/gururise/AlpacaDataCleaned/main/alpaca_data.json @@ -48,7 +44,7 @@ async function prepareWorkspaceChatsForExport(format = "jsonl") { id: "asc", }); - if (format === "csv") { + if (format === "csv" || format === "json") { const preparedData = chats.map((chat) => { const responseJson = JSON.parse(chat.response); return { From 4731ec8be81f88b6e4e26a74eb4e01953d8c93c9 Mon Sep 17 00:00:00 2001 From: Gabriel Koo Date: Thu, 7 Mar 2024 01:14:36 +0800 Subject: [PATCH 4/8] [FIX] : missing import for `parseAuthHeader` in `server/utils/vectorDbProviders/chroma/index.js` (#869) fix: import parseAuthHeader in chroma/index.js --- server/utils/vectorDbProviders/chroma/index.js | 1 + 1 file changed, 1 insertion(+) diff --git a/server/utils/vectorDbProviders/chroma/index.js b/server/utils/vectorDbProviders/chroma/index.js index 23f173dd..9e3caa7a 100644 --- a/server/utils/vectorDbProviders/chroma/index.js +++ b/server/utils/vectorDbProviders/chroma/index.js @@ -7,6 +7,7 @@ const { getLLMProvider, getEmbeddingEngineSelection, } = require("../../helpers"); +const { parseAuthHeader } = require("../../http"); const Chroma = { name: "Chroma", From 063401378819c68fac2352cd9b5adfdfa742448a Mon Sep 17 00:00:00 2001 From: Sean Hatfield Date: Wed, 6 Mar 2024 14:48:38 -0800 Subject: [PATCH 5/8] [FEAT] Groq LLM support (#865) * Groq LLM support complete * update useGetProvidersModels for groq models * Add definiations update comments and error log reports add example envs --------- Co-authored-by: timothycarambat --- .vscode/settings.json | 3 + docker/.env.example | 4 + .../LLMSelection/GroqAiOptions/index.jsx | 41 ++++ frontend/src/hooks/useGetProvidersModels.js | 1 + frontend/src/media/llmprovider/groq.png | Bin 0 -> 1450 bytes .../GeneralSettings/LLMPreference/index.jsx | 14 +- .../Steps/DataHandling/index.jsx | 9 + .../Steps/LLMPreference/index.jsx | 12 +- server/.env.example | 4 + server/models/systemSettings.js | 15 +- server/utils/AiProviders/groq/index.js | 207 ++++++++++++++++++ server/utils/helpers/index.js | 3 + server/utils/helpers/updateENV.js | 11 + 13 files changed, 320 insertions(+), 4 deletions(-) create mode 100644 frontend/src/components/LLMSelection/GroqAiOptions/index.jsx create mode 100644 frontend/src/media/llmprovider/groq.png create mode 100644 server/utils/AiProviders/groq/index.js diff --git a/.vscode/settings.json b/.vscode/settings.json index 096f1c9f..02e25cee 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -4,12 +4,15 @@ "Astra", "Dockerized", "Embeddable", + "GROQ", "hljs", + "inferencing", "Langchain", "Milvus", "Mintplex", "Ollama", "openai", + "openrouter", "Qdrant", "vectordbs", "Weaviate", diff --git a/docker/.env.example b/docker/.env.example index ba33bd5c..ae4913dc 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -61,6 +61,10 @@ GID='1000' # HUGGING_FACE_LLM_API_KEY=hf_xxxxxx # HUGGING_FACE_LLM_TOKEN_LIMIT=8000 +# LLM_PROVIDER='groq' +# GROQ_API_KEY=gsk_abcxyz +# GROQ_MODEL_PREF=llama2-70b-4096 + ########################################### ######## Embedding API SElECTION ########## ########################################### diff --git a/frontend/src/components/LLMSelection/GroqAiOptions/index.jsx b/frontend/src/components/LLMSelection/GroqAiOptions/index.jsx new file mode 100644 index 00000000..cc6fbbcc --- /dev/null +++ b/frontend/src/components/LLMSelection/GroqAiOptions/index.jsx @@ -0,0 +1,41 @@ +export default function GroqAiOptions({ settings }) { + return ( +
+
+ + +
+ +
+ + +
+
+ ); +} diff --git a/frontend/src/hooks/useGetProvidersModels.js b/frontend/src/hooks/useGetProvidersModels.js index 1f8cce98..b40ca01d 100644 --- a/frontend/src/hooks/useGetProvidersModels.js +++ b/frontend/src/hooks/useGetProvidersModels.js @@ -19,6 +19,7 @@ const PROVIDER_DEFAULT_MODELS = { localai: [], ollama: [], togetherai: [], + groq: ["llama2-70b-4096", "mixtral-8x7b-32768"], native: [], }; diff --git a/frontend/src/media/llmprovider/groq.png b/frontend/src/media/llmprovider/groq.png new file mode 100644 index 0000000000000000000000000000000000000000..31564145e1068131f3cf0a49766efbd68a0b4d54 GIT binary patch literal 1450 zcmb7^dotQT$Rm%*&tNegE6tFx zX~(OSS86M*Ff-*bWJY9AB4p4GJG+1FUwh7;bMO7$d%x%2KR;Pc4mL7Uho!{C#AI+F z)C%l{0cSnPN$C?LbJuzsrmhz*qR>*$4wYt2$y}dOzvyq>& zo)P_{qe&3q`RD6y;l$`#fWt;PNp#!cIN&Y%CK2u8;F{Ovd()hO4f|a*!*N&(xA^jf z^3Rles?r(!-?BNJ5YNSJ!lO)dG|ma>xACQ5QGoxX5DA%2OicJ#s)R4HLuc6_j|jx0 z;O7)!mAT@M)~Uxdo)zTtdcmdtSd+ESw}f(9$38TNdkzn8R2Jn?jgJb>(uZWORTk$= zBT@1`oHOpek&s)%o~ZGd`S&W66sobV?>Q~&3ac{WHPfVvd+D-6T6JiIY@JD{sYekt z1a8zYf4aCts~1A4$todQY7yjB#)0Ku3n(^28dnh_x7q0(SbbeDr+WH%Gf24QMe`Tf z0&qE+M2%%+{!Q(>_o1Ui)#3Puyj%mY-%l_`YDj8d+T?MNwg}}ksa~Bl3zi_Faq-1H zCsmj@+;q$&6NP8gy~gcIGS$7?B})_@n?%ZkBP|B~rO029cEiObRKB%BIg~jPJeRg* zhwo|U-d^e+9db65(}A7RbiR70%yu-hxG>EDN9+!CI?M#qfWG+4as2xaUZ1c6G0$V$ z_f$JL7Rvtt_7jn?;NA9-SbZyNOcYPPzDW2;PZ8#>8azpQoRe;J8nF-N?v-We8LF$d z5BANgPH}gse2M^`FK0t}(%)MzCw~zOY$23fldH%M^97>bh&7e6{Yi5@wh)rLWgl0) zwaj^_oO}BP_x447>E+Va1f)9L%)rsGAWL`fJHxqvNNU^68cX17s(q_iqDYb~s`aIk zy-6=X+sw$8^h-(oB zpyN{1OEZ?{p1C$f+=Jz$nJFp^tpq&-F*S`A(*c0_bS-0GReH@J(5|l1SIk zlb9^Pzwdj?3d_e%yyuAH-dS~rL3s8xW>#4G^h;FN%T~+COa3>s+1$gu=lMy}Z~In) zgfvMol0Eozac0k5=H+rvCe$VM$nN97@TSjkJjO6^Xxq&X&W%wpX4a{4@#2EiH)IGy z=?bw?>p-%^>lIPY2 zV!qI2!{>Svz3FRiQ!m@XE9yUU+OA$XWVV4b@3fRzumHCa+Y}Ccj!O>4$HFRXE zZBT5<`L3Jtb;7_M%;xnN?IxvM7wTO4e%X)XntK<_Sft!Kd*c>k?Fnkx1MJU?E*j@2 z!r6lkL4oz0ZG%_E1mdkp--dRAVrP;+MKg{R5h*Lseb|K=B&j4 literal 0 HcmV?d00001 diff --git a/frontend/src/pages/GeneralSettings/LLMPreference/index.jsx b/frontend/src/pages/GeneralSettings/LLMPreference/index.jsx index ac4c1e3d..c7b6fb7b 100644 --- a/frontend/src/pages/GeneralSettings/LLMPreference/index.jsx +++ b/frontend/src/pages/GeneralSettings/LLMPreference/index.jsx @@ -16,6 +16,7 @@ import MistralLogo from "@/media/llmprovider/mistral.jpeg"; import HuggingFaceLogo from "@/media/llmprovider/huggingface.png"; import PerplexityLogo from "@/media/llmprovider/perplexity.png"; import OpenRouterLogo from "@/media/llmprovider/openrouter.jpeg"; +import GroqLogo from "@/media/llmprovider/groq.png"; import PreLoader from "@/components/Preloader"; import OpenAiOptions from "@/components/LLMSelection/OpenAiOptions"; import AzureAiOptions from "@/components/LLMSelection/AzureAiOptions"; @@ -28,11 +29,12 @@ import OllamaLLMOptions from "@/components/LLMSelection/OllamaLLMOptions"; import TogetherAiOptions from "@/components/LLMSelection/TogetherAiOptions"; import MistralOptions from "@/components/LLMSelection/MistralOptions"; import HuggingFaceOptions from "@/components/LLMSelection/HuggingFaceOptions"; +import PerplexityOptions from "@/components/LLMSelection/PerplexityOptions"; +import OpenRouterOptions from "@/components/LLMSelection/OpenRouterOptions"; +import GroqAiOptions from "@/components/LLMSelection/GroqAiOptions"; import LLMItem from "@/components/LLMSelection/LLMItem"; import { MagnifyingGlass } from "@phosphor-icons/react"; -import PerplexityOptions from "@/components/LLMSelection/PerplexityOptions"; -import OpenRouterOptions from "@/components/LLMSelection/OpenRouterOptions"; export default function GeneralLLMPreference() { const [saving, setSaving] = useState(false); @@ -173,6 +175,14 @@ export default function GeneralLLMPreference() { options: , description: "A unified interface for LLMs.", }, + { + name: "Groq", + value: "groq", + logo: GroqLogo, + options: , + description: + "The fastest LLM inferencing available for real-time AI applications.", + }, { name: "Native", value: "native", diff --git a/frontend/src/pages/OnboardingFlow/Steps/DataHandling/index.jsx b/frontend/src/pages/OnboardingFlow/Steps/DataHandling/index.jsx index 5beec3c1..af3b3a9d 100644 --- a/frontend/src/pages/OnboardingFlow/Steps/DataHandling/index.jsx +++ b/frontend/src/pages/OnboardingFlow/Steps/DataHandling/index.jsx @@ -13,6 +13,7 @@ import MistralLogo from "@/media/llmprovider/mistral.jpeg"; import HuggingFaceLogo from "@/media/llmprovider/huggingface.png"; import PerplexityLogo from "@/media/llmprovider/perplexity.png"; import OpenRouterLogo from "@/media/llmprovider/openrouter.jpeg"; +import GroqLogo from "@/media/llmprovider/groq.png"; import ZillizLogo from "@/media/vectordbs/zilliz.png"; import AstraDBLogo from "@/media/vectordbs/astraDB.png"; import ChromaLogo from "@/media/vectordbs/chroma.png"; @@ -127,6 +128,14 @@ const LLM_SELECTION_PRIVACY = { ], logo: OpenRouterLogo, }, + groq: { + name: "Groq", + description: [ + "Your chats will not be used for training", + "Your prompts and document text used in response creation are visible to Groq", + ], + logo: GroqLogo, + }, }; const VECTOR_DB_PRIVACY = { diff --git a/frontend/src/pages/OnboardingFlow/Steps/LLMPreference/index.jsx b/frontend/src/pages/OnboardingFlow/Steps/LLMPreference/index.jsx index 433914ae..33883dc7 100644 --- a/frontend/src/pages/OnboardingFlow/Steps/LLMPreference/index.jsx +++ b/frontend/src/pages/OnboardingFlow/Steps/LLMPreference/index.jsx @@ -13,6 +13,7 @@ import MistralLogo from "@/media/llmprovider/mistral.jpeg"; import HuggingFaceLogo from "@/media/llmprovider/huggingface.png"; import PerplexityLogo from "@/media/llmprovider/perplexity.png"; import OpenRouterLogo from "@/media/llmprovider/openrouter.jpeg"; +import GroqLogo from "@/media/llmprovider/groq.png"; import OpenAiOptions from "@/components/LLMSelection/OpenAiOptions"; import AzureAiOptions from "@/components/LLMSelection/AzureAiOptions"; import AnthropicAiOptions from "@/components/LLMSelection/AnthropicAiOptions"; @@ -25,12 +26,13 @@ import MistralOptions from "@/components/LLMSelection/MistralOptions"; import HuggingFaceOptions from "@/components/LLMSelection/HuggingFaceOptions"; import TogetherAiOptions from "@/components/LLMSelection/TogetherAiOptions"; import PerplexityOptions from "@/components/LLMSelection/PerplexityOptions"; +import OpenRouterOptions from "@/components/LLMSelection/OpenRouterOptions"; +import GroqAiOptions from "@/components/LLMSelection/GroqAiOptions"; import LLMItem from "@/components/LLMSelection/LLMItem"; import System from "@/models/system"; import paths from "@/utils/paths"; import showToast from "@/utils/toast"; import { useNavigate } from "react-router-dom"; -import OpenRouterOptions from "@/components/LLMSelection/OpenRouterOptions"; const TITLE = "LLM Preference"; const DESCRIPTION = @@ -147,6 +149,14 @@ export default function LLMPreference({ options: , description: "A unified interface for LLMs.", }, + { + name: "Groq", + value: "groq", + logo: GroqLogo, + options: , + description: + "The fastest LLM inferencing available for real-time AI applications.", + }, { name: "Native", value: "native", diff --git a/server/.env.example b/server/.env.example index 0ca826e8..88e60182 100644 --- a/server/.env.example +++ b/server/.env.example @@ -58,6 +58,10 @@ JWT_SECRET="my-random-string-for-seeding" # Please generate random string at lea # HUGGING_FACE_LLM_API_KEY=hf_xxxxxx # HUGGING_FACE_LLM_TOKEN_LIMIT=8000 +# LLM_PROVIDER='groq' +# GROQ_API_KEY=gsk_abcxyz +# GROQ_MODEL_PREF=llama2-70b-4096 + ########################################### ######## Embedding API SElECTION ########## ########################################### diff --git a/server/models/systemSettings.js b/server/models/systemSettings.js index 31d5c59a..b06fe123 100644 --- a/server/models/systemSettings.js +++ b/server/models/systemSettings.js @@ -219,12 +219,25 @@ const SystemSettings = { AzureOpenAiEmbeddingModelPref: process.env.EMBEDDING_MODEL_PREF, } : {}), + + ...(llmProvider === "groq" + ? { + GroqApiKey: !!process.env.GROQ_API_KEY, + GroqModelPref: process.env.GROQ_MODEL_PREF, + + // For embedding credentials when groq is selected. + OpenAiKey: !!process.env.OPEN_AI_KEY, + AzureOpenAiEndpoint: process.env.AZURE_OPENAI_ENDPOINT, + AzureOpenAiKey: !!process.env.AZURE_OPENAI_KEY, + AzureOpenAiEmbeddingModelPref: process.env.EMBEDDING_MODEL_PREF, + } + : {}), ...(llmProvider === "native" ? { NativeLLMModelPref: process.env.NATIVE_LLM_MODEL_PREF, NativeLLMTokenLimit: process.env.NATIVE_LLM_MODEL_TOKEN_LIMIT, - // For embedding credentials when ollama is selected. + // For embedding credentials when native is selected. OpenAiKey: !!process.env.OPEN_AI_KEY, AzureOpenAiEndpoint: process.env.AZURE_OPENAI_ENDPOINT, AzureOpenAiKey: !!process.env.AZURE_OPENAI_KEY, diff --git a/server/utils/AiProviders/groq/index.js b/server/utils/AiProviders/groq/index.js new file mode 100644 index 00000000..1b15fe1f --- /dev/null +++ b/server/utils/AiProviders/groq/index.js @@ -0,0 +1,207 @@ +const { NativeEmbedder } = require("../../EmbeddingEngines/native"); +const { chatPrompt } = require("../../chats"); +const { handleDefaultStreamResponse } = require("../../helpers/chat/responses"); + +class GroqLLM { + constructor(embedder = null, modelPreference = null) { + const { Configuration, OpenAIApi } = require("openai"); + if (!process.env.GROQ_API_KEY) throw new Error("No Groq API key was set."); + + const config = new Configuration({ + basePath: "https://api.groq.com/openai/v1", + apiKey: process.env.GROQ_API_KEY, + }); + + this.openai = new OpenAIApi(config); + this.model = + modelPreference || process.env.GROQ_MODEL_PREF || "llama2-70b-4096"; + this.limits = { + history: this.promptWindowLimit() * 0.15, + system: this.promptWindowLimit() * 0.15, + user: this.promptWindowLimit() * 0.7, + }; + + this.embedder = !embedder ? new NativeEmbedder() : embedder; + this.defaultTemp = 0.7; + } + + #appendContext(contextTexts = []) { + if (!contextTexts || !contextTexts.length) return ""; + return ( + "\nContext:\n" + + contextTexts + .map((text, i) => { + return `[CONTEXT ${i}]:\n${text}\n[END CONTEXT ${i}]\n\n`; + }) + .join("") + ); + } + + streamingEnabled() { + return "streamChat" in this && "streamGetChatCompletion" in this; + } + + promptWindowLimit() { + switch (this.model) { + case "llama2-70b-4096": + return 4096; + case "mixtral-8x7b-32768": + return 32_768; + default: + return 4096; + } + } + + async isValidChatCompletionModel(modelName = "") { + const validModels = ["llama2-70b-4096", "mixtral-8x7b-32768"]; + const isPreset = validModels.some((model) => modelName === model); + if (isPreset) return true; + + const model = await this.openai + .retrieveModel(modelName) + .then((res) => res.data) + .catch(() => null); + return !!model; + } + + constructPrompt({ + systemPrompt = "", + contextTexts = [], + chatHistory = [], + userPrompt = "", + }) { + const prompt = { + role: "system", + content: `${systemPrompt}${this.#appendContext(contextTexts)}`, + }; + return [prompt, ...chatHistory, { role: "user", content: userPrompt }]; + } + + async isSafe(_input = "") { + // Not implemented so must be stubbed + return { safe: true, reasons: [] }; + } + + async sendChat(chatHistory = [], prompt, workspace = {}, rawHistory = []) { + if (!(await this.isValidChatCompletionModel(this.model))) + throw new Error( + `Groq chat: ${this.model} is not valid for chat completion!` + ); + + const textResponse = await this.openai + .createChatCompletion({ + model: this.model, + temperature: Number(workspace?.openAiTemp ?? this.defaultTemp), + n: 1, + messages: await this.compressMessages( + { + systemPrompt: chatPrompt(workspace), + userPrompt: prompt, + chatHistory, + }, + rawHistory + ), + }) + .then((json) => { + const res = json.data; + if (!res.hasOwnProperty("choices")) + throw new Error("GroqAI chat: No results!"); + if (res.choices.length === 0) + throw new Error("GroqAI chat: No results length!"); + return res.choices[0].message.content; + }) + .catch((error) => { + throw new Error( + `GroqAI::createChatCompletion failed with: ${error.message}` + ); + }); + + return textResponse; + } + + async streamChat(chatHistory = [], prompt, workspace = {}, rawHistory = []) { + if (!(await this.isValidChatCompletionModel(this.model))) + throw new Error( + `GroqAI:streamChat: ${this.model} is not valid for chat completion!` + ); + + const streamRequest = await this.openai.createChatCompletion( + { + model: this.model, + stream: true, + temperature: Number(workspace?.openAiTemp ?? this.defaultTemp), + n: 1, + messages: await this.compressMessages( + { + systemPrompt: chatPrompt(workspace), + userPrompt: prompt, + chatHistory, + }, + rawHistory + ), + }, + { responseType: "stream" } + ); + return streamRequest; + } + + async getChatCompletion(messages = null, { temperature = 0.7 }) { + if (!(await this.isValidChatCompletionModel(this.model))) + throw new Error( + `GroqAI:chatCompletion: ${this.model} is not valid for chat completion!` + ); + + const { data } = await this.openai + .createChatCompletion({ + model: this.model, + messages, + temperature, + }) + .catch((e) => { + throw new Error(e.response.data.error.message); + }); + + if (!data.hasOwnProperty("choices")) return null; + return data.choices[0].message.content; + } + + async streamGetChatCompletion(messages = null, { temperature = 0.7 }) { + if (!(await this.isValidChatCompletionModel(this.model))) + throw new Error( + `GroqAI:streamChatCompletion: ${this.model} is not valid for chat completion!` + ); + + const streamRequest = await this.openai.createChatCompletion( + { + model: this.model, + stream: true, + messages, + temperature, + }, + { responseType: "stream" } + ); + return streamRequest; + } + + handleStream(response, stream, responseProps) { + return handleDefaultStreamResponse(response, stream, responseProps); + } + + // Simple wrapper for dynamic embedder & normalize interface for all LLM implementations + async embedTextInput(textInput) { + return await this.embedder.embedTextInput(textInput); + } + async embedChunks(textChunks = []) { + return await this.embedder.embedChunks(textChunks); + } + + async compressMessages(promptArgs = {}, rawHistory = []) { + const { messageArrayCompressor } = require("../../helpers/chat"); + const messageArray = this.constructPrompt(promptArgs); + return await messageArrayCompressor(this, messageArray, rawHistory); + } +} + +module.exports = { + GroqLLM, +}; diff --git a/server/utils/helpers/index.js b/server/utils/helpers/index.js index a31a3e4f..78360972 100644 --- a/server/utils/helpers/index.js +++ b/server/utils/helpers/index.js @@ -73,6 +73,9 @@ function getLLMProvider(modelPreference = null) { case "huggingface": const { HuggingFaceLLM } = require("../AiProviders/huggingface"); return new HuggingFaceLLM(embedder, modelPreference); + case "groq": + const { GroqLLM } = require("../AiProviders/groq"); + return new GroqLLM(embedder, modelPreference); default: throw new Error("ENV: No LLM_PROVIDER value found in environment!"); } diff --git a/server/utils/helpers/updateENV.js b/server/utils/helpers/updateENV.js index 1ca93682..575f80ab 100644 --- a/server/utils/helpers/updateENV.js +++ b/server/utils/helpers/updateENV.js @@ -259,6 +259,16 @@ const KEY_MAPPING = { checks: [isNotEmpty], }, + // Groq Options + GroqApiKey: { + envKey: "GROQ_API_KEY", + checks: [isNotEmpty], + }, + GroqModelPref: { + envKey: "GROQ_MODEL_PREF", + checks: [isNotEmpty], + }, + // System Settings AuthToken: { envKey: "AUTH_TOKEN", @@ -336,6 +346,7 @@ function supportedLLM(input = "") { "huggingface", "perplexity", "openrouter", + "groq", ].includes(input); return validSelection ? null : `${input} is not a valid LLM provider.`; } From e0d5d8039aebc68fad8554d21facfabc7b7e6d08 Mon Sep 17 00:00:00 2001 From: Sean Hatfield Date: Wed, 6 Mar 2024 14:57:47 -0800 Subject: [PATCH 6/8] [FEAT] Claude 3 support and implement new version of Anthropic SDK (#863) * implement new version of anthropic sdk and support new models * remove handleAnthropicStream and move to handleStream inside anthropic provider * update useGetProvidersModels for new anthropic models --- .../LLMSelection/AnthropicAiOptions/index.jsx | 8 +- frontend/src/hooks/useGetProvidersModels.js | 8 +- server/package.json | 2 +- server/utils/AiProviders/anthropic/index.js | 194 +++++++++++------- server/utils/helpers/updateENV.js | 8 +- server/yarn.lock | 8 +- 6 files changed, 146 insertions(+), 82 deletions(-) diff --git a/frontend/src/components/LLMSelection/AnthropicAiOptions/index.jsx b/frontend/src/components/LLMSelection/AnthropicAiOptions/index.jsx index 3d493f1c..6bc18a5a 100644 --- a/frontend/src/components/LLMSelection/AnthropicAiOptions/index.jsx +++ b/frontend/src/components/LLMSelection/AnthropicAiOptions/index.jsx @@ -48,7 +48,13 @@ export default function AnthropicAiOptions({ settings, showAlert = false }) { required={true} className="bg-zinc-900 border-gray-500 text-white text-sm rounded-lg block w-full p-2.5" > - {["claude-2", "claude-instant-1"].map((model) => { + {[ + "claude-instant-1.2", + "claude-2.0", + "claude-2.1", + "claude-3-opus-20240229", + "claude-3-sonnet-20240229", + ].map((model) => { return (
-
+
{users .filter((user) => user.role !== "admin") .map((user) => { From 9ce3d1150d7c688d3c08bfadf8e59007fb1538c4 Mon Sep 17 00:00:00 2001 From: Francisco Bischoff <984592+franzbischoff@users.noreply.github.com> Date: Thu, 7 Mar 2024 00:34:45 +0000 Subject: [PATCH 8/8] Update Ubuntu base image and improve Dockerfile (#609) * Update Ubuntu base image and improve Dockerfile * Add unzip to Docker image dependencies Needed for the arm64 build * reset tabs * formalized lint rules for hadolint. however the Docker formatting is being handled by MS Docker extension which doesn't indent code as expected. WIP. * found a workaround to keep formatting --------- Co-authored-by: timothycarambat --- .hadolint.yaml | 8 +++++ .vscode/settings.json | 5 ++-- docker/Dockerfile | 58 +++++++++++++++++++++++++----------- docker/docker-entrypoint.sh | 11 +++---- docker/docker-healthcheck.sh | 10 +++---- 5 files changed, 62 insertions(+), 30 deletions(-) create mode 100644 .hadolint.yaml diff --git a/.hadolint.yaml b/.hadolint.yaml new file mode 100644 index 00000000..b76a5107 --- /dev/null +++ b/.hadolint.yaml @@ -0,0 +1,8 @@ +failure-threshold: warning +ignored: + - DL3008 + - DL3013 +format: tty +trustedRegistries: + - docker.io + - gcr.io diff --git a/.vscode/settings.json b/.vscode/settings.json index 02e25cee..72b612b8 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -18,5 +18,6 @@ "Weaviate", "Zilliz" ], - "eslint.experimental.useFlatConfig": true -} \ No newline at end of file + "eslint.experimental.useFlatConfig": true, + "docker.languageserver.formatter.ignoreMultilineInstructions": true +} diff --git a/docker/Dockerfile b/docker/Dockerfile index b1ea62a6..2edbadb2 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,12 +1,17 @@ # Setup base image -FROM ubuntu:jammy-20230522 AS base +FROM ubuntu:jammy-20230916 AS base +# Build arguments ARG ARG_UID=1000 ARG ARG_GID=1000 FROM base AS build-arm64 RUN echo "Preparing build of AnythingLLM image for arm64 architecture" +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +# Install system dependencies +# hadolint ignore=DL3008,DL3013 RUN DEBIAN_FRONTEND=noninteractive apt-get update && \ DEBIAN_FRONTEND=noninteractive apt-get install -yq --no-install-recommends \ unzip curl gnupg libgfortran5 libgbm1 tzdata netcat \ @@ -25,8 +30,8 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get update && \ && rm yarn_1.22.19_all.deb # Create a group and user with specific UID and GID -RUN groupadd -g $ARG_GID anythingllm && \ - useradd -u $ARG_UID -m -d /app -s /bin/bash -g anythingllm anythingllm && \ +RUN groupadd -g "$ARG_GID" anythingllm && \ + useradd -l -u "$ARG_UID" -m -d /app -s /bin/bash -g anythingllm anythingllm && \ mkdir -p /app/frontend/ /app/server/ /app/collector/ && chown -R anythingllm:anythingllm /app # Copy docker helper scripts @@ -61,6 +66,10 @@ RUN echo "Done running arm64 specific installtion steps" FROM base AS build-amd64 RUN echo "Preparing build of AnythingLLM image for non-ARM architecture" +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +# Install system dependencies +# hadolint ignore=DL3008,DL3013 RUN DEBIAN_FRONTEND=noninteractive apt-get update && \ DEBIAN_FRONTEND=noninteractive apt-get install -yq --no-install-recommends \ curl gnupg libgfortran5 libgbm1 tzdata netcat \ @@ -79,8 +88,8 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get update && \ && rm yarn_1.22.19_all.deb # Create a group and user with specific UID and GID -RUN groupadd -g $ARG_GID anythingllm && \ - useradd -u $ARG_UID -m -d /app -s /bin/bash -g anythingllm anythingllm && \ +RUN groupadd -g "$ARG_GID" anythingllm && \ + useradd -l -u "$ARG_UID" -m -d /app -s /bin/bash -g anythingllm anythingllm && \ mkdir -p /app/frontend/ /app/server/ /app/collector/ && chown -R anythingllm:anythingllm /app # Copy docker helper scripts @@ -95,6 +104,8 @@ RUN chmod +x /usr/local/bin/docker-entrypoint.sh && \ ############################################# # COMMON BUILD FLOW FOR ALL ARCHS ############################################# + +# hadolint ignore=DL3006 FROM build-${TARGETARCH} AS build RUN echo "Running common build flow of AnythingLLM image for all architectures" @@ -102,43 +113,54 @@ USER anythingllm WORKDIR /app # Install frontend dependencies -FROM build as frontend-deps +FROM build AS frontend-deps COPY ./frontend/package.json ./frontend/yarn.lock ./frontend/ -RUN cd ./frontend/ && yarn install --network-timeout 100000 && yarn cache clean +WORKDIR /app/frontend +RUN yarn install --network-timeout 100000 && yarn cache clean +WORKDIR /app # Install server dependencies -FROM build as server-deps +FROM build AS server-deps COPY ./server/package.json ./server/yarn.lock ./server/ -RUN cd ./server/ && yarn install --production --network-timeout 100000 && yarn cache clean +WORKDIR /app/server +RUN yarn install --production --network-timeout 100000 && yarn cache clean +WORKDIR /app # Compile Llama.cpp bindings for node-llama-cpp for this operating system. USER root -RUN cd ./server && npx --no node-llama-cpp download +WORKDIR /app/server +RUN npx --no node-llama-cpp download +WORKDIR /app USER anythingllm # Build the frontend -FROM frontend-deps as build-stage +FROM frontend-deps AS build-stage COPY ./frontend/ ./frontend/ -RUN cd ./frontend/ && yarn build && yarn cache clean +WORKDIR /app/frontend +RUN yarn build && yarn cache clean +WORKDIR /app # Setup the server -FROM server-deps as production-stage +FROM server-deps AS production-stage COPY --chown=anythingllm:anythingllm ./server/ ./server/ # Copy built static frontend files to the server public directory -COPY --from=build-stage /app/frontend/dist ./server/public +COPY --chown=anythingllm:anythingllm --from=build-stage /app/frontend/dist ./server/public # Copy the collector COPY --chown=anythingllm:anythingllm ./collector/ ./collector/ # Install collector dependencies +WORKDIR /app/collector ENV PUPPETEER_DOWNLOAD_BASE_URL=https://storage.googleapis.com/chrome-for-testing-public -RUN cd /app/collector && yarn install --production --network-timeout 100000 && yarn cache clean +RUN yarn install --production --network-timeout 100000 && yarn cache clean # Migrate and Run Prisma against known schema -RUN cd ./server && npx prisma generate --schema=./prisma/schema.prisma -RUN cd ./server && npx prisma migrate deploy --schema=./prisma/schema.prisma +WORKDIR /app/server +RUN npx prisma generate --schema=./prisma/schema.prisma && \ + npx prisma migrate deploy --schema=./prisma/schema.prisma +WORKDIR /app # Setup the environment ENV NODE_ENV=production @@ -152,4 +174,4 @@ HEALTHCHECK --interval=1m --timeout=10s --start-period=1m \ CMD /bin/bash /usr/local/bin/docker-healthcheck.sh || exit 1 # Run the server -ENTRYPOINT ["/bin/bash", "/usr/local/bin/docker-entrypoint.sh"] \ No newline at end of file +ENTRYPOINT ["/bin/bash", "/usr/local/bin/docker-entrypoint.sh"] diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index 3d890db1..1ac69e5b 100755 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -1,9 +1,10 @@ #!/bin/bash -{ cd /app/server/ &&\ - npx prisma generate --schema=./prisma/schema.prisma &&\ - npx prisma migrate deploy --schema=./prisma/schema.prisma &&\ - node /app/server/index.js +{ + cd /app/server/ && + npx prisma generate --schema=./prisma/schema.prisma && + npx prisma migrate deploy --schema=./prisma/schema.prisma && + node /app/server/index.js } & { node /app/collector/index.js; } & wait -n -exit $? \ No newline at end of file +exit $? diff --git a/docker/docker-healthcheck.sh b/docker/docker-healthcheck.sh index 45a88477..49bee3e1 100644 --- a/docker/docker-healthcheck.sh +++ b/docker/docker-healthcheck.sh @@ -4,10 +4,10 @@ response=$(curl --write-out '%{http_code}' --silent --output /dev/null http://localhost:3001/api/ping) # If the HTTP response code is 200 (OK), the server is up -if [ $response -eq 200 ]; then - echo "Server is up" - exit 0 +if [ "$response" -eq 200 ]; then + echo "Server is up" + exit 0 else - echo "Server is down" - exit 1 + echo "Server is down" + exit 1 fi