diff --git a/.github/workflows/dev-build.yaml b/.github/workflows/dev-build.yaml index e81d99c58..a7632dfd0 100644 --- a/.github/workflows/dev-build.yaml +++ b/.github/workflows/dev-build.yaml @@ -6,7 +6,7 @@ concurrency: on: push: - branches: ['vex'] # put your current branch to create a build. Core team only. + branches: ['558-multi-modal-support'] # put your current branch to create a build. Core team only. paths-ignore: - '**.md' - 'cloud-deployments/*' diff --git a/.vscode/settings.json b/.vscode/settings.json index 5e26e4778..3fcc79cd5 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -29,6 +29,7 @@ "mbox", "Milvus", "Mintplex", + "mixtral", "moderations", "numpages", "Ollama", @@ -38,11 +39,13 @@ "openrouter", "pagerender", "Qdrant", + "royalblue", "searxng", "Serper", "Serply", "textgenwebui", "togetherai", + "Unembed", "vectordbs", "Weaviate", "Zilliz" diff --git a/README.md b/README.md index a0b41a762..d7812265d 100644 --- a/README.md +++ b/README.md @@ -53,19 +53,19 @@ AnythingLLM is a full-stack application where you can use commercial off-the-she AnythingLLM divides your documents into objects called `workspaces`. A Workspace functions a lot like a thread, but with the addition of containerization of your documents. Workspaces can share documents, but they do not talk to each other so you can keep your context for each workspace clean. -Some cool features of AnythingLLM +## Cool features of AnythingLLM -- **Multi-user instance support and permissioning** -- Agents inside your workspace (browse the web, run code, etc) -- [Custom Embeddable Chat widget for your website](./embed/README.md) -- Multiple document type support (PDF, TXT, DOCX, etc) -- Manage documents in your vector database from a simple UI -- Two chat modes `conversation` and `query`. Conversation retains previous questions and amendments. Query is simple QA against your documents -- In-chat citations +- 🆕 **Multi-modal support (both closed and open-source LLMs!)** +- 👤 Multi-user instance support and permissioning _Docker version only_ +- 🦾 Agents inside your workspace (browse the web, run code, etc) +- 💬 [Custom Embeddable Chat widget for your website](./embed/README.md) _Docker version only_ +- 📖 Multiple document type support (PDF, TXT, DOCX, etc) +- Simple chat UI with Drag-n-Drop funcitonality and clear citations. - 100% Cloud deployment ready. -- "Bring your own LLM" model. -- Extremely efficient cost-saving measures for managing very large documents. You'll never pay to embed a massive document or transcript more than once. 90% more cost effective than other document chatbot solutions. +- Works with all popular [closed and open-source LLM providers](#supported-llms-embedder-models-speech-models-and-vector-databases). +- Built-in cost & time-saving measures for managing very large documents compared to any other chat UI. - Full Developer API for custom integrations! +- Much more...install and find out! ### Supported LLMs, Embedder Models, Speech models, and Vector Databases diff --git a/cloud-deployments/aws/cloudformation/cloudformation_create_anythingllm.json b/cloud-deployments/aws/cloudformation/cloudformation_create_anythingllm.json index a500e54cd..b10bbc2db 100644 --- a/cloud-deployments/aws/cloudformation/cloudformation_create_anythingllm.json +++ b/cloud-deployments/aws/cloudformation/cloudformation_create_anythingllm.json @@ -89,7 +89,7 @@ "mkdir -p /home/ec2-user/anythingllm\n", "touch /home/ec2-user/anythingllm/.env\n", "sudo chown ec2-user:ec2-user -R /home/ec2-user/anythingllm\n", - "docker pull mintplexlabs/anythingllm:master\n", + "docker pull mintplexlabs/anythingllm\n", "docker run -d -p 3001:3001 --cap-add SYS_ADMIN -v /home/ec2-user/anythingllm:/app/server/storage -v /home/ec2-user/anythingllm/.env:/app/server/.env -e STORAGE_DIR=\"/app/server/storage\" mintplexlabs/anythingllm\n", "echo \"Container ID: $(sudo docker ps --latest --quiet)\"\n", "export ONLINE=$(curl -Is http://localhost:3001/api/ping | head -n 1|cut -d$' ' -f2)\n", diff --git a/cloud-deployments/digitalocean/terraform/user_data.tp1 b/cloud-deployments/digitalocean/terraform/user_data.tp1 index d23d566eb..cd239c6b4 100644 --- a/cloud-deployments/digitalocean/terraform/user_data.tp1 +++ b/cloud-deployments/digitalocean/terraform/user_data.tp1 @@ -10,9 +10,8 @@ sudo systemctl start docker mkdir -p /home/anythingllm touch /home/anythingllm/.env -sudo chown -R ubuntu:ubuntu /home/anythingllm -sudo docker pull mintplexlabs/anythingllm:master +sudo docker pull mintplexlabs/anythingllm sudo docker run -d -p 3001:3001 --cap-add SYS_ADMIN -v /home/anythingllm:/app/server/storage -v /home/anythingllm/.env:/app/server/.env -e STORAGE_DIR="/app/server/storage" mintplexlabs/anythingllm echo "Container ID: $(sudo docker ps --latest --quiet)" diff --git a/cloud-deployments/gcp/deployment/gcp_deploy_anything_llm.yaml b/cloud-deployments/gcp/deployment/gcp_deploy_anything_llm.yaml index 5a4381022..3f44d5af8 100644 --- a/cloud-deployments/gcp/deployment/gcp_deploy_anything_llm.yaml +++ b/cloud-deployments/gcp/deployment/gcp_deploy_anything_llm.yaml @@ -34,7 +34,7 @@ resources: touch /home/anythingllm/.env sudo chown -R ubuntu:ubuntu /home/anythingllm - sudo docker pull mintplexlabs/anythingllm:master + sudo docker pull mintplexlabs/anythingllm sudo docker run -d -p 3001:3001 --cap-add SYS_ADMIN -v /home/anythingllm:/app/server/storage -v /home/anythingllm/.env:/app/server/.env -e STORAGE_DIR="/app/server/storage" mintplexlabs/anythingllm echo "Container ID: $(sudo docker ps --latest --quiet)" diff --git a/frontend/src/components/LLMSelection/GroqAiOptions/index.jsx b/frontend/src/components/LLMSelection/GroqAiOptions/index.jsx index c9b38ed27..4dd923d61 100644 --- a/frontend/src/components/LLMSelection/GroqAiOptions/index.jsx +++ b/frontend/src/components/LLMSelection/GroqAiOptions/index.jsx @@ -89,13 +89,16 @@ function GroqAIModelSelection({ apiKey, settings }) { name="GroqModelPref" required={true} className="border-none bg-zinc-900 border-gray-500 text-white text-sm rounded-lg block w-full p-2.5" - defaultValue={settings?.GroqModelPref} > {customModels.length > 0 && ( {customModels.map((model) => { return ( - ); diff --git a/frontend/src/components/LLMSelection/OpenRouterOptions/index.jsx b/frontend/src/components/LLMSelection/OpenRouterOptions/index.jsx index 470340d2f..f19956645 100644 --- a/frontend/src/components/LLMSelection/OpenRouterOptions/index.jsx +++ b/frontend/src/components/LLMSelection/OpenRouterOptions/index.jsx @@ -1,27 +1,70 @@ import System from "@/models/system"; +import { CaretDown, CaretUp } from "@phosphor-icons/react"; import { useState, useEffect } from "react"; export default function OpenRouterOptions({ settings }) { return ( -
-
- - +
+
+
+ + +
+ {!settings?.credentialsOnly && ( + + )} +
+ +
+ ); +} + +function AdvancedControls({ settings }) { + const [showAdvancedControls, setShowAdvancedControls] = useState(false); + + return ( +
+ + - {!settings?.credentialsOnly && ( - - )}
); } diff --git a/frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/HistoricalMessage/Actions/EditMessage/index.jsx b/frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/HistoricalMessage/Actions/EditMessage/index.jsx index 34a77dea8..811de87d6 100644 --- a/frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/HistoricalMessage/Actions/EditMessage/index.jsx +++ b/frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/HistoricalMessage/Actions/EditMessage/index.jsx @@ -69,6 +69,7 @@ export function EditMessageForm({ role, chatId, message, + attachments = [], adjustTextArea, saveChanges, }) { @@ -77,15 +78,15 @@ export function EditMessageForm({ e.preventDefault(); const form = new FormData(e.target); const editedMessage = form.get("editedMessage"); - saveChanges({ editedMessage, chatId, role }); + saveChanges({ editedMessage, chatId, role, attachments }); window.dispatchEvent( - new CustomEvent(EDIT_EVENT, { detail: { chatId, role } }) + new CustomEvent(EDIT_EVENT, { detail: { chatId, role, attachments } }) ); } function cancelEdits() { window.dispatchEvent( - new CustomEvent(EDIT_EVENT, { detail: { chatId, role } }) + new CustomEvent(EDIT_EVENT, { detail: { chatId, role, attachments } }) ); return false; } diff --git a/frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/HistoricalMessage/index.jsx b/frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/HistoricalMessage/index.jsx index 7446b166c..f6920fa15 100644 --- a/frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/HistoricalMessage/index.jsx +++ b/frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/HistoricalMessage/index.jsx @@ -19,6 +19,7 @@ const HistoricalMessage = ({ role, workspace, sources = [], + attachments = [], error = false, feedbackScore = null, chatId = null, @@ -92,16 +93,20 @@ const HistoricalMessage = ({ role={role} chatId={chatId} message={message} + attachments={attachments} adjustTextArea={adjustTextArea} saveChanges={saveEditedMessage} /> ) : ( - +
+ + +
)}
@@ -160,3 +165,18 @@ export default memo( ); } ); + +function ChatAttachments({ attachments = [] }) { + if (!attachments.length) return null; + return ( +
+ {attachments.map((item) => ( + + ))} +
+ ); +} diff --git a/frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/index.jsx b/frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/index.jsx index 53cbeb64f..8a5697f94 100644 --- a/frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/index.jsx +++ b/frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/index.jsx @@ -17,6 +17,7 @@ export default function ChatHistory({ sendCommand, updateHistory, regenerateAssistantMessage, + hasAttachments = false, }) { const { user } = useUser(); const { threadSlug = null } = useParams(); @@ -92,7 +93,12 @@ export default function ChatHistory({ sendCommand(`${heading} ${message}`, true); }; - const saveEditedMessage = async ({ editedMessage, chatId, role }) => { + const saveEditedMessage = async ({ + editedMessage, + chatId, + role, + attachments = [], + }) => { if (!editedMessage) return; // Don't save empty edits. // if the edit was a user message, we will auto-regenerate the response and delete all @@ -109,7 +115,7 @@ export default function ChatHistory({ updatedHistory[updatedHistory.length - 1].content = editedMessage; // remove all edited messages after the edited message in backend await Workspace.deleteEditedChats(workspace.slug, threadSlug, chatId); - sendCommand(editedMessage, true, updatedHistory); + sendCommand(editedMessage, true, updatedHistory, attachments); return; } @@ -144,7 +150,7 @@ export default function ChatHistory({ ); }; - if (history.length === 0) { + if (history.length === 0 && !hasAttachments) { return (
@@ -227,6 +233,7 @@ export default function ChatHistory({ feedbackScore={props.feedbackScore} chatId={props.chatId} error={props.error} + attachments={props.attachments} regenerateMessage={regenerateAssistantMessage} isLastMessage={isLastBotReply} saveEditedMessage={saveEditedMessage} diff --git a/frontend/src/components/WorkspaceChat/ChatContainer/DnDWrapper/dnd-icon.png b/frontend/src/components/WorkspaceChat/ChatContainer/DnDWrapper/dnd-icon.png new file mode 100644 index 000000000..9cb0cd7cc Binary files /dev/null and b/frontend/src/components/WorkspaceChat/ChatContainer/DnDWrapper/dnd-icon.png differ diff --git a/frontend/src/components/WorkspaceChat/ChatContainer/DnDWrapper/index.jsx b/frontend/src/components/WorkspaceChat/ChatContainer/DnDWrapper/index.jsx new file mode 100644 index 000000000..6d6f34535 --- /dev/null +++ b/frontend/src/components/WorkspaceChat/ChatContainer/DnDWrapper/index.jsx @@ -0,0 +1,215 @@ +import { useState, useEffect, createContext, useContext } from "react"; +import { v4 } from "uuid"; +import System from "@/models/system"; +import { useDropzone } from "react-dropzone"; +import DndIcon from "./dnd-icon.png"; +import Workspace from "@/models/workspace"; +import useUser from "@/hooks/useUser"; + +export const DndUploaderContext = createContext(); +export const REMOVE_ATTACHMENT_EVENT = "ATTACHMENT_REMOVE"; +export const CLEAR_ATTACHMENTS_EVENT = "ATTACHMENT_CLEAR"; + +/** + * File Attachment for automatic upload on the chat container page. + * @typedef Attachment + * @property {string} uid - unique file id. + * @property {File} file - native File object + * @property {string|null} contentString - base64 encoded string of file + * @property {('in_progress'|'failed'|'success')} status - the automatic upload status. + * @property {string|null} error - Error message + * @property {{id:string, location:string}|null} document - uploaded document details + * @property {('attachment'|'upload')} type - The type of upload. Attachments are chat-specific, uploads go to the workspace. + */ + +export function DnDFileUploaderProvider({ workspace, children }) { + const [files, setFiles] = useState([]); + const [ready, setReady] = useState(false); + const [dragging, setDragging] = useState(false); + const { user } = useUser(); + + useEffect(() => { + if (!!user && user.role === "default") return false; + System.checkDocumentProcessorOnline().then((status) => setReady(status)); + }, [user]); + + useEffect(() => { + window.addEventListener(REMOVE_ATTACHMENT_EVENT, handleRemove); + window.addEventListener(CLEAR_ATTACHMENTS_EVENT, resetAttachments); + + return () => { + window.removeEventListener(REMOVE_ATTACHMENT_EVENT, handleRemove); + window.removeEventListener(CLEAR_ATTACHMENTS_EVENT, resetAttachments); + }; + }, []); + + /** + * Remove file from uploader queue. + * @param {CustomEvent<{uid: string}>} event + */ + async function handleRemove(event) { + /** @type {{uid: Attachment['uid'], document: Attachment['document']}} */ + const { uid, document } = event.detail; + setFiles((prev) => prev.filter((prevFile) => prevFile.uid !== uid)); + if (!document?.location) return; + await Workspace.deleteAndUnembedFile(workspace.slug, document.location); + } + + /** + * Clear queue of attached files currently in prompt box + */ + function resetAttachments() { + setFiles([]); + } + + /** + * Turns files into attachments we can send as body request to backend + * for a chat. + * @returns {{name:string,mime:string,contentString:string}[]} + */ + function parseAttachments() { + return ( + files + ?.filter((file) => file.type === "attachment") + ?.map( + ( + /** @type {Attachment} */ + attachment + ) => { + return { + name: attachment.file.name, + mime: attachment.file.type, + contentString: attachment.contentString, + }; + } + ) || [] + ); + } + + /** + * Handle dropped files. + * @param {Attachment[]} acceptedFiles + * @param {any[]} _rejections + */ + async function onDrop(acceptedFiles, _rejections) { + setDragging(false); + + /** @type {Attachment[]} */ + const newAccepted = []; + for (const file of acceptedFiles) { + if (file.type.startsWith("image/")) { + newAccepted.push({ + uid: v4(), + file, + contentString: await toBase64(file), + status: "success", + error: null, + type: "attachment", + }); + } else { + newAccepted.push({ + uid: v4(), + file, + contentString: null, + status: "in_progress", + error: null, + type: "upload", + }); + } + } + + setFiles((prev) => [...prev, ...newAccepted]); + + for (const attachment of newAccepted) { + // Images/attachments are chat specific. + if (attachment.type === "attachment") continue; + + const formData = new FormData(); + formData.append("file", attachment.file, attachment.file.name); + Workspace.uploadAndEmbedFile(workspace.slug, formData).then( + ({ response, data }) => { + const updates = { + status: response.ok ? "success" : "failed", + error: data?.error ?? null, + document: data?.document, + }; + + setFiles((prev) => { + return prev.map( + ( + /** @type {Attachment} */ + prevFile + ) => { + if (prevFile.uid !== attachment.uid) return prevFile; + return { ...prevFile, ...updates }; + } + ); + }); + } + ); + } + } + + return ( + + {children} + + ); +} + +export default function DnDFileUploaderWrapper({ children }) { + const { onDrop, ready, dragging, setDragging } = + useContext(DndUploaderContext); + const { getRootProps, getInputProps } = useDropzone({ + onDrop, + disabled: !ready, + noClick: true, + noKeyboard: true, + onDragEnter: () => setDragging(true), + onDragLeave: () => setDragging(false), + }); + + return ( +
+ + + {children} +
+ ); +} + +/** + * Convert image types into Base64 strings for requests. + * @param {File} file + * @returns {string} + */ +async function toBase64(file) { + return new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.onload = () => { + const base64String = reader.result.split(",")[1]; + resolve(`data:${file.type};base64,${base64String}`); + }; + reader.onerror = (error) => reject(error); + reader.readAsDataURL(file); + }); +} diff --git a/frontend/src/components/WorkspaceChat/ChatContainer/PromptInput/AttachItem/index.jsx b/frontend/src/components/WorkspaceChat/ChatContainer/PromptInput/AttachItem/index.jsx new file mode 100644 index 000000000..74f22f90c --- /dev/null +++ b/frontend/src/components/WorkspaceChat/ChatContainer/PromptInput/AttachItem/index.jsx @@ -0,0 +1,34 @@ +import { PaperclipHorizontal } from "@phosphor-icons/react"; +import { Tooltip } from "react-tooltip"; + +/** + * This is a simple proxy component that clicks on the DnD file uploader for the user. + * @returns + */ +export default function AttachItem() { + return ( + <> + + + ); +} diff --git a/frontend/src/components/WorkspaceChat/ChatContainer/PromptInput/Attachments/index.jsx b/frontend/src/components/WorkspaceChat/ChatContainer/PromptInput/Attachments/index.jsx new file mode 100644 index 000000000..8cfc3c535 --- /dev/null +++ b/frontend/src/components/WorkspaceChat/ChatContainer/PromptInput/Attachments/index.jsx @@ -0,0 +1,222 @@ +import { + CircleNotch, + FileCode, + FileCsv, + FileDoc, + FileHtml, + FileText, + FileImage, + FilePdf, + WarningOctagon, + X, +} from "@phosphor-icons/react"; +import { humanFileSize } from "@/utils/numbers"; +import { REMOVE_ATTACHMENT_EVENT } from "../../DnDWrapper"; +import { Tooltip } from "react-tooltip"; + +/** + * @param {{attachments: import("../../DnDWrapper").Attachment[]}} + * @returns + */ +export default function AttachmentManager({ attachments }) { + if (attachments.length === 0) return null; + return ( +
+ {attachments.map((attachment) => ( + + ))} +
+ ); +} + +/** + * @param {{attachment: import("../../DnDWrapper").Attachment}} + */ +function AttachmentItem({ attachment }) { + const { uid, file, status, error, document, type } = attachment; + const { iconBgColor, Icon } = displayFromFile(file); + + function removeFileFromQueue() { + window.dispatchEvent( + new CustomEvent(REMOVE_ATTACHMENT_EVENT, { detail: { uid, document } }) + ); + } + + if (status === "in_progress") { + return ( +
+
+ +
+
+

{file.name}

+

+ {humanFileSize(file.size)} +

+
+
+ ); + } + + if (status === "failed") { + return ( + <> +
+
+ +
+
+ +
+
+

+ {file.name} +

+

+ {error ?? "this file failed to upload"}. It will not be available + in the workspace. +

+
+
+ + + ); + } + + if (type === "attachment") { + return ( + <> +
+
+ +
+
+ +
+
+

+ {file.name} +

+

Image attached!

+
+
+ + + ); + } + + return ( + <> +
+
+ +
+
+ +
+
+

{file.name}

+

File embedded!

+
+
+ + + ); +} + +/** + * @param {File} file + * @returns {{iconBgColor:string, Icon: React.Component}} + */ +function displayFromFile(file) { + const extension = file?.name?.split(".")?.pop()?.toLowerCase() ?? "txt"; + switch (extension) { + case "pdf": + return { iconBgColor: "bg-magenta", Icon: FilePdf }; + case "doc": + case "docx": + return { iconBgColor: "bg-royalblue", Icon: FileDoc }; + case "html": + return { iconBgColor: "bg-purple", Icon: FileHtml }; + case "csv": + case "xlsx": + return { iconBgColor: "bg-success", Icon: FileCsv }; + case "json": + case "sql": + case "js": + case "jsx": + case "cpp": + case "c": + return { iconBgColor: "bg-warn", Icon: FileCode }; + case "png": + case "jpg": + case "jpeg": + return { iconBgColor: "bg-royalblue", Icon: FileImage }; + default: + return { iconBgColor: "bg-royalblue", Icon: FileText }; + } +} diff --git a/frontend/src/components/WorkspaceChat/ChatContainer/PromptInput/index.jsx b/frontend/src/components/WorkspaceChat/ChatContainer/PromptInput/index.jsx index fc46fbe9c..031d71ae1 100644 --- a/frontend/src/components/WorkspaceChat/ChatContainer/PromptInput/index.jsx +++ b/frontend/src/components/WorkspaceChat/ChatContainer/PromptInput/index.jsx @@ -13,6 +13,8 @@ import AvailableAgentsButton, { import TextSizeButton from "./TextSizeMenu"; import SpeechToText from "./SpeechToText"; import { Tooltip } from "react-tooltip"; +import AttachmentManager from "./Attachments"; +import AttachItem from "./AttachItem"; export const PROMPT_INPUT_EVENT = "set_prompt_input"; export default function PromptInput({ @@ -21,6 +23,7 @@ export default function PromptInput({ inputDisabled, buttonDisabled, sendCommand, + attachments = [], }) { const [promptInput, setPromptInput] = useState(""); const { showAgents, setShowAgents } = useAvailableAgents(); @@ -106,10 +109,11 @@ export default function PromptInput({ />
-
+
+