Merge branch 'master' of github.com:Mintplex-Labs/anything-llm into render

collector/processSingleFile/convert/asPDF.js

@@ -40,9 +40,9 @@ async function asPDF({ fullFilePath = "", filename = "" }) {
   const data = {
     id: v4(),
     url: "file://" + fullFilePath,
-    title: docs[0]?.metadata?.pdf?.info?.Title || filename,
+    title: filename,
     docAuthor: docs[0]?.metadata?.pdf?.info?.Creator || "no author found",
-    description: "No description found.",
+    description: docs[0]?.metadata?.pdf?.info?.Title || "No description found.",
     docSource: "pdf file uploaded by the user.",
     chunkSource: "",
     published: createdDate(fullFilePath),

docker/.env.example

@@ -27,6 +27,7 @@ GID='1000'
 
 # LLM_PROVIDER='lmstudio'
 # LMSTUDIO_BASE_PATH='http://your-server:1234/v1'
+# LMSTUDIO_MODEL_PREF='Loaded from Chat UI' # this is a bug in LMStudio 0.2.17
 # LMSTUDIO_MODEL_TOKEN_LIMIT=4096
 
 # LLM_PROVIDER='localai'

docker/HOW_TO_USE_DOCKER.md

@@ -109,29 +109,34 @@ container rebuilds or pulls from Docker Hub.
 
 Your docker host will show the image as online once the build process is completed. This will build the app to `http://localhost:3001`.
 
-## ⚠️ Vector DB support ⚠️
+## Integrations and one-click setups
 
-Out of the box, all vector databases are supported. Any vector databases requiring special configuration are listed below.
+The integrations below are templates or tooling built by the community to make running the docker experience of AnythingLLM easier.
 
-### Using local ChromaDB with Dockerized AnythingLLM
+### Use the Midori AI Subsystem to Manage AnythingLLM
 
-- Ensure in your `./docker/.env` file that you have
+Follow the setup found on [Midori AI Subsystem Site](https://io.midori-ai.xyz/subsystem/manager/) for your host OS
+After setting that up install the AnythingLLM docker backend to the Midori AI Subsystem.
 
-```
+Once that is done, you are all set!
-#./docker/.env
-...other configs
-
-VECTOR_DB="chroma"
-CHROMA_ENDPOINT='http://host.docker.internal:8000' # Allow docker to look on host port, not container.
-# CHROMA_API_HEADER="X-Api-Key" // If you have an Auth middleware on your instance.
-# CHROMA_API_KEY="sk-123abc"
-
-...other configs
-
-```
 
 ## Common questions and fixes
 
+### Cannot connect to service running on localhost!
+
+If you are in docker and cannot connect to a service running on your host machine running on a local interface or loopback:
+
+- `localhost`
+- `127.0.0.1`
+- `0.0.0.0`
+
+> [!IMPORTANT]
+> On linux `http://host.docker.internal:xxxx` does not work.
+> Use `http://172.17.0.1:xxxx` instead to emulate this functionality.
+
+Then in docker you need to replace that localhost part with `host.docker.internal`. For example, if running Ollama on the host machine, bound to http://127.0.0.1:11434 you should put `http://host.docker.internal:11434` into the connection URL in AnythingLLM.
+
+
 ### API is not working, cannot login, LLM is "offline"?
 
 You are likely running the docker container on a remote machine like EC2 or some other instance where the reachable URL

frontend/src/components/LLMSelection/LMStudioOptions/index.jsx

@@ -1,7 +1,14 @@
+import { useEffect, useState } from "react";
 import { Info } from "@phosphor-icons/react";
 import paths from "@/utils/paths";
+import System from "@/models/system";
 
 export default function LMStudioOptions({ settings, showAlert = false }) {
+  const [basePathValue, setBasePathValue] = useState(
+    settings?.LMStudioBasePath
+  );
+  const [basePath, setBasePath] = useState(settings?.LMStudioBasePath);
+
   return (
     <div className="w-full flex flex-col">
       {showAlert && (
@@ -35,8 +42,11 @@ export default function LMStudioOptions({ settings, showAlert = false }) {
             required={true}
             autoComplete="off"
             spellCheck={false}
+            onChange={(e) => setBasePathValue(e.target.value)}
+            onBlur={() => setBasePath(basePathValue)}
           />
         </div>
+        <LMStudioModelSelection settings={settings} basePath={basePath} />
         <div className="flex flex-col w-60">
           <label className="text-white text-sm font-semibold block mb-4">
             Token context window
@@ -57,3 +67,73 @@ export default function LMStudioOptions({ settings, showAlert = false }) {
     </div>
   );
 }
+
+function LMStudioModelSelection({ settings, basePath = null }) {
+  const [customModels, setCustomModels] = useState([]);
+  const [loading, setLoading] = useState(true);
+
+  useEffect(() => {
+    async function findCustomModels() {
+      if (!basePath || !basePath.includes("/v1")) {
+        setCustomModels([]);
+        setLoading(false);
+        return;
+      }
+      setLoading(true);
+      const { models } = await System.customModels("lmstudio", null, basePath);
+      setCustomModels(models || []);
+      setLoading(false);
+    }
+    findCustomModels();
+  }, [basePath]);
+
+  if (loading || customModels.length == 0) {
+    return (
+      <div className="flex flex-col w-60">
+        <label className="text-white text-sm font-semibold block mb-4">
+          Chat Model Selection
+        </label>
+        <select
+          name="LMStudioModelPref"
+          disabled={true}
+          className="bg-zinc-900 border-gray-500 text-white text-sm rounded-lg block w-full p-2.5"
+        >
+          <option disabled={true} selected={true}>
+            {basePath?.includes("/v1")
+              ? "-- loading available models --"
+              : "-- waiting for URL --"}
+          </option>
+        </select>
+      </div>
+    );
+  }
+
+  return (
+    <div className="flex flex-col w-60">
+      <label className="text-white text-sm font-semibold block mb-4">
+        Chat Model Selection
+      </label>
+      <select
+        name="LMStudioModelPref"
+        required={true}
+        className="bg-zinc-900 border-gray-500 text-white text-sm rounded-lg block w-full p-2.5"
+      >
+        {customModels.length > 0 && (
+          <optgroup label="Your loaded models">
+            {customModels.map((model) => {
+              return (
+                <option
+                  key={model.id}
+                  value={model.id}
+                  selected={settings.LMStudioModelPref === model.id}
+                >
+                  {model.id}
+                </option>
+              );
+            })}
+          </optgroup>
+        )}
+      </select>
+    </div>
+  );
+}

frontend/src/components/Modals/MangeWorkspace/Documents/Directory/FileRow/index.jsx

@@ -4,44 +4,12 @@ import {
   getFileExtension,
   middleTruncate,
 } from "@/utils/directories";
-import { File, Trash } from "@phosphor-icons/react";
+import { File } from "@phosphor-icons/react";
-import System from "@/models/system";
 import debounce from "lodash.debounce";
 
-export default function FileRow({
+export default function FileRow({ item, selected, toggleSelection }) {
-  item,
-  folderName,
-  selected,
-  toggleSelection,
-  fetchKeys,
-  setLoading,
-  setLoadingMessage,
-}) {
   const [showTooltip, setShowTooltip] = useState(false);
 
-  const onTrashClick = async (event) => {
-    event.stopPropagation();
-    if (
-      !window.confirm(
-        "Are you sure you want to delete this document?\nThis will require you to re-upload and re-embed it.\nThis document will be removed from any workspace that is currently referencing it.\nThis action is not reversible."
-      )
-    ) {
-      return false;
-    }
-
-    try {
-      setLoading(true);
-      setLoadingMessage("This may take a while for large documents");
-      await System.deleteDocument(`${folderName}/${item.name}`);
-      await fetchKeys(true);
-    } catch (error) {
-      console.error("Failed to delete the document:", error);
-    }
-
-    if (selected) toggleSelection(item);
-    setLoading(false);
-  };
-
   const handleShowTooltip = () => {
     setShowTooltip(true);
   };
@@ -56,11 +24,11 @@ export default function FileRow({
   return (
     <tr
       onClick={() => toggleSelection(item)}
-      className={`transition-all duration-200 text-white/80 text-xs grid grid-cols-12 py-2 pl-3.5 pr-8 hover:bg-sky-500/20 cursor-pointer file-row ${
+      className={`text-white/80 text-xs grid grid-cols-12 py-2 pl-3.5 pr-8 hover:bg-sky-500/20 cursor-pointer file-row ${
         selected ? "selected" : ""
       }`}
     >
-      <div className="pl-2 col-span-5 flex gap-x-[4px] items-center">
+      <div className="pl-2 col-span-6 flex gap-x-[4px] items-center">
         <div
           className="shrink-0 w-3 h-3 rounded border-[1px] border-white flex justify-center items-center cursor-pointer"
           role="checkbox"
@@ -94,16 +62,12 @@ export default function FileRow({
       <p className="col-span-2 pl-2 uppercase overflow-x-hidden">
         {getFileExtension(item.url)}
       </p>
-      <div className="col-span-2 flex justify-end items-center">
+      <div className="-col-span-2 flex justify-end items-center">
         {item?.cached && (
           <div className="bg-white/10 rounded-3xl">
             <p className="text-xs px-2 py-0.5">Cached</p>
           </div>
         )}
-        <Trash
-          onClick={onTrashClick}
-          className="text-base font-bold w-4 h-4 ml-2 flex-shrink-0 cursor-pointer"
-        />
       </div>
     </tr>
   );

frontend/src/components/Modals/MangeWorkspace/Documents/Directory/FolderRow/index.jsx

@@ -1,8 +1,7 @@
 import { useState } from "react";
 import FileRow from "../FileRow";
-import { CaretDown, FolderNotch, Trash } from "@phosphor-icons/react";
+import { CaretDown, FolderNotch } from "@phosphor-icons/react";
 import { middleTruncate } from "@/utils/directories";
-import System from "@/models/system";
 
 export default function FolderRow({
   item,
@@ -10,36 +9,10 @@ export default function FolderRow({
   onRowClick,
   toggleSelection,
   isSelected,
-  fetchKeys,
-  setLoading,
-  setLoadingMessage,
   autoExpanded = false,
 }) {
   const [expanded, setExpanded] = useState(autoExpanded);
 
-  const onTrashClick = async (event) => {
-    event.stopPropagation();
-    if (
-      !window.confirm(
-        "Are you sure you want to delete this folder?\nThis will require you to re-upload and re-embed it.\nAny documents in this folder will be removed from any workspace that is currently referencing it.\nThis action is not reversible."
-      )
-    ) {
-      return false;
-    }
-
-    try {
-      setLoading(true);
-      setLoadingMessage("This may take a while for large folders");
-      await System.deleteFolder(item.name);
-      await fetchKeys(true);
-    } catch (error) {
-      console.error("Failed to delete the document:", error);
-    }
-
-    if (selected) toggleSelection(item);
-    setLoading(false);
-  };
-
   const handleExpandClick = (event) => {
     event.stopPropagation();
     setExpanded(!expanded);
@@ -49,7 +22,7 @@ export default function FolderRow({
     <>
       <tr
         onClick={onRowClick}
-        className={`transition-all duration-200 text-white/80 text-xs grid grid-cols-12 py-2 pl-3.5 pr-8 bg-[#2C2C2C] hover:bg-sky-500/20 cursor-pointer w-full file-row:0 ${
+        className={`text-white/80 text-xs grid grid-cols-12 py-2 pl-3.5 pr-8 bg-[#1C1E21] hover:bg-sky-500/20 cursor-pointer w-full file-row ${
           selected ? "selected" : ""
         }`}
       >
@@ -59,6 +32,10 @@ export default function FolderRow({
             role="checkbox"
             aria-checked={selected}
             tabIndex={0}
+            onClick={(event) => {
+              event.stopPropagation();
+              toggleSelection(item);
+            }}
           >
             {selected && <div className="w-2 h-2 bg-white rounded-[2px]" />}
           </div>
@@ -75,35 +52,23 @@ export default function FolderRow({
             weight="fill"
           />
           <p className="whitespace-nowrap overflow-show">
-            {middleTruncate(item.name, 40)}
+            {middleTruncate(item.name, 35)}
           </p>
         </div>
         <p className="col-span-2 pl-3.5" />
         <p className="col-span-2 pl-2" />
-        <div className="col-span-2 flex justify-end items-center">
-          {item.name !== "custom-documents" && (
-            <Trash
-              onClick={onTrashClick}
-              className="text-base font-bold w-4 h-4 ml-2 flex-shrink-0 cursor-pointer"
-            />
-          )}
-        </div>
       </tr>
       {expanded && (
-        <div className="col-span-full">
+        <>
           {item.items.map((fileItem) => (
             <FileRow
               key={fileItem.id}
               item={fileItem}
-              folderName={item.name}
               selected={isSelected(fileItem.id)}
               toggleSelection={toggleSelection}
-              fetchKeys={fetchKeys}
-              setLoading={setLoading}
-              setLoadingMessage={setLoadingMessage}
             />
           ))}
-        </div>
+        </>
       )}
     </>
   );

frontend/src/components/Modals/MangeWorkspace/Documents/Directory/FolderSelectionPopup/index.jsx

@@ -0,0 +1,24 @@
+import { middleTruncate } from "@/utils/directories";
+
+export default function FolderSelectionPopup({ folders, onSelect, onClose }) {
+  const handleFolderSelect = (folder) => {
+    onSelect(folder);
+    onClose();
+  };
+
+  return (
+    <div className="absolute bottom-full left-0 mb-2 bg-white rounded-lg shadow-lg">
+      <ul>
+        {folders.map((folder) => (
+          <li
+            key={folder.name}
+            onClick={() => handleFolderSelect(folder)}
+            className="px-4 py-2 text-xs text-gray-700 hover:bg-gray-200 rounded-lg cursor-pointer whitespace-nowrap"
+          >
+            {middleTruncate(folder.name, 25)}
+          </li>
+        ))}
+      </ul>
+    </div>
+  );
+}

frontend/src/components/Modals/MangeWorkspace/Documents/Directory/MoveToFolderIcon.jsx

@@ -0,0 +1,44 @@
+export default function MoveToFolderIcon({
+  className,
+  width = 18,
+  height = 18,
+}) {
+  return (
+    <svg
+      width={width}
+      height={height}
+      viewBox="0 0 17 19"
+      fill="none"
+      xmlns="http://www.w3.org/2000/svg"
+      className={className}
+    >
+      <path
+        d="M1.46092 17.9754L3.5703 12.7019C3.61238 12.5979 3.68461 12.5088 3.7777 12.4462C3.8708 12.3836 3.98051 12.3502 4.09272 12.3504H7.47897C7.59001 12.3502 7.69855 12.3174 7.79116 12.2562L9.19741 11.3196C9.29001 11.2583 9.39855 11.2256 9.50959 11.2254H15.5234C15.6126 11.2254 15.7004 11.2465 15.7798 11.2872C15.8591 11.3278 15.9277 11.3867 15.9798 11.459C16.0319 11.5313 16.0661 11.6149 16.0795 11.703C16.093 11.7912 16.0853 11.8812 16.0571 11.9658L14.0532 17.9754H1.46092Z"
+        stroke="currentColor"
+        strokeWidth="1.5"
+        strokeLinecap="round"
+        strokeLinejoin="round"
+      />
+      <path
+        fillRule="evenodd"
+        clipRule="evenodd"
+        d="M2.25331 6.53891H2.02342C1.67533 6.53891 1.34149 6.67719 1.09534 6.92333C0.849204 7.16947 0.710922 7.50331 0.710922 7.85141V17.9764C0.710922 18.3906 1.04671 18.7264 1.46092 18.7264C1.87514 18.7264 2.21092 18.3906 2.21092 17.9764V8.03891H2.25331V6.53891ZM13.0859 9.98714V11.2264C13.0859 11.6406 13.4217 11.9764 13.8359 11.9764C14.2501 11.9764 14.5859 11.6406 14.5859 11.2264V9.53891C14.5859 9.19081 14.4476 8.85698 14.2015 8.61083C13.9554 8.36469 13.6215 8.22641 13.2734 8.22641H13.0863V9.98714H13.0859Z"
+        fill="currentColor"
+      />
+      <path
+        d="M7.53416 1.62906L7.53416 7.70406"
+        stroke="currentColor"
+        strokeWidth="1.5"
+        strokeLinecap="round"
+        strokeLinejoin="round"
+      />
+      <path
+        d="M10.6411 5.21854L7.53456 7.70376L4.42803 5.21854"
+        stroke="currentColor"
+        strokeWidth="1.5"
+        strokeLinecap="round"
+        strokeLinejoin="round"
+      />
+    </svg>
+  );
+}

frontend/src/components/Modals/MangeWorkspace/Documents/Directory/index.jsx

@@ -2,11 +2,16 @@ import UploadFile from "../UploadFile";
 import PreLoader from "@/components/Preloader";
 import { memo, useEffect, useState } from "react";
 import FolderRow from "./FolderRow";
-import pluralize from "pluralize";
 import System from "@/models/system";
+import { Plus, Trash } from "@phosphor-icons/react";
+import Document from "@/models/document";
+import showToast from "@/utils/toast";
+import FolderSelectionPopup from "./FolderSelectionPopup";
+import MoveToFolderIcon from "./MoveToFolderIcon";
 
 function Directory({
   files,
+  setFiles,
   loading,
   setLoading,
   workspace,
@@ -19,12 +24,19 @@ function Directory({
   loadingMessage,
 }) {
   const [amountSelected, setAmountSelected] = useState(0);
+  const [newFolderName, setNewFolderName] = useState("");
+  const [showNewFolderInput, setShowNewFolderInput] = useState(false);
+  const [showFolderSelection, setShowFolderSelection] = useState(false);
+
+  useEffect(() => {
+    setAmountSelected(Object.keys(selectedItems).length);
+  }, [selectedItems]);
 
   const deleteFiles = async (event) => {
     event.stopPropagation();
     if (
       !window.confirm(
-        "Are you sure you want to delete these files?\nThis will remove the files from the system and remove them from any existing workspaces automatically.\nThis action is not reversible."
+        "Are you sure you want to delete these files and folders?\nThis will remove the files from the system and remove them from any existing workspaces automatically.\nThis action is not reversible."
       )
     ) {
       return false;
@@ -32,6 +44,8 @@ function Directory({
 
     try {
       const toRemove = [];
+      const foldersToRemove = [];
+
       for (const itemId of Object.keys(selectedItems)) {
         for (const folder of files.items) {
           const foundItem = folder.items.find((file) => file.id === itemId);
@@ -41,13 +55,29 @@ function Directory({
           }
         }
       }
+      for (const folder of files.items) {
+        if (folder.name === "custom-documents") {
+          continue;
+        }
+
+        if (isSelected(folder.id, folder)) {
+          foldersToRemove.push(folder.name);
+        }
+      }
+
       setLoading(true);
-      setLoadingMessage(`Removing ${toRemove.length} documents. Please wait.`);
+      setLoadingMessage(
+        `Removing ${toRemove.length} documents and ${foldersToRemove.length} folders. Please wait.`
+      );
       await System.deleteDocuments(toRemove);
+      for (const folderName of foldersToRemove) {
+        await System.deleteFolder(folderName);
+      }
+
       await fetchKeys(true);
       setSelectedItems({});
     } catch (error) {
-      console.error("Failed to delete the document:", error);
+      console.error("Failed to delete files and folders:", error);
     } finally {
       setLoading(false);
       setSelectedItems({});
@@ -57,15 +87,17 @@ function Directory({
   const toggleSelection = (item) => {
     setSelectedItems((prevSelectedItems) => {
       const newSelectedItems = { ...prevSelectedItems };
-
       if (item.type === "folder") {
-        const isCurrentlySelected = isFolderCompletelySelected(item);
+        // select all files in the folder
-        if (isCurrentlySelected) {
+        if (newSelectedItems[item.name]) {
+          delete newSelectedItems[item.name];
           item.items.forEach((file) => delete newSelectedItems[file.id]);
         } else {
+          newSelectedItems[item.name] = true;
           item.items.forEach((file) => (newSelectedItems[file.id] = true));
         }
       } else {
+        // single file selections
         if (newSelectedItems[item.id]) {
           delete newSelectedItems[item.id];
         } else {
@@ -77,44 +109,124 @@ function Directory({
     });
   };
 
-  const isFolderCompletelySelected = (folder) => {
+  // check if item is selected based on selectedItems state
-    if (folder.items.length === 0) {
-      return false;
-    }
-    return folder.items.every((file) => selectedItems[file.id]);
-  };
-
   const isSelected = (id, item) => {
     if (item && item.type === "folder") {
-      return isFolderCompletelySelected(item);
+      if (!selectedItems[item.name]) {
+        return false;
+      }
+      return item.items.every((file) => selectedItems[file.id]);
     }
 
     return !!selectedItems[id];
   };
 
-  useEffect(() => {
+  const createNewFolder = () => {
-    setAmountSelected(Object.keys(selectedItems).length);
+    setShowNewFolderInput(true);
-  }, [selectedItems]);
+  };
+
+  const confirmNewFolder = async () => {
+    if (newFolderName.trim() !== "") {
+      const newFolder = {
+        name: newFolderName,
+        type: "folder",
+        items: [],
+      };
+
+      // If folder failed to create - silently fail.
+      const { success } = await Document.createFolder(newFolderName);
+      if (success) {
+        setFiles({
+          ...files,
+          items: [...files.items, newFolder],
+        });
+      }
+
+      setNewFolderName("");
+      setShowNewFolderInput(false);
+    }
+  };
+
+  const moveToFolder = async (folder) => {
+    const toMove = [];
+    for (const itemId of Object.keys(selectedItems)) {
+      for (const currentFolder of files.items) {
+        const foundItem = currentFolder.items.find(
+          (file) => file.id === itemId
+        );
+        if (foundItem) {
+          toMove.push({ ...foundItem, folderName: currentFolder.name });
+          break;
+        }
+      }
+    }
+    setLoading(true);
+    setLoadingMessage(`Moving ${toMove.length} documents. Please wait.`);
+    const { success, message } = await Document.moveToFolder(
+      toMove,
+      folder.name
+    );
+    if (!success) {
+      showToast(`Error moving files: ${message}`, "error");
+      setLoading(false);
+      return;
+    }
+
+    if (success && message) {
+      // show info if some files were not moved due to being embedded
+      showToast(message, "info");
+    } else {
+      showToast(`Successfully moved ${toMove.length} documents.`, "success");
+    }
+    await fetchKeys(true);
+    setSelectedItems({});
+    setLoading(false);
+  };
 
   return (
     <div className="px-8 pb-8">
       <div className="flex flex-col gap-y-6">
-        <div className="flex items-center justify-between w-[560px] px-5">
+        <div className="flex items-center justify-between w-[560px] px-5 relative">
           <h3 className="text-white text-base font-bold">My Documents</h3>
+          {showNewFolderInput ? (
+            <div className="flex items-center gap-x-2 z-50">
+              <input
+                type="text"
+                placeholder="Folder name"
+                value={newFolderName}
+                onChange={(e) => setNewFolderName(e.target.value)}
+                className="bg-zinc-900 text-white placeholder-white/20 text-sm rounded-md p-2.5 w-[150px] h-[32px]"
+              />
+              <div className="flex gap-x-2">
+                <button
+                  onClick={confirmNewFolder}
+                  className="text-sky-400 rounded-md text-sm font-bold hover:text-sky-500"
+                >
+                  Create
+                </button>
+              </div>
+            </div>
+          ) : (
+            <button
+              className="flex items-center gap-x-2 cursor-pointer px-[14px] py-[7px] -mr-[14px] rounded-lg hover:bg-[#222628]/60"
+              onClick={createNewFolder}
+            >
+              <Plus size={18} weight="bold" color="#D3D4D4" />
+              <div className="text-[#D3D4D4] text-xs font-bold leading-[18px]">
+                New Folder
+              </div>
+            </button>
+          )}
         </div>
 
-        <div className="relative w-[560px] h-[310px] bg-zinc-900 rounded-2xl">
+        <div className="relative w-[560px] h-[310px] bg-zinc-900 rounded-2xl overflow-hidden">
-          <div className="rounded-t-2xl text-white/80 text-xs grid grid-cols-12 py-2 px-8 border-b border-white/20 shadow-lg bg-zinc-900 sticky top-0 z-10">
+          <div className="absolute top-0 left-0 right-0 z-10 rounded-t-2xl text-white/80 text-xs grid grid-cols-12 py-2 px-8 border-b border-white/20 shadow-lg bg-zinc-900">
-            <p className="col-span-5">Name</p>
+            <p className="col-span-6">Name</p>
             <p className="col-span-3">Date</p>
             <p className="col-span-2">Kind</p>
-            <p className="col-span-2">Cached</p>
           </div>
 
-          <div
+          <div className="overflow-y-auto h-full pt-8">
-            className="overflow-y-auto pb-9"
-            style={{ height: "calc(100% - 40px)" }}
-          >
             {loading ? (
               <div className="w-full h-full flex items-center justify-center flex-col gap-y-5">
                 <PreLoader />
@@ -122,11 +234,10 @@ function Directory({
                   {loadingMessage}
                 </p>
               </div>
-            ) : !!files.items ? (
+            ) : files.items ? (
               files.items.map(
                 (item, index) =>
-                  (item.name === "custom-documents" ||
+                  item.type === "folder" && (
-                    (item.type === "folder" && item.items.length > 0)) && (
                     <FolderRow
                       key={index}
                       item={item}
@@ -134,12 +245,9 @@ function Directory({
                         item.id,
                         item.type === "folder" ? item : null
                       )}
-                      fetchKeys={fetchKeys}
                       onRowClick={() => toggleSelection(item)}
                       toggleSelection={toggleSelection}
                       isSelected={isSelected}
-                      setLoading={setLoading}
-                      setLoadingMessage={setLoadingMessage}
                       autoExpanded={index === 0}
                     />
                   )
@@ -152,27 +260,46 @@ function Directory({
               </div>
             )}
           </div>
-
           {amountSelected !== 0 && (
-            <div className="absolute bottom-0 left-0 w-full flex justify-between items-center h-9 bg-white rounded-b-2xl">
+            <div className="absolute bottom-[12px] left-0 right-0 flex justify-center">
-              <div className="flex gap-x-5 w-[80%] justify-center">
+              <div className="mx-auto bg-white/40 rounded-lg py-1 px-2">
+                <div className="flex flex-row items-center gap-x-2">
                   <button
+                    onClick={moveToWorkspace}
                     onMouseEnter={() => setHighlightWorkspace(true)}
                     onMouseLeave={() => setHighlightWorkspace(false)}
-                  onClick={moveToWorkspace}
+                    className="border-none text-sm font-semibold bg-white h-[30px] px-2.5 rounded-lg hover:text-white hover:bg-neutral-800/80"
-                  className="border-none text-sm font-semibold h-7 px-2.5 rounded-lg hover:text-white hover:bg-neutral-800/80 flex items-center"
                   >
-                  Move {amountSelected} {pluralize("file", amountSelected)} to
+                    Move to Workspace
-                  workspace
                   </button>
+                  <div className="relative">
+                    <button
+                      onClick={() =>
+                        setShowFolderSelection(!showFolderSelection)
+                      }
+                      className="border-none text-sm font-semibold bg-white h-[32px] w-[32px] rounded-lg text-[#222628] hover:bg-neutral-800/80 flex justify-center items-center group"
+                    >
+                      <MoveToFolderIcon className="text-[#222628] group-hover:text-white" />
+                    </button>
+                    {showFolderSelection && (
+                      <FolderSelectionPopup
+                        folders={files.items.filter(
+                          (item) => item.type === "folder"
+                        )}
+                        onSelect={moveToFolder}
+                        onClose={() => setShowFolderSelection(false)}
+                      />
+                    )}
                   </div>
                   <button
                     onClick={deleteFiles}
-                className="border-none text-red-500/50 text-sm font-semibold h-7 px-2.5 rounded-lg hover:text-red-500/80 flex items-center"
+                    className="border-none text-sm font-semibold bg-white h-[32px] w-[32px] rounded-lg text-[#222628] hover:text-white hover:bg-neutral-800/80 flex justify-center items-center"
                   >
-                Delete
+                    <Trash size={18} weight="bold" />
                   </button>
                 </div>
+              </div>
+            </div>
           )}
         </div>
         <UploadFile

frontend/src/components/Modals/MangeWorkspace/Documents/UploadFile/index.jsx

@@ -76,7 +76,7 @@ export default function UploadFile({ workspace, fetchKeys, setLoading }) {
   return (
     <div>
       <div
-        className={`transition-all duration-300 w-[560px] border-2 border-dashed rounded-2xl bg-zinc-900/50 p-3 ${
+        className={`w-[560px] border-2 border-dashed rounded-2xl bg-zinc-900/50 p-3 ${
           ready ? "cursor-pointer" : "cursor-not-allowed"
         } hover:bg-zinc-900/90`}
         {...getRootProps()}
@@ -134,7 +134,7 @@ export default function UploadFile({ workspace, fetchKeys, setLoading }) {
         <button
           disabled={fetchingUrl}
           type="submit"
-          className="disabled:bg-white/20 disabled:text-slate-300 disabled:border-slate-400 disabled:cursor-wait bg bg-transparent hover:bg-slate-200 hover:text-slate-800 w-auto border border-white text-sm text-white p-2.5 rounded-lg transition-all duration-300"
+          className="disabled:bg-white/20 disabled:text-slate-300 disabled:border-slate-400 disabled:cursor-wait bg bg-transparent hover:bg-slate-200 hover:text-slate-800 w-auto border border-white text-sm text-white p-2.5 rounded-lg"
         >
           {fetchingUrl ? "Fetching..." : "Fetch website"}
         </button>

frontend/src/components/Modals/MangeWorkspace/Documents/WorkspaceDirectory/WorkspaceFileRow/index.jsx

@@ -53,7 +53,7 @@ export default function WorkspaceFileRow({
   const handleMouseLeave = debounce(handleHideTooltip, 500);
   return (
     <div
-      className={`items-center transition-all duration-200 text-white/80 text-xs grid grid-cols-12 py-2 pl-3.5 pr-8 hover:bg-sky-500/20 cursor-pointer
+      className={`items-center text-white/80 text-xs grid grid-cols-12 py-2 pl-3.5 pr-8 hover:bg-sky-500/20 cursor-pointer
           ${isMovedItem ? "bg-green-800/40" : "file-row"}`}
     >
       <div className="col-span-5 flex gap-x-[4px] items-center">

frontend/src/components/Modals/MangeWorkspace/Documents/WorkspaceDirectory/index.jsx

@@ -55,7 +55,7 @@ function WorkspaceDirectory({
           </h3>
         </div>
         <div
-          className={`relative w-[560px] h-[445px] bg-zinc-900 rounded-2xl mt-5 overflow-y-auto border-4 transition-all duration-300 ${
+          className={`relative w-[560px] h-[445px] bg-zinc-900 rounded-2xl mt-5 overflow-y-auto border-4 ${
             highlightWorkspace ? "border-cyan-300/80" : "border-transparent"
           }`}
         >
@@ -96,7 +96,7 @@ function WorkspaceDirectory({
           </div>
         </div>
         {hasChanges && (
-          <div className="flex items-center justify-between py-6 transition-all duration-300">
+          <div className="flex items-center justify-between py-6">
             <div className="text-white/80">
               <p className="text-sm font-semibold">
                 {embeddingCosts === 0
@@ -114,7 +114,7 @@ function WorkspaceDirectory({
 
             <button
               onClick={saveChanges}
-              className="transition-all duration-300 border border-slate-200 px-5 py-2.5 rounded-lg text-white text-sm items-center flex gap-x-2 hover:bg-slate-200 hover:text-slate-800 focus:ring-gray-800"
+              className="border border-slate-200 px-5 py-2.5 rounded-lg text-white text-sm items-center flex gap-x-2 hover:bg-slate-200 hover:text-slate-800 focus:ring-gray-800"
             >
               Save and Embed
             </button>
@@ -177,7 +177,7 @@ const PinAlert = memo(() => {
             <button disabled={true} className="invisible" />
             <button
               onClick={dismissAlert}
-              className="transition-all duration-300 border border-slate-200 px-4 py-2 rounded-lg text-white text-sm items-center flex gap-x-2 hover:bg-slate-200 hover:text-slate-800 focus:ring-gray-800"
+              className="border border-slate-200 px-4 py-2 rounded-lg text-white text-sm items-center flex gap-x-2 hover:bg-slate-200 hover:text-slate-800 focus:ring-gray-800"
             >
               Okay, got it
             </button>

frontend/src/components/Modals/MangeWorkspace/Documents/index.jsx

@@ -191,9 +191,10 @@ export default function DocumentSettings({ workspace, systemSettings }) {
   };
 
   return (
-    <div className="flex gap-x-6 justify-center -mt-6 z-10 relative">
+    <div className="flex upload-modal -mt-6 z-10 relative">
       <Directory
         files={availableDocs}
+        setFiles={setAvailableDocs}
         loading={loading}
         loadingMessage={loadingMessage}
         setLoading={setLoading}
@@ -207,7 +208,7 @@ export default function DocumentSettings({ workspace, systemSettings }) {
         moveToWorkspace={moveSelectedItemsToWorkspace}
         setLoadingMessage={setLoadingMessage}
       />
-      <div className="flex items-center">
+      <div className="upload-modal-arrow">
         <ArrowsDownUp className="text-white text-base font-bold rotate-90 w-11 h-11" />
       </div>
       <WorkspaceDirectory

frontend/src/components/Modals/MangeWorkspace/index.jsx

@@ -64,13 +64,14 @@ const ManageWorkspace = ({ hideModal = noop, providedSlug = null }) => {
   return (
     <div className="w-screen h-screen fixed top-0 left-0 flex justify-center items-center z-99">
       <div className="backdrop h-full w-full absolute top-0 z-10" />
-      <div className={`absolute max-h-full w-fit transition duration-300 z-20`}>
+      <div className="absolute max-h-full w-fit transition duration-300 z-20 md:overflow-y-auto py-10">
         <div className="relative bg-main-gradient rounded-[12px] shadow border-2 border-slate-300/10">
-          <div className="flex items-start justify-between p-2 rounded-t border-gray-500/50 z-40 relative">
+          <div className="flex items-start justify-between p-2 rounded-t border-gray-500/50 relative">
+            <div />
             <button
               onClick={hideModal}
               type="button"
-              className="transition-all duration-300 text-gray-400 bg-transparent rounded-lg text-sm p-1.5 ml-auto inline-flex items-center hover:border-white/60 bg-sidebar-button hover:bg-menu-item-selected-gradient hover:border-slate-100 hover:border-opacity-50 border-transparent border"
+              className="z-50 text-gray-400 bg-transparent rounded-lg text-sm p-1.5 ml-auto inline-flex items-center hover:border-white/60 bg-sidebar-button hover:bg-menu-item-selected-gradient hover:border-slate-100 hover:border-opacity-50 border-transparent border"
             >
               <X className="text-gray-300 text-lg" />
             </button>

frontend/src/components/WorkspaceChat/ChatContainer/PromptInput/index.jsx

@@ -1,4 +1,4 @@
-import React, { useState, useRef } from "react";
+import React, { useState, useRef, useEffect } from "react";
 import SlashCommandsButton, {
   SlashCommands,
   useSlashCommands,
@@ -7,9 +7,7 @@ import { isMobile } from "react-device-detect";
 import debounce from "lodash.debounce";
 import { PaperPlaneRight } from "@phosphor-icons/react";
 import StopGenerationButton from "./StopGenerationButton";
-
 export default function PromptInput({
-  workspace,
   message,
   submit,
   onChange,
@@ -19,13 +17,27 @@ export default function PromptInput({
 }) {
   const { showSlashCommand, setShowSlashCommand } = useSlashCommands();
   const formRef = useRef(null);
+  const textareaRef = useRef(null);
   const [_, setFocused] = useState(false);
 
+  useEffect(() => {
+    if (!inputDisabled && textareaRef.current) {
+      textareaRef.current.focus();
+    }
+    resetTextAreaHeight();
+  }, [inputDisabled]);
+
   const handleSubmit = (e) => {
     setFocused(false);
     submit(e);
   };
 
+  const resetTextAreaHeight = () => {
+    if (textareaRef.current) {
+      textareaRef.current.style.height = "auto";
+    }
+  };
+
   const checkForSlash = (e) => {
     const input = e.target.value;
     if (input === "/") setShowSlashCommand(true);
@@ -44,14 +56,12 @@ export default function PromptInput({
   const adjustTextArea = (event) => {
     if (isMobile) return false;
     const element = event.target;
-    element.style.height = "1px";
+    element.style.height = "auto";
-    element.style.height =
+    element.style.height = `${element.scrollHeight}px`;
-      event.target.value.length !== 0
-        ? 25 + element.scrollHeight + "px"
-        : "1px";
   };
 
   const watchForSlash = debounce(checkForSlash, 300);
+
   return (
     <div className="w-full fixed md:absolute bottom-0 left-0 z-10 md:z-0 flex justify-center items-center">
       <SlashCommands
@@ -67,12 +77,13 @@ export default function PromptInput({
           <div className="w-[600px] bg-main-gradient shadow-2xl border border-white/50 rounded-2xl flex flex-col px-4 overflow-hidden">
             <div className="flex items-center w-full border-b-2 border-gray-500/50">
               <textarea
-                onKeyUp={adjustTextArea}
+                ref={textareaRef}
-                onKeyDown={captureEnter}
                 onChange={(e) => {
                   onChange(e);
                   watchForSlash(e);
+                  adjustTextArea(e);
                 }}
+                onKeyDown={captureEnter}
                 required={true}
                 disabled={inputDisabled}
                 onFocus={() => setFocused(true)}

frontend/src/components/WorkspaceChat/ChatContainer/index.jsx

@@ -114,7 +114,6 @@ export default function ChatContainer({ workspace, knownHistory = [] }) {
           sendCommand={sendCommand}
         />
         <PromptInput
-          workspace={workspace}
           message={message}
           submit={handleSubmit}
           onChange={handleMessageChange}

frontend/src/index.css

@@ -598,18 +598,38 @@ dialog::backdrop {
   color: #fff;
 }
 
-.file-row:nth-child(odd) {
+.file-row:nth-child(even) {
   @apply bg-[#1C1E21];
 }
 
-.file-row:nth-child(even) {
+.file-row:nth-child(odd) {
   @apply bg-[#2C2C2C];
 }
 
-.file-row.selected:nth-child(odd) {
+.file-row.selected:nth-child(even) {
   @apply bg-sky-500/20;
 }
 
-.file-row.selected:nth-child(even) {
+.file-row.selected:nth-child(odd) {
   @apply bg-sky-500/10;
 }
+
+/* Flex upload modal to be a column when on small screens so that the UI
+does not extend the close button beyond the viewport. */
+@media (max-width: 1330px) {
+  .upload-modal {
+    @apply !flex-col !items-center !py-4 no-scroll;
+  }
+
+  .upload-modal-arrow {
+    margin-top: 0px !important;
+  }
+}
+
+.upload-modal {
+  @apply flex-row items-start gap-x-6 justify-center;
+}
+
+.upload-modal-arrow {
+  margin-top: 25%;
+}

frontend/src/models/document.js

@@ -0,0 +1,38 @@
+import { API_BASE } from "@/utils/constants";
+import { baseHeaders } from "@/utils/request";
+
+const Document = {
+  createFolder: async (name) => {
+    return await fetch(`${API_BASE}/document/create-folder`, {
+      method: "POST",
+      headers: baseHeaders(),
+      body: JSON.stringify({ name }),
+    })
+      .then((res) => res.json())
+      .catch((e) => {
+        console.error(e);
+        return { success: false, error: e.message };
+      });
+  },
+  moveToFolder: async (files, folderName) => {
+    const data = {
+      files: files.map((file) => ({
+        from: file.folderName ? `${file.folderName}/${file.name}` : file.name,
+        to: `${folderName}/${file.name}`,
+      })),
+    };
+
+    return await fetch(`${API_BASE}/document/move-files`, {
+      method: "POST",
+      headers: baseHeaders(),
+      body: JSON.stringify(data),
+    })
+      .then((res) => res.json())
+      .catch((e) => {
+        console.error(e);
+        return { success: false, error: e.message };
+      });
+  },
+};
+
+export default Document;

server/.env.example

@@ -24,6 +24,7 @@ JWT_SECRET="my-random-string-for-seeding" # Please generate random string at lea
 
 # LLM_PROVIDER='lmstudio'
 # LMSTUDIO_BASE_PATH='http://your-server:1234/v1'
+# LMSTUDIO_MODEL_PREF='Loaded from Chat UI' # this is a bug in LMStudio 0.2.17
 # LMSTUDIO_MODEL_TOKEN_LIMIT=4096
 
 # LLM_PROVIDER='localai'

server/endpoints/api/document/index.js

@@ -4,11 +4,19 @@ const { setupMulter } = require("../../../utils/files/multer");
 const {
   viewLocalFiles,
   findDocumentInDocuments,
+  normalizePath,
 } = require("../../../utils/files");
 const { reqBody } = require("../../../utils/http");
 const { EventLogs } = require("../../../models/eventLogs");
 const { CollectorApi } = require("../../../utils/collectorApi");
 const { handleUploads } = setupMulter();
+const fs = require("fs");
+const path = require("path");
+const { Document } = require("../../../models/documents");
+const documentsPath =
+  process.env.NODE_ENV === "development"
+    ? path.resolve(__dirname, "../../../storage/documents")
+    : path.resolve(process.env.STORAGE_DIR, `documents`);
 
 function apiDocumentEndpoints(app) {
   if (!app) return;
@@ -552,6 +560,169 @@ function apiDocumentEndpoints(app) {
       response.sendStatus(500).end();
     }
   });
+
+  app.post(
+    "/v1/document/create-folder",
+    [validApiKey],
+    async (request, response) => {
+      /*
+      #swagger.tags = ['Documents']
+      #swagger.description = 'Create a new folder inside the documents storage directory.'
+      #swagger.requestBody = {
+        description: 'Name of the folder to create.',
+        required: true,
+        type: 'object',
+        content: {
+          "application/json": {
+            schema: {
+              type: 'object',
+              example: {
+                "name": "new-folder"
+              }
+            }
+          }
+        }
+      }
+      #swagger.responses[200] = {
+        content: {
+          "application/json": {
+            schema: {
+              type: 'object',
+              example: {
+                success: true,
+                message: null
+              }
+            }
+          }
+        }
+      }
+      #swagger.responses[403] = {
+        schema: {
+          "$ref": "#/definitions/InvalidAPIKey"
+        }
+      }
+      */
+      try {
+        const { name } = reqBody(request);
+        const storagePath = path.join(documentsPath, normalizePath(name));
+
+        if (fs.existsSync(storagePath)) {
+          response.status(500).json({
+            success: false,
+            message: "Folder by that name already exists",
+          });
+          return;
+        }
+
+        fs.mkdirSync(storagePath, { recursive: true });
+        response.status(200).json({ success: true, message: null });
+      } catch (e) {
+        console.error(e);
+        response.status(500).json({
+          success: false,
+          message: `Failed to create folder: ${e.message}`,
+        });
+      }
+    }
+  );
+
+  app.post(
+    "/v1/document/move-files",
+    [validApiKey],
+    async (request, response) => {
+      /*
+      #swagger.tags = ['Documents']
+      #swagger.description = 'Move files within the documents storage directory.'
+      #swagger.requestBody = {
+        description: 'Array of objects containing source and destination paths of files to move.',
+        required: true,
+        type: 'object',
+        content: {
+          "application/json": {
+            schema: {
+              type: 'object',
+              example: {
+                "files": [
+                  {
+                    "from": "custom-documents/file.txt-fc4beeeb-e436-454d-8bb4-e5b8979cb48f.json",
+                    "to": "folder/file.txt-fc4beeeb-e436-454d-8bb4-e5b8979cb48f.json"
+                  }
+                ]
+              }
+            }
+          }
+        }
+      }
+      #swagger.responses[200] = {
+        content: {
+          "application/json": {
+            schema: {
+              type: 'object',
+              example: {
+                success: true,
+                message: null
+              }
+            }
+          }
+        }
+      }
+      #swagger.responses[403] = {
+        schema: {
+          "$ref": "#/definitions/InvalidAPIKey"
+        }
+      }
+      */
+      try {
+        const { files } = reqBody(request);
+        const docpaths = files.map(({ from }) => from);
+        const documents = await Document.where({ docpath: { in: docpaths } });
+        const embeddedFiles = documents.map((doc) => doc.docpath);
+        const moveableFiles = files.filter(
+          ({ from }) => !embeddedFiles.includes(from)
+        );
+        const movePromises = moveableFiles.map(({ from, to }) => {
+          const sourcePath = path.join(documentsPath, normalizePath(from));
+          const destinationPath = path.join(documentsPath, normalizePath(to));
+          return new Promise((resolve, reject) => {
+            fs.rename(sourcePath, destinationPath, (err) => {
+              if (err) {
+                console.error(`Error moving file ${from} to ${to}:`, err);
+                reject(err);
+              } else {
+                resolve();
+              }
+            });
+          });
+        });
+        Promise.all(movePromises)
+          .then(() => {
+            const unmovableCount = files.length - moveableFiles.length;
+            if (unmovableCount > 0) {
+              response.status(200).json({
+                success: true,
+                message: `${unmovableCount}/${files.length} files not moved. Unembed them from all workspaces.`,
+              });
+            } else {
+              response.status(200).json({
+                success: true,
+                message: null,
+              });
+            }
+          })
+          .catch((err) => {
+            console.error("Error moving files:", err);
+            response
+              .status(500)
+              .json({ success: false, message: "Failed to move some files." });
+          });
+      } catch (e) {
+        console.error(e);
+        response
+          .status(500)
+          .json({ success: false, message: "Failed to move files." });
+      }
+    }
+  );
 }
 
 module.exports = { apiDocumentEndpoints };

server/endpoints/document.js

@@ -0,0 +1,103 @@
+const { Document } = require("../models/documents");
+const { normalizePath, documentsPath } = require("../utils/files");
+const { reqBody } = require("../utils/http");
+const {
+  flexUserRoleValid,
+  ROLES,
+} = require("../utils/middleware/multiUserProtected");
+const { validatedRequest } = require("../utils/middleware/validatedRequest");
+const fs = require("fs");
+const path = require("path");
+
+function documentEndpoints(app) {
+  if (!app) return;
+  app.post(
+    "/document/create-folder",
+    [validatedRequest, flexUserRoleValid([ROLES.admin, ROLES.manager])],
+    async (request, response) => {
+      try {
+        const { name } = reqBody(request);
+        const storagePath = path.join(documentsPath, normalizePath(name));
+
+        if (fs.existsSync(storagePath)) {
+          response.status(500).json({
+            success: false,
+            message: "Folder by that name already exists",
+          });
+          return;
+        }
+
+        fs.mkdirSync(storagePath, { recursive: true });
+        response.status(200).json({ success: true, message: null });
+      } catch (e) {
+        console.error(e);
+        response.status(500).json({
+          success: false,
+          message: `Failed to create folder: ${e.message} `,
+        });
+      }
+    }
+  );
+
+  app.post(
+    "/document/move-files",
+    [validatedRequest, flexUserRoleValid([ROLES.admin, ROLES.manager])],
+    async (request, response) => {
+      try {
+        const { files } = reqBody(request);
+        const docpaths = files.map(({ from }) => from);
+        const documents = await Document.where({ docpath: { in: docpaths } });
+
+        const embeddedFiles = documents.map((doc) => doc.docpath);
+        const moveableFiles = files.filter(
+          ({ from }) => !embeddedFiles.includes(from)
+        );
+
+        const movePromises = moveableFiles.map(({ from, to }) => {
+          const sourcePath = path.join(documentsPath, normalizePath(from));
+          const destinationPath = path.join(documentsPath, normalizePath(to));
+
+          return new Promise((resolve, reject) => {
+            fs.rename(sourcePath, destinationPath, (err) => {
+              if (err) {
+                console.error(`Error moving file ${from} to ${to}:`, err);
+                reject(err);
+              } else {
+                resolve();
+              }
+            });
+          });
+        });
+
+        Promise.all(movePromises)
+          .then(() => {
+            const unmovableCount = files.length - moveableFiles.length;
+            if (unmovableCount > 0) {
+              response.status(200).json({
+                success: true,
+                message: `${unmovableCount}/${files.length} files not moved. Unembed them from all workspaces.`,
+              });
+            } else {
+              response.status(200).json({
+                success: true,
+                message: null,
+              });
+            }
+          })
+          .catch((err) => {
+            console.error("Error moving files:", err);
+            response
+              .status(500)
+              .json({ success: false, message: "Failed to move some files." });
+          });
+      } catch (e) {
+        console.error(e);
+        response
+          .status(500)
+          .json({ success: false, message: "Failed to move files." });
+      }
+    }
+  );
+}
+
+module.exports = { documentEndpoints };

server/endpoints/workspaces.js

@@ -87,7 +87,7 @@ function workspaceEndpoints(app) {
           response.sendStatus(400).end();
           return;
         }
-
+        await Workspace.trackChange(currWorkspace, data, user);
         const { workspace, message } = await Workspace.update(
           currWorkspace.id,
           data

server/index.js

@@ -24,6 +24,7 @@ const { developerEndpoints } = require("./endpoints/api");
 const { extensionEndpoints } = require("./endpoints/extensions");
 const { bootHTTP, bootSSL } = require("./utils/boot");
 const { workspaceThreadEndpoints } = require("./endpoints/workspaceThreads");
+const { documentEndpoints } = require("./endpoints/document");
 const app = express();
 const apiRouter = express.Router();
 const FILE_LIMIT = "3GB";
@@ -48,6 +49,7 @@ adminEndpoints(apiRouter);
 inviteEndpoints(apiRouter);
 embedManagementEndpoints(apiRouter);
 utilEndpoints(apiRouter);
+documentEndpoints(apiRouter);
 developerEndpoints(app, apiRouter);
 
 // Externally facing embedder endpoints

server/models/systemSettings.js

@@ -141,6 +141,7 @@ const SystemSettings = {
         ? {
             LMStudioBasePath: process.env.LMSTUDIO_BASE_PATH,
             LMStudioTokenLimit: process.env.LMSTUDIO_MODEL_TOKEN_LIMIT,
+            LMStudioModelPref: process.env.LMSTUDIO_MODEL_PREF,
 
             // For embedding credentials when lmstudio is selected.
             OpenAiKey: !!process.env.OPEN_AI_KEY,

server/models/workspace.js

@@ -6,6 +6,8 @@ const { ROLES } = require("../utils/middleware/multiUserProtected");
 const { v4: uuidv4 } = require("uuid");
 
 const Workspace = {
+  defaultPrompt:
+    "Given the following conversation, relevant context, and a follow up question, reply with an answer to the current question the user is asking. Return only your response to the question given the above information following the users instructions as needed.",
   writable: [
     // Used for generic updates so we can validate keys in request body
     "name",
@@ -213,6 +215,34 @@ const Workspace = {
       return { success: false, error: error.message };
     }
   },
+
+  trackChange: async function (prevData, newData, user) {
+    try {
+      const { Telemetry } = require("./telemetry");
+      const { EventLogs } = require("./eventLogs");
+      if (
+        !newData?.openAiPrompt ||
+        newData?.openAiPrompt === this.defaultPrompt ||
+        newData?.openAiPrompt === prevData?.openAiPrompt
+      )
+        return;
+
+      await Telemetry.sendTelemetry("workspace_prompt_changed");
+      await EventLogs.logEvent(
+        "workspace_prompt_changed",
+        {
+          workspaceName: prevData?.name,
+          prevSystemPrompt: prevData?.openAiPrompt || this.defaultPrompt,
+          newSystemPrompt: newData?.openAiPrompt,
+        },
+        user?.id
+      );
+      return;
+    } catch (error) {
+      console.error("Error tracking workspace change:", error.message);
+      return;
+    }
+  },
 };
 
 module.exports = { Workspace };

server/swagger/openapi.json

@@ -1340,6 +1340,143 @@
         }
       }
     },
+    "/v1/document/create-folder": {
+      "post": {
+        "tags": [
+          "Documents"
+        ],
+        "description": "Create a new folder inside the documents storage directory.",
+        "parameters": [
+          {
+            "name": "Authorization",
+            "in": "header",
+            "schema": {
+              "type": "string"
+            }
+          }
+        ],
+        "responses": {
+          "200": {
+            "description": "OK",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "type": "object",
+                  "example": {
+                    "success": true,
+                    "message": null
+                  }
+                }
+              }
+            }
+          },
+          "403": {
+            "description": "Forbidden",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/InvalidAPIKey"
+                }
+              },
+              "application/xml": {
+                "schema": {
+                  "$ref": "#/components/schemas/InvalidAPIKey"
+                }
+              }
+            }
+          },
+          "500": {
+            "description": "Internal Server Error"
+          }
+        },
+        "requestBody": {
+          "description": "Name of the folder to create.",
+          "required": true,
+          "type": "object",
+          "content": {
+            "application/json": {
+              "schema": {
+                "type": "object",
+                "example": {
+                  "name": "new-folder"
+                }
+              }
+            }
+          }
+        }
+      }
+    },
+    "/v1/document/move-files": {
+      "post": {
+        "tags": [
+          "Documents"
+        ],
+        "description": "Move files within the documents storage directory.",
+        "parameters": [
+          {
+            "name": "Authorization",
+            "in": "header",
+            "schema": {
+              "type": "string"
+            }
+          }
+        ],
+        "responses": {
+          "200": {
+            "description": "OK",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "type": "object",
+                  "example": {
+                    "success": true,
+                    "message": null
+                  }
+                }
+              }
+            }
+          },
+          "403": {
+            "description": "Forbidden",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/InvalidAPIKey"
+                }
+              },
+              "application/xml": {
+                "schema": {
+                  "$ref": "#/components/schemas/InvalidAPIKey"
+                }
+              }
+            }
+          },
+          "500": {
+            "description": "Internal Server Error"
+          }
+        },
+        "requestBody": {
+          "description": "Array of objects containing source and destination paths of files to move.",
+          "required": true,
+          "type": "object",
+          "content": {
+            "application/json": {
+              "schema": {
+                "type": "object",
+                "example": {
+                  "files": [
+                    {
+                      "from": "custom-documents/file.txt-fc4beeeb-e436-454d-8bb4-e5b8979cb48f.json",
+                      "to": "folder/file.txt-fc4beeeb-e436-454d-8bb4-e5b8979cb48f.json"
+                    }
+                  ]
+                }
+              }
+            }
+          }
+        }
+      }
+    },
     "/v1/workspace/new": {
       "post": {
         "tags": [

server/utils/AiProviders/lmStudio/index.js

@@ -12,9 +12,14 @@ class LMStudioLLM {
       basePath: process.env.LMSTUDIO_BASE_PATH?.replace(/\/+$/, ""), // here is the URL to your LMStudio instance
     });
     this.lmstudio = new OpenAIApi(config);
-    // When using LMStudios inference server - the model param is not required so
+
-    // we can stub it here. LMStudio can only run one model at a time.
+    // Prior to LMStudio 0.2.17 the `model` param was not required and you could pass anything
-    this.model = "model-placeholder";
+    // into that field and it would work. On 0.2.17 LMStudio introduced multi-model chat
+    // which now has a bug that reports the server model id as "Loaded from Chat UI"
+    // and any other value will crash inferencing. So until this is patched we will
+    // try to fetch the `/models` and have the user set it, or just fallback to "Loaded from Chat UI"
+    // which will not impact users with <v0.2.17 and should work as well once the bug is fixed.
+    this.model = process.env.LMSTUDIO_MODEL_PREF || "Loaded from Chat UI";
     this.limits = {
       history: this.promptWindowLimit() * 0.15,
       system: this.promptWindowLimit() * 0.15,

server/utils/files/index.js

@@ -66,6 +66,12 @@ async function viewLocalFiles() {
     }
   }
 
+  // Make sure custom-documents is always the first folder in picker
+  directory.items = [
+    directory.items.find((folder) => folder.name === "custom-documents"),
+    ...directory.items.filter((folder) => folder.name !== "custom-documents"),
+  ].filter((i) => !!i);
+
   return directory;
 }
 

server/utils/helpers/customModels.js

@@ -10,6 +10,7 @@ const SUPPORT_CUSTOM_MODELS = [
   "mistral",
   "perplexity",
   "openrouter",
+  "lmstudio",
 ];
 
 async function getCustomModels(provider = "", apiKey = null, basePath = null) {
@@ -33,6 +34,8 @@ async function getCustomModels(provider = "", apiKey = null, basePath = null) {
       return await getPerplexityModels();
     case "openrouter":
       return await getOpenRouterModels();
+    case "lmstudio":
+      return await getLMStudioModels(basePath);
     default:
       return { models: [], error: "Invalid provider for custom models" };
   }
@@ -81,6 +84,28 @@ async function localAIModels(basePath = null, apiKey = null) {
   return { models, error: null };
 }
 
+async function getLMStudioModels(basePath = null) {
+  try {
+    const { Configuration, OpenAIApi } = require("openai");
+    const config = new Configuration({
+      basePath: basePath || process.env.LMSTUDIO_BASE_PATH,
+    });
+    const openai = new OpenAIApi(config);
+    const models = await openai
+      .listModels()
+      .then((res) => res.data.data)
+      .catch((e) => {
+        console.error(`LMStudio:listModels`, e.message);
+        return [];
+      });
+
+    return { models, error: null };
+  } catch (e) {
+    console.error(`LMStudio:getLMStudioModels`, e.message);
+    return { models: [], error: "Could not fetch LMStudio Models" };
+  }
+}
+
 async function ollamaAIModels(basePath = null) {
   let url;
   try {

server/utils/helpers/updateENV.js

@@ -59,6 +59,10 @@ const KEY_MAPPING = {
     envKey: "LMSTUDIO_BASE_PATH",
     checks: [isNotEmpty, validLLMExternalBasePath, validDockerizedUrl],
   },
+  LMStudioModelPref: {
+    envKey: "LMSTUDIO_MODEL_PREF",
+    checks: [],
+  },
   LMStudioTokenLimit: {
     envKey: "LMSTUDIO_MODEL_TOKEN_LIMIT",
     checks: [nonZero],