Merge branch 'master' into 1606-feat-rename-new-thread-when-thread-is-created

frontend/package.json

@@ -19,6 +19,7 @@
     "file-saver": "^2.0.5",
     "he": "^1.2.0",
     "highlight.js": "^11.9.0",
+    "js-levenshtein": "^1.1.6",
     "lodash.debounce": "^4.0.8",
     "markdown-it": "^13.0.1",
     "pluralize": "^8.0.0",

frontend/src/components/Modals/ManageWorkspace/Documents/Directory/NewFolderModal/index.jsx

@@ -0,0 +1,90 @@
+import React, { useState } from "react";
+import { X } from "@phosphor-icons/react";
+import Document from "@/models/document";
+
+export default function NewFolderModal({ closeModal, files, setFiles }) {
+  const [error, setError] = useState(null);
+  const [folderName, setFolderName] = useState("");
+
+  const handleCreate = async (e) => {
+    e.preventDefault();
+    setError(null);
+    if (folderName.trim() !== "") {
+      const newFolder = {
+        name: folderName,
+        type: "folder",
+        items: [],
+      };
+      const { success } = await Document.createFolder(folderName);
+      if (success) {
+        setFiles({
+          ...files,
+          items: [...files.items, newFolder],
+        });
+        closeModal();
+      } else {
+        setError("Failed to create folder");
+      }
+    }
+  };
+
+  return (
+    <div className="relative w-full max-w-xl max-h-full">
+      <div className="relative bg-main-gradient rounded-lg shadow">
+        <div className="flex items-start justify-between p-4 border-b rounded-t border-gray-500/50">
+          <h3 className="text-xl font-semibold text-white">
+            Create New Folder
+          </h3>
+          <button
+            onClick={closeModal}
+            type="button"
+            className="transition-all duration-300 text-gray-400 bg-transparent hover:border-white/60 rounded-lg text-sm p-1.5 ml-auto inline-flex items-center bg-sidebar-button hover:bg-menu-item-selected-gradient hover:border-slate-100 hover:border-opacity-50 border-transparent border"
+            data-modal-hide="staticModal"
+          >
+            <X className="text-gray-300 text-lg" />
+          </button>
+        </div>
+        <form onSubmit={handleCreate}>
+          <div className="p-6 space-y-6 flex h-full w-full">
+            <div className="w-full flex flex-col gap-y-4">
+              <div>
+                <label
+                  htmlFor="folderName"
+                  className="block mb-2 text-sm font-medium text-white"
+                >
+                  Folder Name
+                </label>
+                <input
+                  name="folderName"
+                  type="text"
+                  className="bg-zinc-900 placeholder:text-white/20 border-gray-500 text-white text-sm rounded-lg focus:ring-blue-500 focus:border-blue-500 block w-full p-2.5"
+                  placeholder="Enter folder name"
+                  required={true}
+                  autoComplete="off"
+                  value={folderName}
+                  onChange={(e) => setFolderName(e.target.value)}
+                />
+              </div>
+              {error && <p className="text-red-400 text-sm">Error: {error}</p>}
+            </div>
+          </div>
+          <div className="flex w-full justify-between items-center p-6 space-x-2 border-t rounded-b border-gray-500/50">
+            <button
+              onClick={closeModal}
+              type="button"
+              className="px-4 py-2 rounded-lg text-white hover:bg-stone-900 transition-all duration-300"
+            >
+              Cancel
+            </button>
+            <button
+              type="submit"
+              className="transition-all duration-300 border border-slate-200 px-4 py-2 rounded-lg text-white text-sm items-center flex gap-x-2 hover:bg-slate-200 hover:text-slate-800 focus:ring-gray-800"
+            >
+              Create Folder
+            </button>
+          </div>
+        </form>
+      </div>
+    </div>
+  );
+}

frontend/src/components/Modals/ManageWorkspace/Documents/Directory/index.jsx

@@ -3,11 +3,16 @@ import PreLoader from "@/components/Preloader";
 import { memo, useEffect, useState } from "react";
 import FolderRow from "./FolderRow";
 import System from "@/models/system";
-import { Plus, Trash } from "@phosphor-icons/react";
+import { MagnifyingGlass, Plus, Trash } from "@phosphor-icons/react";
 import Document from "@/models/document";
 import showToast from "@/utils/toast";
 import FolderSelectionPopup from "./FolderSelectionPopup";
 import MoveToFolderIcon from "./MoveToFolderIcon";
+import { useModal } from "@/hooks/useModal";
+import ModalWrapper from "@/components/ModalWrapper";
+import NewFolderModal from "./NewFolderModal";
+import debounce from "lodash.debounce";
+import { filterFileSearchResults } from "./utils";
 
 function Directory({
   files,
@@ -24,9 +29,13 @@ function Directory({
   loadingMessage,
 }) {
   const [amountSelected, setAmountSelected] = useState(0);
-  const [newFolderName, setNewFolderName] = useState("");
-  const [showNewFolderInput, setShowNewFolderInput] = useState(false);
   const [showFolderSelection, setShowFolderSelection] = useState(false);
+  const [searchTerm, setSearchTerm] = useState("");
+  const {
+    isOpen: isFolderModalOpen,
+    openModal: openFolderModal,
+    closeModal: closeFolderModal,
+  } = useModal();
 
   useEffect(() => {
     setAmountSelected(Object.keys(selectedItems).length);
@@ -121,32 +130,6 @@ function Directory({
     return !!selectedItems[id];
   };
 
-  const createNewFolder = () => {
-    setShowNewFolderInput(true);
-  };
-
-  const confirmNewFolder = async () => {
-    if (newFolderName.trim() !== "") {
-      const newFolder = {
-        name: newFolderName,
-        type: "folder",
-        items: [],
-      };
-
-      // If folder failed to create - silently fail.
-      const { success } = await Document.createFolder(newFolderName);
-      if (success) {
-        setFiles({
-          ...files,
-          items: [...files.items, newFolder],
-        });
-      }
-
-      setNewFolderName("");
-      setShowNewFolderInput(false);
-    }
-  };
-
   const moveToFolder = async (folder) => {
     const toMove = [];
     for (const itemId of Object.keys(selectedItems)) {
@@ -183,40 +166,39 @@ function Directory({
     setLoading(false);
   };
 
+  const handleSearch = debounce((e) => {
+    const searchValue = e.target.value;
+    setSearchTerm(searchValue);
+  }, 500);
+
+  const filteredFiles = filterFileSearchResults(files, searchTerm);
   return (
     <div className="px-8 pb-8">
       <div className="flex flex-col gap-y-6">
         <div className="flex items-center justify-between w-[560px] px-5 relative">
           <h3 className="text-white text-base font-bold">My Documents</h3>
-          {showNewFolderInput ? (
+          <div className="relative">
-            <div className="flex items-center gap-x-2 z-50">
             <input
-                type="text"
+              type="search"
-                placeholder="Folder name"
+              placeholder="Search for document"
-                value={newFolderName}
+              onChange={handleSearch}
-                onChange={(e) => setNewFolderName(e.target.value)}
+              className="search-input bg-zinc-900 text-white placeholder-white/40 text-sm rounded-lg pl-9 pr-2.5 py-2 w-[250px] h-[32px]"
-                className="bg-zinc-900 text-white placeholder-white/20 text-sm rounded-md p-2.5 w-[150px] h-[32px]"
+            />
+            <MagnifyingGlass
+              size={14}
+              className="absolute left-3 top-1/2 transform -translate-y-1/2 text-white"
+              weight="bold"
             />
-              <div className="flex gap-x-2">
-                <button
-                  onClick={confirmNewFolder}
-                  className="text-sky-400 rounded-md text-sm font-bold hover:text-sky-500"
-                >
-                  Create
-                </button>
           </div>
-            </div>
-          ) : (
           <button
             className="flex items-center gap-x-2 cursor-pointer px-[14px] py-[7px] -mr-[14px] rounded-lg hover:bg-[#222628]/60"
-              onClick={createNewFolder}
+            onClick={openFolderModal}
           >
             <Plus size={18} weight="bold" color="#D3D4D4" />
             <div className="text-[#D3D4D4] text-xs font-bold leading-[18px]">
               New Folder
             </div>
           </button>
-          )}
         </div>
 
         <div className="relative w-[560px] h-[310px] bg-zinc-900 rounded-2xl overflow-hidden">
@@ -234,8 +216,8 @@ function Directory({
                   {loadingMessage}
                 </p>
               </div>
-            ) : files.items ? (
+            ) : filteredFiles.length > 0 ? (
-              files.items.map(
+              filteredFiles.map(
                 (item, index) =>
                   item.type === "folder" && (
                     <FolderRow
@@ -302,6 +284,7 @@ function Directory({
             </div>
           )}
         </div>
+
         <UploadFile
           workspace={workspace}
           fetchKeys={fetchKeys}
@@ -309,6 +292,14 @@ function Directory({
           setLoadingMessage={setLoadingMessage}
         />
       </div>
+
+      <ModalWrapper isOpen={isFolderModalOpen}>
+        <NewFolderModal
+          closeModal={closeFolderModal}
+          files={files}
+          setFiles={setFiles}
+        />
+      </ModalWrapper>
     </div>
   );
 }

frontend/src/components/Modals/ManageWorkspace/Documents/Directory/utils.js

@@ -0,0 +1,49 @@
+import strDistance from "js-levenshtein";
+
+const LEVENSHTEIN_MIN = 8;
+
+// Regular expression pattern to match the v4 UUID and the ending .json
+const uuidPattern =
+  /-[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}/;
+const jsonPattern = /\.json$/;
+
+// Function to strip UUID v4 and JSON from file names as that will impact search results.
+const stripUuidAndJsonFromString = (input = "") => {
+  return input
+    ?.replace(uuidPattern, "") // remove v4 uuid
+    ?.replace(jsonPattern, "") // remove trailing .json
+    ?.replace("-", " "); // turn slugged names into spaces
+};
+
+export function filterFileSearchResults(files = [], searchTerm = "") {
+  if (!searchTerm) return files?.items || [];
+
+  const searchResult = [];
+  for (const folder of files?.items) {
+    // If folder is a good match then add all its children
+    if (strDistance(folder.name, searchTerm) <= LEVENSHTEIN_MIN) {
+      searchResult.push(folder);
+      continue;
+    }
+
+    // Otherwise check children for good results
+    const fileSearchResults = [];
+    for (const file of folder?.items) {
+      if (
+        strDistance(stripUuidAndJsonFromString(file.name), searchTerm) <=
+        LEVENSHTEIN_MIN
+      ) {
+        fileSearchResults.push(file);
+      }
+    }
+
+    if (fileSearchResults.length > 0) {
+      searchResult.push({
+        ...folder,
+        items: fileSearchResults,
+      });
+    }
+  }
+
+  return searchResult;
+}

frontend/src/index.css

@@ -742,3 +742,7 @@ does not extend the close button beyond the viewport. */
     opacity: 0;
   }
 }
+
+.search-input::-webkit-search-cancel-button {
+  filter: grayscale(100%) invert(1) brightness(100) opacity(0.5);
+}

frontend/src/pages/WorkspaceSettings/AgentConfig/SQLConnectorSelection/DBConnection.jsx

@@ -9,7 +9,7 @@ export const DB_LOGOS = {
   "sql-server": MSSQLLogo,
 };
 
-export default function DBConnection({ connection, onRemove }) {
+export default function DBConnection({ connection, onRemove, setHasChanges }) {
   const { database_id, engine } = connection;
   function removeConfirmation() {
     if (
@@ -20,6 +20,7 @@ export default function DBConnection({ connection, onRemove }) {
       return false;
     }
     onRemove(database_id);
+    setHasChanges(true);
   }
 
   return (

frontend/src/pages/WorkspaceSettings/AgentConfig/SQLConnectorSelection/index.jsx

@@ -9,6 +9,7 @@ export default function AgentSQLConnectorSelection({
   settings,
   toggleSkill,
   enabled = false,
+  setHasChanges,
 }) {
   const { isOpen, openModal, closeModal } = useModal();
   const [connections, setConnections] = useState(
@@ -72,6 +73,7 @@ export default function AgentSQLConnectorSelection({
                           })
                         );
                       }}
+                      setHasChanges={setHasChanges}
                     />
                   ))}
                 <button

frontend/src/pages/WorkspaceSettings/AgentConfig/index.jsx

@@ -100,6 +100,7 @@ export default function WorkspaceAgentConfiguration({ workspace }) {
           skills={agentSkills}
           toggleAgentSkill={toggleAgentSkill}
           settings={settings}
+          setHasChanges={setHasChanges}
         />
         {hasChanges && (
           <button
@@ -143,7 +144,12 @@ function LoadingSkeleton() {
   );
 }
 
-function AvailableAgentSkills({ skills, settings, toggleAgentSkill }) {
+function AvailableAgentSkills({
+  skills,
+  settings,
+  toggleAgentSkill,
+  setHasChanges,
+}) {
   return (
     <div>
       <div className="flex flex-col mb-8">
@@ -211,6 +217,7 @@ function AvailableAgentSkills({ skills, settings, toggleAgentSkill }) {
           settings={settings}
           toggleSkill={toggleAgentSkill}
           enabled={skills.includes("sql-agent")}
+          setHasChanges={setHasChanges}
         />
       </div>
     </div>

frontend/yarn.lock

@@ -2260,6 +2260,11 @@ jiti@^1.19.1:
   resolved "https://registry.yarnpkg.com/jiti/-/jiti-1.21.0.tgz#7c97f8fe045724e136a397f7340475244156105d"
   integrity sha512-gFqAIbuKyyso/3G2qhiO2OM6shY6EPP/R0+mkDbyspxKazh8BXDC5FiFsUjlczgdNz/vfra0da2y+aHrusLG/Q==
 
+js-levenshtein@^1.1.6:
+  version "1.1.6"
+  resolved "https://registry.yarnpkg.com/js-levenshtein/-/js-levenshtein-1.1.6.tgz#c6cee58eb3550372df8deb85fad5ce66ce01d59d"
+  integrity sha512-X2BB11YZtrRqY4EnQcLX5Rh373zbK4alC1FW7D7MBhL2gtcC17cTnr6DmfHZeS0s2rTHjUTMMHfG7gO8SSdw+g==
+
 "js-tokens@^3.0.0 || ^4.0.0", js-tokens@^4.0.0:
   version "4.0.0"
   resolved "https://registry.yarnpkg.com/js-tokens/-/js-tokens-4.0.0.tgz#19203fb59991df98e3a287050d4647cdeaf32499"

server/endpoints/api/system/index.js

@@ -206,6 +206,72 @@ function apiSystemEndpoints(app) {
       }
     }
   );
+  app.delete(
+    "/v1/system/remove-documents",
+    [validApiKey],
+    async (request, response) => {
+      /*
+      #swagger.tags = ['System Settings']
+      #swagger.description = 'Permanently remove documents from the system.'
+      #swagger.requestBody = {
+        description: 'Array of document names to be removed permanently.',
+        required: true,
+        content: {
+          "application/json": {
+            schema: {
+              type: 'object',
+              properties: {
+                names: {
+                  type: 'array',
+                  items: {
+                    type: 'string'
+                  },
+                  example: [
+                    "custom-documents/file.txt-fc4beeeb-e436-454d-8bb4-e5b8979cb48f.json"
+                  ]
+                }
+              }
+            }
+          }
+        }
+      }
+      #swagger.responses[200] = {
+        description: 'Documents removed successfully.',
+        content: {
+          "application/json": {
+            schema: {
+              type: 'object',
+              example: {
+                success: true,
+                message: 'Documents removed successfully'
+              }
+            }
+          }
+        }
+      }
+      #swagger.responses[403] = {
+        description: 'Forbidden',
+        schema: {
+          "$ref": "#/definitions/InvalidAPIKey"
+        }
+      }
+      #swagger.responses[500] = {
+        description: 'Internal Server Error'
+      }
+      */
+      try {
+        const { names } = reqBody(request);
+        for await (const name of names) await purgeDocument(name);
+        response
+          .status(200)
+          .json({ success: true, message: "Documents removed successfully" })
+          .end();
+      } catch (e) {
+        console.log(e.message, e);
+        response.sendStatus(500).end();
+      }
+    }
+  );
 }
 
 module.exports = { apiSystemEndpoints };

server/swagger/openapi.json

@@ -2241,6 +2241,71 @@
           }
         }
       }
+    },
+    "/v1/system/remove-documents": {
+      "delete": {
+        "tags": [
+          "System Settings"
+        ],
+        "description": "Permanently remove documents from the system.",
+        "parameters": [],
+        "responses": {
+          "200": {
+            "description": "Documents removed successfully.",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "type": "object",
+                  "example": {
+                    "success": true,
+                    "message": "Documents removed successfully"
+                  }
+                }
+              }
+            }
+          },
+          "403": {
+            "description": "Forbidden",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/InvalidAPIKey"
+                }
+              },
+              "application/xml": {
+                "schema": {
+                  "$ref": "#/components/schemas/InvalidAPIKey"
+                }
+              }
+            }
+          },
+          "500": {
+            "description": "Internal Server Error"
+          }
+        },
+        "requestBody": {
+          "description": "Array of document names to be removed permanently.",
+          "required": true,
+          "content": {
+            "application/json": {
+              "schema": {
+                "type": "object",
+                "properties": {
+                  "names": {
+                    "type": "array",
+                    "items": {
+                      "type": "string"
+                    },
+                    "example": [
+                      "custom-documents/file.txt-fc4beeeb-e436-454d-8bb4-e5b8979cb48f.json"
+                    ]
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
     }
   },
   "components": {