Refactor file search method and implementation

frontend/package.json

@@ -19,6 +19,7 @@
     "file-saver": "^2.0.5",
     "he": "^1.2.0",
     "highlight.js": "^11.9.0",
+    "js-levenshtein": "^1.1.6",
     "lodash.debounce": "^4.0.8",
     "markdown-it": "^13.0.1",
     "pluralize": "^8.0.0",
@@ -63,4 +64,4 @@
     "tailwindcss": "^3.3.1",
     "vite": "^4.3.0"
   }
-}
+}

frontend/src/components/Modals/ManageWorkspace/Documents/Directory/index.jsx

@@ -12,6 +12,7 @@ import { useModal } from "@/hooks/useModal";
 import ModalWrapper from "@/components/ModalWrapper";
 import NewFolderModal from "./NewFolderModal";
 import debounce from "lodash.debounce";
+import { filterFileSearchResults } from "./utils";
 
 function Directory({
   files,
@@ -30,7 +31,6 @@ function Directory({
   const [amountSelected, setAmountSelected] = useState(0);
   const [showFolderSelection, setShowFolderSelection] = useState(false);
   const [searchTerm, setSearchTerm] = useState("");
-  const [isSearching, setIsSearching] = useState(false);
   const {
     isOpen: isFolderModalOpen,
     openModal: openFolderModal,
@@ -166,34 +166,12 @@ function Directory({
     setLoading(false);
   };
 
-  const filterFiles = (item) => {
-    if (item.type === "folder") {
-      return (
-        item.name.toLowerCase().includes(searchTerm.toLowerCase()) ||
-        item.items.some((file) =>
-          file.name.toLowerCase().includes(searchTerm.toLowerCase())
-        )
-      );
-    } else {
-      return item.name.toLowerCase().includes(searchTerm.toLowerCase());
-    }
-  };
-
-  const debouncedSearch = debounce((value) => {
-    setSearchTerm(value);
-    setIsSearching(false);
-  }, 500);
-
-  const handleSearch = (e) => {
+  const handleSearch = debounce((e) => {
     const searchValue = e.target.value;
     setSearchTerm(searchValue);
-    setIsSearching(true);
-    debouncedSearch(searchValue);
-  };
-
-  const filteredFiles =
-    files && files.items ? files.items.filter((item) => filterFiles(item)) : [];
+  }, 500);
 
+  const filteredFiles = filterFileSearchResults(files, searchTerm);
   return (
     <div className="px-8 pb-8">
       <div className="flex flex-col gap-y-6">
@@ -201,10 +179,10 @@ function Directory({
           <h3 className="text-white text-base font-bold">My Documents</h3>
           <div className="relative">
             <input
-              type="text"
-              placeholder="Search"
+              type="search"
+              placeholder="Search for document"
               onChange={handleSearch}
-              className="bg-zinc-900 text-white placeholder-white/80 text-sm rounded-lg pl-9 pr-2.5 py-2 w-[250px] h-[32px]"
+              className="search-input bg-zinc-900 text-white placeholder-white/40 text-sm rounded-lg pl-9 pr-2.5 py-2 w-[250px] h-[32px]"
             />
             <MagnifyingGlass
               size={14}
@@ -238,10 +216,6 @@ function Directory({
                   {loadingMessage}
                 </p>
               </div>
-            ) : isSearching && searchTerm !== "" ? (
-              <div className="w-full h-full flex items-center justify-center">
-                <PreLoader />
-              </div>
             ) : filteredFiles.length > 0 ? (
               filteredFiles.map(
                 (item, index) =>

frontend/src/components/Modals/ManageWorkspace/Documents/Directory/utils.js

@@ -0,0 +1,49 @@
+import strDistance from "js-levenshtein";
+
+const LEVENSHTEIN_MIN = 8;
+
+// Regular expression pattern to match the v4 UUID and the ending .json
+const uuidPattern =
+  /-[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}/;
+const jsonPattern = /\.json$/;
+
+// Function to strip UUID v4 and JSON from file names as that will impact search results.
+const stripUuidAndJsonFromString = (input = "") => {
+  return input
+    ?.replace(uuidPattern, "") // remove v4 uuid
+    ?.replace(jsonPattern, "") // remove trailing .json
+    ?.replace("-", " "); // turn slugged names into spaces
+};
+
+export function filterFileSearchResults(files = [], searchTerm = "") {
+  if (!searchTerm) return files?.items || [];
+
+  const searchResult = [];
+  for (const folder of files?.items) {
+    // If folder is a good match then add all its children
+    if (strDistance(folder.name, searchTerm) <= LEVENSHTEIN_MIN) {
+      searchResult.push(folder);
+      continue;
+    }
+
+    // Otherwise check children for good results
+    const fileSearchResults = [];
+    for (const file of folder?.items) {
+      if (
+        strDistance(stripUuidAndJsonFromString(file.name), searchTerm) <=
+        LEVENSHTEIN_MIN
+      ) {
+        fileSearchResults.push(file);
+      }
+    }
+
+    if (fileSearchResults.length > 0) {
+      searchResult.push({
+        ...folder,
+        items: fileSearchResults,
+      });
+    }
+  }
+
+  return searchResult;
+}

frontend/src/components/Modals/ManageWorkspace/index.jsx

@@ -8,7 +8,7 @@ import useUser from "../../../hooks/useUser";
 import DocumentSettings from "./Documents";
 import DataConnectors from "./DataConnectors";
 
-const noop = () => { };
+const noop = () => {};
 const ManageWorkspace = ({ hideModal = noop, providedSlug = null }) => {
   const { slug } = useParams();
   const { user } = useUser();
@@ -105,19 +105,21 @@ const ModalTabSwitcher = ({ selectedTab, setSelectedTab }) => {
       <div className="gap-x-2 flex justify-center -mt-[68px] mb-10 bg-sidebar-button p-1 rounded-xl shadow border-2 border-slate-300/10 w-fit">
         <button
           onClick={() => setSelectedTab("documents")}
-          className={`px-4 py-2 rounded-[8px] font-semibold text-white hover:bg-switch-selected hover:bg-opacity-60 ${selectedTab === "documents"
+          className={`px-4 py-2 rounded-[8px] font-semibold text-white hover:bg-switch-selected hover:bg-opacity-60 ${
+            selectedTab === "documents"
               ? "bg-switch-selected shadow-md font-bold"
               : "bg-sidebar-button text-white/20 font-medium hover:text-white"
-            }`}
+          }`}
         >
           Documents
         </button>
         <button
           onClick={() => setSelectedTab("dataConnectors")}
-          className={`px-4 py-2 rounded-[8px] font-semibold text-white hover:bg-switch-selected hover:bg-opacity-60 ${selectedTab === "dataConnectors"
+          className={`px-4 py-2 rounded-[8px] font-semibold text-white hover:bg-switch-selected hover:bg-opacity-60 ${
+            selectedTab === "dataConnectors"
               ? "bg-switch-selected shadow-md font-bold"
               : "bg-sidebar-button text-white/20 font-medium hover:text-white"
-            }`}
+          }`}
         >
           Data Connectors
         </button>

frontend/src/index.css

@@ -742,3 +742,7 @@ does not extend the close button beyond the viewport. */
     opacity: 0;
   }
 }
+
+.search-input::-webkit-search-cancel-button {
+  filter: grayscale(100%) invert(1) brightness(100) opacity(0.5);
+}

frontend/src/pages/WorkspaceSettings/AgentConfig/index.jsx

@@ -144,7 +144,12 @@ function LoadingSkeleton() {
   );
 }
 
-function AvailableAgentSkills({ skills, settings, toggleAgentSkill, setHasChanges }) {
+function AvailableAgentSkills({
+  skills,
+  settings,
+  toggleAgentSkill,
+  setHasChanges,
+}) {
   return (
     <div>
       <div className="flex flex-col mb-8">

frontend/yarn.lock

@@ -2260,6 +2260,11 @@ jiti@^1.19.1:
   resolved "https://registry.yarnpkg.com/jiti/-/jiti-1.21.0.tgz#7c97f8fe045724e136a397f7340475244156105d"
   integrity sha512-gFqAIbuKyyso/3G2qhiO2OM6shY6EPP/R0+mkDbyspxKazh8BXDC5FiFsUjlczgdNz/vfra0da2y+aHrusLG/Q==
 
+js-levenshtein@^1.1.6:
+  version "1.1.6"
+  resolved "https://registry.yarnpkg.com/js-levenshtein/-/js-levenshtein-1.1.6.tgz#c6cee58eb3550372df8deb85fad5ce66ce01d59d"
+  integrity sha512-X2BB11YZtrRqY4EnQcLX5Rh373zbK4alC1FW7D7MBhL2gtcC17cTnr6DmfHZeS0s2rTHjUTMMHfG7gO8SSdw+g==
+
 "js-tokens@^3.0.0 || ^4.0.0", js-tokens@^4.0.0:
   version "4.0.0"
   resolved "https://registry.yarnpkg.com/js-tokens/-/js-tokens-4.0.0.tgz#19203fb59991df98e3a287050d4647cdeaf32499"