[FEAT] Implement new data connectors UI (#1034)

* WIP data connector redesign * new UI for data connectors complete * remove old data connector page/cleanup imports * cleanup of UI and imports --------- Co-authored-by: timothycarambat <rambat1010@gmail.com>
2024-11-10 17:00:11 +01:00 · 2024-04-05 14:25:41 -07:00 · 2024-04-05 14:25:41 -07:00 · 004b1f8db5
commit 004b1f8db5
parent 657be7ecfc
18 changed files with 531 additions and 540 deletions
--- a/frontend/src/App.jsx
+++ b/frontend/src/App.jsx
@ -39,12 +39,6 @@ const GeneralVectorDatabase = lazy(
  () => import("@/pages/GeneralSettings/VectorDatabase")
 );
 const GeneralSecurity = lazy(() => import("@/pages/GeneralSettings/Security"));
-const DataConnectors = lazy(
-  () => import("@/pages/GeneralSettings/DataConnectors")
-);
-const DataConnectorSetup = lazy(
-  () => import("@/pages/GeneralSettings/DataConnectors/Connectors")
-);
 const WorkspaceSettings = lazy(() => import("@/pages/WorkspaceSettings"));
 const EmbedConfigSetup = lazy(
  () => import("@/pages/GeneralSettings/EmbedConfigs")
@ -145,15 +139,6 @@ export default function App() {
                path="/settings/workspaces"
                element={<ManagerRoute Component={AdminWorkspaces} />}
              />
-              <Route
-                path="/settings/data-connectors"
-                element={<ManagerRoute Component={DataConnectors} />}
-              />
-              <Route
-                path="/settings/data-connectors/:connector"
-                element={<ManagerRoute Component={DataConnectorSetup} />}
-              />
-
              {/* Onboarding Flow */}
              <Route path="/onboarding" element={<OnboardingFlow />} />
              <Route path="/onboarding/:step" element={<OnboardingFlow />} />
--- a/frontend/src/components/DataConnectorOption/index.jsx
+++ b/frontend/src/components/DataConnectorOption/index.jsx
@ -1,6 +1,3 @@
-import paths from "@/utils/paths";
-import ConnectorImages from "./media";
-
 export default function DataConnectorOption({ slug }) {
  if (!DATA_CONNECTORS.hasOwnProperty(slug)) return null;
  const { path, image, name, description, link } = DATA_CONNECTORS[slug];
@ -26,22 +23,3 @@ export default function DataConnectorOption({ slug }) {
    </a>
  );
 }
-
-export const DATA_CONNECTORS = {
-  github: {
-    name: "GitHub Repo",
-    path: paths.settings.dataConnectors.github(),
-    image: ConnectorImages.github,
-    description:
-      "Import an entire public or private Github repository in a single click.",
-    link: "https://github.com",
-  },
-  "youtube-transcript": {
-    name: "YouTube Transcript",
-    path: paths.settings.dataConnectors.youtubeTranscript(),
-    image: ConnectorImages.youtube,
-    description:
-      "Import the transcription of an entire YouTube video from a link.",
-    link: "https://youtube.com",
-  },
-};
--- a/frontend/src/components/DataConnectorOption/media/github.png
+++ b/frontend/src/components/DataConnectorOption/media/github.png
--- a/frontend/src/components/DataConnectorOption/media/github.svg
+++ b/frontend/src/components/DataConnectorOption/media/github.svg
@ -0,0 +1,4 @@
+<svg width="38" height="39" viewBox="0 0 38 39" fill="none" xmlns="http://www.w3.org/2000/svg">
+<rect y="0.696777" width="37.9922" height="37.9922" rx="5.42746" fill="white"/>
+<path d="M27.9829 16.8445V17.6583C27.9812 19.0353 27.4813 20.3652 26.5756 21.4024C25.6699 22.4395 24.4194 23.114 23.0552 23.3012C23.6121 24.0137 23.9143 24.8922 23.9138 25.7965V29.8656C23.9138 30.0815 23.8281 30.2885 23.6754 30.4411C23.5228 30.5937 23.3158 30.6794 23.1 30.6794H16.5894C16.3736 30.6794 16.1666 30.5937 16.014 30.4411C15.8613 30.2885 15.7756 30.0815 15.7756 29.8656V28.238H13.3341C12.255 28.238 11.22 27.8093 10.4569 27.0462C9.69375 26.2831 9.26505 25.2481 9.26505 24.1689C9.26505 23.5214 9.00782 22.9004 8.54996 22.4425C8.0921 21.9847 7.4711 21.7274 6.82359 21.7274C6.60775 21.7274 6.40075 21.6417 6.24813 21.4891C6.09551 21.3364 6.00977 21.1294 6.00977 20.9136C6.00977 20.6978 6.09551 20.4908 6.24813 20.3381C6.40075 20.1855 6.60775 20.0998 6.82359 20.0998C7.35795 20.0998 7.88708 20.205 8.38076 20.4095C8.87445 20.614 9.32302 20.9137 9.70087 21.2916C10.0787 21.6694 10.3785 22.118 10.5829 22.6117C10.7874 23.1054 10.8927 23.6345 10.8927 24.1689C10.8927 24.8164 11.1499 25.4374 11.6078 25.8953C12.0656 26.3531 12.6866 26.6103 13.3341 26.6103H15.7756V25.7965C15.7751 24.8922 16.0773 24.0137 16.6342 23.3012C15.27 23.114 14.0196 22.4395 13.1138 21.4024C12.2081 20.3652 11.7082 19.0353 11.7065 17.6583V16.8445C11.7166 15.8331 11.986 14.8412 12.4888 13.9636C12.24 13.1612 12.1602 12.3159 12.2544 11.4811C12.3486 10.6463 12.6148 9.84005 13.0361 9.11322C13.1075 8.98948 13.2103 8.88673 13.334 8.8153C13.4578 8.74387 13.5982 8.70628 13.7411 8.70631C14.689 8.70433 15.6242 8.92407 16.472 9.34799C17.3199 9.77191 18.0568 10.3883 18.624 11.1478H21.0654C21.6326 10.3883 22.3695 9.77191 23.2174 9.34799C24.0652 8.92407 25.0005 8.70433 25.9484 8.70631C26.0912 8.70628 26.2316 8.74387 26.3554 8.8153C26.4791 8.88673 26.5819 8.98948 26.6533 9.11322C27.0747 9.84003 27.3408 10.6463 27.4348 11.4812C27.5289 12.316 27.4488 13.1613 27.1996 13.9636C27.7034 14.8409 27.9731 15.8329 27.9829 16.8445Z" fill="#222628"/>
+</svg>
--- a/frontend/src/components/DataConnectorOption/media/index.js
+++ b/frontend/src/components/DataConnectorOption/media/index.js
@ -1,5 +1,5 @@
-import Github from "./github.png";
-import YouTube from "./youtube.png";
+import Github from "./github.svg";
+import YouTube from "./youtube.svg";

 const ConnectorImages = {
  github: Github,
--- a/frontend/src/components/DataConnectorOption/media/youtube.png
+++ b/frontend/src/components/DataConnectorOption/media/youtube.png
--- a/frontend/src/components/DataConnectorOption/media/youtube.svg
+++ b/frontend/src/components/DataConnectorOption/media/youtube.svg
--- a/frontend/src/components/Modals/MangeWorkspace/DataConnectors/ConnectorOption/index.jsx
+++ b/frontend/src/components/Modals/MangeWorkspace/DataConnectors/ConnectorOption/index.jsx
@ -0,0 +1,25 @@
+export default function ConnectorOption({
+  slug,
+  selectedConnector,
+  setSelectedConnector,
+  image,
+  name,
+  description,
+}) {
+  return (
+    <button
+      onClick={() => setSelectedConnector(slug)}
+      className={`flex text-left gap-x-3.5 items-center py-2 px-4 hover:bg-white/10 ${
+        selectedConnector === slug ? "bg-white/10" : ""
+      } rounded-lg cursor-pointer w-full`}
+    >
+      <img src={image} alt={name} className="w-[40px] h-[40px] rounded-md" />
+      <div className="flex flex-col">
+        <div className="text-white font-bold text-[14px]">{name}</div>
+        <div>
+          <p className="text-[12px] text-white/60">{description}</p>
+        </div>
+      </div>
+    </button>
+  );
+}
--- a/frontend/src/components/Modals/MangeWorkspace/DataConnectors/Connectors/Github/index.jsx
+++ b/frontend/src/components/Modals/MangeWorkspace/DataConnectors/Connectors/Github/index.jsx
@ -0,0 +1,271 @@
+import React, { useEffect, useState } from "react";
+import System from "@/models/system";
+import showToast from "@/utils/toast";
+import pluralize from "pluralize";
+import { TagsInput } from "react-tag-input-component";
+import { Warning } from "@phosphor-icons/react";
+import { Tooltip } from "react-tooltip";
+
+const DEFAULT_BRANCHES = ["main", "master"];
+export default function GithubOptions() {
+  const [loading, setLoading] = useState(false);
+  const [repo, setRepo] = useState(null);
+  const [accessToken, setAccessToken] = useState(null);
+  const [ignores, setIgnores] = useState([]);
+
+  const [settings, setSettings] = useState({
+    repo: null,
+    accessToken: null,
+  });
+
+  const handleSubmit = async (e) => {
+    e.preventDefault();
+    const form = new FormData(e.target);
+
+    try {
+      setLoading(true);
+      showToast(
+        "Fetching all files for repo - this may take a while.",
+        "info",
+        { clear: true, autoClose: false }
+      );
+      const { data, error } = await System.dataConnectors.github.collect({
+        repo: form.get("repo"),
+        accessToken: form.get("accessToken"),
+        branch: form.get("branch"),
+        ignorePaths: ignores,
+      });
+
+      if (!!error) {
+        showToast(error, "error", { clear: true });
+        setLoading(false);
+        return;
+      }
+
+      showToast(
+        `${data.files} ${pluralize("file", data.files)} collected from ${
+          data.author
+        }/${data.repo}:${data.branch}. Output folder is ${data.destination}.`,
+        "success",
+        { clear: true }
+      );
+      e.target.reset();
+      setLoading(false);
+      return;
+    } catch (e) {
+      console.error(e);
+      showToast(e.message, "error", { clear: true });
+      setLoading(false);
+    }
+  };
+
+  return (
+    <div className="flex w-full">
+      <div className="flex flex-col w-full px-1 md:pb-6 pb-16">
+        <form className="w-full" onSubmit={handleSubmit}>
+          <div className="w-full flex flex-col py-2">
+            <div className="w-full flex flex-col gap-4">
+              <div className="flex flex-col pr-10">
+                <div className="flex flex-col gap-y-1 mb-4">
+                  <label className="text-white text-sm font-bold">
+                    GitHub Repo URL
+                  </label>
+                  <p className="text-xs font-normal text-white/50">
+                    Url of the GitHub repo you wish to collect.
+                  </p>
+                </div>
+                <input
+                  type="url"
+                  name="repo"
+                  className="bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:border-white block w-full p-2.5"
+                  placeholder="https://github.com/Mintplex-Labs/anything-llm"
+                  required={true}
+                  autoComplete="off"
+                  onChange={(e) => setRepo(e.target.value)}
+                  onBlur={() => setSettings({ ...settings, repo })}
+                  spellCheck={false}
+                />
+              </div>
+              <div className="flex flex-col pr-10">
+                <div className="flex flex-col gap-y-1 mb-4">
+                  <label className="text-white font-bold text-sm flex gap-x-2 items-center">
+                    <p className="font-bold text-white">Github Access Token</p>{" "}
+                    <p className="text-xs text-white/50 font-light flex items-center">
+                      optional
+                      {!accessToken && (
+                        <Warning
+                          size={14}
+                          className="ml-1 text-orange-500 cursor-pointer"
+                          data-tooltip-id="access-token-tooltip"
+                          data-tooltip-place="right"
+                        />
+                      )}
+                      <Tooltip
+                        delayHide={300}
+                        id="access-token-tooltip"
+                        className="max-w-xs"
+                        clickable={true}
+                      >
+                        <p className="text-sm">
+                          Without a{" "}
+                          <a
+                            href="https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens"
+                            rel="noreferrer"
+                            target="_blank"
+                            className="underline"
+                            onClick={(e) => e.stopPropagation()}
+                          >
+                            Personal Access Token
+                          </a>
+                          , the GitHub API may limit the number of files that
+                          can be collected due to rate limits. You can{" "}
+                          <a
+                            href="https://github.com/settings/personal-access-tokens/new"
+                            rel="noreferrer"
+                            target="_blank"
+                            className="underline"
+                            onClick={(e) => e.stopPropagation()}
+                          >
+                            create a temporary Access Token
+                          </a>{" "}
+                          to avoid this issue.
+                        </p>
+                      </Tooltip>
+                    </p>
+                  </label>
+                  <p className="text-xs font-normal text-white/50">
+                    Access Token to prevent rate limiting.
+                  </p>
+                </div>
+                <input
+                  type="text"
+                  name="accessToken"
+                  className="bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:border-white block w-full p-2.5"
+                  placeholder="github_pat_1234_abcdefg"
+                  required={false}
+                  autoComplete="off"
+                  spellCheck={false}
+                  onChange={(e) => setAccessToken(e.target.value)}
+                  onBlur={() => setSettings({ ...settings, accessToken })}
+                />
+              </div>
+              <GitHubBranchSelection
+                repo={settings.repo}
+                accessToken={settings.accessToken}
+              />
+            </div>
+
+            <div className="flex flex-col w-full py-4 pr-10">
+              <div className="flex flex-col gap-y-1 mb-4">
+                <label className="text-white text-sm flex gap-x-2 items-center">
+                  <p className="text-white text-sm font-bold">File Ignores</p>
+                </label>
+                <p className="text-xs font-normal text-white/50">
+                  List in .gitignore format to ignore specific files during
+                  collection. Press enter after each entry you want to save.
+                </p>
+              </div>
+              <TagsInput
+                value={ignores}
+                onChange={setIgnores}
+                name="ignores"
+                placeholder="!*.js, images/*, .DS_Store, bin/*"
+                classNames={{
+                  tag: "bg-blue-300/10 text-zinc-800",
+                  input:
+                    "flex bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:border-white",
+                }}
+              />
+            </div>
+          </div>
+
+          <div className="flex flex-col gap-y-2 w-full pr-10">
+            <button
+              type="submit"
+              disabled={loading}
+              className="mt-2 w-full justify-center border border-slate-200 px-4 py-2 rounded-lg text-[#222628] text-sm font-bold items-center flex gap-x-2 bg-slate-200 hover:bg-slate-300 hover:text-slate-800 disabled:bg-slate-300 disabled:cursor-not-allowed"
+            >
+              {loading ? "Collecting files..." : "Submit"}
+            </button>
+            {loading && (
+              <p className="text-xs text-white/50">
+                Once complete, all files will be available for embedding into
+                workspaces in the document picker.
+              </p>
+            )}
+          </div>
+        </form>
+      </div>
+    </div>
+  );
+}
+
+function GitHubBranchSelection({ repo, accessToken }) {
+  const [allBranches, setAllBranches] = useState(DEFAULT_BRANCHES);
+  const [loading, setLoading] = useState(true);
+
+  useEffect(() => {
+    async function fetchAllBranches() {
+      if (!repo) {
+        setAllBranches(DEFAULT_BRANCHES);
+        setLoading(false);
+        return;
+      }
+
+      setLoading(true);
+      const { branches } = await System.dataConnectors.github.branches({
+        repo,
+        accessToken,
+      });
+      setAllBranches(branches.length > 0 ? branches : DEFAULT_BRANCHES);
+      setLoading(false);
+    }
+    fetchAllBranches();
+  }, [repo, accessToken]);
+
+  if (loading) {
+    return (
+      <div className="flex flex-col w-60">
+        <div className="flex flex-col gap-y-1 mb-4">
+          <label className="text-white text-sm font-bold">Branch</label>
+          <p className="text-xs font-normal text-white/50">
+            Branch you wish to collect files from.
+          </p>
+        </div>
+        <select
+          name="branch"
+          required={true}
+          className="bg-zinc-900 border-gray-500 text-white text-sm rounded-lg block w-full p-2.5"
+        >
+          <option disabled={true} selected={true}>
+            -- loading available branches --
+          </option>
+        </select>
+      </div>
+    );
+  }
+
+  return (
+    <div className="flex flex-col w-60">
+      <div className="flex flex-col gap-y-1 mb-4">
+        <label className="text-white text-sm font-bold">Branch</label>
+        <p className="text-xs font-normal text-white/50">
+          Branch you wish to collect files from.
+        </p>
+      </div>
+      <select
+        name="branch"
+        required={true}
+        className="bg-zinc-900 border-gray-500 text-white text-sm rounded-lg block w-full p-2.5"
+      >
+        {allBranches.map((branch) => {
+          return (
+            <option key={branch} value={branch}>
+              {branch}
+            </option>
+          );
+        })}
+      </select>
+    </div>
+  );
+}
--- a/frontend/src/components/Modals/MangeWorkspace/DataConnectors/Connectors/Youtube/index.jsx
+++ b/frontend/src/components/Modals/MangeWorkspace/DataConnectors/Connectors/Youtube/index.jsx
@ -0,0 +1,91 @@
+import React, { useState } from "react";
+import System from "@/models/system";
+import showToast from "@/utils/toast";
+
+export default function YoutubeOptions() {
+  const [loading, setLoading] = useState(false);
+
+  const handleSubmit = async (e) => {
+    e.preventDefault();
+    const form = new FormData(e.target);
+
+    try {
+      setLoading(true);
+      showToast("Fetching transcript for YouTube video.", "info", {
+        clear: true,
+        autoClose: false,
+      });
+
+      const { data, error } = await System.dataConnectors.youtube.transcribe({
+        url: form.get("url"),
+      });
+
+      if (!!error) {
+        showToast(error, "error", { clear: true });
+        setLoading(false);
+        return;
+      }
+
+      showToast(
+        `${data.title} by ${data.author} transcription completed. Output folder is ${data.destination}.`,
+        "success",
+        { clear: true }
+      );
+      e.target.reset();
+      setLoading(false);
+      return;
+    } catch (e) {
+      console.error(e);
+      showToast(e.message, "error", { clear: true });
+      setLoading(false);
+    }
+  };
+
+  return (
+    <div className="flex w-full">
+      <div className="flex flex-col w-full px-1 md:pb-6 pb-16">
+        <form className="w-full" onSubmit={handleSubmit}>
+          <div className="w-full flex flex-col py-2">
+            <div className="w-full flex flex-col gap-4">
+              <div className="flex flex-col pr-10">
+                <div className="flex flex-col gap-y-1 mb-4">
+                  <label className="text-white text-sm font-bold">
+                    YouTube Video URL
+                  </label>
+                  <p className="text-xs font-normal text-white/50">
+                    URL of the YouTube video you wish to transcribe.
+                  </p>
+                </div>
+                <input
+                  type="url"
+                  name="url"
+                  className="bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:border-white block w-full p-2.5"
+                  placeholder="https://youtube.com/watch?v=abc123"
+                  required={true}
+                  autoComplete="off"
+                  spellCheck={false}
+                />
+              </div>
+            </div>
+          </div>
+
+          <div className="flex flex-col gap-y-2 w-full pr-10">
+            <button
+              type="submit"
+              disabled={loading}
+              className="mt-2 w-full justify-center border border-slate-200 px-4 py-2 rounded-lg text-[#222628] text-sm font-bold items-center flex gap-x-2 bg-slate-200 hover:bg-slate-300 hover:text-slate-800 disabled:bg-slate-300 disabled:cursor-not-allowed"
+            >
+              {loading ? "Collecting transcript..." : "Collect transcript"}
+            </button>
+            {loading && (
+              <p className="text-xs text-white/50 max-w-sm">
+                Once complete, the transcription will be available for embedding
+                into workspaces in the document picker.
+              </p>
+            )}
+          </div>
+        </form>
+      </div>
+    </div>
+  );
+}
--- a/frontend/src/components/Modals/MangeWorkspace/DataConnectors/index.jsx
+++ b/frontend/src/components/Modals/MangeWorkspace/DataConnectors/index.jsx
@ -0,0 +1,77 @@
+import ConnectorImages from "@/components/DataConnectorOption/media";
+import { MagnifyingGlass } from "@phosphor-icons/react";
+import GithubOptions from "./Connectors/Github";
+import YoutubeOptions from "./Connectors/Youtube";
+import { useState } from "react";
+import ConnectorOption from "./ConnectorOption";
+
+export const DATA_CONNECTORS = {
+  github: {
+    name: "GitHub Repo",
+    image: ConnectorImages.github,
+    description:
+      "Import an entire public or private Github repository in a single click.",
+    options: <GithubOptions />,
+  },
+  "youtube-transcript": {
+    name: "YouTube Transcript",
+    image: ConnectorImages.youtube,
+    description:
+      "Import the transcription of an entire YouTube video from a link.",
+    options: <YoutubeOptions />,
+  },
+};
+
+export default function DataConnectors() {
+  const [selectedConnector, setSelectedConnector] = useState("github");
+  const [searchQuery, setSearchQuery] = useState("");
+
+  const filteredConnectors = Object.keys(DATA_CONNECTORS).filter((slug) =>
+    DATA_CONNECTORS[slug].name.toLowerCase().includes(searchQuery.toLowerCase())
+  );
+
+  return (
+    <div className="flex upload-modal -mt-10 relative min-h-[80vh] w-[70vw]">
+      <div className="w-full p-4 top-0 z-20">
+        <div className="w-full flex items-center sticky top-0 z-50">
+          <MagnifyingGlass
+            size={16}
+            weight="bold"
+            className="absolute left-4 z-30 text-white"
+          />
+          <input
+            type="text"
+            placeholder="Search data connectors"
+            className="border-none bg-zinc-600 z-20 pl-10 h-[38px] rounded-full w-full px-4 py-1 text-sm border-2 border-slate-300/40 outline-none focus:border-white text-white"
+            autoComplete="off"
+            value={searchQuery}
+            onChange={(e) => setSearchQuery(e.target.value)}
+          />
+        </div>
+        <div className="mt-2 flex flex-col gap-y-2">
+          {filteredConnectors.length > 0 ? (
+            filteredConnectors.map((slug, index) => (
+              <ConnectorOption
+                key={index}
+                slug={slug}
+                selectedConnector={selectedConnector}
+                setSelectedConnector={setSelectedConnector}
+                image={DATA_CONNECTORS[slug].image}
+                name={DATA_CONNECTORS[slug].name}
+                description={DATA_CONNECTORS[slug].description}
+              />
+            ))
+          ) : (
+            <div className="text-white text-center mt-4">
+              No data connectors found.
+            </div>
+          )}
+        </div>
+      </div>
+      <div className="xl:block hidden absolute left-1/2 top-0 bottom-0 w-[0.5px] bg-white/20 -translate-x-1/2"></div>
+      <div className="w-full p-4 top-0 text-white min-w-[500px]">
+        {DATA_CONNECTORS[selectedConnector].options}
+      </div>
+    </div>
+  );
+}
--- a/frontend/src/components/Modals/MangeWorkspace/index.jsx
+++ b/frontend/src/components/Modals/MangeWorkspace/index.jsx
@ -6,12 +6,15 @@ import System from "../../../models/system";
 import { isMobile } from "react-device-detect";
 import useUser from "../../../hooks/useUser";
 import DocumentSettings from "./Documents";
+import DataConnectors from "./DataConnectors";

 const noop = () => {};
 const ManageWorkspace = ({ hideModal = noop, providedSlug = null }) => {
  const { slug } = useParams();
+  const { user } = useUser();
  const [workspace, setWorkspace] = useState(null);
  const [settings, setSettings] = useState({});
+  const [selectedTab, setSelectedTab] = useState("documents");

  useEffect(() => {
    async function getSettings() {
@ -67,7 +70,6 @@ const ManageWorkspace = ({ hideModal = noop, providedSlug = null }) => {
      <div className="absolute max-h-full w-fit transition duration-300 z-20 md:overflow-y-auto py-10">
        <div className="relative bg-main-gradient rounded-[12px] shadow border-2 border-slate-300/10">
          <div className="flex items-start justify-between p-2 rounded-t border-gray-500/50 relative">
-            <div />
            <button
              onClick={hideModal}
              type="button"
@ -76,7 +78,19 @@ const ManageWorkspace = ({ hideModal = noop, providedSlug = null }) => {
              <X className="text-gray-300 text-lg" />
            </button>
          </div>
-          <DocumentSettings workspace={workspace} systemSettings={settings} />
+
+          {user?.role !== "default" && (
+            <ModalTabSwitcher
+              selectedTab={selectedTab}
+              setSelectedTab={setSelectedTab}
+            />
+          )}
+
+          {selectedTab === "documents" ? (
+            <DocumentSettings workspace={workspace} systemSettings={settings} />
+          ) : (
+            <DataConnectors workspace={workspace} systemSettings={settings} />
+          )}
        </div>
      </div>
    </div>
@ -84,6 +98,35 @@ const ManageWorkspace = ({ hideModal = noop, providedSlug = null }) => {
 };

 export default memo(ManageWorkspace);
+
+const ModalTabSwitcher = ({ selectedTab, setSelectedTab }) => {
+  return (
+    <div className="w-full flex justify-center z-10 relative">
+      <div className="gap-x-2 flex justify-center -mt-[68px] mb-10 bg-sidebar-button p-1 rounded-xl shadow border-2 border-slate-300/10 w-fit">
+        <button
+          onClick={() => setSelectedTab("documents")}
+          className={`px-4 py-2 rounded-[8px] font-semibold text-white hover:bg-switch-selected hover:bg-opacity-60 ${
+            selectedTab === "documents"
+              ? "bg-switch-selected shadow-md font-bold"
+              : "bg-sidebar-button text-white/20 font-medium hover:text-white"
+          }`}
+        >
+          Documents
+        </button>
+        <button
+          onClick={() => setSelectedTab("dataConnectors")}
+          className={`px-4 py-2 rounded-[8px] font-semibold text-white hover:bg-switch-selected hover:bg-opacity-60 ${
+            selectedTab === "dataConnectors"
+              ? "bg-switch-selected shadow-md font-bold"
+              : "bg-sidebar-button text-white/20 font-medium hover:text-white"
+          }`}
+        >
+          Data Connectors
+        </button>
+      </div>
+    </div>
+  );
+};
 export function useManageWorkspaceModal() {
  const { user } = useUser();
  const [showing, setShowing] = useState(false);
--- a/frontend/src/components/SettingsSidebar/index.jsx
+++ b/frontend/src/components/SettingsSidebar/index.jsx
@ -15,7 +15,6 @@ import {
  House,
  List,
  FileCode,
-  Plugs,
  Notepad,
  CodeBlock,
  Barcode,
@ -75,11 +74,10 @@ export default function SettingsSidebar() {
          className={`z-99 fixed top-0 left-0 transition-all duration-500 w-[100vw] h-[100vh]`}
        >
          <div
-            className={`${
-              showBgOverlay
+            className={`${showBgOverlay
                ? "transition-all opacity-1"
                : "transition-none opacity-0"
-            }  duration-500 fixed top-0 left-0 ${USER_BACKGROUND_COLOR} bg-opacity-75 w-screen h-screen`}
+              }  duration-500 fixed top-0 left-0 ${USER_BACKGROUND_COLOR} bg-opacity-75 w-screen h-screen`}
            onClick={() => setShowSidebar(false)}
          />
          <div
@ -192,11 +190,10 @@ const Option = ({
          transition-all duration-[200ms]
          flex flex-grow w-[75%] gap-x-2 py-[6px] px-[12px] rounded-[4px] justify-start items-center
          hover:bg-workspace-item-selected-gradient hover:text-white hover:font-medium
-          ${
-            isActive
+          ${isActive
              ? "bg-menu-item-selected-gradient font-medium border-outline text-white"
              : "hover:bg-menu-item-selected-gradient text-zinc-200"
-          }
+            }
        `}
        >
          {React.cloneElement(icon, { weight: isActive ? "fill" : "regular" })}
@ -207,9 +204,8 @@ const Option = ({
      </div>
      {!!subOptions && (isActive || hasActiveChild) && (
        <div
-          className={`ml-4 ${
-            hasActiveChild ? "" : "border-l-2 border-slate-400"
-          } rounded-r-lg`}
+          className={`ml-4 ${hasActiveChild ? "" : "border-l-2 border-slate-400"
+            } rounded-r-lg`}
        >
          {subOptions}
        </div>
@ -304,14 +300,6 @@ const SidebarOptions = ({ user = null }) => (
      flex={true}
      allowedRole={["admin"]}
    />
-    <Option
-      href={paths.settings.dataConnectors.list()}
-      btnText="Data Connectors"
-      icon={<Plugs className="h-5 w-5 flex-shrink-0" />}
-      user={user}
-      flex={true}
-      allowedRole={["admin", "manager"]}
-    />
    <Option
      href={paths.settings.embedSetup()}
      childLinks={[paths.settings.embedChats()]}
--- a/frontend/src/pages/GeneralSettings/DataConnectors/Connectors/Github/index.jsx
+++ b/frontend/src/pages/GeneralSettings/DataConnectors/Connectors/Github/index.jsx
@ -1,293 +0,0 @@
-import React, { useEffect, useState } from "react";
-import Sidebar from "@/components/SettingsSidebar";
-import { isMobile } from "react-device-detect";
-import { DATA_CONNECTORS } from "@/components/DataConnectorOption";
-import System from "@/models/system";
-import showToast from "@/utils/toast";
-import pluralize from "pluralize";
-import { TagsInput } from "react-tag-input-component";
-import { Info } from "@phosphor-icons/react";
-
-const DEFAULT_BRANCHES = ["main", "master"];
-export default function GithubConnectorSetup() {
-  const { image } = DATA_CONNECTORS.github;
-  const [loading, setLoading] = useState(false);
-  const [repo, setRepo] = useState(null);
-  const [accessToken, setAccessToken] = useState(null);
-  const [ignores, setIgnores] = useState([]);
-
-  const [settings, setSettings] = useState({
-    repo: null,
-    accessToken: null,
-  });
-
-  const handleSubmit = async (e) => {
-    e.preventDefault();
-    const form = new FormData(e.target);
-
-    try {
-      setLoading(true);
-      showToast(
-        "Fetching all files for repo - this may take a while.",
-        "info",
-        { clear: true, autoClose: false }
-      );
-      const { data, error } = await System.dataConnectors.github.collect({
-        repo: form.get("repo"),
-        accessToken: form.get("accessToken"),
-        branch: form.get("branch"),
-        ignorePaths: ignores,
-      });
-
-      if (!!error) {
-        showToast(error, "error", { clear: true });
-        setLoading(false);
-        return;
-      }
-
-      showToast(
-        `${data.files} ${pluralize("file", data.files)} collected from ${
-          data.author
-        }/${data.repo}:${data.branch}. Output folder is ${data.destination}.`,
-        "success",
-        { clear: true }
-      );
-      e.target.reset();
-      setLoading(false);
-      return;
-    } catch (e) {
-      console.error(e);
-      showToast(e.message, "error", { clear: true });
-      setLoading(false);
-    }
-  };
-
-  return (
-    <div className="w-screen h-screen overflow-hidden bg-sidebar flex">
-      <Sidebar />
-      <div
-        style={{ height: isMobile ? "100%" : "calc(100% - 32px)" }}
-        className="relative md:ml-[2px] md:mr-[16px] md:my-[16px] md:rounded-[16px] bg-main-gradient w-full h-full overflow-y-scroll"
-      >
-        <div className="flex w-full">
-          <div className="flex flex-col w-full px-1 md:pl-6 md:pr-[86px] md:py-6 py-16">
-            <div className="flex w-full gap-x-4 items-center pb-6 border-white border-b-2 border-opacity-10">
-              <img src={image} alt="Github" className="rounded-lg h-16 w-16" />
-              <div className="w-full flex flex-col gap-y-1">
-                <div className="items-center">
-                  <p className="text-lg leading-6 font-bold text-white">
-                    Import GitHub Repository
-                  </p>
-                </div>
-                <p className="text-xs leading-[18px] font-base text-white text-opacity-60">
-                  Import all files from a public or private Github repository
-                  and have its files be available in your workspace.
-                </p>
-              </div>
-            </div>
-
-            <form className="w-full" onSubmit={handleSubmit}>
-              {!accessToken && (
-                <div className="flex flex-col gap-y-1 py-4">
-                  <div className="flex flex-col w-fit gap-y-2 bg-blue-600/20 rounded-lg px-4 py-2">
-                    <div className="flex items-center gap-x-2">
-                      <Info size={20} className="shrink-0 text-blue-400" />
-                      <p className="text-blue-400 text-sm">
-                        Trying to collect a GitHub repo without a{" "}
-                        <a
-                          href="https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens"
-                          rel="noreferrer"
-                          target="_blank"
-                          className="underline"
-                        >
-                          Personal Access Token
-                        </a>{" "}
-                        will fail to collect all files due to GitHub API limits.
-                      </p>
-                    </div>
-                    <a
-                      href="https://github.com/settings/personal-access-tokens/new"
-                      rel="noreferrer"
-                      target="_blank"
-                      className="text-blue-400 hover:underline"
-                    >
-                      Create a temporary Access Token for this data connector
-                      &rarr;
-                    </a>
-                  </div>
-                </div>
-              )}
-
-              <div className="w-full flex flex-col py-2">
-                <div className="w-full flex items-center gap-4">
-                  <div className="flex flex-col w-60">
-                    <div className="flex flex-col gap-y-1 mb-4">
-                      <label className="text-white text-sm font-semibold block">
-                        GitHub Repo URL
-                      </label>
-                      <p className="text-xs text-zinc-300">
-                        Url of the GitHub repo you wish to collect.
-                      </p>
-                    </div>
-                    <input
-                      type="url"
-                      name="repo"
-                      className="bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:border-white block w-full p-2.5"
-                      placeholder="https://github.com/Mintplex-Labs/anything-llm"
-                      required={true}
-                      autoComplete="off"
-                      onChange={(e) => setRepo(e.target.value)}
-                      onBlur={() => setSettings({ ...settings, repo })}
-                      spellCheck={false}
-                    />
-                  </div>
-                  <div className="flex flex-col w-60">
-                    <div className="flex flex-col gap-y-1 mb-4">
-                      <label className="text-white text-sm block flex gap-x-2 items-center">
-                        <p className="font-semibold ">Github Access Token</p>{" "}
-                        <p className="text-xs text-zinc-300 font-base!">
-                          <i>optional</i>
-                        </p>
-                      </label>
-                      <p className="text-xs text-zinc-300 flex gap-x-2">
-                        Access Token to prevent rate limiting.
-                      </p>
-                    </div>
-                    <input
-                      type="text"
-                      name="accessToken"
-                      className="bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:border-white block w-full p-2.5"
-                      placeholder="github_pat_1234_abcdefg"
-                      required={false}
-                      autoComplete="off"
-                      spellCheck={false}
-                      onChange={(e) => setAccessToken(e.target.value)}
-                      onBlur={() => setSettings({ ...settings, accessToken })}
-                    />
-                  </div>
-                  <GitHubBranchSelection
-                    repo={settings.repo}
-                    accessToken={settings.accessToken}
-                  />
-                </div>
-
-                <div className="flex flex-col w-1/2 py-4">
-                  <div className="flex flex-col gap-y-1 mb-4">
-                    <label className="text-white text-sm block flex gap-x-2 items-center">
-                      <p className="font-semibold ">File Ignores</p>
-                    </label>
-                    <p className="text-xs text-zinc-300 flex gap-x-2">
-                      List in .gitignore format to ignore specific files during
-                      collection. Press enter after each entry you want to save.
-                    </p>
-                  </div>
-                  <TagsInput
-                    value={ignores}
-                    onChange={setIgnores}
-                    name="ignores"
-                    placeholder="!*.js, images/*, .DS_Store, bin/*"
-                    classNames={{
-                      tag: "bg-blue-300/10 text-zinc-800 m-1",
-                      input:
-                        "flex bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:border-white p-2.5",
-                    }}
-                  />
-                </div>
-              </div>
-
-              <div className="flex flex-col gap-y-2 w-fit">
-                <button
-                  type="submit"
-                  disabled={loading}
-                  className="mt-2 text-lg w-fit border border-slate-200 px-4 py-1 rounded-lg text-slate-200 items-center flex gap-x-2 hover:bg-slate-200 hover:text-slate-800 disabled:bg-slate-200 disabled:text-slate-800"
-                >
-                  {loading
-                    ? "Collecting files..."
-                    : "Collect all files from GitHub repo"}
-                </button>
-                {loading && (
-                  <p className="text-xs text-zinc-300">
-                    Once complete, all files will be available for embedding
-                    into workspaces in the document picker.
-                  </p>
-                )}
-              </div>
-            </form>
-          </div>
-        </div>
-      </div>
-    </div>
-  );
-}
-
-function GitHubBranchSelection({ repo, accessToken }) {
-  const [allBranches, setAllBranches] = useState(DEFAULT_BRANCHES);
-  const [loading, setLoading] = useState(true);
-
-  useEffect(() => {
-    async function fetchAllBranches() {
-      if (!repo) {
-        setAllBranches(DEFAULT_BRANCHES);
-        setLoading(false);
-        return;
-      }
-
-      setLoading(true);
-      const { branches } = await System.dataConnectors.github.branches({
-        repo,
-        accessToken,
-      });
-      setAllBranches(branches.length > 0 ? branches : DEFAULT_BRANCHES);
-      setLoading(false);
-    }
-    fetchAllBranches();
-  }, [repo, accessToken]);
-
-  if (loading) {
-    return (
-      <div className="flex flex-col w-60">
-        <div className="flex flex-col gap-y-1 mb-4">
-          <label className="text-white text-sm font-semibold block">
-            Branch
-          </label>
-          <p className="text-xs text-zinc-300">
-            Branch you wish to collect files of
-          </p>
-        </div>
-        <select
-          name="branch"
-          required={true}
-          className="bg-zinc-900 border-gray-500 text-white text-sm rounded-lg block w-full p-2.5"
-        >
-          <option disabled={true} selected={true}>
-            -- loading available models --
-          </option>
-        </select>
-      </div>
-    );
-  }
-
-  return (
-    <div className="flex flex-col w-60">
-      <div className="flex flex-col gap-y-1 mb-4">
-        <label className="text-white text-sm font-semibold block">Branch</label>
-        <p className="text-xs text-zinc-300">
-          Branch you wish to collect files of
-        </p>
-      </div>
-      <select
-        name="branch"
-        required={true}
-        className="bg-zinc-900 border-gray-500 text-white text-sm rounded-lg block w-full p-2.5"
-      >
-        {allBranches.map((branch) => {
-          return (
-            <option key={branch} value={branch}>
-              {branch}
-            </option>
-          );
-        })}
-      </select>
-    </div>
-  );
-}
--- a/frontend/src/pages/GeneralSettings/DataConnectors/Connectors/Youtube/index.jsx
+++ b/frontend/src/pages/GeneralSettings/DataConnectors/Connectors/Youtube/index.jsx
@ -1,113 +0,0 @@
-import React, { useState } from "react";
-import Sidebar from "@/components/SettingsSidebar";
-import { isMobile } from "react-device-detect";
-import { DATA_CONNECTORS } from "@/components/DataConnectorOption";
-import System from "@/models/system";
-import showToast from "@/utils/toast";
-
-export default function YouTubeTranscriptConnectorSetup() {
-  const { image } = DATA_CONNECTORS["youtube-transcript"];
-  const [loading, setLoading] = useState(false);
-  const handleSubmit = async (e) => {
-    e.preventDefault();
-    const form = new FormData(e.target);
-
-    try {
-      setLoading(true);
-      showToast("Fetching transcript for YouTube video.", "info", {
-        clear: true,
-        autoClose: false,
-      });
-      const { data, error } = await System.dataConnectors.youtube.transcribe({
-        url: form.get("url"),
-      });
-
-      if (!!error) {
-        showToast(error, "error", { clear: true });
-        setLoading(false);
-        return;
-      }
-
-      showToast(
-        `${data.title} by ${data.author} transcription completed. Output folder is ${data.destination}.`,
-        "success",
-        { clear: true }
-      );
-      e.target.reset();
-      setLoading(false);
-      return;
-    } catch (e) {
-      console.error(e);
-      showToast(e.message, "error", { clear: true });
-      setLoading(false);
-    }
-  };
-
-  return (
-    <div className="w-screen h-screen overflow-hidden bg-sidebar flex">
-      <Sidebar />
-      <div
-        style={{ height: isMobile ? "100%" : "calc(100% - 32px)" }}
-        className="relative md:ml-[2px] md:mr-[16px] md:my-[16px] md:rounded-[16px] bg-main-gradient w-full h-full overflow-y-scroll"
-      >
-        <div className="flex w-full">
-          <div className="flex flex-col w-full px-1 md:pl-6 md:pr-[86px] md:py-6 py-16">
-            <div className="flex w-full gap-x-4 items-center pb-6 border-white border-b-2 border-opacity-10">
-              <img src={image} alt="YouTube" className="rounded-lg h-16 w-16" />
-              <div className="w-full flex flex-col gap-y-1">
-                <div className="items-center">
-                  <p className="text-lg leading-6 font-bold text-white">
-                    Import YouTube transcription
-                  </p>
-                </div>
-                <p className="text-xs leading-[18px] font-base text-white text-opacity-60">
-                  From a youtube link, import the entire transcript of that
-                  video for embedding.
-                </p>
-              </div>
-            </div>
-
-            <form className="w-full" onSubmit={handleSubmit}>
-              <div className="w-full flex flex-col py-2">
-                <div className="w-full flex items-center gap-4">
-                  <div className="flex flex-col w-60">
-                    <div className="flex flex-col gap-y-1 mb-4">
-                      <label className="text-white text-sm font-semibold block">
-                        YouTube video URL
-                      </label>
-                    </div>
-                    <input
-                      type="url"
-                      name="url"
-                      className="bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:border-white block w-full p-2.5"
-                      placeholder="https://youtube.com/watch?v=abc123"
-                      required={true}
-                      autoComplete="off"
-                      spellCheck={false}
-                    />
-                  </div>
-                </div>
-              </div>
-
-              <div className="flex flex-col gap-y-2 w-fit">
-                <button
-                  type="submit"
-                  disabled={loading}
-                  className="mt-2 text-lg w-fit border border-slate-200 px-4 py-1 rounded-lg text-slate-200 items-center flex gap-x-2 hover:bg-slate-200 hover:text-slate-800 disabled:bg-slate-200 disabled:text-slate-800"
-                >
-                  {loading ? "Collecting transcript..." : "Collect transcript"}
-                </button>
-                {loading && (
-                  <p className="text-xs text-zinc-300">
-                    Once complete, the transcription will be available for
-                    embedding into workspaces in the document picker.
-                  </p>
-                )}
-              </div>
-            </form>
-          </div>
-        </div>
-      </div>
-    </div>
-  );
-}
--- a/frontend/src/pages/GeneralSettings/DataConnectors/Connectors/index.jsx
+++ b/frontend/src/pages/GeneralSettings/DataConnectors/Connectors/index.jsx
@ -1,21 +0,0 @@
-import paths from "@/utils/paths";
-import { lazy } from "react";
-import { useParams } from "react-router-dom";
-const Github = lazy(() => import("./Github"));
-const YouTubeTranscript = lazy(() => import("./Youtube"));
-
-const CONNECTORS = {
-  github: Github,
-  "youtube-transcript": YouTubeTranscript,
-};
-
-export default function DataConnectorSetup() {
-  const { connector } = useParams();
-  if (!connector || !CONNECTORS.hasOwnProperty(connector)) {
-    window.location = paths.home();
-    return;
-  }
-
-  const Page = CONNECTORS[connector];
-  return <Page />;
-}
--- a/frontend/src/pages/GeneralSettings/DataConnectors/index.jsx
+++ b/frontend/src/pages/GeneralSettings/DataConnectors/index.jsx
@ -1,43 +0,0 @@
-import React from "react";
-import Sidebar from "@/components/SettingsSidebar";
-import { isMobile } from "react-device-detect";
-import DataConnectorOption from "@/components/DataConnectorOption";
-
-export default function DataConnectors() {
-  return (
-    <div className="w-screen h-screen overflow-hidden bg-sidebar flex">
-      <Sidebar />
-      <div
-        style={{ height: isMobile ? "100%" : "calc(100% - 32px)" }}
-        className="relative md:ml-[2px] md:mr-[16px] md:my-[16px] md:rounded-[16px] bg-main-gradient w-full h-full overflow-y-scroll"
-      >
-        <div className="flex w-full">
-          <div className="flex flex-col w-full px-1 md:pl-6 md:pr-[86px] md:py-6 py-16">
-            <div className="w-full flex flex-col gap-y-1 pb-6 border-white border-b-2 border-opacity-10">
-              <div className="items-center">
-                <p className="text-lg leading-6 font-bold text-white">
-                  Data Connectors
-                </p>
-              </div>
-              <p className="text-xs leading-[18px] font-base text-white text-opacity-60">
-                Verified data connectors allow you to add more content to your
-                AnythingLLM workspaces with no custom code or complexity.
-                <br />
-                Guaranteed to work with your AnythingLLM instance.
-              </p>
-            </div>
-            <div className="text-sm font-medium text-white mt-6 mb-4">
-              Available Data Connectors
-            </div>
-            <div className="w-full">
-              <div className="py-4 w-full flex md:flex-wrap overflow-x-scroll gap-4 max-w-full">
-                <DataConnectorOption slug="github" />
-                <DataConnectorOption slug="youtube-transcript" />
-              </div>
-            </div>
-          </div>
-        </div>
-      </div>
-    </div>
-  );
-}
--- a/frontend/src/utils/paths.js
+++ b/frontend/src/utils/paths.js
@ -125,16 +125,5 @@ export default {
    embedChats: () => {
      return `/settings/embed-chats`;
    },
-    dataConnectors: {
-      list: () => {
-        return "/settings/data-connectors";
-      },
-      github: () => {
-        return "/settings/data-connectors/github";
-      },
-      youtubeTranscript: () => {
-        return "/settings/data-connectors/youtube-transcript";
-      },
-    },
  },
 };