[FEAT] Remove vectordb and embedder settings in onboarding flow (#1138)

* default to AnythingLLM embedder and LanceDB on onboarding for simpler setup

* remove unused onboarding flow components

* add text telling user settings can be reconfigured later

* remove space

* update copy
patch back-path navigation on onboarding

---------

Co-authored-by: timothycarambat <rambat1010@gmail.com>

frontend/src/pages/GeneralSettings/EmbeddingPreference/index.jsx

@@ -30,7 +30,7 @@ const EMBEDDERS = [
     logo: AnythingLLMIcon,
     options: (settings) => <NativeEmbeddingOptions settings={settings} />,
     description:
-      "Use the built-in embedding engine for AnythingLLM. Zero setup!",
+      "Use the built-in embedding provider for AnythingLLM. Zero setup!",
   },
   {
     name: "OpenAI",

frontend/src/pages/GeneralSettings/PrivacyAndData/index.jsx

@@ -88,7 +88,9 @@ function ThirdParty({ settings }) {
           </ul>
         </div>
         <div className="flex flex-col gap-y-2 border-b border-zinc-500/50 pb-4">
-          <div className="text-white text-base font-bold">Embedding Engine</div>
+          <div className="text-white text-base font-bold">
+            Embedding Preference
+          </div>
           <div className="flex items-center gap-2.5">
             <img
               src={EMBEDDING_ENGINE_PRIVACY[embeddingEngine].logo}

frontend/src/pages/OnboardingFlow/Steps/CustomLogo/index.jsx

@@ -18,7 +18,7 @@ export default function CustomLogo({ setHeader, setForwardBtn, setBackBtn }) {
   }
 
   function handleBack() {
-    navigate(paths.onboarding.vectorDatabase());
+    navigate(paths.onboarding.llmPreference());
   }
 
   useEffect(() => {

frontend/src/pages/OnboardingFlow/Steps/DataHandling/index.jsx

@@ -305,7 +305,9 @@ export default function DataHandling({ setHeader, setForwardBtn, setBackBtn }) {
           </ul>
         </div>
         <div className="flex flex-col gap-y-2 border-b border-zinc-500/50 pb-4">
-          <div className="text-white text-base font-bold">Embedding Engine</div>
+          <div className="text-white text-base font-bold">
+            Embedding Preference
+          </div>
           <div className="flex items-center gap-2.5">
             <img
               src={EMBEDDING_ENGINE_PRIVACY[embeddingEngine].logo}
@@ -344,6 +346,9 @@ export default function DataHandling({ setHeader, setForwardBtn, setBackBtn }) {
           </ul>
         </div>
       </div>
+      <p className="text-white/60 text-sm font-medium py-1">
+        These settings can be reconfigured at any time in the settings.
+      </p>
     </div>
   );
 }

frontend/src/pages/OnboardingFlow/Steps/EmbeddingPreference/index.jsx

@@ -1,186 +0,0 @@
-import { MagnifyingGlass } from "@phosphor-icons/react";
-import { useEffect, useState, useRef } from "react";
-import AnythingLLMIcon from "@/media/logo/anything-llm-icon.png";
-import OpenAiLogo from "@/media/llmprovider/openai.png";
-import AzureOpenAiLogo from "@/media/llmprovider/azure.png";
-import LocalAiLogo from "@/media/llmprovider/localai.png";
-import OllamaLogo from "@/media/llmprovider/ollama.png";
-import LMStudioLogo from "@/media/llmprovider/lmstudio.png";
-import NativeEmbeddingOptions from "@/components/EmbeddingSelection/NativeEmbeddingOptions";
-import OpenAiOptions from "@/components/EmbeddingSelection/OpenAiOptions";
-import AzureAiOptions from "@/components/EmbeddingSelection/AzureAiOptions";
-import LocalAiOptions from "@/components/EmbeddingSelection/LocalAiOptions";
-import OllamaEmbeddingOptions from "@/components/EmbeddingSelection/OllamaOptions";
-import LMStudioEmbeddingOptions from "@/components/EmbeddingSelection/LMStudioOptions";
-import EmbedderItem from "@/components/EmbeddingSelection/EmbedderItem";
-import System from "@/models/system";
-import paths from "@/utils/paths";
-import showToast from "@/utils/toast";
-import { useNavigate } from "react-router-dom";
-
-const TITLE = "Embedding Preference";
-const DESCRIPTION =
-  "AnythingLLM can work with many embedding models. This will be the model which turns documents into vectors.";
-const EMBEDDERS = [
-  {
-    name: "AnythingLLM Embedder",
-    value: "native",
-    logo: AnythingLLMIcon,
-    options: (settings) => <NativeEmbeddingOptions settings={settings} />,
-    description:
-      "Use the built-in embedding engine for AnythingLLM. Zero setup!",
-  },
-  {
-    name: "OpenAI",
-    value: "openai",
-    logo: OpenAiLogo,
-    options: (settings) => <OpenAiOptions settings={settings} />,
-    description: "The standard option for most non-commercial use.",
-  },
-  {
-    name: "Azure OpenAI",
-    value: "azure",
-    logo: AzureOpenAiLogo,
-    options: (settings) => <AzureAiOptions settings={settings} />,
-    description: "The enterprise option of OpenAI hosted on Azure services.",
-  },
-  {
-    name: "Local AI",
-    value: "localai",
-    logo: LocalAiLogo,
-    options: (settings) => <LocalAiOptions settings={settings} />,
-    description: "Run embedding models locally on your own machine.",
-  },
-  {
-    name: "Ollama",
-    value: "ollama",
-    logo: OllamaLogo,
-    options: (settings) => <OllamaEmbeddingOptions settings={settings} />,
-    description: "Run embedding models locally on your own machine.",
-  },
-  {
-    name: "LM Studio",
-    value: "lmstudio",
-    logo: LMStudioLogo,
-    options: (settings) => <LMStudioEmbeddingOptions settings={settings} />,
-    description:
-      "Discover, download, and run thousands of cutting edge LLMs in a few clicks.",
-  },
-];
-
-export default function EmbeddingPreference({
-  setHeader,
-  setForwardBtn,
-  setBackBtn,
-}) {
-  const [searchQuery, setSearchQuery] = useState("");
-  const [filteredEmbedders, setFilteredEmbedders] = useState([]);
-  const [selectedEmbedder, setSelectedEmbedder] = useState(null);
-  const [settings, setSettings] = useState(null);
-  const formRef = useRef(null);
-  const hiddenSubmitButtonRef = useRef(null);
-  const navigate = useNavigate();
-
-  useEffect(() => {
-    async function fetchKeys() {
-      const _settings = await System.keys();
-      setSettings(_settings);
-      setSelectedEmbedder(_settings?.EmbeddingEngine || "native");
-    }
-    fetchKeys();
-  }, []);
-
-  function handleForward() {
-    if (hiddenSubmitButtonRef.current) {
-      hiddenSubmitButtonRef.current.click();
-    }
-  }
-
-  function handleBack() {
-    navigate(paths.onboarding.llmPreference());
-  }
-
-  const handleSubmit = async (e) => {
-    e.preventDefault();
-    const form = e.target;
-    const data = {};
-    const formData = new FormData(form);
-    data.EmbeddingEngine = selectedEmbedder;
-    for (var [key, value] of formData.entries()) data[key] = value;
-
-    const { error } = await System.updateSystem(data);
-    if (error) {
-      showToast(`Failed to save embedding settings: ${error}`, "error");
-      return;
-    }
-    navigate(paths.onboarding.vectorDatabase());
-  };
-
-  useEffect(() => {
-    setHeader({ title: TITLE, description: DESCRIPTION });
-    setForwardBtn({ showing: true, disabled: false, onClick: handleForward });
-    setBackBtn({ showing: true, disabled: false, onClick: handleBack });
-  }, []);
-
-  useEffect(() => {
-    const filtered = EMBEDDERS.filter((embedder) =>
-      embedder.name.toLowerCase().includes(searchQuery.toLowerCase())
-    );
-    setFilteredEmbedders(filtered);
-  }, [searchQuery, selectedEmbedder]);
-
-  return (
-    <div>
-      <form ref={formRef} onSubmit={handleSubmit} className="w-full">
-        <div className="w-full relative border-slate-300/40 shadow border-2 rounded-lg text-white">
-          <div className="w-full p-4 absolute top-0 rounded-t-lg backdrop-blur-sm">
-            <div className="w-full flex items-center sticky top-0 z-20">
-              <MagnifyingGlass
-                size={16}
-                weight="bold"
-                className="absolute left-4 z-30 text-white"
-              />
-              <input
-                type="text"
-                placeholder="Search Embedding providers"
-                className="bg-zinc-600 z-20 pl-10 h-[38px] rounded-full w-full px-4 py-1 text-sm border-2 border-slate-300/40 outline-none focus:border-white text-white"
-                onChange={(e) => setSearchQuery(e.target.value)}
-                autoComplete="off"
-                onKeyDown={(e) => {
-                  if (e.key === "Enter") e.preventDefault();
-                }}
-              />
-            </div>
-          </div>
-          <div className="px-4 pt-[70px] flex flex-col gap-y-1 max-h-[390px] overflow-y-auto no-scroll pb-4">
-            {filteredEmbedders.map((embedder) => {
-              return (
-                <EmbedderItem
-                  key={embedder.name}
-                  name={embedder.name}
-                  value={embedder.value}
-                  image={embedder.logo}
-                  description={embedder.description}
-                  checked={selectedEmbedder === embedder.value}
-                  onClick={() => setSelectedEmbedder(embedder.value)}
-                />
-              );
-            })}
-          </div>
-        </div>
-        <div className="mt-4 flex flex-col gap-y-1">
-          {selectedEmbedder &&
-            EMBEDDERS.find(
-              (embedder) => embedder.value === selectedEmbedder
-            )?.options(settings)}
-        </div>
-        <button
-          type="submit"
-          ref={hiddenSubmitButtonRef}
-          hidden
-          aria-hidden="true"
-        ></button>
-      </form>
-    </div>
-  );
-}

frontend/src/pages/OnboardingFlow/Steps/LLMPreference/index.jsx

@@ -183,6 +183,9 @@ export default function LLMPreference({
     const data = {};
     const formData = new FormData(form);
     data.LLMProvider = selectedLLM;
+    // Default to AnythingLLM embedder and LanceDB
+    data.EmbeddingEngine = "native";
+    data.VectorDB = "lancedb";
     for (var [key, value] of formData.entries()) data[key] = value;
 
     const { error } = await System.updateSystem(data);
@@ -190,7 +193,7 @@ export default function LLMPreference({
       showToast(`Failed to save LLM settings: ${error}`, "error");
       return;
     }
-    navigate(paths.onboarding.embeddingPreference());
+    navigate(paths.onboarding.customLogo());
   };
 
   useEffect(() => {

frontend/src/pages/OnboardingFlow/Steps/VectorDatabaseConnection/index.jsx

@@ -1,202 +0,0 @@
-import React, { useEffect, useState, useRef } from "react";
-import { MagnifyingGlass } from "@phosphor-icons/react";
-import ChromaLogo from "@/media/vectordbs/chroma.png";
-import PineconeLogo from "@/media/vectordbs/pinecone.png";
-import LanceDbLogo from "@/media/vectordbs/lancedb.png";
-import WeaviateLogo from "@/media/vectordbs/weaviate.png";
-import QDrantLogo from "@/media/vectordbs/qdrant.png";
-import MilvusLogo from "@/media/vectordbs/milvus.png";
-import ZillizLogo from "@/media/vectordbs/zilliz.png";
-import AstraDBLogo from "@/media/vectordbs/astraDB.png";
-import System from "@/models/system";
-import paths from "@/utils/paths";
-import PineconeDBOptions from "@/components/VectorDBSelection/PineconeDBOptions";
-import ChromaDBOptions from "@/components/VectorDBSelection/ChromaDBOptions";
-import QDrantDBOptions from "@/components/VectorDBSelection/QDrantDBOptions";
-import WeaviateDBOptions from "@/components/VectorDBSelection/WeaviateDBOptions";
-import LanceDBOptions from "@/components/VectorDBSelection/LanceDBOptions";
-import MilvusOptions from "@/components/VectorDBSelection/MilvusDBOptions";
-import ZillizCloudOptions from "@/components/VectorDBSelection/ZillizCloudOptions";
-import AstraDBOptions from "@/components/VectorDBSelection/AstraDBOptions";
-import showToast from "@/utils/toast";
-import { useNavigate } from "react-router-dom";
-import VectorDBItem from "@/components/VectorDBSelection/VectorDBItem";
-
-const TITLE = "Vector Database Connection";
-const DESCRIPTION =
-  "These are the credentials and settings for your vector database of choice.";
-
-export default function VectorDatabaseConnection({
-  setHeader,
-  setForwardBtn,
-  setBackBtn,
-}) {
-  const [searchQuery, setSearchQuery] = useState("");
-  const [filteredVDBs, setFilteredVDBs] = useState([]);
-  const [selectedVDB, setSelectedVDB] = useState(null);
-  const [settings, setSettings] = useState(null);
-  const formRef = useRef(null);
-  const hiddenSubmitButtonRef = useRef(null);
-  const navigate = useNavigate();
-
-  useEffect(() => {
-    async function fetchKeys() {
-      const _settings = await System.keys();
-      setSettings(_settings);
-      setSelectedVDB(_settings?.VectorDB || "lancedb");
-    }
-    fetchKeys();
-  }, []);
-
-  const VECTOR_DBS = [
-    {
-      name: "LanceDB",
-      value: "lancedb",
-      logo: LanceDbLogo,
-      options: <LanceDBOptions />,
-      description:
-        "100% local vector DB that runs on the same instance as AnythingLLM.",
-    },
-    {
-      name: "Chroma",
-      value: "chroma",
-      logo: ChromaLogo,
-      options: <ChromaDBOptions settings={settings} />,
-      description:
-        "Open source vector database you can host yourself or on the cloud.",
-    },
-    {
-      name: "Pinecone",
-      value: "pinecone",
-      logo: PineconeLogo,
-      options: <PineconeDBOptions settings={settings} />,
-      description: "100% cloud-based vector database for enterprise use cases.",
-    },
-    {
-      name: "Zilliz Cloud",
-      value: "zilliz",
-      logo: ZillizLogo,
-      options: <ZillizCloudOptions settings={settings} />,
-      description:
-        "Cloud hosted vector database built for enterprise with SOC 2 compliance.",
-    },
-    {
-      name: "QDrant",
-      value: "qdrant",
-      logo: QDrantLogo,
-      options: <QDrantDBOptions settings={settings} />,
-      description: "Open source local and distributed cloud vector database.",
-    },
-    {
-      name: "Weaviate",
-      value: "weaviate",
-      logo: WeaviateLogo,
-      options: <WeaviateDBOptions settings={settings} />,
-      description:
-        "Open source local and cloud hosted multi-modal vector database.",
-    },
-    {
-      name: "Milvus",
-      value: "milvus",
-      logo: MilvusLogo,
-      options: <MilvusOptions settings={settings} />,
-      description: "Open-source, highly scalable, and blazing fast.",
-    },
-    {
-      name: "AstraDB",
-      value: "astra",
-      logo: AstraDBLogo,
-      options: <AstraDBOptions settings={settings} />,
-      description: "Vector Search for Real-world GenAI.",
-    },
-  ];
-
-  function handleForward() {
-    if (hiddenSubmitButtonRef.current) {
-      hiddenSubmitButtonRef.current.click();
-    }
-  }
-
-  function handleBack() {
-    navigate(paths.onboarding.embeddingPreference());
-  }
-
-  const handleSubmit = async (e) => {
-    e.preventDefault();
-    const form = e.target;
-    const data = {};
-    const formData = new FormData(form);
-    data.VectorDB = selectedVDB;
-    for (var [key, value] of formData.entries()) data[key] = value;
-    const { error } = await System.updateSystem(data);
-    if (error) {
-      showToast(`Failed to save Vector Database settings: ${error}`, "error");
-      return;
-    }
-    navigate(paths.onboarding.customLogo());
-  };
-
-  useEffect(() => {
-    setHeader({ title: TITLE, description: DESCRIPTION });
-    setForwardBtn({ showing: true, disabled: false, onClick: handleForward });
-    setBackBtn({ showing: true, disabled: false, onClick: handleBack });
-  }, []);
-
-  useEffect(() => {
-    const filtered = VECTOR_DBS.filter((vdb) =>
-      vdb.name.toLowerCase().includes(searchQuery.toLowerCase())
-    );
-    setFilteredVDBs(filtered);
-  }, [searchQuery, selectedVDB]);
-
-  return (
-    <>
-      <form ref={formRef} onSubmit={handleSubmit} className="w-full">
-        <div className="w-full relative border-slate-300/40 shadow border-2 rounded-lg text-white pb-4">
-          <div className="w-full p-4 absolute top-0 rounded-t-lg backdrop-blur-sm">
-            <div className="w-full flex items-center sticky top-0 z-20">
-              <MagnifyingGlass
-                size={16}
-                weight="bold"
-                className="absolute left-4 z-30 text-white"
-              />
-              <input
-                type="text"
-                placeholder="Search vector databases"
-                className="bg-zinc-600 z-20 pl-10 h-[38px] rounded-full w-full px-4 py-1 text-sm border-2 border-slate-300/40 outline-none focus:border-white text-white"
-                onChange={(e) => setSearchQuery(e.target.value)}
-                autoComplete="off"
-                onKeyDown={(e) => {
-                  if (e.key === "Enter") e.preventDefault();
-                }}
-              />
-            </div>
-          </div>
-          <div className="px-4 pt-[70px] flex flex-col gap-y-1 max-h-[390px] overflow-y-auto no-scroll">
-            {filteredVDBs.map((vdb) => (
-              <VectorDBItem
-                key={vdb.name}
-                name={vdb.name}
-                value={vdb.value}
-                image={vdb.logo}
-                description={vdb.description}
-                checked={selectedVDB === vdb.value}
-                onClick={setSelectedVDB}
-              />
-            ))}
-          </div>
-        </div>
-        <div className="mt-4 flex flex-col gap-y-1">
-          {selectedVDB &&
-            VECTOR_DBS.find((vdb) => vdb.value === selectedVDB)?.options}
-        </div>
-        <button
-          type="submit"
-          ref={hiddenSubmitButtonRef}
-          hidden
-          aria-hidden="true"
-        ></button>
-      </form>
-    </>
-  );
-}

frontend/src/pages/OnboardingFlow/Steps/index.jsx

@@ -3,8 +3,6 @@ import { useState } from "react";
 import { isMobile } from "react-device-detect";
 import Home from "./Home";
 import LLMPreference from "./LLMPreference";
-import EmbeddingPreference from "./EmbeddingPreference";
-import VectorDatabaseConnection from "./VectorDatabaseConnection";
 import CustomLogo from "./CustomLogo";
 import UserSetup from "./UserSetup";
 import DataHandling from "./DataHandling";
@@ -14,8 +12,6 @@ import CreateWorkspace from "./CreateWorkspace";
 const OnboardingSteps = {
   home: Home,
   "llm-preference": LLMPreference,
-  "embedding-preference": EmbeddingPreference,
-  "vector-database": VectorDatabaseConnection,
   "custom-logo": CustomLogo,
   "user-setup": UserSetup,
   "data-handling": DataHandling,