Merge branch 'master' of github.com:Mintplex-Labs/anything-llm into render

2024-11-19 20:50:09 +01:00 · 2024-04-12 14:57:46 -07:00 · 2024-04-12 14:57:46 -07:00 · fde4e5400f
commit fde4e5400f
parent 75ced7e65a af84b01482
14 changed files with 172 additions and 100 deletions
--- a/collector/utils/extensions/GithubRepo/RepoLoader/index.js
+++ b/collector/utils/extensions/GithubRepo/RepoLoader/index.js
@ -13,7 +13,9 @@ class RepoLoader {

  #validGithubUrl() {
    const UrlPattern = require("url-pattern");
-    const pattern = new UrlPattern("https\\://github.com/(:author)/(:project)");
+    const pattern = new UrlPattern(
+      "https\\://github.com/(:author)/(:project(*))"
+    );
    const match = pattern.match(this.repo);
    if (!match) return false;

--- a/collector/utils/files/mime.js
+++ b/collector/utils/files/mime.js
@ -21,7 +21,21 @@ class MimeDetector {
    // which has had this extension far before TS was invented. So need to force re-map this MIME map.
    this.lib.define(
      {
-        "text/plain": ["ts", "py", "opts", "lock", "jsonl", "qml", "sh"],
+        "text/plain": [
+          "ts",
+          "py",
+          "opts",
+          "lock",
+          "jsonl",
+          "qml",
+          "sh",
+          "c",
+          "cs",
+          "h",
+          "js",
+          "lua",
+          "pas",
+        ],
      },
      true
    );
--- a/embed/README.md
+++ b/embed/README.md
@ -14,6 +14,7 @@ This folder of AnythingLLM contains the source code for how the embedded version
 The AnythingLLM Embedded chat widget allows you to expose a workspace and its embedded knowledge base as a chat bubble via a `<script>` or `<iframe>` element that you can embed in a website or HTML.

 ### Security
+
 - Users will _not_ be able to view or read context snippets like they can in the core AnythingLLM application
 - Users are assigned a random session ID that they use to persist a chat session.
 - **Recommended** You can limit both the number of chats an embedding can process **and** per-session.
@ -21,6 +22,7 @@ The AnythingLLM Embedded chat widget allows you to expose a workspace and its em
 _by using the AnythingLLM embedded chat widget you are responsible for securing and configuration of the embed as to not allow excessive chat model abuse of your instance_

 ### Developer Setup
+
 - `cd embed` from the root of the repo
 - `yarn` to install all dev and script dependencies
 - `yarn dev` to boot up an example HTML page to use the chat embed widget.
@ -34,6 +36,7 @@ While in development mode (`yarn dev`) the script will rebuild on any changes to
 ### `<script>` tag HTML embed

 The primary way of embedding a workspace as a chat widget is via a simple `<script>`
+
 ```html
 <!--
 An example of a script tag embed
@ -44,13 +47,14 @@ REQUIRED data attributes:
 <script
  data-embed-id="5fc05aaf-2f2c-4c84-87a3-367a4692c1ee"
  data-base-api-url="http://localhost:3001/api/embed"
-  src="http://localhost:3000/embed/anythingllm-chat-widget.min.js">
-</script>
+  src="http://localhost:3000/embed/anythingllm-chat-widget.min.js"
+></script>
 ```

 ### `<script>` Customization Options

 **LLM Overrides**
+
 - `data-prompt` — Override the chat window with a custom system prompt. This is not visible to the user. If undefined it will use the embeds attached workspace system prompt.

 - `data-model` — Override the chat model used for responses. This must be a valid model string for your AnythingLLM LLM provider. If unset it will use the embeds attached workspace model selection or the system setting.
@ -58,6 +62,7 @@ REQUIRED data attributes:
 - `data-temperature` — Override the chat model temperature. This must be a valid value for your AnythingLLM LLM provider. If unset it will use the embeds attached workspace model temperature or the system setting.

 **Style Overrides**
+
 - `data-chat-icon` — The chat bubble icon show when chat is closed. Options are `plus`, `chatCircle`, `support`, `search2`, `search`, `magic`.

 - `data-button-color` — The chat bubble background color shown when chat is closed. Value must be hex color code.
@ -78,15 +83,20 @@ REQUIRED data attributes:

 - `data-position` - Adjust the positioning of the embed chat widget and open chat button. Default `bottom-right`. Options are `bottom-right`, `bottom-left`, `top-right`, `top-left`.

+- `data-assistant-name` - Set the chat assistant name that appears above each chat message. Default `AnythingLLM Chat Assistant`
+
+- `data-assistant-icon` - Set the icon of the chat assistant.

 **Behavior Overrides**
+
 - `data-open-on-load` — Once loaded, open the chat as default. It can still be closed by the user.

 - `data-support-email` — Shows a support email that the user can used to draft an email via the "three dot" menu in the top right. Option will not appear if it is not set.

-
 ### `<iframe>` tag HTML embed
+
 _work in progress_

 ### `<iframe>` Customization Options
+
 _work in progress_
--- a/embed/src/components/ChatWindow/ChatContainer/ChatHistory/HistoricalMessage/index.jsx
+++ b/embed/src/components/ChatWindow/ChatContainer/ChatHistory/HistoricalMessage/index.jsx
@ -19,7 +19,8 @@ const HistoricalMessage = forwardRef(
          <div
            className={`text-[10px] font-medium text-gray-400 ml-[54px] mr-6 mb-2 text-left`}
          >
-            AnythingLLM Chat Assistant
+            {embedderSettings.settings.assistantName ||
+              "Anything LLM Chat Assistant"}
          </div>
        )}
        <div
@ -31,7 +32,7 @@ const HistoricalMessage = forwardRef(
        >
          {role === "assistant" && (
            <img
-              src={AnythingLLMIcon}
+              src={embedderSettings.settings.assistantIcon || AnythingLLMIcon}
              alt="Anything LLM Icon"
              className="w-9 h-9 flex-shrink-0 ml-2 mt-2"
              id="anything-llm-icon"
--- a/embed/src/components/ChatWindow/ChatContainer/ChatHistory/PromptReply/index.jsx
+++ b/embed/src/components/ChatWindow/ChatContainer/ChatHistory/PromptReply/index.jsx
@ -13,7 +13,7 @@ const PromptReply = forwardRef(
      return (
        <div className={`flex items-start w-full h-fit justify-start`}>
          <img
-            src={AnythingLLMIcon}
+            src={embedderSettings.settings.assistantIcon || AnythingLLMIcon}
            alt="Anything LLM Icon"
            className="w-9 h-9 flex-shrink-0 ml-2"
          />
@ -33,7 +33,7 @@ const PromptReply = forwardRef(
      return (
        <div className={`flex items-end w-full h-fit justify-start`}>
          <img
-            src={AnythingLLMIcon}
+            src={embedderSettings.settings.assistantIcon || AnythingLLMIcon}
            alt="Anything LLM Icon"
            className="w-9 h-9 flex-shrink-0 ml-2"
          />
@ -60,7 +60,8 @@ const PromptReply = forwardRef(
        <div
          className={`text-[10px] font-medium text-gray-400 ml-[54px] mr-6 mb-2 text-left`}
        >
-          AnythingLLM Chat Assistant
+          {embedderSettings.settings.assistantName ||
+            "Anything LLM Chat Assistant"}
        </div>
        <div
          key={uuid}
@ -68,7 +69,7 @@ const PromptReply = forwardRef(
          className={`flex items-start w-full h-fit justify-start`}
        >
          <img
-            src={AnythingLLMIcon}
+            src={embedderSettings.settings.assistantIcon || AnythingLLMIcon}
            alt="Anything LLM Icon"
            className="w-9 h-9 flex-shrink-0 ml-2"
          />
--- a/embed/src/hooks/useScriptAttributes.js
+++ b/embed/src/hooks/useScriptAttributes.js
@ -21,6 +21,8 @@ const DEFAULT_SETTINGS = {
  sponsorText: "Powered by AnythingLLM", // default sponsor text
  sponsorLink: "https://useanything.com", // default sponsor link
  position: "bottom-right", // position of chat button/window
+  assistantName: "AnythingLLM Chat Assistant", // default assistant name
+  assistantIcon: null, // default assistant icon

  // behaviors
  openOnLoad: "off", // or "on"
--- a/frontend/public/embed/anythingllm-chat-widget.min.js
+++ b/frontend/public/embed/anythingllm-chat-widget.min.js
--- a/frontend/src/models/workspace.js
+++ b/frontend/src/models/workspace.js
@ -173,6 +173,14 @@ const Workspace = {

    return result;
  },
+  wipeVectorDb: async function (slug) {
+    return await fetch(`${API_BASE}/workspace/${slug}/reset-vector-db`, {
+      method: "DELETE",
+      headers: baseHeaders(),
+    })
+      .then((res) => res.ok)
+      .catch(() => false);
+  },
  uploadFile: async function (slug, formData) {
    const response = await fetch(`${API_BASE}/workspace/${slug}/upload`, {
      method: "POST",
--- a/frontend/src/pages/OnboardingFlow/Steps/Survey/index.jsx
+++ b/frontend/src/pages/OnboardingFlow/Steps/Survey/index.jsx
@ -8,7 +8,11 @@ const TITLE = "Welcome to AnythingLLM";
 const DESCRIPTION = "Help us make AnythingLLM built for your needs. Optional.";

 async function sendQuestionnaire({ email, useCase, comment }) {
-  if (import.meta.env.DEV) return;
+  if (import.meta.env.DEV) {
+    console.log("sendQuestionnaire", { email, useCase, comment });
+    return;
+  }
+
  return fetch(`https://onboarding-wxich7363q-uc.a.run.app`, {
    method: "POST",
    body: JSON.stringify({
@ -38,9 +42,25 @@ export default function Survey({ setHeader, setForwardBtn, setBackBtn }) {
      navigate(paths.onboarding.createWorkspace());
      return;
    }
-    if (submitRef.current) {
-      submitRef.current.click();
+
+    if (!formRef.current) {
+      skipSurvey();
+      return;
    }
+
+    // Check if any inputs are not empty. If that is the case, trigger form validation.
+    // via the requestSubmit() handler
+    const formData = new FormData(formRef.current);
+    if (
+      !!formData.get("email") ||
+      !!formData.get("use_case") ||
+      !!formData.get("comment")
+    ) {
+      formRef.current.requestSubmit();
+      return;
+    }
+
+    skipSurvey();
  }

  function skipSurvey() {
@ -116,26 +136,24 @@ export default function Survey({ setHeader, setForwardBtn, setBackBtn }) {
          <div className="mt-2 gap-y-3 flex flex-col">
            <label
              className={`transition-all duration-300 w-full h-11 p-2.5 bg-white/10 rounded-lg flex justify-start items-center gap-2.5 cursor-pointer border border-transparent ${
-                selectedOption === "business"
-                  ? "border-white border-opacity-40"
-                  : ""
+                selectedOption === "job" ? "border-white border-opacity-40" : ""
              } hover:border-white/60`}
            >
              <input
                type="radio"
                name="use_case"
-                value={"business"}
-                checked={selectedOption === "business"}
+                value={"job"}
+                checked={selectedOption === "job"}
                onChange={(e) => setSelectedOption(e.target.value)}
                className="hidden"
              />
              <div
                className={`w-4 h-4 rounded-full border-2 border-white mr-2 ${
-                  selectedOption === "business" ? "bg-white" : ""
+                  selectedOption === "job" ? "bg-white" : ""
                }`}
              ></div>
              <div className="text-white text-sm font-medium font-['Plus Jakarta Sans'] leading-tight">
-                For my business
+                For work
              </div>
            </label>
            <label
@ -159,77 +177,7 @@ export default function Survey({ setHeader, setForwardBtn, setBackBtn }) {
                }`}
              ></div>
              <div className="text-white text-sm font-medium font-['Plus Jakarta Sans'] leading-tight">
-                For personal use
-              </div>
-            </label>
-            <label
-              className={`transition-all duration-300 w-full h-11 p-2.5 bg-white/10 rounded-lg flex justify-start items-center gap-2.5 cursor-pointer border border-transparent ${
-                selectedOption === "education"
-                  ? "border-white border-opacity-40"
-                  : ""
-              } hover:border-white/60`}
-            >
-              <input
-                type="radio"
-                name="use_case"
-                value={"education"}
-                checked={selectedOption === "education"}
-                onChange={(e) => setSelectedOption(e.target.value)}
-                className="hidden"
-              />
-              <div
-                className={`w-4 h-4 rounded-full border-2 border-white mr-2 ${
-                  selectedOption === "education" ? "bg-white" : ""
-                }`}
-              ></div>
-              <div className="text-white text-sm font-medium font-['Plus Jakarta Sans'] leading-tight">
-                For my education
-              </div>
-            </label>
-            <label
-              className={`transition-all duration-300 w-full h-11 p-2.5 bg-white/10 rounded-lg flex justify-start items-center gap-2.5 cursor-pointer border border-transparent ${
-                selectedOption === "side_hustle"
-                  ? "border-white border-opacity-40"
-                  : ""
-              } hover:border-white/60`}
-            >
-              <input
-                type="radio"
-                name="use_case"
-                value={"side_hustle"}
-                checked={selectedOption === "side_hustle"}
-                onChange={(e) => setSelectedOption(e.target.value)}
-                className="hidden"
-              />
-              <div
-                className={`w-4 h-4 rounded-full border-2 border-white mr-2 ${
-                  selectedOption === "side_hustle" ? "bg-white" : ""
-                }`}
-              ></div>
-              <div className="text-white text-sm font-medium font-['Plus Jakarta Sans'] leading-tight">
-                For my side-hustle
-              </div>
-            </label>
-            <label
-              className={`transition-all duration-300 w-full h-11 p-2.5 bg-white/10 rounded-lg flex justify-start items-center gap-2.5 cursor-pointer border border-transparent ${
-                selectedOption === "job" ? "border-white border-opacity-40" : ""
-              } hover:border-white/60`}
-            >
-              <input
-                type="radio"
-                name="use_case"
-                value={"job"}
-                checked={selectedOption === "job"}
-                onChange={(e) => setSelectedOption(e.target.value)}
-                className="hidden"
-              />
-              <div
-                className={`w-4 h-4 rounded-full border-2 border-white mr-2 ${
-                  selectedOption === "job" ? "bg-white" : ""
-                }`}
-              ></div>
-              <div className="text-white text-sm font-medium font-['Plus Jakarta Sans'] leading-tight">
-                For my job
+                For my personal use
              </div>
            </label>
            <label
--- a/frontend/src/pages/WorkspaceSettings/ChatSettings/WorkspaceLLMSelection/index.jsx
+++ b/frontend/src/pages/WorkspaceSettings/ChatSettings/WorkspaceLLMSelection/index.jsx
@ -15,6 +15,10 @@ const LLM_DEFAULT = {
  requiredConfig: [],
 };

+const LLMS = [LLM_DEFAULT, ...AVAILABLE_LLM_PROVIDERS].filter(
+  (llm) => !DISABLED_PROVIDERS.includes(llm.value)
+);
+
 export default function WorkspaceLLMSelection({
  settings,
  workspace,
@ -27,9 +31,6 @@ export default function WorkspaceLLMSelection({
  const [searchQuery, setSearchQuery] = useState("");
  const [searchMenuOpen, setSearchMenuOpen] = useState(false);
  const searchInputRef = useRef(null);
-  const LLMS = [LLM_DEFAULT, ...AVAILABLE_LLM_PROVIDERS].filter(
-    (llm) => !DISABLED_PROVIDERS.includes(llm.value)
-  );

  function updateLLMChoice(selection) {
    setSearchQuery("");
--- a/frontend/src/pages/WorkspaceSettings/VectorDatabase/ResetDatabase/index.jsx
+++ b/frontend/src/pages/WorkspaceSettings/VectorDatabase/ResetDatabase/index.jsx
@ -0,0 +1,42 @@
+import { useState } from "react";
+import Workspace from "@/models/workspace";
+import showToast from "@/utils/toast";
+
+export default function ResetDatabase({ workspace }) {
+  const [deleting, setDeleting] = useState(false);
+
+  const resetVectorDatabase = async () => {
+    if (
+      !window.confirm(
+        `You are about to reset this workspace's vector database. This will remove all vector embeddings currently embedded.\n\nThe original source files will remain untouched. This action is irreversible.`
+      )
+    )
+      return false;
+
+    setDeleting(true);
+    const success = await Workspace.wipeVectorDb(workspace.slug);
+    if (!success) {
+      showToast("Workspace vector database could not be reset!", "error", {
+        clear: true,
+      });
+      setDeleting(false);
+      return;
+    }
+
+    showToast("Workspace vector database was reset!", "success", {
+      clear: true,
+    });
+    setDeleting(false);
+  };
+
+  return (
+    <button
+      disabled={deleting}
+      onClick={resetVectorDatabase}
+      type="button"
+      className="border-none w-fit transition-all duration-300 border border-transparent rounded-lg whitespace-nowrap text-sm px-5 py-2.5 focus:z-10 bg-red-500/25 text-red-200 hover:text-white hover:bg-red-600 disabled:bg-red-600 disabled:text-red-200 disabled:animate-pulse"
+    >
+      {deleting ? "Clearing vectors..." : "Reset Workspace Vector Database"}
+    </button>
+  );
+}
--- a/frontend/src/pages/WorkspaceSettings/VectorDatabase/index.jsx
+++ b/frontend/src/pages/WorkspaceSettings/VectorDatabase/index.jsx
@ -5,6 +5,7 @@ import { useRef, useState } from "react";
 import VectorDBIdentifier from "./VectorDBIdentifier";
 import MaxContextSnippets from "./MaxContextSnippets";
 import DocumentSimilarityThreshold from "./DocumentSimilarityThreshold";
+import ResetDatabase from "./ResetDatabase";

 export default function VectorDatabase({ workspace }) {
  const [hasChanges, setHasChanges] = useState(false);
@ -43,6 +44,7 @@ export default function VectorDatabase({ workspace }) {
        workspace={workspace}
        setHasChanges={setHasChanges}
      />
+      <ResetDatabase workspace={workspace} />
      {hasChanges && (
        <button
          type="submit"
--- a/server/endpoints/workspaces.js
+++ b/server/endpoints/workspaces.js
@ -266,6 +266,47 @@ function workspaceEndpoints(app) {
    }
  );

+  app.delete(
+    "/workspace/:slug/reset-vector-db",
+    [validatedRequest, flexUserRoleValid([ROLES.admin, ROLES.manager])],
+    async (request, response) => {
+      try {
+        const { slug = "" } = request.params;
+        const user = await userFromSession(request, response);
+        const VectorDb = getVectorDbClass();
+        const workspace = multiUserMode(response)
+          ? await Workspace.getWithUser(user, { slug })
+          : await Workspace.get({ slug });
+
+        if (!workspace) {
+          response.sendStatus(400).end();
+          return;
+        }
+
+        await DocumentVectors.deleteForWorkspace(workspace.id);
+        await Document.delete({ workspaceId: Number(workspace.id) });
+
+        await EventLogs.logEvent(
+          "workspace_vectors_reset",
+          {
+            workspaceName: workspace?.name || "Unknown Workspace",
+          },
+          response.locals?.user?.id
+        );
+
+        try {
+          await VectorDb["delete-namespace"]({ namespace: slug });
+        } catch (e) {
+          console.error(e.message);
+        }
+        response.sendStatus(200).end();
+      } catch (e) {
+        console.log(e.message, e);
+        response.sendStatus(500).end();
+      }
+    }
+  );
+
  app.get(
    "/workspaces",
    [validatedRequest, flexUserRoleValid([ROLES.all])],
--- a/server/utils/helpers/updateENV.js
+++ b/server/utils/helpers/updateENV.js
@ -425,8 +425,8 @@ function validChromaURL(input = "") {
 function validAzureURL(input = "") {
  try {
    new URL(input);
-    if (!input.includes("openai.azure.com"))
-      return "URL must include openai.azure.com";
+    if (!input.includes("openai.azure.com") && !input.includes("microsoft.com"))
+      return "Valid Azure endpoints must contain openai.azure.com OR microsoft.com";
    return null;
  } catch {
    return "Not a valid URL";