merge to master

.devcontainer/devcontainer.json

@@ -76,10 +76,10 @@
       ]
     }
   },
-  "updateContentCommand": "yarn setup",
+  "updateContentCommand": "cd server && yarn && cd ../collector && PUPPETEER_DOWNLOAD_BASE_URL=https://storage.googleapis.com/chrome-for-testing-public yarn && cd ../frontend && yarn && cd .. && yarn setup:envs && yarn prisma:setup && echo \"Please run yarn dev:server, yarn dev:collector, and yarn dev:frontend in separate terminal tabs.\"",
   // Use 'postCreateCommand' to run commands after the container is created.
-  // This configures VITE for github codespaces
-  "postCreateCommand": "if [ \"${CODESPACES}\" = \"true\" ]; then echo 'VITE_API_BASE=\"https://$CODESPACE_NAME-3001.$GITHUB_CODESPACES_PORT_FORWARDING_DOMAIN/api\"' > ./frontend/.env; fi",
+  // This configures VITE for github codespaces and installs gh cli
+  "postCreateCommand": "if [ \"${CODESPACES}\" = \"true\" ]; then echo 'VITE_API_BASE=\"https://$CODESPACE_NAME-3001.$GITHUB_CODESPACES_PORT_FORWARDING_DOMAIN/api\"' > ./frontend/.env && (type -p wget >/dev/null || (sudo apt update && sudo apt-get install wget -y)) && sudo mkdir -p -m 755 /etc/apt/keyrings && wget -qO- https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null && sudo chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg && echo \"deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main\" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null && sudo apt update && sudo apt install gh -y; fi",
   "portsAttributes": {
     "3001": {
       "label": "Backend",

.dockerignore

@@ -4,7 +4,6 @@
 **/server/storage/*.db
 **/server/storage/lancedb
 **/collector/hotdir/**
-**/collector/v-env/**
 **/collector/outputs/**
 **/node_modules/
 **/dist/
@@ -13,6 +12,7 @@
 **/.env
 **/.env.*
 **/bundleinspector.html
+**/tmp/**
+**/.log
 !docker/.env.example
 !frontend/.env.production
-**/tmp/**

.github/workflows/build-and-push-image-semver.yaml

@@ -0,0 +1,115 @@
+name: Publish AnythingLLM Docker image on Release (amd64 & arm64)
+
+concurrency:
+  group: build-${{ github.ref }}
+  cancel-in-progress: true
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  push_multi_platform_to_registries:
+    name: Push Docker multi-platform image to multiple registries
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+      contents: read
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v4
+
+      - name: Check if DockerHub build needed
+        shell: bash
+        run: |
+          # Check if the secret for USERNAME is set (don't even check for the password)
+          if [[ -z "${{ secrets.DOCKER_USERNAME }}" ]]; then
+            echo "DockerHub build not needed"
+            echo "enabled=false" >> $GITHUB_OUTPUT
+          else
+            echo "DockerHub build needed"
+            echo "enabled=true" >> $GITHUB_OUTPUT
+          fi
+        id: dockerhub
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      
+      - name: Log in to Docker Hub
+        uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a
+        # Only login to the Docker Hub if the repo is mintplex/anythingllm, to allow for forks to build on GHCR
+        if: steps.dockerhub.outputs.enabled == 'true' 
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      
+      - name: Log in to the Container registry
+        uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
+        with:
+          images: |
+            ${{ steps.dockerhub.outputs.enabled == 'true' && 'mintplexlabs/anythingllm' || '' }}
+            ghcr.io/${{ github.repository }}
+          tags: |
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+
+      - name: Build and push multi-platform Docker image
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: ./docker/Dockerfile
+          push: true
+          sbom: true
+          provenance: mode=max
+          platforms: linux/amd64,linux/arm64
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+      
+      # For Docker scout there are some intermediary reported CVEs which exists outside
+      # of execution content or are unreachable by an attacker but exist in image.
+      # We create VEX files for these so they don't show in scout summary. 
+      - name: Collect known and verified CVE exceptions
+        id: cve-list
+        run: |
+          # Collect CVEs from filenames in vex folder
+          CVE_NAMES=""
+          for file in ./docker/vex/*.vex.json; do
+            [ -e "$file" ] || continue
+            filename=$(basename "$file")
+            stripped_filename=${filename%.vex.json}
+            CVE_NAMES+=" $stripped_filename"
+          done
+          echo "CVE_EXCEPTIONS=$CVE_NAMES" >> $GITHUB_OUTPUT
+        shell: bash
+
+      # About VEX attestations https://docs.docker.com/scout/explore/exceptions/
+      # Justifications https://github.com/openvex/spec/blob/main/OPENVEX-SPEC.md#status-justifications
+      - name: Add VEX attestations
+        env:
+          CVE_EXCEPTIONS: ${{ steps.cve-list.outputs.CVE_EXCEPTIONS }}
+        run: |
+          echo $CVE_EXCEPTIONS
+          curl -sSfL https://raw.githubusercontent.com/docker/scout-cli/main/install.sh | sh -s --
+          for cve in $CVE_EXCEPTIONS; do
+            for tag in "${{ join(fromJSON(steps.meta.outputs.json).tags, ' ') }}"; do
+              echo "Attaching VEX exception $cve to $tag"
+              docker scout attestation add \
+              --file "./docker/vex/$cve.vex.json" \
+              --predicate-type https://openvex.dev/ns/v0.2.0 \
+              $tag
+            done
+          done
+        shell: bash

.github/workflows/build-and-push-image.yaml

@@ -15,11 +15,12 @@ on:
     branches: ['lancedb-revert']
     paths-ignore:
       - '**.md'
-      - 'cloud-deployments/*'
+      - 'cloud-deployments/**/*'
       - 'images/**/*'
       - '.vscode/**/*'
       - '**/.env.example'
       - '.github/ISSUE_TEMPLATE/**/*'
+      - '.devcontainer/**/*'
       - 'embed/**/*' # Embed should be published to frontend (yarn build:publish) if any changes are introduced
       - 'server/utils/agents/aibitat/example/**/*' # Do not push new image for local dev testing of new aibitat images.
 
@@ -78,13 +79,51 @@ jobs:
             type=raw,value=lancedb_revert
 
       - name: Build and push multi-platform Docker image
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         with:
           context: .
           file: ./docker/Dockerfile
           push: true
+          sbom: true
+          provenance: mode=max
           platforms: linux/amd64,linux/arm64
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
           cache-from: type=gha
           cache-to: type=gha,mode=max
+      
+      # For Docker scout there are some intermediary reported CVEs which exists outside
+      # of execution content or are unreachable by an attacker but exist in image.
+      # We create VEX files for these so they don't show in scout summary. 
+      - name: Collect known and verified CVE exceptions
+        id: cve-list
+        run: |
+          # Collect CVEs from filenames in vex folder
+          CVE_NAMES=""
+          for file in ./docker/vex/*.vex.json; do
+            [ -e "$file" ] || continue
+            filename=$(basename "$file")
+            stripped_filename=${filename%.vex.json}
+            CVE_NAMES+=" $stripped_filename"
+          done
+          echo "CVE_EXCEPTIONS=$CVE_NAMES" >> $GITHUB_OUTPUT
+        shell: bash
+
+      # About VEX attestations https://docs.docker.com/scout/explore/exceptions/
+      # Justifications https://github.com/openvex/spec/blob/main/OPENVEX-SPEC.md#status-justifications
+      - name: Add VEX attestations
+        env:
+          CVE_EXCEPTIONS: ${{ steps.cve-list.outputs.CVE_EXCEPTIONS }}
+        run: |
+          echo $CVE_EXCEPTIONS
+          curl -sSfL https://raw.githubusercontent.com/docker/scout-cli/main/install.sh | sh -s --
+          for cve in $CVE_EXCEPTIONS; do
+            for tag in "${{ join(fromJSON(steps.meta.outputs.json).tags, ' ') }}"; do
+              echo "Attaching VEX exception $cve to $tag"
+              docker scout attestation add \
+              --file "./docker/vex/$cve.vex.json" \
+              --predicate-type https://openvex.dev/ns/v0.2.0 \
+              $tag
+            done
+          done
+        shell: bash

.github/workflows/check-translations.yaml

@@ -0,0 +1,37 @@
+# This Github action is for validation of all languages which translations are offered for
+# in the locales folder in `frontend/src`. All languages are compared to the EN translation
+# schema since that is the fallback language setting. This workflow will run on all PRs that
+# modify any files in the translation directory
+name: Verify translations files
+
+concurrency:
+  group: build-${{ github.ref }}
+  cancel-in-progress: true
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened]
+    paths:
+      - "frontend/src/locales/**.js"
+
+jobs:
+  run-script:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v3
+        with:
+          node-version: '18'
+
+      - name: Run verifyTranslations.mjs script
+        run: |
+          cd frontend/src/locales
+          node verifyTranslations.mjs
+
+      - name: Fail job on error
+        if: failure()
+        run: exit 1

.github/workflows/dev-build.yaml

@@ -0,0 +1,114 @@
+name: AnythingLLM Development Docker image (amd64)
+
+concurrency:
+  group: build-${{ github.ref }}
+  cancel-in-progress: true
+
+on:
+  push:
+    branches: ['encrypt-jwt-value'] # put your current branch to create a build. Core team only.
+    paths-ignore:
+      - '**.md'
+      - 'cloud-deployments/*'
+      - 'images/**/*'
+      - '.vscode/**/*'
+      - '**/.env.example'
+      - '.github/ISSUE_TEMPLATE/**/*'
+      - 'embed/**/*' # Embed should be published to frontend (yarn build:publish) if any changes are introduced
+      - 'server/utils/agents/aibitat/example/**/*' # Do not push new image for local dev testing of new aibitat images.
+
+jobs:
+  push_multi_platform_to_registries:
+    name: Push Docker multi-platform image to multiple registries
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+      contents: read
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v4
+
+      - name: Check if DockerHub build needed
+        shell: bash
+        run: |
+          # Check if the secret for USERNAME is set (don't even check for the password)
+          if [[ -z "${{ secrets.DOCKER_USERNAME }}" ]]; then
+            echo "DockerHub build not needed"
+            echo "enabled=false" >> $GITHUB_OUTPUT
+          else
+            echo "DockerHub build needed"
+            echo "enabled=true" >> $GITHUB_OUTPUT
+          fi
+        id: dockerhub
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      
+      - name: Log in to Docker Hub
+        uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a
+        # Only login to the Docker Hub if the repo is mintplex/anythingllm, to allow for forks to build on GHCR
+        if: steps.dockerhub.outputs.enabled == 'true' 
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+   
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
+        with:
+          images: |
+            ${{ steps.dockerhub.outputs.enabled == 'true' && 'mintplexlabs/anythingllm' || '' }}
+          tags: |
+            type=raw,value=dev
+
+      - name: Build and push multi-platform Docker image
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: ./docker/Dockerfile
+          push: true
+          sbom: true
+          provenance: mode=max
+          platforms: linux/amd64
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      # For Docker scout there are some intermediary reported CVEs which exists outside
+      # of execution content or are unreachable by an attacker but exist in image.
+      # We create VEX files for these so they don't show in scout summary. 
+      - name: Collect known and verified CVE exceptions
+        id: cve-list
+        run: |
+          # Collect CVEs from filenames in vex folder
+          CVE_NAMES=""
+          for file in ./docker/vex/*.vex.json; do
+            [ -e "$file" ] || continue
+            filename=$(basename "$file")
+            stripped_filename=${filename%.vex.json}
+            CVE_NAMES+=" $stripped_filename"
+          done
+          echo "CVE_EXCEPTIONS=$CVE_NAMES" >> $GITHUB_OUTPUT
+        shell: bash
+
+      # About VEX attestations https://docs.docker.com/scout/explore/exceptions/
+      # Justifications https://github.com/openvex/spec/blob/main/OPENVEX-SPEC.md#status-justifications
+      - name: Add VEX attestations
+        env:
+          CVE_EXCEPTIONS: ${{ steps.cve-list.outputs.CVE_EXCEPTIONS }}
+        run: |
+          echo $CVE_EXCEPTIONS
+          curl -sSfL https://raw.githubusercontent.com/docker/scout-cli/main/install.sh | sh -s --
+          for cve in $CVE_EXCEPTIONS; do
+            for tag in "${{ join(fromJSON(steps.meta.outputs.json).tags, ' ') }}"; do
+              echo "Attaching VEX exception $cve to $tag"
+              docker scout attestation add \
+              --file "./docker/vex/$cve.vex.json" \
+              --predicate-type https://openvex.dev/ns/v0.2.0 \
+              $tag
+            done
+          done
+        shell: bash
+
+      

.vscode/settings.json

@@ -3,14 +3,18 @@
     "adoc",
     "aibitat",
     "AIbitat",
+    "allm",
     "anythingllm",
     "Astra",
     "Chartable",
+    "cleancss",
     "comkey",
     "cooldown",
     "cooldowns",
+    "datafile",
     "Deduplicator",
     "Dockerized",
+    "docpath",
     "elevenlabs",
     "Embeddable",
     "epub",
@@ -25,16 +29,23 @@
     "mbox",
     "Milvus",
     "Mintplex",
+    "mixtral",
     "moderations",
+    "numpages",
     "Ollama",
     "Oobabooga",
     "openai",
     "opendocument",
     "openrouter",
+    "pagerender",
     "Qdrant",
+    "royalblue",
+    "searxng",
     "Serper",
+    "Serply",
     "textgenwebui",
     "togetherai",
+    "Unembed",
     "vectordbs",
     "Weaviate",
     "Zilliz"

.vscode/tasks.json

@@ -46,7 +46,7 @@
           }
         }
       },
-      "command": "cd ${workspaceFolder}/server/ && yarn dev",
+      "command": "if [ \"${CODESPACES}\" = \"true\" ]; then while ! gh codespace ports -c $CODESPACE_NAME | grep 3001; do sleep 1; done; gh codespace ports visibility 3001:public -c $CODESPACE_NAME; fi & cd ${workspaceFolder}/server/ && yarn dev",
       "runOptions": {
         "instanceLimit": 1,
         "reevaluateOnRerun": true

README.md

@@ -1,7 +1,7 @@
 <a name="readme-top"></a>
 
 <p align="center">
-  <a href="https://useanything.com"><img src="https://github.com/Mintplex-Labs/anything-llm/blob/master/images/wordmark.png?raw=true" alt="AnythingLLM logo"></a>
+  <a href="https://anythingllm.com"><img src="https://github.com/Mintplex-Labs/anything-llm/blob/master/images/wordmark.png?raw=true" alt="AnythingLLM logo"></a>
 </p>
 
 <div align='center'>
@@ -10,7 +10,7 @@
 
 <p align="center">
     <b>AnythingLLM:</b> The all-in-one AI app you were looking for.<br />
-    Chat with your docs, use AI Agents, hyper-configurable, multi-user, & no fustrating set up required.
+    Chat with your docs, use AI Agents, hyper-configurable, multi-user, & no frustrating set up required.
 </p>
 
 <p align="center">
@@ -20,7 +20,7 @@
   <a href="https://github.com/Mintplex-Labs/anything-llm/blob/master/LICENSE" target="_blank">
       <img src="https://img.shields.io/static/v1?label=license&message=MIT&color=white" alt="License">
   </a> |
-  <a href="https://docs.useanything.com" target="_blank">
+  <a href="https://docs.anythingllm.com" target="_blank">
     Docs
   </a> |
    <a href="https://my.mintplexlabs.com/aio-checkout?product=anythingllm" target="_blank">
@@ -33,7 +33,7 @@
 </p>
 
 <p align="center">
-👉 AnythingLLM for desktop (Mac, Windows, & Linux)! <a href="https://useanything.com/download" target="_blank"> Download Now</a>
+👉 AnythingLLM for desktop (Mac, Windows, & Linux)! <a href="https://anythingllm.com/download" target="_blank"> Download Now</a>
 </p>
 
 A full-stack application that enables you to turn any document, resource, or piece of content into context that any LLM can use as references during chatting. This application allows you to pick and choose which LLM or Vector Database you want to use as well as supporting multi-user management and permissions.
@@ -53,19 +53,19 @@ AnythingLLM is a full-stack application where you can use commercial off-the-she
 
 AnythingLLM divides your documents into objects called `workspaces`. A Workspace functions a lot like a thread, but with the addition of containerization of your documents. Workspaces can share documents, but they do not talk to each other so you can keep your context for each workspace clean.
 
-Some cool features of AnythingLLM
+## Cool features of AnythingLLM
 
-- **Multi-user instance support and permissioning**
-- Agents inside your workspace (browse the web, run code, etc)
-- [Custom Embeddable Chat widget for your website](./embed/README.md)
-- Multiple document type support (PDF, TXT, DOCX, etc)
-- Manage documents in your vector database from a simple UI
-- Two chat modes `conversation` and `query`. Conversation retains previous questions and amendments. Query is simple QA against your documents
-- In-chat citations
+- 🆕 **Multi-modal support (both closed and open-source LLMs!)**
+- 👤 Multi-user instance support and permissioning _Docker version only_
+- 🦾 Agents inside your workspace (browse the web, run code, etc)
+- 💬 [Custom Embeddable Chat widget for your website](./embed/README.md) _Docker version only_
+- 📖 Multiple document type support (PDF, TXT, DOCX, etc)
+- Simple chat UI with Drag-n-Drop funcitonality and clear citations.
 - 100% Cloud deployment ready.
-- "Bring your own LLM" model.
-- Extremely efficient cost-saving measures for managing very large documents. You'll never pay to embed a massive document or transcript more than once. 90% more cost effective than other document chatbot solutions.
+- Works with all popular [closed and open-source LLM providers](#supported-llms-embedder-models-speech-models-and-vector-databases).
+- Built-in cost & time-saving measures for managing very large documents compared to any other chat UI.
 - Full Developer API for custom integrations!
+- Much more...install and find out!
 
 ### Supported LLMs, Embedder Models, Speech models, and Vector Databases
 
@@ -75,6 +75,7 @@ Some cool features of AnythingLLM
 - [OpenAI](https://openai.com)
 - [OpenAI (Generic)](https://openai.com)
 - [Azure OpenAI](https://azure.microsoft.com/en-us/products/ai-services/openai-service)
+- [AWS Bedrock](https://aws.amazon.com/bedrock/)
 - [Anthropic](https://www.anthropic.com/)
 - [Google Gemini Pro](https://ai.google.dev/)
 - [Hugging Face (chat models)](https://huggingface.co/)
@@ -109,6 +110,7 @@ Some cool features of AnythingLLM
 **TTS (text-to-speech) support:**
 
 - Native Browser Built-in (default)
+- [PiperTTSLocal - runs in browser](https://github.com/rhasspy/piper)
 - [OpenAI TTS](https://platform.openai.com/docs/guides/text-to-speech/voice-options)
 - [ElevenLabs](https://elevenlabs.io/)
 
@@ -162,12 +164,6 @@ Mintplex Labs & the community maintain a number of deployment methods, scripts,
 
 [Learn about vector caching](./server/storage/vector-cache/VECTOR_CACHE.md)
 
-## Contributing
-
-- create issue
-- create PR with branch name format of `<issue number>-<short name>`
-- yee haw let's merge
-
 ## Telemetry & Privacy
 
 AnythingLLM by Mintplex Labs Inc contains a telemetry feature that collects anonymous usage information.
@@ -199,6 +195,19 @@ You can verify these claims by finding all locations `Telemetry.sendTelemetry` i
 
 </details>
 
+
+## 👋 Contributing
+
+- create issue
+- create PR with branch name format of `<issue number>-<short name>`
+- LGTM from core-team
+
+## 🌟 Contributors
+
+[![anythingllm contributors](https://contrib.rocks/image?repo=mintplex-labs/anything-llm)](https://github.com/mintplex-labs/anything-llm/graphs/contributors)
+
+[![Star History Chart](https://api.star-history.com/svg?repos=mintplex-labs/anything-llm&type=Timeline)](https://star-history.com/#mintplex-labs/anything-llm&Date)
+
 ## 🔗 More Products
 
 - **[VectorAdmin][vector-admin]:** An all-in-one GUI & tool-suite for managing vector databases.

cloud-deployments/aws/cloudformation/cloudformation_create_anythingllm.json

@@ -89,8 +89,8 @@
                 "mkdir -p /home/ec2-user/anythingllm\n",
                 "touch /home/ec2-user/anythingllm/.env\n",
                 "sudo chown ec2-user:ec2-user -R /home/ec2-user/anythingllm\n",
-                "docker pull mintplexlabs/anythingllm:master\n",
-                "docker run -d -p 3001:3001 --cap-add SYS_ADMIN -v /home/ec2-user/anythingllm:/app/server/storage -v /home/ec2-user/anythingllm/.env:/app/server/.env -e STORAGE_DIR=\"/app/server/storage\" mintplexlabs/anythingllm:master\n",
+                "docker pull mintplexlabs/anythingllm\n",
+                "docker run -d -p 3001:3001 --cap-add SYS_ADMIN -v /home/ec2-user/anythingllm:/app/server/storage -v /home/ec2-user/anythingllm/.env:/app/server/.env -e STORAGE_DIR=\"/app/server/storage\" mintplexlabs/anythingllm\n",
                 "echo \"Container ID: $(sudo docker ps --latest --quiet)\"\n",
                 "export ONLINE=$(curl -Is http://localhost:3001/api/ping | head -n 1|cut -d$' ' -f2)\n",
                 "echo \"Health check: $ONLINE\"\n",

cloud-deployments/digitalocean/terraform/DEPLOY.md

@@ -12,16 +12,18 @@ The output of this Terraform configuration will be:
   - Follow the instructions in the [official Terraform documentation](https://developer.hashicorp.com/terraform/tutorials/aws-get-started/install-cli) for your operating system.
 
 ## How to deploy on DigitalOcean
-Open your terminal and navigate to the `digitalocean/terraform` folder
-1. Replace the token value in the provider "digitalocean" block in main.tf with your DigitalOcean API token.
-2. Run the following commands to initialize Terraform, review the infrastructure changes, and apply them:
+Open your terminal and navigate to the `docker` folder
+1. Create a `.env` file by cloning the `.env.example`. 
+2. Navigate to `digitalocean/terraform` folder.
+3. Replace the token value in the provider "digitalocean" block in main.tf with your DigitalOcean API token.
+4. Run the following commands to initialize Terraform, review the infrastructure changes, and apply them:
     ```
     terraform init  
     terraform plan  
     terraform apply  
     ```
 Confirm the changes by typing yes when prompted.
-4. Once the deployment is complete, Terraform will output the public IP address of your droplet. You can access your application using this IP address.
+5. Once the deployment is complete, Terraform will output the public IP address of your droplet. You can access your application using this IP address.
 
 ## How to deploy on DigitalOcean
 To delete the resources created by Terraform, run the following command in the terminal:

cloud-deployments/digitalocean/terraform/main.tf

@@ -16,7 +16,7 @@ provider "digitalocean" {
 
   
 resource "digitalocean_droplet" "anything_llm_instance" {  
-  image  = "ubuntu-22-10-x64"  
+  image  = "ubuntu-24-04-x64"  
   name   = "anything-llm-instance"  
   region = "nyc3"  
   size   = "s-2vcpu-2gb"  

cloud-deployments/digitalocean/terraform/user_data.tp1

@@ -9,10 +9,12 @@ sudo systemctl enable docker
 sudo systemctl start docker  
   
 mkdir -p /home/anythingllm
-touch /home/anythingllm/.env
+cat <<EOF >/home/anythingllm/.env
+${env_content}
+EOF
 
-sudo docker pull mintplexlabs/anythingllm:master
-sudo docker run -d -p 3001:3001 --cap-add SYS_ADMIN -v /home/anythingllm:/app/server/storage -v /home/anythingllm/.env:/app/server/.env -e STORAGE_DIR="/app/server/storage" mintplexlabs/anythingllm:master
+sudo docker pull mintplexlabs/anythingllm
+sudo docker run -d -p 3001:3001 --cap-add SYS_ADMIN -v /home/anythingllm:/app/server/storage -v /home/anythingllm/.env:/app/server/.env -e STORAGE_DIR="/app/server/storage" mintplexlabs/anythingllm
 echo "Container ID: $(sudo docker ps --latest --quiet)"  
   
 export ONLINE=$(curl -Is http://localhost:3001/api/ping | head -n 1|cut -d$' ' -f2)  

cloud-deployments/gcp/deployment/gcp_deploy_anything_llm.yaml

@@ -34,8 +34,8 @@ resources:
               touch /home/anythingllm/.env
               sudo chown -R ubuntu:ubuntu /home/anythingllm
                
-              sudo docker pull mintplexlabs/anythingllm:master
-              sudo docker run -d -p 3001:3001 --cap-add SYS_ADMIN -v /home/anythingllm:/app/server/storage -v /home/anythingllm/.env:/app/server/.env -e STORAGE_DIR="/app/server/storage" mintplexlabs/anythingllm:master
+              sudo docker pull mintplexlabs/anythingllm
+              sudo docker run -d -p 3001:3001 --cap-add SYS_ADMIN -v /home/anythingllm:/app/server/storage -v /home/anythingllm/.env:/app/server/.env -e STORAGE_DIR="/app/server/storage" mintplexlabs/anythingllm
               echo "Container ID: $(sudo docker ps --latest --quiet)"  
 
               export ONLINE=$(curl -Is http://localhost:3001/api/ping | head -n 1|cut -d$' ' -f2)  

cloud-deployments/huggingface-spaces/Dockerfile

@@ -0,0 +1,31 @@
+# With this dockerfile in a Huggingface space you will get an entire AnythingLLM instance running
+# in your space with all features you would normally get from the docker based version of AnythingLLM.
+#
+# How to use
+# - Login to https://huggingface.co/spaces
+# - Click on "Create new Space"
+# - Name the space and select "Docker" as the SDK w/ a blank template
+# - The default 2vCPU/16GB machine is OK. The more the merrier.
+# - Decide if you want your AnythingLLM Space public or private.
+#   **You might want to stay private until you at least set a password or enable multi-user mode**
+# - Click "Create Space"
+# - Click on "Settings" on top of page (https://huggingface.co/spaces/<username>/<space-name>/settings)
+# - Scroll to "Persistent Storage" and select the lowest tier of now - you can upgrade if you run out.
+# - Confirm and continue storage upgrade
+# - Go to "Files" Tab (https://huggingface.co/spaces/<username>/<space-name>/tree/main)
+# - Click "Add Files"
+# - Upload this file or create a file named `Dockerfile` and copy-paste this content into it. "Commit to main" and save.
+# - Your container will build and boot. You now have AnythingLLM on HuggingFace. Your data is stored in the persistent storage attached.
+# Have Fun 🤗 
+# Have issues? Check the logs on HuggingFace for clues.
+FROM mintplexlabs/anythingllm:render
+
+USER root
+RUN mkdir -p /data/storage
+RUN ln -s /data/storage /storage
+USER anythingllm
+
+ENV STORAGE_DIR="/data/storage"
+ENV SERVER_PORT=7860
+
+ENTRYPOINT ["/bin/bash", "/usr/local/bin/render-entrypoint.sh"]

collector/extensions/index.js

@@ -1,19 +1,44 @@
+const { setDataSigner } = require("../middleware/setDataSigner");
 const { verifyPayloadIntegrity } = require("../middleware/verifyIntegrity");
+const { resolveRepoLoader, resolveRepoLoaderFunction } = require("../utils/extensions/RepoLoader");
 const { reqBody } = require("../utils/http");
 const { validURL } = require("../utils/url");
+const RESYNC_METHODS = require("./resync");
 
 function extensions(app) {
   if (!app) return;
 
   app.post(
-    "/ext/github-repo",
-    [verifyPayloadIntegrity],
+    "/ext/resync-source-document",
+    [verifyPayloadIntegrity, setDataSigner],
     async function (request, response) {
       try {
-        const loadGithubRepo = require("../utils/extensions/GithubRepo");
-        const { success, reason, data } = await loadGithubRepo(
-          reqBody(request)
+        const { type, options } = reqBody(request);
+        if (!RESYNC_METHODS.hasOwnProperty(type)) throw new Error(`Type "${type}" is not a valid type to sync.`);
+        return await RESYNC_METHODS[type](options, response);
+      } catch (e) {
+        console.error(e);
+        response.status(200).json({
+          success: false,
+          content: null,
+          reason: e.message || "A processing error occurred.",
+        });
+      }
+      return;
+    }
+  )
+
+  app.post(
+    "/ext/:repo_platform-repo",
+    [verifyPayloadIntegrity, setDataSigner],
+    async function (request, response) {
+      try {
+        const loadRepo = resolveRepoLoaderFunction(request.params.repo_platform);
+        const { success, reason, data } = await loadRepo(
+          reqBody(request),
+          response,
         );
+        console.log({ success, reason, data })
         response.status(200).json({
           success,
           reason,
@@ -33,12 +58,12 @@ function extensions(app) {
 
   // gets all branches for a specific repo
   app.post(
-    "/ext/github-repo/branches",
+    "/ext/:repo_platform-repo/branches",
     [verifyPayloadIntegrity],
     async function (request, response) {
       try {
-        const GithubRepoLoader = require("../utils/extensions/GithubRepo/RepoLoader");
-        const allBranches = await new GithubRepoLoader(
+        const RepoLoader = resolveRepoLoader(request.params.repo_platform);
+        const allBranches = await new RepoLoader(
           reqBody(request)
         ).getRepoBranches();
         response.status(200).json({
@@ -67,7 +92,7 @@ function extensions(app) {
     [verifyPayloadIntegrity],
     async function (request, response) {
       try {
-        const loadYouTubeTranscript = require("../utils/extensions/YoutubeTranscript");
+        const { loadYouTubeTranscript } = require("../utils/extensions/YoutubeTranscript");
         const { success, reason, data } = await loadYouTubeTranscript(
           reqBody(request)
         );
@@ -108,12 +133,13 @@ function extensions(app) {
 
   app.post(
     "/ext/confluence",
-    [verifyPayloadIntegrity],
+    [verifyPayloadIntegrity, setDataSigner],
     async function (request, response) {
       try {
-        const loadConfluence = require("../utils/extensions/Confluence");
+        const { loadConfluence } = require("../utils/extensions/Confluence");
         const { success, reason, data } = await loadConfluence(
-          reqBody(request)
+          reqBody(request),
+          response
         );
         response.status(200).json({ success, reason, data });
       } catch (e) {

collector/extensions/resync/index.js

@@ -0,0 +1,113 @@
+const { getLinkText } = require("../../processLink");
+
+/**
+ * Fetches the content of a raw link. Returns the content as a text string of the link in question.
+ * @param {object} data - metadata from document (eg: link) 
+ * @param {import("../../middleware/setDataSigner").ResponseWithSigner} response
+ */
+async function resyncLink({ link }, response) {
+  if (!link) throw new Error('Invalid link provided');
+  try {
+    const { success, content = null } = await getLinkText(link);
+    if (!success) throw new Error(`Failed to sync link content. ${reason}`);
+    response.status(200).json({ success, content });
+  } catch (e) {
+    console.error(e);
+    response.status(200).json({
+      success: false,
+      content: null,
+    });
+  }
+}
+
+/**
+ * Fetches the content of a YouTube link. Returns the content as a text string of the video in question.
+ * We offer this as there may be some videos where a transcription could be manually edited after initial scraping
+ * but in general - transcriptions often never change.
+ * @param {object} data - metadata from document (eg: link) 
+ * @param {import("../../middleware/setDataSigner").ResponseWithSigner} response
+ */
+async function resyncYouTube({ link }, response) {
+  if (!link) throw new Error('Invalid link provided');
+  try {
+    const { fetchVideoTranscriptContent } = require("../../utils/extensions/YoutubeTranscript");
+    const { success, reason, content } = await fetchVideoTranscriptContent({ url: link });
+    if (!success) throw new Error(`Failed to sync YouTube video transcript. ${reason}`);
+    response.status(200).json({ success, content });
+  } catch (e) {
+    console.error(e);
+    response.status(200).json({
+      success: false,
+      content: null,
+    });
+  }
+}
+
+/**
+ * Fetches the content of a specific confluence page via its chunkSource. 
+ * Returns the content as a text string of the page in question and only that page.
+ * @param {object} data - metadata from document (eg: chunkSource) 
+ * @param {import("../../middleware/setDataSigner").ResponseWithSigner} response
+ */
+async function resyncConfluence({ chunkSource }, response) {
+  if (!chunkSource) throw new Error('Invalid source property provided');
+  try {
+    // Confluence data is `payload` encrypted. So we need to expand its
+    // encrypted payload back into query params so we can reFetch the page with same access token/params.
+    const source = response.locals.encryptionWorker.expandPayload(chunkSource);
+    const { fetchConfluencePage } = require("../../utils/extensions/Confluence");
+    const { success, reason, content } = await fetchConfluencePage({
+      pageUrl: `https:${source.pathname}`, // need to add back the real protocol
+      baseUrl: source.searchParams.get('baseUrl'),
+      accessToken: source.searchParams.get('token'),
+      username: source.searchParams.get('username'),
+    });
+
+    if (!success) throw new Error(`Failed to sync Confluence page content. ${reason}`);
+    response.status(200).json({ success, content });
+  } catch (e) {
+    console.error(e);
+    response.status(200).json({
+      success: false,
+      content: null,
+    });
+  }
+}
+
+/**
+ * Fetches the content of a specific confluence page via its chunkSource. 
+ * Returns the content as a text string of the page in question and only that page.
+ * @param {object} data - metadata from document (eg: chunkSource) 
+ * @param {import("../../middleware/setDataSigner").ResponseWithSigner} response
+ */
+async function resyncGithub({ chunkSource }, response) {
+  if (!chunkSource) throw new Error('Invalid source property provided');
+  try {
+    // Github file data is `payload` encrypted (might contain PAT). So we need to expand its
+    // encrypted payload back into query params so we can reFetch the page with same access token/params.
+    const source = response.locals.encryptionWorker.expandPayload(chunkSource);
+    const { fetchGithubFile } = require("../../utils/extensions/RepoLoader/GithubRepo");
+    const { success, reason, content } = await fetchGithubFile({
+      repoUrl: `https:${source.pathname}`, // need to add back the real protocol
+      branch: source.searchParams.get('branch'),
+      accessToken: source.searchParams.get('pat'),
+      sourceFilePath: source.searchParams.get('path'),
+    });
+
+    if (!success) throw new Error(`Failed to sync Github file content. ${reason}`);
+    response.status(200).json({ success, content });
+  } catch (e) {
+    console.error(e);
+    response.status(200).json({
+      success: false,
+      content: null,
+    });
+  }
+}
+
+module.exports = {
+  link: resyncLink,
+  youtube: resyncYouTube,
+  confluence: resyncConfluence,
+  github: resyncGithub,
+}

collector/index.js

@@ -2,6 +2,7 @@ process.env.NODE_ENV === "development"
   ? require("dotenv").config({ path: `.env.${process.env.NODE_ENV}` })
   : require("dotenv").config();
 
+require("./utils/logger")();
 const express = require("express");
 const bodyParser = require("body-parser");
 const cors = require("cors");

collector/middleware/setDataSigner.js

@@ -0,0 +1,41 @@
+const { EncryptionWorker } = require("../utils/EncryptionWorker");
+const { CommunicationKey } = require("../utils/comKey");
+
+/** 
+ * Express Response Object interface with defined encryptionWorker attached to locals property.
+ * @typedef {import("express").Response & import("express").Response['locals'] & {encryptionWorker: EncryptionWorker} } ResponseWithSigner
+*/
+
+// You can use this middleware to assign the EncryptionWorker to the response locals
+// property so that if can be used to encrypt/decrypt arbitrary data via response object.
+// eg: Encrypting API keys in chunk sources.
+
+// The way this functions is that the rolling RSA Communication Key is used server-side to private-key encrypt the raw
+// key of the persistent EncryptionManager credentials. Since EncryptionManager credentials do _not_ roll, we should not send them
+// even between server<>collector in plaintext because if the user configured the server/collector to be public they could technically
+// be exposing the key in transit via the X-Payload-Signer header. Even if this risk is minimal we should not do this.
+
+// This middleware uses the CommunicationKey public key to first decrypt the base64 representation of the EncryptionManager credentials
+// and then loads that in to the EncryptionWorker as a buffer so we can use the same credentials across the system. Should we ever break the
+// collector out into its own service this would still work without SSL/TLS.
+
+/**
+ * 
+ * @param {import("express").Request} request 
+ * @param {import("express").Response} response 
+ * @param {import("express").NextFunction} next 
+ */
+function setDataSigner(request, response, next) {
+  const comKey = new CommunicationKey();
+  const encryptedPayloadSigner = request.header("X-Payload-Signer");
+  if (!encryptedPayloadSigner) console.log('Failed to find signed-payload to set encryption worker! Encryption calls will fail.');
+
+  const decryptedPayloadSignerKey = comKey.decrypt(encryptedPayloadSigner);
+  const encryptionWorker = new EncryptionWorker(decryptedPayloadSignerKey);
+  response.locals.encryptionWorker = encryptionWorker;
+  next();
+}
+
+module.exports = {
+  setDataSigner
+}

collector/package.json

@@ -15,7 +15,6 @@
     "lint": "yarn prettier --ignore-path ../.prettierignore --write ./processSingleFile ./processLink ./utils index.js"
   },
   "dependencies": {
-    "@googleapis/youtube": "^9.0.0",
     "@xenova/transformers": "^2.11.0",
     "bcrypt": "^5.1.0",
     "body-parser": "^1.20.2",
@@ -23,7 +22,6 @@
     "dotenv": "^16.0.3",
     "epub2": "^3.0.2",
     "express": "^4.18.2",
-    "extract-zip": "^2.0.1",
     "fluent-ffmpeg": "^2.1.2",
     "html-to-text": "^9.0.5",
     "ignore": "^5.3.0",
@@ -32,8 +30,8 @@
     "mammoth": "^1.6.0",
     "mbox-parser": "^1.0.1",
     "mime": "^3.0.0",
+    "minimatch": "5.1.0",
     "moment": "^2.29.4",
-    "multer": "^1.4.5-lts.1",
     "node-html-parser": "^6.1.13",
     "officeparser": "^4.0.5",
     "openai": "4.38.5",
@@ -43,6 +41,7 @@
     "url-pattern": "^1.0.3",
     "uuid": "^9.0.0",
     "wavefile": "^11.0.0",
+    "winston": "^3.13.0",
     "youtubei.js": "^9.1.0"
   },
   "devDependencies": {

collector/processLink/convert/generic.js

@@ -57,6 +57,7 @@ async function getPageContent(link) {
     const loader = new PuppeteerWebBaseLoader(link, {
       launchOptions: {
         headless: "new",
+        ignoreHTTPSErrors: true,
       },
       gotoOptions: {
         waitUntil: "domcontentloaded",

collector/processSingleFile/convert/asDocx.js

@@ -15,7 +15,6 @@ async function asDocX({ fullFilePath = "", filename = "" }) {
   let pageContent = [];
   const docs = await loader.load();
   for (const doc of docs) {
-    console.log(doc.metadata);
     console.log(`-- Parsing content from docx page --`);
     if (!doc.pageContent.length) continue;
     pageContent.push(doc.pageContent);

collector/processSingleFile/convert/asPDF/PDFLoader/index.js

@@ -0,0 +1,97 @@
+const fs = require("fs").promises;
+
+class PDFLoader {
+  constructor(filePath, { splitPages = true } = {}) {
+    this.filePath = filePath;
+    this.splitPages = splitPages;
+  }
+
+  async load() {
+    const buffer = await fs.readFile(this.filePath);
+    const { getDocument, version } = await this.getPdfJS();
+
+    const pdf = await getDocument({
+      data: new Uint8Array(buffer),
+      useWorkerFetch: false,
+      isEvalSupported: false,
+      useSystemFonts: true,
+    }).promise;
+
+    const meta = await pdf.getMetadata().catch(() => null);
+    const documents = [];
+
+    for (let i = 1; i <= pdf.numPages; i += 1) {
+      const page = await pdf.getPage(i);
+      const content = await page.getTextContent();
+
+      if (content.items.length === 0) {
+        continue;
+      }
+
+      let lastY;
+      const textItems = [];
+      for (const item of content.items) {
+        if ("str" in item) {
+          if (lastY === item.transform[5] || !lastY) {
+            textItems.push(item.str);
+          } else {
+            textItems.push(`\n${item.str}`);
+          }
+          lastY = item.transform[5];
+        }
+      }
+
+      const text = textItems.join("");
+      documents.push({
+        pageContent: text.trim(),
+        metadata: {
+          source: this.filePath,
+          pdf: {
+            version,
+            info: meta?.info,
+            metadata: meta?.metadata,
+            totalPages: pdf.numPages,
+          },
+          loc: { pageNumber: i },
+        },
+      });
+    }
+
+    if (this.splitPages) {
+      return documents;
+    }
+
+    if (documents.length === 0) {
+      return [];
+    }
+
+    return [
+      {
+        pageContent: documents.map((doc) => doc.pageContent).join("\n\n"),
+        metadata: {
+          source: this.filePath,
+          pdf: {
+            version,
+            info: meta?.info,
+            metadata: meta?.metadata,
+            totalPages: pdf.numPages,
+          },
+        },
+      },
+    ];
+  }
+
+  async getPdfJS() {
+    try {
+      const pdfjs = await import("pdf-parse/lib/pdf.js/v1.10.100/build/pdf.js");
+      return { getDocument: pdfjs.getDocument, version: pdfjs.version };
+    } catch (e) {
+      console.error(e);
+      throw new Error(
+        "Failed to load pdf-parse. Please install it with eg. `npm install pdf-parse`."
+      );
+    }
+  }
+}
+
+module.exports = PDFLoader;

collector/processSingleFile/convert/asPDF/index.js

@@ -1,14 +1,14 @@
 const { v4 } = require("uuid");
-const { PDFLoader } = require("langchain/document_loaders/fs/pdf");
 const {
   createdDate,
   trashFile,
   writeToServerDocuments,
-} = require("../../utils/files");
-const { tokenizeString } = require("../../utils/tokenizer");
+} = require("../../../utils/files");
+const { tokenizeString } = require("../../../utils/tokenizer");
 const { default: slugify } = require("slugify");
+const PDFLoader = require("./PDFLoader");
 
-async function asPDF({ fullFilePath = "", filename = "" }) {
+async function asPdf({ fullFilePath = "", filename = "" }) {
   const pdfLoader = new PDFLoader(fullFilePath, {
     splitPages: true,
   });
@@ -16,6 +16,7 @@ async function asPDF({ fullFilePath = "", filename = "" }) {
   console.log(`-- Working ${filename} --`);
   const pageContent = [];
   const docs = await pdfLoader.load();
+
   for (const doc of docs) {
     console.log(
       `-- Parsing content from pg ${
@@ -60,4 +61,4 @@ async function asPDF({ fullFilePath = "", filename = "" }) {
   return { success: true, reason: null, documents: [document] };
 }
 
-module.exports = asPDF;
+module.exports = asPdf;

collector/utils/EncryptionWorker/index.js

@@ -0,0 +1,77 @@
+const crypto = require("crypto");
+
+// Differs from EncryptionManager in that is does not set or define the keys that will be used
+// to encrypt or read data and it must be told the key (as base64 string) explicitly that will be used and is provided to
+// the class on creation. This key should be the same `key` that is used by the EncryptionManager class.
+class EncryptionWorker {
+  constructor(presetKeyBase64 = "") {
+    this.key = Buffer.from(presetKeyBase64, "base64");
+    this.algorithm = "aes-256-cbc";
+    this.separator = ":";
+  }
+
+  log(text, ...args) {
+    console.log(`\x1b[36m[EncryptionManager]\x1b[0m ${text}`, ...args);
+  }
+
+  /**
+   * Give a chunk source, parse its payload query param and expand that object back into the URL
+   * as additional query params
+   * @param {string} chunkSource
+   * @returns {URL} Javascript URL object with query params decrypted from payload query param.
+   */
+  expandPayload(chunkSource = "") {
+    try {
+      const url = new URL(chunkSource);
+      if (!url.searchParams.has("payload")) return url;
+
+      const decryptedPayload = this.decrypt(url.searchParams.get("payload"));
+      const encodedParams = JSON.parse(decryptedPayload);
+      url.searchParams.delete("payload"); // remove payload prop
+
+      // Add all query params needed to replay as query params
+      Object.entries(encodedParams).forEach(([key, value]) =>
+        url.searchParams.append(key, value)
+      );
+      return url;
+    } catch (e) {
+      console.error(e);
+    }
+    return new URL(chunkSource);
+  }
+
+  encrypt(plainTextString = null) {
+    try {
+      if (!plainTextString)
+        throw new Error("Empty string is not valid for this method.");
+      const iv = crypto.randomBytes(16);
+      const cipher = crypto.createCipheriv(this.algorithm, this.key, iv);
+      const encrypted = cipher.update(plainTextString, "utf8", "hex");
+      return [
+        encrypted + cipher.final("hex"),
+        Buffer.from(iv).toString("hex"),
+      ].join(this.separator);
+    } catch (e) {
+      this.log(e);
+      return null;
+    }
+  }
+
+  decrypt(encryptedString) {
+    try {
+      const [encrypted, iv] = encryptedString.split(this.separator);
+      if (!iv) throw new Error("IV not found");
+      const decipher = crypto.createDecipheriv(
+        this.algorithm,
+        this.key,
+        Buffer.from(iv, "hex")
+      );
+      return decipher.update(encrypted, "hex", "utf8") + decipher.final("utf8");
+    } catch (e) {
+      this.log(e);
+      return null;
+    }
+  }
+}
+
+module.exports = { EncryptionWorker };

collector/utils/comKey/index.js

@@ -40,6 +40,15 @@ class CommunicationKey {
     } catch {}
     return false;
   }
+
+  // Use the rolling public-key to decrypt arbitrary data that was encrypted via the private key on the server side CommunicationKey class
+  // that we know was done with the same key-pair and the given input is in base64 format already.
+  // Returns plaintext string of the data that was encrypted.
+  decrypt(base64String = "") {
+    return crypto
+      .publicDecrypt(this.#readPublicKey(), Buffer.from(base64String, "base64"))
+      .toString();
+  }
 }
 
 module.exports = { CommunicationKey };

collector/utils/constants.js

@@ -33,7 +33,7 @@ const SUPPORTED_FILETYPE_CONVERTERS = {
   ".rst": "./convert/asTxt.js",
 
   ".html": "./convert/asTxt.js",
-  ".pdf": "./convert/asPDF.js",
+  ".pdf": "./convert/asPDF/index.js",
 
   ".docx": "./convert/asDocx.js",
   ".pptx": "./convert/asOfficeMime.js",

collector/utils/extensions/Confluence/ConfluenceLoader/index.js

@@ -0,0 +1,134 @@
+/*
+ * This is a custom implementation of the Confluence langchain loader. There was an issue where
+ * code blocks were not being extracted. This is a temporary fix until this issue is resolved.*/
+
+const { htmlToText } = require("html-to-text");
+
+class ConfluencePagesLoader {
+  constructor({
+    baseUrl,
+    spaceKey,
+    username,
+    accessToken,
+    limit = 25,
+    expand = "body.storage,version",
+    personalAccessToken,
+  }) {
+    this.baseUrl = baseUrl;
+    this.spaceKey = spaceKey;
+    this.username = username;
+    this.accessToken = accessToken;
+    this.limit = limit;
+    this.expand = expand;
+    this.personalAccessToken = personalAccessToken;
+  }
+
+  get authorizationHeader() {
+    if (this.personalAccessToken) {
+      return `Bearer ${this.personalAccessToken}`;
+    } else if (this.username && this.accessToken) {
+      const authToken = Buffer.from(
+        `${this.username}:${this.accessToken}`
+      ).toString("base64");
+      return `Basic ${authToken}`;
+    }
+    return undefined;
+  }
+
+  async load(options) {
+    try {
+      const pages = await this.fetchAllPagesInSpace(
+        options?.start,
+        options?.limit
+      );
+      return pages.map((page) => this.createDocumentFromPage(page));
+    } catch (error) {
+      console.error("Error:", error);
+      return [];
+    }
+  }
+
+  async fetchConfluenceData(url) {
+    try {
+      const initialHeaders = {
+        "Content-Type": "application/json",
+        Accept: "application/json",
+      };
+      const authHeader = this.authorizationHeader;
+      if (authHeader) {
+        initialHeaders.Authorization = authHeader;
+      }
+      const response = await fetch(url, {
+        headers: initialHeaders,
+      });
+      if (!response.ok) {
+        throw new Error(
+          `Failed to fetch ${url} from Confluence: ${response.status}`
+        );
+      }
+      return await response.json();
+    } catch (error) {
+      throw new Error(`Failed to fetch ${url} from Confluence: ${error}`);
+    }
+  }
+
+  async fetchAllPagesInSpace(start = 0, limit = this.limit) {
+    const url = `${this.baseUrl}/rest/api/content?spaceKey=${this.spaceKey}&limit=${limit}&start=${start}&expand=${this.expand}`;
+    const data = await this.fetchConfluenceData(url);
+    if (data.size === 0) {
+      return [];
+    }
+    const nextPageStart = start + data.size;
+    const nextPageResults = await this.fetchAllPagesInSpace(
+      nextPageStart,
+      limit
+    );
+    return data.results.concat(nextPageResults);
+  }
+
+  createDocumentFromPage(page) {
+    // Function to extract code blocks
+    const extractCodeBlocks = (content) => {
+      const codeBlockRegex =
+        /<ac:structured-macro ac:name="code"[^>]*>[\s\S]*?<ac:plain-text-body><!\[CDATA\[([\s\S]*?)\]\]><\/ac:plain-text-body>[\s\S]*?<\/ac:structured-macro>/g;
+      const languageRegex =
+        /<ac:parameter ac:name="language">(.*?)<\/ac:parameter>/;
+
+      return content.replace(codeBlockRegex, (match) => {
+        const language = match.match(languageRegex)?.[1] || "";
+        const code =
+          match.match(
+            /<ac:plain-text-body><!\[CDATA\[([\s\S]*?)\]\]><\/ac:plain-text-body>/
+          )?.[1] || "";
+        return `\n\`\`\`${language}\n${code.trim()}\n\`\`\`\n`;
+      });
+    };
+
+    const contentWithCodeBlocks = extractCodeBlocks(page.body.storage.value);
+    const plainTextContent = htmlToText(contentWithCodeBlocks, {
+      wordwrap: false,
+      preserveNewlines: true,
+    });
+    const textWithPreservedStructure = plainTextContent.replace(
+      /\n{3,}/g,
+      "\n\n"
+    );
+    const pageUrl = `${this.baseUrl}/spaces/${this.spaceKey}/pages/${page.id}`;
+
+    return {
+      pageContent: textWithPreservedStructure,
+      metadata: {
+        id: page.id,
+        status: page.status,
+        title: page.title,
+        type: page.type,
+        url: pageUrl,
+        version: page.version?.number,
+        updated_by: page.version?.by?.displayName,
+        updated_at: page.version?.when,
+      },
+    };
+  }
+}
+
+module.exports = { ConfluencePagesLoader };

collector/utils/extensions/Confluence/index.js

@@ -3,44 +3,17 @@ const path = require("path");
 const { default: slugify } = require("slugify");
 const { v4 } = require("uuid");
 const UrlPattern = require("url-pattern");
-const { writeToServerDocuments } = require("../../files");
+const { writeToServerDocuments, sanitizeFileName } = require("../../files");
 const { tokenizeString } = require("../../tokenizer");
-const {
-  ConfluencePagesLoader,
-} = require("langchain/document_loaders/web/confluence");
+const { ConfluencePagesLoader } = require("./ConfluenceLoader");
 
-function validSpaceUrl(spaceUrl = "") {
-  // Atlassian default URL match
-  const atlassianPattern = new UrlPattern(
-    "https\\://(:subdomain).atlassian.net/wiki/spaces/(:spaceKey)*"
-  );
-  const atlassianMatch = atlassianPattern.match(spaceUrl);
-  if (atlassianMatch) {
-    return { valid: true, result: atlassianMatch };
-  }
-
-  let customMatch = null;
-  [
-    "https\\://(:subdomain.):domain.:tld/wiki/spaces/(:spaceKey)*", // Custom Confluence space
-    "https\\://(:subdomain.):domain.:tld/display/(:spaceKey)*", // Custom Confluence space + Human-readable space tag.
-  ].forEach((matchPattern) => {
-    if (!!customMatch) return;
-    const pattern = new UrlPattern(matchPattern);
-    customMatch = pattern.match(spaceUrl);
-  });
-
-  if (customMatch) {
-    customMatch.customDomain =
-      (customMatch.subdomain ? `${customMatch.subdomain}.` : "") + //
-      `${customMatch.domain}.${customMatch.tld}`;
-    return { valid: true, result: customMatch, custom: true };
-  }
-
-  // No match
-  return { valid: false, result: null };
-}
-
-async function loadConfluence({ pageUrl, username, accessToken }) {
+/**
+ * Load Confluence documents from a spaceID and Confluence credentials
+ * @param {object} args - forwarded request body params
+ * @param {import("../../../middleware/setDataSigner").ResponseWithSigner} response - Express response object with encryptionWorker
+ * @returns
+ */
+async function loadConfluence({ pageUrl, username, accessToken }, response) {
   if (!pageUrl || !username || !accessToken) {
     return {
       success: false,
@@ -49,21 +22,16 @@ async function loadConfluence({ pageUrl, username, accessToken }) {
     };
   }
 
-  const validSpace = validSpaceUrl(pageUrl);
-  if (!validSpace.result) {
+  const { valid, result } = validSpaceUrl(pageUrl);
+  if (!valid) {
     return {
       success: false,
       reason:
-        "Confluence space URL is not in the expected format of https://domain.atlassian.net/wiki/space/~SPACEID/* or https://customDomain/wiki/space/~SPACEID/*",
+        "Confluence space URL is not in the expected format of one of https://domain.atlassian.net/wiki/space/~SPACEID/* or https://customDomain/wiki/space/~SPACEID/* or https://customDomain/display/~SPACEID/*",
     };
   }
 
-  const { subdomain, customDomain, spaceKey } = validSpace.result;
-  let baseUrl = `https://${subdomain}.atlassian.net/wiki`;
-  if (customDomain) {
-    baseUrl = `https://${customDomain}/wiki`;
-  }
-
+  const { apiBase: baseUrl, spaceKey, subdomain } = result;
   console.log(`-- Working Confluence ${baseUrl} --`);
   const loader = new ConfluencePagesLoader({
     baseUrl,
@@ -115,7 +83,10 @@ async function loadConfluence({ pageUrl, username, accessToken }) {
       docAuthor: subdomain,
       description: doc.metadata.title,
       docSource: `${subdomain} Confluence`,
-      chunkSource: `confluence://${doc.metadata.url}`,
+      chunkSource: generateChunkSource(
+        { doc, baseUrl, accessToken, username },
+        response.locals.encryptionWorker
+      ),
       published: new Date().toLocaleString(),
       wordCount: doc.pageContent.split(" ").length,
       pageContent: doc.pageContent,
@@ -125,11 +96,11 @@ async function loadConfluence({ pageUrl, username, accessToken }) {
     console.log(
       `[Confluence Loader]: Saving ${doc.metadata.title} to ${outFolder}`
     );
-    writeToServerDocuments(
-      data,
-      `${slugify(doc.metadata.title)}-${data.id}`,
-      outFolderPath
+
+    const fileName = sanitizeFileName(
+      `${slugify(doc.metadata.title)}-${data.id}`
     );
+    writeToServerDocuments(data, fileName, outFolderPath);
   });
 
   return {
@@ -142,4 +113,194 @@ async function loadConfluence({ pageUrl, username, accessToken }) {
   };
 }
 
-module.exports = loadConfluence;
+/**
+ * Gets the page content from a specific Confluence page, not all pages in a workspace.
+ * @returns
+ */
+async function fetchConfluencePage({
+  pageUrl,
+  baseUrl,
+  username,
+  accessToken,
+}) {
+  if (!pageUrl || !baseUrl || !username || !accessToken) {
+    return {
+      success: false,
+      content: null,
+      reason:
+        "You need either a username and access token, or a personal access token (PAT), to use the Confluence connector.",
+    };
+  }
+
+  const { valid, result } = validSpaceUrl(pageUrl);
+  if (!valid) {
+    return {
+      success: false,
+      content: null,
+      reason:
+        "Confluence space URL is not in the expected format of https://domain.atlassian.net/wiki/space/~SPACEID/* or https://customDomain/wiki/space/~SPACEID/*",
+    };
+  }
+
+  console.log(`-- Working Confluence Page ${pageUrl} --`);
+  const { spaceKey } = result;
+  const loader = new ConfluencePagesLoader({
+    baseUrl,
+    spaceKey,
+    username,
+    accessToken,
+  });
+
+  const { docs, error } = await loader
+    .load()
+    .then((docs) => {
+      return { docs, error: null };
+    })
+    .catch((e) => {
+      return {
+        docs: [],
+        error: e.message?.split("Error:")?.[1] || e.message,
+      };
+    });
+
+  if (!docs.length || !!error) {
+    return {
+      success: false,
+      reason: error ?? "No pages found for that Confluence space.",
+      content: null,
+    };
+  }
+
+  const targetDocument = docs.find(
+    (doc) => doc.pageContent && doc.metadata.url === pageUrl
+  );
+  if (!targetDocument) {
+    return {
+      success: false,
+      reason: "Target page could not be found in Confluence space.",
+      content: null,
+    };
+  }
+
+  return {
+    success: true,
+    reason: null,
+    content: targetDocument.pageContent,
+  };
+}
+
+/**
+ * A match result for a url-pattern of a Confluence URL
+ * @typedef {Object} ConfluenceMatchResult
+ * @property {string} subdomain - the subdomain of an organization's Confluence space
+ * @property {string} spaceKey - the spaceKey of an organization that determines the documents to collect.
+ * @property {string} apiBase - the correct REST API url to use for loader.
+ */
+
+/**
+ * Generates the correct API base URL for interfacing with the Confluence REST API
+ * depending on the URL pattern being used since there are various ways to host/access a
+ * Confluence space.
+ * @param {ConfluenceMatchResult} matchResult - result from `url-pattern`.match
+ * @param {boolean} isCustomDomain - determines if we need to coerce the subpath of the provided URL
+ * @returns {string} - the resulting REST API URL
+ */
+function generateAPIBaseUrl(matchResult = {}, isCustomDomain = false) {
+  const { subdomain } = matchResult;
+  let subpath = isCustomDomain ? `` : `/wiki`;
+  if (isCustomDomain) return `https://${customDomain}${subpath}`;
+  return `https://${subdomain}.atlassian.net${subpath}`;
+}
+
+/**
+ * Validates and parses the correct information from a given Confluence URL
+ * @param {string} spaceUrl - The organization's Confluence URL to parse
+ * @returns {{
+ *  valid: boolean,
+ *  result: (ConfluenceMatchResult|null),
+ * }}
+ */
+function validSpaceUrl(spaceUrl = "") {
+  let matchResult;
+  const patterns = {
+    default: new UrlPattern(
+      "https\\://(:subdomain).atlassian.net/wiki/spaces/(:spaceKey)*"
+    ),
+    subdomain: new UrlPattern(
+      "https\\://(:subdomain.):domain.:tld/wiki/spaces/(:spaceKey)*"
+    ),
+    custom: new UrlPattern(
+      "https\\://(:subdomain.):domain.:tld/display/(:spaceKey)*"
+    ),
+  };
+
+  // If using the default Atlassian Confluence URL pattern.
+  // We can proceed because the Library/API can use this base url scheme.
+  matchResult = patterns.default.match(spaceUrl);
+  if (matchResult)
+    return {
+      valid: matchResult.hasOwnProperty("spaceKey"),
+      result: {
+        ...matchResult,
+        apiBase: generateAPIBaseUrl(matchResult),
+      },
+    };
+
+  // If using a custom subdomain Confluence URL pattern.
+  // We need to attach the customDomain as a property to the match result
+  // so we can form the correct REST API base from the subdomain.
+  matchResult = patterns.subdomain.match(spaceUrl);
+  if (matchResult) {
+    return {
+      valid: matchResult.hasOwnProperty("spaceKey"),
+      result: {
+        ...matchResult,
+        apiBase: generateAPIBaseUrl(matchResult),
+      },
+    };
+  }
+
+  // If using a base FQDN Confluence URL pattern.
+  // We need to attach the customDomain as a property to the match result
+  // so we can form the correct REST API base from the root domain since /display/ is basically a URL mask.
+  matchResult = patterns.custom.match(spaceUrl);
+  if (matchResult) {
+    return {
+      valid: matchResult.hasOwnProperty("spaceKey"),
+      result: {
+        ...matchResult,
+        apiBase: generateAPIBaseUrl(matchResult, true),
+      },
+    };
+  }
+
+  // No match
+  return { valid: false, result: null };
+}
+
+/**
+ * Generate the full chunkSource for a specific Confluence page so that we can resync it later.
+ * This data is encrypted into a single `payload` query param so we can replay credentials later
+ * since this was encrypted with the systems persistent password and salt.
+ * @param {object} chunkSourceInformation
+ * @param {import("../../EncryptionWorker").EncryptionWorker} encryptionWorker
+ * @returns {string}
+ */
+function generateChunkSource(
+  { doc, baseUrl, accessToken, username },
+  encryptionWorker
+) {
+  const payload = {
+    baseUrl,
+    token: accessToken,
+    username,
+  };
+  return `confluence://${doc.metadata.url}?payload=${encryptionWorker.encrypt(
+    JSON.stringify(payload)
+  )}`;
+}
+
+module.exports = {
+  loadConfluence,
+  fetchConfluencePage,
+};

collector/utils/extensions/GithubRepo/index.js

@@ -1,84 +0,0 @@
-const RepoLoader = require("./RepoLoader");
-const fs = require("fs");
-const path = require("path");
-const { default: slugify } = require("slugify");
-const { v4 } = require("uuid");
-const { writeToServerDocuments } = require("../../files");
-const { tokenizeString } = require("../../tokenizer");
-
-async function loadGithubRepo(args) {
-  const repo = new RepoLoader(args);
-  await repo.init();
-
-  if (!repo.ready)
-    return {
-      success: false,
-      reason: "Could not prepare Github repo for loading! Check URL",
-    };
-
-  console.log(
-    `-- Working Github ${repo.author}/${repo.project}:${repo.branch} --`
-  );
-  const docs = await repo.recursiveLoader();
-  if (!docs.length) {
-    return {
-      success: false,
-      reason: "No files were found for those settings.",
-    };
-  }
-
-  console.log(`[Github Loader]: Found ${docs.length} source files. Saving...`);
-  const outFolder = slugify(
-    `${repo.author}-${repo.project}-${repo.branch}-${v4().slice(0, 4)}`
-  ).toLowerCase();
-
-  const outFolderPath =
-    process.env.NODE_ENV === "development"
-      ? path.resolve(
-          __dirname,
-          `../../../../server/storage/documents/${outFolder}`
-        )
-      : path.resolve(process.env.STORAGE_DIR, `documents/${outFolder}`);
-
-  if (!fs.existsSync(outFolderPath))
-    fs.mkdirSync(outFolderPath, { recursive: true });
-
-  for (const doc of docs) {
-    if (!doc.pageContent) continue;
-    const data = {
-      id: v4(),
-      url: "github://" + doc.metadata.source,
-      title: doc.metadata.source,
-      docAuthor: repo.author,
-      description: "No description found.",
-      docSource: doc.metadata.source,
-      chunkSource: `link://${doc.metadata.repository}/blob/${doc.metadata.branch}/${doc.metadata.source}`,
-      published: new Date().toLocaleString(),
-      wordCount: doc.pageContent.split(" ").length,
-      pageContent: doc.pageContent,
-      token_count_estimate: tokenizeString(doc.pageContent).length,
-    };
-    console.log(
-      `[Github Loader]: Saving ${doc.metadata.source} to ${outFolder}`
-    );
-    writeToServerDocuments(
-      data,
-      `${slugify(doc.metadata.source)}-${data.id}`,
-      outFolderPath
-    );
-  }
-
-  return {
-    success: true,
-    reason: null,
-    data: {
-      author: repo.author,
-      repo: repo.project,
-      branch: repo.branch,
-      files: docs.length,
-      destination: outFolder,
-    },
-  };
-}
-
-module.exports = loadGithubRepo;

collector/utils/extensions/RepoLoader/GithubRepo/RepoLoader/index.js

@@ -1,4 +1,21 @@
-class RepoLoader {
+/**
+ * @typedef {Object} RepoLoaderArgs
+ * @property {string} repo - The GitHub repository URL.
+ * @property {string} [branch] - The branch to load from (optional).
+ * @property {string} [accessToken] - GitHub access token for authentication (optional).
+ * @property {string[]} [ignorePaths] - Array of paths to ignore when loading (optional).
+ */
+
+/**
+ * @class
+ * @classdesc Loads and manages GitHub repository content.
+ */
+class GitHubRepoLoader {
+  /**
+   * Creates an instance of RepoLoader.
+   * @param {RepoLoaderArgs} [args] - The configuration options.
+   * @returns {GitHubRepoLoader}
+   */
   constructor(args = {}) {
     this.ready = false;
     this.repo = args?.repo;
@@ -67,6 +84,10 @@ class RepoLoader {
     return;
   }
 
+  /**
+   * Initializes the RepoLoader instance.
+   * @returns {Promise<RepoLoader>} The initialized RepoLoader instance.
+   */
   async init() {
     if (!this.#validGithubUrl()) return;
     await this.#validBranch();
@@ -75,6 +96,11 @@ class RepoLoader {
     return this;
   }
 
+  /**
+   * Recursively loads the repository content.
+   * @returns {Promise<Array<Object>>} An array of loaded documents.
+   * @throws {Error} If the RepoLoader is not in a ready state.
+   */
   async recursiveLoader() {
     if (!this.ready) throw new Error("[Github Loader]: not in ready state!");
     const {
@@ -109,7 +135,10 @@ class RepoLoader {
     }, []);
   }
 
-  // Get all branches for a given repo.
+  /**
+   * Retrieves all branches for the repository.
+   * @returns {Promise<string[]>} An array of branch names.
+   */
   async getRepoBranches() {
     if (!this.#validGithubUrl() || !this.author || !this.project) return [];
     await this.#validateAccessToken(); // Ensure API access token is valid for pre-flight
@@ -150,6 +179,41 @@ class RepoLoader {
     this.branches = [...new Set(branches.flat())];
     return this.#branchPrefSort(this.branches);
   }
+
+  /**
+   * Fetches the content of a single file from the repository.
+   * @param {string} sourceFilePath - The path to the file in the repository.
+   * @returns {Promise<string|null>} The content of the file, or null if fetching fails.
+   */
+  async fetchSingleFile(sourceFilePath) {
+    try {
+      return fetch(
+        `https://api.github.com/repos/${this.author}/${this.project}/contents/${sourceFilePath}?ref=${this.branch}`,
+        {
+          method: "GET",
+          headers: {
+            Accept: "application/vnd.github+json",
+            "X-GitHub-Api-Version": "2022-11-28",
+            ...(!!this.accessToken
+              ? { Authorization: `Bearer ${this.accessToken}` }
+              : {}),
+          },
+        }
+      )
+        .then((res) => {
+          if (res.ok) return res.json();
+          throw new Error(`Failed to fetch from Github API: ${res.statusText}`);
+        })
+        .then((json) => {
+          if (json.hasOwnProperty("status") || !json.hasOwnProperty("content"))
+            throw new Error(json?.message || "missing content");
+          return atob(json.content);
+        });
+    } catch (e) {
+      console.error(`RepoLoader.fetchSingleFile`, e);
+      return null;
+    }
+  }
 }
 
-module.exports = RepoLoader;
+module.exports = GitHubRepoLoader;

collector/utils/extensions/RepoLoader/GithubRepo/index.js

@@ -0,0 +1,159 @@
+const RepoLoader = require("./RepoLoader");
+const fs = require("fs");
+const path = require("path");
+const { default: slugify } = require("slugify");
+const { v4 } = require("uuid");
+const { writeToServerDocuments } = require("../../../files");
+const { tokenizeString } = require("../../../tokenizer");
+
+/**
+ * Load in a Github Repo recursively or just the top level if no PAT is provided
+ * @param {object} args - forwarded request body params
+ * @param {import("../../../middleware/setDataSigner").ResponseWithSigner} response - Express response object with encryptionWorker
+ * @returns
+ */
+async function loadGithubRepo(args, response) {
+  const repo = new RepoLoader(args);
+  await repo.init();
+
+  if (!repo.ready)
+    return {
+      success: false,
+      reason: "Could not prepare Github repo for loading! Check URL",
+    };
+
+  console.log(
+    `-- Working Github ${repo.author}/${repo.project}:${repo.branch} --`
+  );
+  const docs = await repo.recursiveLoader();
+  if (!docs.length) {
+    return {
+      success: false,
+      reason: "No files were found for those settings.",
+    };
+  }
+
+  console.log(`[Github Loader]: Found ${docs.length} source files. Saving...`);
+  const outFolder = slugify(
+    `${repo.author}-${repo.project}-${repo.branch}-${v4().slice(0, 4)}`
+  ).toLowerCase();
+
+  const outFolderPath =
+    process.env.NODE_ENV === "development"
+      ? path.resolve(
+          __dirname,
+          `../../../../../server/storage/documents/${outFolder}`
+        )
+      : path.resolve(process.env.STORAGE_DIR, `documents/${outFolder}`);
+
+  if (!fs.existsSync(outFolderPath))
+    fs.mkdirSync(outFolderPath, { recursive: true });
+
+  for (const doc of docs) {
+    if (!doc.pageContent) continue;
+    const data = {
+      id: v4(),
+      url: "github://" + doc.metadata.source,
+      title: doc.metadata.source,
+      docAuthor: repo.author,
+      description: "No description found.",
+      docSource: doc.metadata.source,
+      chunkSource: generateChunkSource(
+        repo,
+        doc,
+        response.locals.encryptionWorker
+      ),
+      published: new Date().toLocaleString(),
+      wordCount: doc.pageContent.split(" ").length,
+      pageContent: doc.pageContent,
+      token_count_estimate: tokenizeString(doc.pageContent).length,
+    };
+    console.log(
+      `[Github Loader]: Saving ${doc.metadata.source} to ${outFolder}`
+    );
+    writeToServerDocuments(
+      data,
+      `${slugify(doc.metadata.source)}-${data.id}`,
+      outFolderPath
+    );
+  }
+
+  return {
+    success: true,
+    reason: null,
+    data: {
+      author: repo.author,
+      repo: repo.project,
+      branch: repo.branch,
+      files: docs.length,
+      destination: outFolder,
+    },
+  };
+}
+
+/**
+ * Gets the page content from a specific source file in a give Github Repo, not all items in a repo.
+ * @returns
+ */
+async function fetchGithubFile({
+  repoUrl,
+  branch,
+  accessToken = null,
+  sourceFilePath,
+}) {
+  const repo = new RepoLoader({
+    repo: repoUrl,
+    branch,
+    accessToken,
+  });
+  await repo.init();
+
+  if (!repo.ready)
+    return {
+      success: false,
+      content: null,
+      reason: "Could not prepare Github repo for loading! Check URL or PAT.",
+    };
+
+  console.log(
+    `-- Working Github ${repo.author}/${repo.project}:${repo.branch} file:${sourceFilePath} --`
+  );
+  const fileContent = await repo.fetchSingleFile(sourceFilePath);
+  if (!fileContent) {
+    return {
+      success: false,
+      reason: "Target file returned a null content response.",
+      content: null,
+    };
+  }
+
+  return {
+    success: true,
+    reason: null,
+    content: fileContent,
+  };
+}
+
+/**
+ * Generate the full chunkSource for a specific file so that we can resync it later.
+ * This data is encrypted into a single `payload` query param so we can replay credentials later
+ * since this was encrypted with the systems persistent password and salt.
+ * @param {RepoLoader} repo
+ * @param {import("@langchain/core/documents").Document} doc
+ * @param {import("../../EncryptionWorker").EncryptionWorker} encryptionWorker
+ * @returns {string}
+ */
+function generateChunkSource(repo, doc, encryptionWorker) {
+  const payload = {
+    owner: repo.author,
+    project: repo.project,
+    branch: repo.branch,
+    path: doc.metadata.source,
+    pat: !!repo.accessToken ? repo.accessToken : null,
+  };
+  return `github://${repo.repo}?payload=${encryptionWorker.encrypt(
+    JSON.stringify(payload)
+  )}`;
+}
+
+module.exports = { loadGithubRepo, fetchGithubFile };

collector/utils/extensions/RepoLoader/GitlabRepo/RepoLoader/index.js

@@ -0,0 +1,285 @@
+const minimatch = require("minimatch");
+
+/**
+ * @typedef {Object} RepoLoaderArgs
+ * @property {string} repo - The GitLab repository URL.
+ * @property {string} [branch] - The branch to load from (optional).
+ * @property {string} [accessToken] - GitLab access token for authentication (optional).
+ * @property {string[]} [ignorePaths] - Array of paths to ignore when loading (optional).
+ */
+
+/**
+ * @typedef {Object} FileTreeObject
+ * @property {string} id - The file object ID.
+ * @property {string} name - name of file.
+ * @property {('blob'|'tree')} type - type of file object.
+ * @property {string} path - path + name of file.
+ * @property {string} mode - Linux permission code.
+ */
+
+/**
+ * @class
+ * @classdesc Loads and manages GitLab repository content.
+ */
+class GitLabRepoLoader {
+  /**
+   * Creates an instance of RepoLoader.
+   * @param {RepoLoaderArgs} [args] - The configuration options.
+   * @returns {GitLabRepoLoader}
+   */
+  constructor(args = {}) {
+    this.ready = false;
+    this.repo = args?.repo;
+    this.branch = args?.branch;
+    this.accessToken = args?.accessToken || null;
+    this.ignorePaths = args?.ignorePaths || [];
+
+    this.projectId = null;
+    this.apiBase = "https://gitlab.com";
+    this.author = null;
+    this.project = null;
+    this.branches = [];
+  }
+
+  #validGitlabUrl() {
+    const UrlPattern = require("url-pattern");
+    const validPatterns = [
+      new UrlPattern("https\\://gitlab.com/(:projectId(*))", {
+        segmentValueCharset: "a-zA-Z0-9-._~%/+",
+      }),
+      // This should even match the regular hosted URL, but we may want to know
+      // if this was a hosted GitLab (above) or a self-hosted (below) instance
+      // since the API interface could be different.
+      new UrlPattern(
+        "(:protocol(http|https))\\://(:hostname*)/(:projectId(*))",
+        {
+          segmentValueCharset: "a-zA-Z0-9-._~%/+",
+        }
+      ),
+    ];
+
+    let match = null;
+    for (const pattern of validPatterns) {
+      if (match !== null) continue;
+      match = pattern.match(this.repo);
+    }
+    if (!match) return false;
+    const [author, project] = match.projectId.split("/");
+
+    this.projectId = encodeURIComponent(match.projectId);
+    this.apiBase = new URL(this.repo).origin;
+    this.author = author;
+    this.project = project;
+    return true;
+  }
+
+  async #validBranch() {
+    await this.getRepoBranches();
+    if (!!this.branch && this.branches.includes(this.branch)) return;
+
+    console.log(
+      "[Gitlab Loader]: Branch not set! Auto-assigning to a default branch."
+    );
+    this.branch = this.branches.includes("main") ? "main" : "master";
+    console.log(`[Gitlab Loader]: Branch auto-assigned to ${this.branch}.`);
+    return;
+  }
+
+  async #validateAccessToken() {
+    if (!this.accessToken) return;
+    try {
+      await fetch(`${this.apiBase}/api/v4/user`, {
+        method: "GET",
+        headers: this.accessToken ? { "PRIVATE-TOKEN": this.accessToken } : {},
+      }).then((res) => res.ok);
+    } catch (e) {
+      console.error(
+        "Invalid Gitlab Access Token provided! Access token will not be used",
+        e.message
+      );
+      this.accessToken = null;
+    }
+  }
+
+  /**
+   * Initializes the RepoLoader instance.
+   * @returns {Promise<RepoLoader>} The initialized RepoLoader instance.
+   */
+  async init() {
+    if (!this.#validGitlabUrl()) return;
+    await this.#validBranch();
+    await this.#validateAccessToken();
+    this.ready = true;
+    return this;
+  }
+
+  /**
+   * Recursively loads the repository content.
+   * @returns {Promise<Array<Object>>} An array of loaded documents.
+   * @throws {Error} If the RepoLoader is not in a ready state.
+   */
+  async recursiveLoader() {
+    if (!this.ready) throw new Error("[Gitlab Loader]: not in ready state!");
+
+    if (this.accessToken)
+      console.log(
+        `[Gitlab Loader]: Access token set! Recursive loading enabled!`
+      );
+
+    const files = await this.fetchFilesRecursive();
+    const docs = [];
+
+    for (const file of files) {
+      if (this.ignorePaths.some((path) => file.path.includes(path))) continue;
+
+      const content = await this.fetchSingleFileContents(file.path);
+      if (content) {
+        docs.push({
+          pageContent: content,
+          metadata: { source: file.path },
+        });
+      }
+    }
+
+    return docs;
+  }
+
+  #branchPrefSort(branches = []) {
+    const preferredSort = ["main", "master"];
+    return branches.reduce((acc, branch) => {
+      if (preferredSort.includes(branch)) return [branch, ...acc];
+      return [...acc, branch];
+    }, []);
+  }
+
+  /**
+   * Retrieves all branches for the repository.
+   * @returns {Promise<string[]>} An array of branch names.
+   */
+  async getRepoBranches() {
+    if (!this.#validGitlabUrl() || !this.projectId) return [];
+    await this.#validateAccessToken();
+
+    try {
+      this.branches = await fetch(
+        `${this.apiBase}/api/v4/projects/${this.projectId}/repository/branches`,
+        {
+          method: "GET",
+          headers: {
+            Accepts: "application/json",
+            ...(this.accessToken ? { "PRIVATE-TOKEN": this.accessToken } : {}),
+          },
+        }
+      )
+        .then((res) => res.json())
+        .then((branches) => {
+          return branches.map((b) => b.name);
+        })
+        .catch((e) => {
+          console.error(e);
+          return [];
+        });
+
+      return this.#branchPrefSort(this.branches);
+    } catch (err) {
+      console.log(`RepoLoader.getRepoBranches`, err);
+      this.branches = [];
+      return [];
+    }
+  }
+
+  /**
+   * Returns list of all file objects from tree API for GitLab
+   * @returns {Promise<FileTreeObject[]>}
+   */
+  async fetchFilesRecursive() {
+    const files = [];
+    let perPage = 100;
+    let fetching = true;
+    let page = 1;
+
+    while (fetching) {
+      try {
+        const params = new URLSearchParams({
+          ref: this.branch,
+          recursive: true,
+          per_page: perPage,
+          page,
+        });
+        const queryUrl = `${this.apiBase}/api/v4/projects/${
+          this.projectId
+        }/repository/tree?${params.toString()}`;
+        const response = await fetch(queryUrl, {
+          method: "GET",
+          headers: this.accessToken
+            ? { "PRIVATE-TOKEN": this.accessToken }
+            : {},
+        });
+        const totalPages = Number(response.headers.get("x-total-pages"));
+        const nextPage = Number(response.headers.get("x-next-page"));
+        const data = await response.json();
+
+        /** @type {FileTreeObject[]} */
+        const objects = Array.isArray(data)
+          ? data.filter((item) => item.type === "blob")
+          : []; // only get files, not paths or submodules
+
+        // Apply ignore path rules to found objects. If any rules match it is an invalid file path.
+        console.log(
+          `Found ${objects.length} blobs from repo from pg ${page}/${totalPages}`
+        );
+        for (const file of objects) {
+          const isIgnored = this.ignorePaths.some((ignorePattern) =>
+            minimatch(file.path, ignorePattern, { matchBase: true })
+          );
+          if (!isIgnored) files.push(file);
+        }
+
+        if (page === totalPages) {
+          fetching = false;
+          break;
+        }
+
+        page = Number(nextPage);
+      } catch (e) {
+        console.error(`RepoLoader.getRepositoryTree`, e);
+        fetching = false;
+        break;
+      }
+    }
+    return files;
+  }
+
+  /**
+   * Fetches the content of a single file from the repository.
+   * @param {string} sourceFilePath - The path to the file in the repository.
+   * @returns {Promise<string|null>} The content of the file, or null if fetching fails.
+   */
+  async fetchSingleFileContents(sourceFilePath) {
+    try {
+      const data = await fetch(
+        `${this.apiBase}/api/v4/projects/${
+          this.projectId
+        }/repository/files/${encodeURIComponent(sourceFilePath)}/raw?ref=${
+          this.branch
+        }`,
+        {
+          method: "GET",
+          headers: this.accessToken
+            ? { "PRIVATE-TOKEN": this.accessToken }
+            : {},
+        }
+      ).then((res) => {
+        if (res.ok) return res.text();
+        throw new Error(`Failed to fetch single file ${sourceFilePath}`);
+      });
+
+      return data;
+    } catch (e) {
+      console.error(`RepoLoader.fetchSingleFileContents`, e);
+      return null;
+    }
+  }
+}
+
+module.exports = GitLabRepoLoader;

collector/utils/extensions/RepoLoader/GitlabRepo/index.js

@@ -0,0 +1,145 @@
+const RepoLoader = require("./RepoLoader");
+const fs = require("fs");
+const path = require("path");
+const { default: slugify } = require("slugify");
+const { v4 } = require("uuid");
+const { writeToServerDocuments } = require("../../../files");
+const { tokenizeString } = require("../../../tokenizer");
+
+/**
+ * Load in a Gitlab Repo recursively or just the top level if no PAT is provided
+ * @param {object} args - forwarded request body params
+ * @param {import("../../../middleware/setDataSigner").ResponseWithSigner} response - Express response object with encryptionWorker
+ * @returns
+ */
+async function loadGitlabRepo(args, response) {
+  const repo = new RepoLoader(args);
+  await repo.init();
+
+  if (!repo.ready)
+    return {
+      success: false,
+      reason: "Could not prepare Gitlab repo for loading! Check URL",
+    };
+
+  console.log(
+    `-- Working GitLab ${repo.author}/${repo.project}:${repo.branch} --`
+  );
+  const docs = await repo.recursiveLoader();
+  if (!docs.length) {
+    return {
+      success: false,
+      reason: "No files were found for those settings.",
+    };
+  }
+
+  console.log(`[GitLab Loader]: Found ${docs.length} source files. Saving...`);
+  const outFolder = slugify(
+    `${repo.author}-${repo.project}-${repo.branch}-${v4().slice(0, 4)}`
+  ).toLowerCase();
+
+  const outFolderPath =
+    process.env.NODE_ENV === "development"
+      ? path.resolve(
+          __dirname,
+          `../../../../../server/storage/documents/${outFolder}`
+        )
+      : path.resolve(process.env.STORAGE_DIR, `documents/${outFolder}`);
+
+  if (!fs.existsSync(outFolderPath))
+    fs.mkdirSync(outFolderPath, { recursive: true });
+
+  for (const doc of docs) {
+    if (!doc.pageContent) continue;
+    const data = {
+      id: v4(),
+      url: "gitlab://" + doc.metadata.source,
+      title: doc.metadata.source,
+      docAuthor: repo.author,
+      description: "No description found.",
+      docSource: doc.metadata.source,
+      chunkSource: generateChunkSource(
+        repo,
+        doc,
+        response.locals.encryptionWorker
+      ),
+      published: new Date().toLocaleString(),
+      wordCount: doc.pageContent.split(" ").length,
+      pageContent: doc.pageContent,
+      token_count_estimate: tokenizeString(doc.pageContent).length,
+    };
+    console.log(
+      `[GitLab Loader]: Saving ${doc.metadata.source} to ${outFolder}`
+    );
+    writeToServerDocuments(
+      data,
+      `${slugify(doc.metadata.source)}-${data.id}`,
+      outFolderPath
+    );
+  }
+
+  return {
+    success: true,
+    reason: null,
+    data: {
+      author: repo.author,
+      repo: repo.project,
+      projectId: repo.projectId,
+      branch: repo.branch,
+      files: docs.length,
+      destination: outFolder,
+    },
+  };
+}
+
+async function fetchGitlabFile({
+  repoUrl,
+  branch,
+  accessToken = null,
+  sourceFilePath,
+}) {
+  const repo = new RepoLoader({
+    repo: repoUrl,
+    branch,
+    accessToken,
+  });
+  await repo.init();
+
+  if (!repo.ready)
+    return {
+      success: false,
+      content: null,
+      reason: "Could not prepare GitLab repo for loading! Check URL or PAT.",
+    };
+  console.log(
+    `-- Working GitLab ${repo.author}/${repo.project}:${repo.branch} file:${sourceFilePath} --`
+  );
+  const fileContent = await repo.fetchSingleFile(sourceFilePath);
+  if (!fileContent) {
+    return {
+      success: false,
+      reason: "Target file returned a null content response.",
+      content: null,
+    };
+  }
+
+  return {
+    success: true,
+    reason: null,
+    content: fileContent,
+  };
+}
+
+function generateChunkSource(repo, doc, encryptionWorker) {
+  const payload = {
+    projectId: decodeURIComponent(repo.projectId),
+    branch: repo.branch,
+    path: doc.metadata.source,
+    pat: !!repo.accessToken ? repo.accessToken : null,
+  };
+  return `gitlab://${repo.repo}?payload=${encryptionWorker.encrypt(
+    JSON.stringify(payload)
+  )}`;
+}
+
+module.exports = { loadGitlabRepo, fetchGitlabFile };

collector/utils/extensions/RepoLoader/index.js

@@ -0,0 +1,41 @@
+/**
+ * Dynamically load the correct repository loader from a specific platform
+ * by default will return Github.
+ * @param {('github'|'gitlab')} platform
+ * @returns {import("./GithubRepo/RepoLoader")|import("./GitlabRepo/RepoLoader")} the repo loader class for provider
+ */
+function resolveRepoLoader(platform = "github") {
+  switch (platform) {
+    case "github":
+      console.log(`Loading GitHub RepoLoader...`);
+      return require("./GithubRepo/RepoLoader");
+    case "gitlab":
+      console.log(`Loading GitLab RepoLoader...`);
+      return require("./GitlabRepo/RepoLoader");
+    default:
+      console.log(`Loading GitHub RepoLoader...`);
+      return require("./GithubRepo/RepoLoader");
+  }
+}
+
+/**
+ * Dynamically load the correct repository loader function from a specific platform
+ * by default will return Github.
+ * @param {('github'|'gitlab')} platform
+ * @returns {import("./GithubRepo")['fetchGithubFile'] | import("./GitlabRepo")['fetchGitlabFile']} the repo loader class for provider
+ */
+function resolveRepoLoaderFunction(platform = "github") {
+  switch (platform) {
+    case "github":
+      console.log(`Loading GitHub loader function...`);
+      return require("./GithubRepo").loadGithubRepo;
+    case "gitlab":
+      console.log(`Loading GitLab loader function...`);
+      return require("./GitlabRepo").loadGitlabRepo;
+    default:
+      console.log(`Loading GitHub loader function...`);
+      return require("./GithubRepo").loadGithubRepo;
+  }
+}
+
+module.exports = { resolveRepoLoader, resolveRepoLoaderFunction };

collector/utils/extensions/WebsiteDepth/index.js

@@ -9,34 +9,36 @@ const { tokenizeString } = require("../../tokenizer");
 const path = require("path");
 const fs = require("fs");
 
-async function discoverLinks(startUrl, depth = 1, maxLinks = 20) {
-  const baseUrl = new URL(startUrl).origin;
-  const discoveredLinks = new Set();
-  const pendingLinks = [startUrl];
-  let currentLevel = 0;
-  depth = depth < 1 ? 1 : depth;
-  maxLinks = maxLinks < 1 ? 1 : maxLinks;
+async function discoverLinks(startUrl, maxDepth = 1, maxLinks = 20) {
+  const baseUrl = new URL(startUrl);
+  const discoveredLinks = new Set([startUrl]);
+  let queue = [[startUrl, 0]]; // [url, currentDepth]
+  const scrapedUrls = new Set();
 
-  // Check depth and if there are any links left to scrape
-  while (currentLevel < depth && pendingLinks.length > 0) {
-    const newLinks = await getPageLinks(pendingLinks[0], baseUrl);
-    pendingLinks.shift();
+  for (let currentDepth = 0; currentDepth < maxDepth; currentDepth++) {
+    const levelSize = queue.length;
+    const nextQueue = [];
+
+    for (let i = 0; i < levelSize && discoveredLinks.size < maxLinks; i++) {
+      const [currentUrl, urlDepth] = queue[i];
+
+      if (!scrapedUrls.has(currentUrl)) {
+        scrapedUrls.add(currentUrl);
+        const newLinks = await getPageLinks(currentUrl, baseUrl);
 
         for (const link of newLinks) {
-      if (!discoveredLinks.has(link)) {
+          if (!discoveredLinks.has(link) && discoveredLinks.size < maxLinks) {
             discoveredLinks.add(link);
-        pendingLinks.push(link);
+            if (urlDepth + 1 < maxDepth) {
+              nextQueue.push([link, urlDepth + 1]);
+            }
+          }
         }
-
-      // Exit out if we reach maxLinks
-      if (discoveredLinks.size >= maxLinks) {
-        return Array.from(discoveredLinks).slice(0, maxLinks);
       }
     }
 
-    if (pendingLinks.length === 0) {
-      currentLevel++;
-    }
+    queue = nextQueue;
+    if (queue.length === 0 || discoveredLinks.size >= maxLinks) break;
   }
 
   return Array.from(discoveredLinks);
@@ -66,8 +68,12 @@ function extractLinks(html, baseUrl) {
   for (const link of links) {
     const href = link.getAttribute("href");
     if (href) {
-      const absoluteUrl = new URL(href, baseUrl).href;
-      if (absoluteUrl.startsWith(baseUrl)) {
+      const absoluteUrl = new URL(href, baseUrl.href).href;
+      if (
+        absoluteUrl.startsWith(
+          baseUrl.origin + baseUrl.pathname.split("/").slice(0, -1).join("/")
+        )
+      ) {
         extractedLinks.add(absoluteUrl);
       }
     }

collector/utils/extensions/YoutubeTranscript/YoutubeLoader/youtube-transcript.js

@@ -47,10 +47,12 @@ class YoutubeTranscript {
       let transcript = "";
       const chunks = transcriptXML.getElementsByTagName("text");
       for (const chunk of chunks) {
-        transcript += chunk.textContent;
+        // Add space after each text chunk
+        transcript += chunk.textContent + " ";
       }
 
-      return transcript;
+      // Trim extra whitespace
+      return transcript.trim().replace(/\s+/g, " ");
     } catch (e) {
       throw new YoutubeTranscriptError(e);
     }

collector/utils/extensions/YoutubeTranscript/index.js

@@ -26,11 +26,13 @@ function validYoutubeVideoUrl(link) {
   return false;
 }
 
-async function loadYouTubeTranscript({ url }) {
+async function fetchVideoTranscriptContent({ url }) {
   if (!validYoutubeVideoUrl(url)) {
     return {
       success: false,
       reason: "Invalid URL. Should be youtu.be or youtube.com/watch.",
+      content: null,
+      metadata: {},
     };
   }
 
@@ -52,6 +54,8 @@ async function loadYouTubeTranscript({ url }) {
     return {
       success: false,
       reason: error ?? "No transcript found for that YouTube video.",
+      content: null,
+      metadata: {},
     };
   }
 
@@ -61,9 +65,30 @@ async function loadYouTubeTranscript({ url }) {
     return {
       success: false,
       reason: "No transcript could be parsed for that YouTube video.",
+      content: null,
+      metadata: {},
     };
   }
 
+  return {
+    success: true,
+    reason: null,
+    content,
+    metadata,
+  };
+}
+
+async function loadYouTubeTranscript({ url }) {
+  const transcriptResults = await fetchVideoTranscriptContent({ url });
+  if (!transcriptResults.success) {
+    return {
+      success: false,
+      reason:
+        transcriptResults.reason ||
+        "An unknown error occurred during transcription retrieval",
+    };
+  }
+  const { content, metadata } = transcriptResults;
   const outFolder = slugify(
     `${metadata.author} YouTube transcripts`
   ).toLowerCase();
@@ -86,7 +111,7 @@ async function loadYouTubeTranscript({ url }) {
     docAuthor: metadata.author,
     description: metadata.description,
     docSource: url,
-    chunkSource: `link://${url}`,
+    chunkSource: `youtube://${url}`,
     published: new Date().toLocaleString(),
     wordCount: content.split(" ").length,
     pageContent: content,
@@ -111,4 +136,7 @@ async function loadYouTubeTranscript({ url }) {
   };
 }
 
-module.exports = loadYouTubeTranscript;
+module.exports = {
+  loadYouTubeTranscript,
+  fetchVideoTranscriptContent,
+};

collector/utils/files/index.js

@@ -129,6 +129,11 @@ function normalizePath(filepath = "") {
   return result;
 }
 
+function sanitizeFileName(fileName) {
+  if (!fileName) return fileName;
+  return fileName.replace(/[<>:"\/\\|?*]/g, "");
+}
+
 module.exports = {
   trashFile,
   isTextType,
@@ -137,4 +142,5 @@ module.exports = {
   wipeCollectorStorage,
   normalizePath,
   isWithin,
+  sanitizeFileName,
 };

collector/utils/files/mime.js

@@ -37,6 +37,7 @@ class MimeDetector {
           "lua",
           "pas",
           "r",
+          "go",
         ],
       },
       true

collector/utils/logger/index.js

@@ -0,0 +1,68 @@
+const winston = require("winston");
+
+class Logger {
+  logger = console;
+  static _instance;
+  constructor() {
+    if (Logger._instance) return Logger._instance;
+    this.logger =
+      process.env.NODE_ENV === "production" ? this.getWinstonLogger() : console;
+    Logger._instance = this;
+  }
+
+  getWinstonLogger() {
+    const logger = winston.createLogger({
+      level: "info",
+      defaultMeta: { service: "collector" },
+      transports: [
+        new winston.transports.Console({
+          format: winston.format.combine(
+            winston.format.colorize(),
+            winston.format.printf(
+              ({ level, message, service, origin = "" }) => {
+                return `\x1b[36m[${service}]\x1b[0m${
+                  origin ? `\x1b[33m[${origin}]\x1b[0m` : ""
+                } ${level}: ${message}`;
+              }
+            )
+          ),
+        }),
+      ],
+    });
+
+    function formatArgs(args) {
+      return args
+        .map((arg) => {
+          if (arg instanceof Error) {
+            return arg.stack; // If argument is an Error object, return its stack trace
+          } else if (typeof arg === "object") {
+            return JSON.stringify(arg); // Convert objects to JSON string
+          } else {
+            return arg; // Otherwise, return as-is
+          }
+        })
+        .join(" ");
+    }
+
+    console.log = function (...args) {
+      logger.info(formatArgs(args));
+    };
+    console.error = function (...args) {
+      logger.error(formatArgs(args));
+    };
+    console.info = function (...args) {
+      logger.warn(formatArgs(args));
+    };
+    return logger;
+  }
+}
+
+/**
+ * Sets and overrides Console methods for logging when called.
+ * This is a singleton method and will not create multiple loggers.
+ * @returns {winston.Logger | console} - instantiated logger interface.
+ */
+function setLogger() {
+  return new Logger().logger;
+}
+module.exports = setLogger;

collector/utils/url/index.js

@@ -1,3 +1,16 @@
+/**  ATTN: SECURITY RESEARCHERS
+ * To Security researchers about to submit an SSRF report CVE - please don't.
+ * We are aware that the code below is does not defend against any of the thousands of ways
+ * you can map a hostname to another IP. The code below does not have intention of blocking this
+ * and is simply to prevent the user from accidentally putting in non-valid websites, which is all this protects
+ * since _all urls must be submitted by the user anyway_ and cannot be done with authentication and manager or admin roles.
+ * If an attacker has those roles then the system is already vulnerable and this is not a primary concern.
+ *
+ * We have gotten this report may times, marked them as duplicate or information and continue to get them. We communicate
+ * already that deployment (and security) of an instance is on the deployer and system admin deploying it. This would include
+ * isolation, firewalls, and the general security of the instance.
+ */
+
 const VALID_PROTOCOLS = ["https:", "http:"];
 const INVALID_OCTETS = [192, 172, 10, 127];
 

collector/yarn.lock

@@ -40,18 +40,25 @@
     js-tokens "^4.0.0"
     picocolors "^1.0.0"
 
+"@colors/colors@1.6.0", "@colors/colors@^1.6.0":
+  version "1.6.0"
+  resolved "https://registry.yarnpkg.com/@colors/colors/-/colors-1.6.0.tgz#ec6cd237440700bc23ca23087f513c75508958b0"
+  integrity sha512-Ir+AOibqzrIsL6ajt3Rz3LskB7OiMVHqltZmspbW/TJuTVuyOMirVqAkjfY6JISiLHgyNqicAC8AyHHGzNd/dA==
+
+"@dabh/diagnostics@^2.0.2":
+  version "2.0.3"
+  resolved "https://registry.yarnpkg.com/@dabh/diagnostics/-/diagnostics-2.0.3.tgz#7f7e97ee9a725dffc7808d93668cc984e1dc477a"
+  integrity sha512-hrlQOIi7hAfzsMqlGSFyVucrx38O+j6wiGOf//H2ecvIEqYN4ADBSS2iLMh5UFyDunCNniUIPk/q3riFv45xRA==
+  dependencies:
+    colorspace "1.1.x"
+    enabled "2.0.x"
+    kuler "^2.0.0"
+
 "@fastify/busboy@^2.0.0":
   version "2.1.1"
   resolved "https://registry.yarnpkg.com/@fastify/busboy/-/busboy-2.1.1.tgz#b9da6a878a371829a0502c9b6c1c143ef6663f4d"
   integrity sha512-vBZP4NlzfOlerQTnba4aqZoMhE/a9HY7HRqoOPaETQcSQuWEIyZMHGfVu6w9wGtGK5fED5qRs2DteVCjOH60sA==
 
-"@googleapis/youtube@^9.0.0":
-  version "9.0.0"
-  resolved "https://registry.yarnpkg.com/@googleapis/youtube/-/youtube-9.0.0.tgz#e45f6f5f7eac198c6391782b94b3ca54bacf0b63"
-  integrity sha512-abCi9o1nfODVsPDqiq/QCHte2rXgMXS7ssHoeJIWiRkbvECNNyYFtmXz96oWGMw8Pnpm3gZzMXsmC9cSdMDnvg==
-  dependencies:
-    googleapis-common "^6.0.3"
-
 "@huggingface/jinja@^0.2.2":
   version "0.2.2"
   resolved "https://registry.yarnpkg.com/@huggingface/jinja/-/jinja-0.2.2.tgz#faeb205a9d6995089bef52655ddd8245d3190627"
@@ -247,6 +254,11 @@
   resolved "https://registry.yarnpkg.com/@types/retry/-/retry-0.12.0.tgz#2b35eccfcee7d38cd72ad99232fbd58bffb3c84d"
   integrity sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA==
 
+"@types/triple-beam@^1.3.2":
+  version "1.3.5"
+  resolved "https://registry.yarnpkg.com/@types/triple-beam/-/triple-beam-1.3.5.tgz#74fef9ffbaa198eb8b588be029f38b00299caa2c"
+  integrity sha512-6WaYesThRMCl19iryMYP7/x2OVgCtbIVflDGFpWnb9irXI3UjYE4AzmYuiUKY1AJstGijoY+MgUszMgRxIYTYw==
+
 "@types/uuid@^9.0.1":
   version "9.0.8"
   resolved "https://registry.yarnpkg.com/@types/uuid/-/uuid-9.0.8.tgz#7545ba4fc3c003d6c756f651f3bf163d8f0f29ba"
@@ -358,11 +370,6 @@ anymatch@~3.1.2:
     normalize-path "^3.0.0"
     picomatch "^2.0.4"
 
-append-field@^1.0.0:
-  version "1.0.0"
-  resolved "https://registry.yarnpkg.com/append-field/-/append-field-1.0.0.tgz#1e3440e915f0b1203d23748e78edd7b9b5b43e56"
-  integrity sha512-klpgFSWLW1ZEs8svjfb7g4qWY0YS5imI82dTg+QahUvJ8YqAY0P10Uk8tTyh9ZGuYEZEMaeJYCF5BFuX552hsw==
-
 "aproba@^1.0.3 || ^2.0.0":
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/aproba/-/aproba-2.0.0.tgz#52520b8ae5b569215b354efc0caa3fe1e45a8adc"
@@ -401,11 +408,6 @@ array-hyper-unique@^2.1.4:
     deep-eql "= 4.0.0"
     lodash "^4.17.21"
 
-arrify@^2.0.0:
-  version "2.0.1"
-  resolved "https://registry.yarnpkg.com/arrify/-/arrify-2.0.1.tgz#c9655e9331e0abcd588d2a7cad7e9956f66701fa"
-  integrity sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug==
-
 ast-types@^0.13.4:
   version "0.13.4"
   resolved "https://registry.yarnpkg.com/ast-types/-/ast-types-0.13.4.tgz#ee0d77b343263965ecc3fb62da16e7222b2b6782"
@@ -413,7 +415,7 @@ ast-types@^0.13.4:
   dependencies:
     tslib "^2.0.1"
 
-async@>=0.2.9:
+async@>=0.2.9, async@^3.2.3:
   version "3.2.5"
   resolved "https://registry.yarnpkg.com/async/-/async-3.2.5.tgz#ebd52a8fdaf7a2289a24df399f8d8485c8a46b66"
   integrity sha512-baNZyqaaLhyLVKm/DlvdW051MSgO6b8eVfIezl9E5PqWxFgzLm/wQntEW4zOytVburDEr0JlALEpdOFwvErLsg==
@@ -471,7 +473,7 @@ base-64@^0.1.0:
   resolved "https://registry.yarnpkg.com/base-64/-/base-64-0.1.0.tgz#780a99c84e7d600260361511c4877613bf24f6bb"
   integrity sha512-Y5gU45svrR5tI2Vt/X9GPd3L0HNIKzGu202EjxrXMpuc2V2CiKgemAbUUsqYmZJvPtCXoUKjNZwBJzsNScUbXA==
 
-base64-js@^1.3.0, base64-js@^1.3.1, base64-js@^1.5.1:
+base64-js@^1.3.1, base64-js@^1.5.1:
   version "1.5.1"
   resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.5.1.tgz#1b1b440160a5bf7ad40b650f095963481903930a"
   integrity sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==
@@ -489,11 +491,6 @@ bcrypt@^5.1.0:
     "@mapbox/node-pre-gyp" "^1.0.11"
     node-addon-api "^5.0.0"
 
-bignumber.js@^9.0.0:
-  version "9.1.2"
-  resolved "https://registry.yarnpkg.com/bignumber.js/-/bignumber.js-9.1.2.tgz#b7c4242259c008903b13707983b5f4bbd31eda0c"
-  integrity sha512-2/mKyZH9K85bzOEfhXDBFZTGd1CTs+5IHpeFQo9luiBG7hghdC851Pj2WAhb6E3R6b9tZj/XKhbg4fum+Kepug==
-
 binary-extensions@^2.0.0, binary-extensions@^2.2.0:
   version "2.3.0"
   resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-2.3.0.tgz#f6e14a97858d327252200242d4ccfe522c445522"
@@ -562,6 +559,13 @@ brace-expansion@^1.1.7:
     balanced-match "^1.0.0"
     concat-map "0.0.1"
 
+brace-expansion@^2.0.1:
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-2.0.1.tgz#1edc459e0f0c548486ecf9fc99f2221364b9a0ae"
+  integrity sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==
+  dependencies:
+    balanced-match "^1.0.0"
+
 braces@~3.0.2:
   version "3.0.2"
   resolved "https://registry.yarnpkg.com/braces/-/braces-3.0.2.tgz#3454e1a462ee8d599e236df336cd9ea4f8afe107"
@@ -587,21 +591,11 @@ buffer-crc32@~0.2.3:
   resolved "https://registry.yarnpkg.com/buffer-crc32/-/buffer-crc32-0.2.13.tgz#0d333e3f00eac50aa1454abd30ef8c2a5d9a7242"
   integrity sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==
 
-buffer-equal-constant-time@1.0.1:
-  version "1.0.1"
-  resolved "https://registry.yarnpkg.com/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz#f8e71132f7ffe6e01a5c9697a4c6f3e48d5cc819"
-  integrity sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==
-
 buffer-fill@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/buffer-fill/-/buffer-fill-1.0.0.tgz#f8f78b76789888ef39f205cd637f68e702122b2c"
   integrity sha512-T7zexNBwiiaCOGDg9xNX9PBmjrubblRkENuptryuI64URkXDFum9il/JGL8Lm8wYfAXpredVXXZz7eMHilimiQ==
 
-buffer-from@^1.0.0:
-  version "1.1.2"
-  resolved "https://registry.yarnpkg.com/buffer-from/-/buffer-from-1.1.2.tgz#2b146a6fd72e80b4f55d255f35ed59a3a9a41bd5"
-  integrity sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==
-
 buffer@^5.2.1, buffer@^5.5.0:
   version "5.7.1"
   resolved "https://registry.yarnpkg.com/buffer/-/buffer-5.7.1.tgz#ba62e7c13133053582197160851a8f648e99eed0"
@@ -610,13 +604,6 @@ buffer@^5.2.1, buffer@^5.5.0:
     base64-js "^1.3.1"
     ieee754 "^1.1.13"
 
-busboy@^1.0.0:
-  version "1.6.0"
-  resolved "https://registry.yarnpkg.com/busboy/-/busboy-1.6.0.tgz#966ea36a9502e43cdb9146962523b92f531f6893"
-  integrity sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA==
-  dependencies:
-    streamsearch "^1.1.0"
-
 bytes@3.1.2:
   version "3.1.2"
   resolved "https://registry.yarnpkg.com/bytes/-/bytes-3.1.2.tgz#8b0beeb98605adf1b128fa4386403c009e0221a5"
@@ -699,7 +686,7 @@ cliui@^8.0.1:
     strip-ansi "^6.0.1"
     wrap-ansi "^7.0.0"
 
-color-convert@^1.9.0:
+color-convert@^1.9.0, color-convert@^1.9.3:
   version "1.9.3"
   resolved "https://registry.yarnpkg.com/color-convert/-/color-convert-1.9.3.tgz#bb71850690e1f136567de629d2d5471deda4c1e8"
   integrity sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==
@@ -723,7 +710,7 @@ color-name@^1.0.0, color-name@~1.1.4:
   resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.4.tgz#c2a09a87acbde69543de6f63fa3995c826c536a2"
   integrity sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==
 
-color-string@^1.9.0:
+color-string@^1.6.0, color-string@^1.9.0:
   version "1.9.1"
   resolved "https://registry.yarnpkg.com/color-string/-/color-string-1.9.1.tgz#4467f9146f036f855b764dfb5bf8582bf342c7a4"
   integrity sha512-shrVawQFojnZv6xM40anx4CkoDP+fZsw/ZerEMsW/pyzsRbElpsL/DBVW7q3ExxwusdNXI3lXpuhEZkzs8p5Eg==
@@ -736,6 +723,14 @@ color-support@^1.1.2:
   resolved "https://registry.yarnpkg.com/color-support/-/color-support-1.1.3.tgz#93834379a1cc9a0c61f82f52f0d04322251bd5a2"
   integrity sha512-qiBjkpbMLO/HL68y+lh4q0/O1MZFj2RX6X/KmMa3+gJD3z+WwI1ZzDHysvqHGS3mP6mznPckpXmw1nI9cJjyRg==
 
+color@^3.1.3:
+  version "3.2.1"
+  resolved "https://registry.yarnpkg.com/color/-/color-3.2.1.tgz#3544dc198caf4490c3ecc9a790b54fe9ff45e164"
+  integrity sha512-aBl7dZI9ENN6fUGC7mWpMTPNHmWUSNan9tuWN6ahh5ZLNk9baLJOnSMlrQkHcrfFgz2/RigjUVAjdx36VcemKA==
+  dependencies:
+    color-convert "^1.9.3"
+    color-string "^1.6.0"
+
 color@^4.2.3:
   version "4.2.3"
   resolved "https://registry.yarnpkg.com/color/-/color-4.2.3.tgz#d781ecb5e57224ee43ea9627560107c0e0c6463a"
@@ -744,6 +739,14 @@ color@^4.2.3:
     color-convert "^2.0.1"
     color-string "^1.9.0"
 
+colorspace@1.1.x:
+  version "1.1.4"
+  resolved "https://registry.yarnpkg.com/colorspace/-/colorspace-1.1.4.tgz#8d442d1186152f60453bf8070cd66eb364e59243"
+  integrity sha512-BgvKJiuVu1igBUF2kEjRCZXol6wiiGbY5ipL/oVPwm0BL9sIpMIzM8IK7vwuxIIzOXMV3Ey5w+vxhm0rR/TN8w==
+  dependencies:
+    color "^3.1.3"
+    text-hex "1.0.x"
+
 combined-stream@^1.0.8:
   version "1.0.8"
   resolved "https://registry.yarnpkg.com/combined-stream/-/combined-stream-1.0.8.tgz#c3d45a8b34fd730631a110a8a2520682b31d5a7f"
@@ -766,16 +769,6 @@ concat-map@0.0.1:
   resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b"
   integrity sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==
 
-concat-stream@^1.5.2:
-  version "1.6.2"
-  resolved "https://registry.yarnpkg.com/concat-stream/-/concat-stream-1.6.2.tgz#904bdf194cd3122fc675c77fc4ac3d4ff0fd1a34"
-  integrity sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==
-  dependencies:
-    buffer-from "^1.0.0"
-    inherits "^2.0.3"
-    readable-stream "^2.2.2"
-    typedarray "^0.0.6"
-
 console-control-strings@^1.0.0, console-control-strings@^1.1.0:
   version "1.1.0"
   resolved "https://registry.yarnpkg.com/console-control-strings/-/console-control-strings-1.1.0.tgz#3d7cf4464db6446ea644bf4b39507f9851008e8e"
@@ -1072,13 +1065,6 @@ duck@^0.1.12:
   dependencies:
     underscore "^1.13.1"
 
-ecdsa-sig-formatter@1.0.11, ecdsa-sig-formatter@^1.0.11:
-  version "1.0.11"
-  resolved "https://registry.yarnpkg.com/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz#ae0f0fa2d85045ef14a817daa3ce9acd0489e5bf"
-  integrity sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==
-  dependencies:
-    safe-buffer "^5.0.1"
-
 ee-first@1.1.1:
   version "1.1.1"
   resolved "https://registry.yarnpkg.com/ee-first/-/ee-first-1.1.1.tgz#590c61156b0ae2f4f0255732a158b266bc56b21d"
@@ -1089,6 +1075,11 @@ emoji-regex@^8.0.0:
   resolved "https://registry.yarnpkg.com/emoji-regex/-/emoji-regex-8.0.0.tgz#e818fd69ce5ccfcb404594f842963bf53164cc37"
   integrity sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==
 
+enabled@2.0.x:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/enabled/-/enabled-2.0.0.tgz#f9dd92ec2d6f4bbc0d5d1e64e21d61cd4665e7c2"
+  integrity sha512-AKrN98kuwOzMIdAizXGI86UFBoo26CL21UM763y1h/GMSJ4/OHU9k2YlsmBpyScFo/wbLzWQJBMCW4+IO3/+OQ==
+
 encodeurl@~1.0.2:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/encodeurl/-/encodeurl-1.0.2.tgz#ad3ff4c86ec2d029322f5a02c3a9a606c95b3f59"
@@ -1250,12 +1241,7 @@ express@^4.18.2:
     utils-merge "1.0.1"
     vary "~1.1.2"
 
-extend@^3.0.2:
-  version "3.0.2"
-  resolved "https://registry.yarnpkg.com/extend/-/extend-3.0.2.tgz#f8b1136b4071fbd8eb140aff858b1019ec2915fa"
-  integrity sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==
-
-extract-zip@2.0.1, extract-zip@^2.0.1:
+extract-zip@2.0.1:
   version "2.0.1"
   resolved "https://registry.yarnpkg.com/extract-zip/-/extract-zip-2.0.1.tgz#663dca56fe46df890d5f131ef4a06d22bb8ba13a"
   integrity sha512-GDhU9ntwuKyGXdZBUgTIe+vXnWj0fppUEtMDL0+idd5Sta8TGpHssn/eusA9mrPr9qNDym6SxAYZjNvCn/9RBg==
@@ -1271,11 +1257,6 @@ fast-fifo@^1.1.0, fast-fifo@^1.2.0:
   resolved "https://registry.yarnpkg.com/fast-fifo/-/fast-fifo-1.3.2.tgz#286e31de96eb96d38a97899815740ba2a4f3640c"
   integrity sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==
 
-fast-text-encoding@^1.0.0:
-  version "1.0.6"
-  resolved "https://registry.yarnpkg.com/fast-text-encoding/-/fast-text-encoding-1.0.6.tgz#0aa25f7f638222e3396d72bf936afcf1d42d6867"
-  integrity sha512-VhXlQgj9ioXCqGstD37E/HBeqEGV/qOD/kmbVG8h5xKBYvM1L3lR1Zn4555cQ8GkYbJa8aJSipLPndE1k6zK2w==
-
 fd-slicer@~1.1.0:
   version "1.1.0"
   resolved "https://registry.yarnpkg.com/fd-slicer/-/fd-slicer-1.1.0.tgz#25c7c89cb1f9077f8891bbe61d8f390eae256f1e"
@@ -1283,6 +1264,11 @@ fd-slicer@~1.1.0:
   dependencies:
     pend "~1.2.0"
 
+fecha@^4.2.0:
+  version "4.2.3"
+  resolved "https://registry.yarnpkg.com/fecha/-/fecha-4.2.3.tgz#4d9ccdbc61e8629b259fdca67e65891448d569fd"
+  integrity sha512-OP2IUU6HeYKJi3i0z4A19kHMQoLVs4Hc+DPqqxI2h/DPZHTm/vjsfC6P0b4jCMy14XizLBqvndQ+UilD7707Jw==
+
 file-type@^16.5.4:
   version "16.5.4"
   resolved "https://registry.yarnpkg.com/file-type/-/file-type-16.5.4.tgz#474fb4f704bee427681f98dd390058a172a6c2fd"
@@ -1345,6 +1331,11 @@ fluent-ffmpeg@^2.1.2:
     async ">=0.2.9"
     which "^1.1.1"
 
+fn.name@1.x.x:
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/fn.name/-/fn.name-1.1.0.tgz#26cad8017967aea8731bc42961d04a3d5988accc"
+  integrity sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw==
+
 form-data-encoder@1.7.2:
   version "1.7.2"
   resolved "https://registry.yarnpkg.com/form-data-encoder/-/form-data-encoder-1.7.2.tgz#1f1ae3dccf58ed4690b86d87e4f57c654fbab040"
@@ -1428,24 +1419,6 @@ gauge@^3.0.0:
     strip-ansi "^6.0.1"
     wide-align "^1.1.2"
 
-gaxios@^5.0.0, gaxios@^5.0.1:
-  version "5.1.3"
-  resolved "https://registry.yarnpkg.com/gaxios/-/gaxios-5.1.3.tgz#f7fa92da0fe197c846441e5ead2573d4979e9013"
-  integrity sha512-95hVgBRgEIRQQQHIbnxBXeHbW4TqFk4ZDJW7wmVtvYar72FdhRIo1UGOLS2eRAKCPEdPBWu+M7+A33D9CdX9rA==
-  dependencies:
-    extend "^3.0.2"
-    https-proxy-agent "^5.0.0"
-    is-stream "^2.0.0"
-    node-fetch "^2.6.9"
-
-gcp-metadata@^5.3.0:
-  version "5.3.0"
-  resolved "https://registry.yarnpkg.com/gcp-metadata/-/gcp-metadata-5.3.0.tgz#6f45eb473d0cb47d15001476b48b663744d25408"
-  integrity sha512-FNTkdNEnBdlqF2oatizolQqNANMrcqJt6AAYt99B3y1aLLC8Hc5IOBb+ZnnzllodEEf6xMBp6wRcBbc16fa65w==
-  dependencies:
-    gaxios "^5.0.0"
-    json-bigint "^1.0.0"
-
 get-caller-file@^2.0.5:
   version "2.0.5"
   resolved "https://registry.yarnpkg.com/get-caller-file/-/get-caller-file-2.0.5.tgz#4f94412a82db32f36e3b0b9741f8a97feb031f7e"
@@ -1511,40 +1484,6 @@ glob@^7.1.3:
     once "^1.3.0"
     path-is-absolute "^1.0.0"
 
-google-auth-library@^8.0.2:
-  version "8.9.0"
-  resolved "https://registry.yarnpkg.com/google-auth-library/-/google-auth-library-8.9.0.tgz#15a271eb2ec35d43b81deb72211bd61b1ef14dd0"
-  integrity sha512-f7aQCJODJFmYWN6PeNKzgvy9LI2tYmXnzpNDHEjG5sDNPgGb2FXQyTBnXeSH+PAtpKESFD+LmHw3Ox3mN7e1Fg==
-  dependencies:
-    arrify "^2.0.0"
-    base64-js "^1.3.0"
-    ecdsa-sig-formatter "^1.0.11"
-    fast-text-encoding "^1.0.0"
-    gaxios "^5.0.0"
-    gcp-metadata "^5.3.0"
-    gtoken "^6.1.0"
-    jws "^4.0.0"
-    lru-cache "^6.0.0"
-
-google-p12-pem@^4.0.0:
-  version "4.0.1"
-  resolved "https://registry.yarnpkg.com/google-p12-pem/-/google-p12-pem-4.0.1.tgz#82841798253c65b7dc2a4e5fe9df141db670172a"
-  integrity sha512-WPkN4yGtz05WZ5EhtlxNDWPhC4JIic6G8ePitwUWy4l+XPVYec+a0j0Ts47PDtW59y3RwAhUd9/h9ZZ63px6RQ==
-  dependencies:
-    node-forge "^1.3.1"
-
-googleapis-common@^6.0.3:
-  version "6.0.4"
-  resolved "https://registry.yarnpkg.com/googleapis-common/-/googleapis-common-6.0.4.tgz#bd968bef2a478bcd3db51b27655502a11eaf8bf4"
-  integrity sha512-m4ErxGE8unR1z0VajT6AYk3s6a9gIMM6EkDZfkPnES8joeOlEtFEJeF8IyZkb0tjPXkktUfYrE4b3Li1DNyOwA==
-  dependencies:
-    extend "^3.0.2"
-    gaxios "^5.0.1"
-    google-auth-library "^8.0.2"
-    qs "^6.7.0"
-    url-template "^2.0.8"
-    uuid "^9.0.0"
-
 gopd@^1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/gopd/-/gopd-1.0.1.tgz#29ff76de69dac7489b7c0918a5788e56477c332c"
@@ -1557,15 +1496,6 @@ graceful-fs@^4.1.10, graceful-fs@^4.1.6, graceful-fs@^4.2.0:
   resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.11.tgz#4183e4e8bf08bb6e05bbb2f7d2e0c8f712ca40e3"
   integrity sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==
 
-gtoken@^6.1.0:
-  version "6.1.2"
-  resolved "https://registry.yarnpkg.com/gtoken/-/gtoken-6.1.2.tgz#aeb7bdb019ff4c3ba3ac100bbe7b6e74dce0e8bc"
-  integrity sha512-4ccGpzz7YAr7lxrT2neugmXQ3hP9ho2gcaityLVkiUecAiwiy60Ii8gRbZeOsXV19fYaRjgBSshs8kXw+NKCPQ==
-  dependencies:
-    gaxios "^5.0.1"
-    google-p12-pem "^4.0.0"
-    jws "^4.0.0"
-
 guid-typescript@^1.0.9:
   version "1.0.9"
   resolved "https://registry.yarnpkg.com/guid-typescript/-/guid-typescript-1.0.9.tgz#e35f77003535b0297ea08548f5ace6adb1480ddc"
@@ -1856,13 +1786,6 @@ jsbn@1.1.0:
   resolved "https://registry.yarnpkg.com/jsbn/-/jsbn-1.1.0.tgz#b01307cb29b618a1ed26ec79e911f803c4da0040"
   integrity sha512-4bYVV3aAMtDTTu4+xsDYa6sy9GyJ69/amsu9sYF2zqjiEoZA5xJi3BrfX3uY+/IekIu7MwdObdbDWpoZdBv3/A==
 
-json-bigint@^1.0.0:
-  version "1.0.0"
-  resolved "https://registry.yarnpkg.com/json-bigint/-/json-bigint-1.0.0.tgz#ae547823ac0cad8398667f8cd9ef4730f5b01ff1"
-  integrity sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==
-  dependencies:
-    bignumber.js "^9.0.0"
-
 json-parse-even-better-errors@^2.3.0:
   version "2.3.1"
   resolved "https://registry.yarnpkg.com/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz#7c47805a94319928e05777405dc12e1f7a4ee02d"
@@ -1892,22 +1815,10 @@ jszip@^3.7.1:
     readable-stream "~2.3.6"
     setimmediate "^1.0.5"
 
-jwa@^2.0.0:
+kuler@^2.0.0:
   version "2.0.0"
-  resolved "https://registry.yarnpkg.com/jwa/-/jwa-2.0.0.tgz#a7e9c3f29dae94027ebcaf49975c9345593410fc"
-  integrity sha512-jrZ2Qx916EA+fq9cEAeCROWPTfCwi1IVHqT2tapuqLEVVDKFDENFw1oL+MwrTvH6msKxsd1YTDVw6uKEcsrLEA==
-  dependencies:
-    buffer-equal-constant-time "1.0.1"
-    ecdsa-sig-formatter "1.0.11"
-    safe-buffer "^5.0.1"
-
-jws@^4.0.0:
-  version "4.0.0"
-  resolved "https://registry.yarnpkg.com/jws/-/jws-4.0.0.tgz#2d4e8cf6a318ffaa12615e9dec7e86e6c97310f4"
-  integrity sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg==
-  dependencies:
-    jwa "^2.0.0"
-    safe-buffer "^5.0.1"
+  resolved "https://registry.yarnpkg.com/kuler/-/kuler-2.0.0.tgz#e2c570a3800388fb44407e851531c1d670b061b3"
+  integrity sha512-Xq9nH7KlWZmXAtodXDDRE7vs6DU1gTU8zYDHDiWLSip45Egwq3plLHzPn27NgvzL2r1LMPC1vdqh98sQxtqj4A==
 
 langchain@0.1.36:
   version "0.1.36"
@@ -2018,6 +1929,18 @@ lodash@^4.17.21:
   resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c"
   integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==
 
+logform@^2.3.2, logform@^2.4.0:
+  version "2.6.0"
+  resolved "https://registry.yarnpkg.com/logform/-/logform-2.6.0.tgz#8c82a983f05d6eaeb2d75e3decae7a768b2bf9b5"
+  integrity sha512-1ulHeNPp6k/LD8H91o7VYFBng5i1BDE7HoKxVbZiGFidS1Rj65qcywLxX+pVfAPoQJEjRdvKcusKwOupHCVOVQ==
+  dependencies:
+    "@colors/colors" "1.6.0"
+    "@types/triple-beam" "^1.3.2"
+    fecha "^4.2.0"
+    ms "^2.1.1"
+    safe-stable-stringify "^2.3.1"
+    triple-beam "^1.3.0"
+
 long@^4.0.0:
   version "4.0.0"
   resolved "https://registry.yarnpkg.com/long/-/long-4.0.0.tgz#9a7b71cfb7d361a194ea555241c92f7468d5bf28"
@@ -2084,9 +2007,9 @@ make-dir@^3.1.0:
     semver "^6.0.0"
 
 mammoth@^1.6.0:
-  version "1.7.2"
-  resolved "https://registry.yarnpkg.com/mammoth/-/mammoth-1.7.2.tgz#e0efd28f46e183d807230e9ce119966dc6b1215e"
-  integrity sha512-MqWU2hcLf1I5QMKyAbfJCvrLxnv5WztrAQyorfZ+WPq7Hk82vZFmvfR2/64ajIPpM4jlq0TXp1xZvp/FFaL1Ug==
+  version "1.8.0"
+  resolved "https://registry.yarnpkg.com/mammoth/-/mammoth-1.8.0.tgz#d8f1b0d3a0355fda129270346e9dc853f223028f"
+  integrity sha512-pJNfxSk9IEGVpau+tsZFz22ofjUsl2mnA5eT8PjPs2n0BP+rhVte4Nez6FdgEuxv3IGI3afiV46ImKqTGDVlbA==
   dependencies:
     "@xmldom/xmldom" "^0.8.6"
     argparse "~1.0.3"
@@ -2159,6 +2082,13 @@ mimic-response@^3.1.0:
   resolved "https://registry.yarnpkg.com/mimic-response/-/mimic-response-3.1.0.tgz#2d1d59af9c1b129815accc2c46a022a5ce1fa3c9"
   integrity sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==
 
+minimatch@5.1.0:
+  version "5.1.0"
+  resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-5.1.0.tgz#1717b464f4971b144f6aabe8f2d0b8e4511e09c7"
+  integrity sha512-9TPBGGak4nHfGZsPBohm9AWg6NoT7QTCehS3BIJABslyZbzxfV78QM2Y6+i741OPZIafFAaiiEMh5OyIrJPgtg==
+  dependencies:
+    brace-expansion "^2.0.1"
+
 minimatch@^3.1.1, minimatch@^3.1.2:
   version "3.1.2"
   resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.1.2.tgz#19cd194bfd3e428f049a70817c038d89ab4be35b"
@@ -2166,7 +2096,7 @@ minimatch@^3.1.1, minimatch@^3.1.2:
   dependencies:
     brace-expansion "^1.1.7"
 
-minimist@^1.2.0, minimist@^1.2.3, minimist@^1.2.6:
+minimist@^1.2.0, minimist@^1.2.3:
   version "1.2.8"
   resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.8.tgz#c1a464e7693302e082a075cee0c057741ac4772c"
   integrity sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==
@@ -2201,13 +2131,6 @@ mkdirp-classic@^0.5.2, mkdirp-classic@^0.5.3:
   resolved "https://registry.yarnpkg.com/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz#fa10c9115cc6d8865be221ba47ee9bed78601113"
   integrity sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==
 
-mkdirp@^0.5.4:
-  version "0.5.6"
-  resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.6.tgz#7def03d2432dcae4ba1d611445c48396062255f6"
-  integrity sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==
-  dependencies:
-    minimist "^1.2.6"
-
 mkdirp@^1.0.3:
   version "1.0.4"
   resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-1.0.4.tgz#3eb5ed62622756d79a5f0e2a221dfebad75c2f7e"
@@ -2269,19 +2192,6 @@ ms@2.1.3, ms@^2.0.0, ms@^2.1.1:
   resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2"
   integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==
 
-multer@^1.4.5-lts.1:
-  version "1.4.5-lts.1"
-  resolved "https://registry.yarnpkg.com/multer/-/multer-1.4.5-lts.1.tgz#803e24ad1984f58edffbc79f56e305aec5cfd1ac"
-  integrity sha512-ywPWvcDMeH+z9gQq5qYHCCy+ethsk4goepZ45GLD63fOu0YcNecQxi64nDs3qluZB+murG3/D4dJ7+dGctcCQQ==
-  dependencies:
-    append-field "^1.0.0"
-    busboy "^1.0.0"
-    concat-stream "^1.5.2"
-    mkdirp "^0.5.4"
-    object-assign "^4.1.1"
-    type-is "^1.6.4"
-    xtend "^4.0.0"
-
 mustache@^4.2.0:
   version "4.2.0"
   resolved "https://registry.yarnpkg.com/mustache/-/mustache-4.2.0.tgz#e5892324d60a12ec9c2a73359edca52972bf6f64"
@@ -2329,18 +2239,13 @@ node-ensure@^0.0.0:
   resolved "https://registry.yarnpkg.com/node-ensure/-/node-ensure-0.0.0.tgz#ecae764150de99861ec5c810fd5d096b183932a7"
   integrity sha512-DRI60hzo2oKN1ma0ckc6nQWlHU69RH6xN0sjQTjMpChPfTYvKZdcQFfdYK2RWbJcKyUizSIy/l8OTGxMAM1QDw==
 
-node-fetch@^2.6.12, node-fetch@^2.6.7, node-fetch@^2.6.9:
+node-fetch@^2.6.12, node-fetch@^2.6.7:
   version "2.7.0"
   resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.7.0.tgz#d0f0fa6e3e2dc1d27efcd8ad99d550bda94d187d"
   integrity sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==
   dependencies:
     whatwg-url "^5.0.0"
 
-node-forge@^1.3.1:
-  version "1.3.1"
-  resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-1.3.1.tgz#be8da2af243b2417d5f646a770663a92b7e9ded3"
-  integrity sha512-dPEtOeMvF9VMcYV/1Wb8CPoVAXtp6MKMlcbAt4ddqmGqUJ6fQZFXkNZNkNlfevtNkGtaSoXf/vNNNSvgrdXwtA==
-
 node-html-parser@^6.1.13:
   version "6.1.13"
   resolved "https://registry.yarnpkg.com/node-html-parser/-/node-html-parser-6.1.13.tgz#a1df799b83df5c6743fcd92740ba14682083b7e4"
@@ -2446,6 +2351,13 @@ once@^1.3.0, once@^1.3.1, once@^1.4.0:
   dependencies:
     wrappy "1"
 
+one-time@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/one-time/-/one-time-1.0.0.tgz#e06bc174aed214ed58edede573b433bbf827cb45"
+  integrity sha512-5DXOiRKwuSEcQ/l0kGCF6Q3jcADFv5tSmRaJck/OqkVFcOzutB134KRSfF0xDrL39MNnqxbHBbUUcjZIhTgb2g==
+  dependencies:
+    fn.name "1.x.x"
+
 onnx-proto@^4.0.4:
   version "4.0.4"
   resolved "https://registry.yarnpkg.com/onnx-proto/-/onnx-proto-4.0.4.tgz#2431a25bee25148e915906dda0687aafe3b9e044"
@@ -2800,13 +2712,6 @@ qs@6.11.0:
   dependencies:
     side-channel "^1.0.4"
 
-qs@^6.7.0:
-  version "6.12.1"
-  resolved "https://registry.yarnpkg.com/qs/-/qs-6.12.1.tgz#39422111ca7cbdb70425541cba20c7d7b216599a"
-  integrity sha512-zWmv4RSuB9r2mYQw3zxQuHWeU+42aKi1wWig/j4ele4ygELZ7PEO6MM7rim9oAQH2A5MWfsAVf/jPvTPgCbvUQ==
-  dependencies:
-    side-channel "^1.0.6"
-
 queue-tick@^1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/queue-tick/-/queue-tick-1.0.1.tgz#f6f07ac82c1fd60f82e098b417a80e52f1f4c142"
@@ -2837,7 +2742,7 @@ rc@^1.2.7:
     minimist "^1.2.0"
     strip-json-comments "~2.0.1"
 
-readable-stream@^2.2.2, readable-stream@^2.3.0, readable-stream@^2.3.5, readable-stream@~2.3.6:
+readable-stream@^2.3.0, readable-stream@^2.3.5, readable-stream@~2.3.6:
   version "2.3.8"
   resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.8.tgz#91125e8042bba1b9887f49345f6277027ce8be9b"
   integrity sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==
@@ -2912,6 +2817,11 @@ safe-buffer@~5.1.0, safe-buffer@~5.1.1:
   resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d"
   integrity sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==
 
+safe-stable-stringify@^2.3.1:
+  version "2.4.3"
+  resolved "https://registry.yarnpkg.com/safe-stable-stringify/-/safe-stable-stringify-2.4.3.tgz#138c84b6f6edb3db5f8ef3ef7115b8f55ccbf886"
+  integrity sha512-e2bDA2WJT0wxseVd4lsDP4+3ONX6HpMXQa1ZhFQ7SU+GjvORCmShbCMltrtIDfkYhVHrOcPtj+KhmDBdPdZD1g==
+
 "safer-buffer@>= 2.1.2 < 3", "safer-buffer@>= 2.1.2 < 3.0.0":
   version "2.1.2"
   resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a"
@@ -3028,7 +2938,7 @@ sharp@^0.32.0:
     tar-fs "^3.0.4"
     tunnel-agent "^0.6.0"
 
-side-channel@^1.0.4, side-channel@^1.0.6:
+side-channel@^1.0.4:
   version "1.0.6"
   resolved "https://registry.yarnpkg.com/side-channel/-/side-channel-1.0.6.tgz#abd25fb7cd24baf45466406b1096b7831c9215f2"
   integrity sha512-fDW/EZ6Q9RiO8eFG8Hj+7u/oW+XrPTIChwCOM2+th2A6OblDtYYIpve9m+KvI9Z4C9qSEXlaGR6bTEYHReuglA==
@@ -3113,16 +3023,16 @@ sprintf-js@~1.0.2:
   resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c"
   integrity sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==
 
+stack-trace@0.0.x:
+  version "0.0.10"
+  resolved "https://registry.yarnpkg.com/stack-trace/-/stack-trace-0.0.10.tgz#547c70b347e8d32b4e108ea1a2a159e5fdde19c0"
+  integrity sha512-KGzahc7puUKkzyMt+IqAep+TVNbKP+k2Lmwhub39m1AsTSkaDutx56aDCo+HLDzf/D26BIHTJWNiTG1KAJiQCg==
+
 statuses@2.0.1:
   version "2.0.1"
   resolved "https://registry.yarnpkg.com/statuses/-/statuses-2.0.1.tgz#55cb000ccf1d48728bd23c685a063998cf1a1b63"
   integrity sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==
 
-streamsearch@^1.1.0:
-  version "1.1.0"
-  resolved "https://registry.yarnpkg.com/streamsearch/-/streamsearch-1.1.0.tgz#404dd1e2247ca94af554e841a8ef0eaa238da764"
-  integrity sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==
-
 streamx@^2.15.0, streamx@^2.16.1:
   version "2.16.1"
   resolved "https://registry.yarnpkg.com/streamx/-/streamx-2.16.1.tgz#2b311bd34832f08aa6bb4d6a80297c9caef89614"
@@ -3265,6 +3175,11 @@ tar@^6.1.11:
     mkdirp "^1.0.3"
     yallist "^4.0.0"
 
+text-hex@1.0.x:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/text-hex/-/text-hex-1.0.0.tgz#69dc9c1b17446ee79a92bf5b884bb4b9127506f5"
+  integrity sha512-uuVGNWzgJ4yhRaNSiubPY7OjISw4sw4E5Uv0wbjp+OzcbmVU/rsT8ujgcXJhn9ypzsgr5vlzpPqP+MBBKcGvbg==
+
 through@^2.3.8:
   version "2.3.8"
   resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5"
@@ -3312,6 +3227,11 @@ tr46@~0.0.3:
   resolved "https://registry.yarnpkg.com/tr46/-/tr46-0.0.3.tgz#8184fd347dac9cdc185992f3a6622e14b9d9ab6a"
   integrity sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==
 
+triple-beam@^1.3.0:
+  version "1.4.1"
+  resolved "https://registry.yarnpkg.com/triple-beam/-/triple-beam-1.4.1.tgz#6fde70271dc6e5d73ca0c3b24e2d92afb7441984"
+  integrity sha512-aZbgViZrg1QNcG+LULa7nhZpJTZSLm/mXnHXnbAbjmN5aSa0y7V+wvv6+4WaBtpISJzThKy+PIPxc1Nq1EJ9mg==
+
 ts-type@>=2:
   version "3.0.1"
   resolved "https://registry.yarnpkg.com/ts-type/-/ts-type-3.0.1.tgz#b52e7623065e0beb43c77c426347d85cf81dff84"
@@ -3338,7 +3258,7 @@ type-detect@^4.0.0:
   resolved "https://registry.yarnpkg.com/type-detect/-/type-detect-4.0.8.tgz#7646fb5f18871cfbb7749e69bd39a6388eb7450c"
   integrity sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g==
 
-type-is@^1.6.4, type-is@~1.6.18:
+type-is@~1.6.18:
   version "1.6.18"
   resolved "https://registry.yarnpkg.com/type-is/-/type-is-1.6.18.tgz#4e552cd05df09467dcbc4ef739de89f2cf37c131"
   integrity sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==
@@ -3351,11 +3271,6 @@ typedarray-dts@^1.0.0:
   resolved "https://registry.yarnpkg.com/typedarray-dts/-/typedarray-dts-1.0.0.tgz#9dec9811386dbfba964c295c2606cf9a6b982d06"
   integrity sha512-Ka0DBegjuV9IPYFT1h0Qqk5U4pccebNIJCGl8C5uU7xtOs+jpJvKGAY4fHGK25hTmXZOEUl9Cnsg5cS6K/b5DA==
 
-typedarray@^0.0.6:
-  version "0.0.6"
-  resolved "https://registry.yarnpkg.com/typedarray/-/typedarray-0.0.6.tgz#867ac74e3864187b1d3d47d996a78ec5c8830777"
-  integrity sha512-/aCDEGatGvZ2BIk+HmLf4ifCJFwvKFNb9/JeZPMulfgFracn9QFcAf5GO8B/mweUjSoblS5In0cWhqpfs/5PQA==
-
 uc.micro@^2.0.0:
   version "2.1.0"
   resolved "https://registry.yarnpkg.com/uc.micro/-/uc.micro-2.1.0.tgz#f8d3f7d0ec4c3dea35a7e3c8efa4cb8b45c9e7ee"
@@ -3406,11 +3321,6 @@ url-pattern@^1.0.3:
   resolved "https://registry.yarnpkg.com/url-pattern/-/url-pattern-1.0.3.tgz#0409292471b24f23c50d65a47931793d2b5acfc1"
   integrity sha512-uQcEj/2puA4aq1R3A2+VNVBgaWYR24FdWjl7VNW83rnWftlhyzOZ/tBjezRiC2UkIzuxC8Top3IekN3vUf1WxA==
 
-url-template@^2.0.8:
-  version "2.0.8"
-  resolved "https://registry.yarnpkg.com/url-template/-/url-template-2.0.8.tgz#fc565a3cccbff7730c775f5641f9555791439f21"
-  integrity sha512-XdVKMF4SJ0nP/O7XIPB0JwAEuT9lDIYnNsK8yGVe43y0AWoKeJNdv3ZNWh7ksJ6KqQFjOO6ox/VEitLnaVNufw==
-
 urlpattern-polyfill@9.0.0:
   version "9.0.0"
   resolved "https://registry.yarnpkg.com/urlpattern-polyfill/-/urlpattern-polyfill-9.0.0.tgz#bc7e386bb12fd7898b58d1509df21d3c29ab3460"
@@ -3478,6 +3388,32 @@ wide-align@^1.1.2:
   dependencies:
     string-width "^1.0.2 || 2 || 3 || 4"
 
+winston-transport@^4.7.0:
+  version "4.7.0"
+  resolved "https://registry.yarnpkg.com/winston-transport/-/winston-transport-4.7.0.tgz#e302e6889e6ccb7f383b926df6936a5b781bd1f0"
+  integrity sha512-ajBj65K5I7denzer2IYW6+2bNIVqLGDHqDw3Ow8Ohh+vdW+rv4MZ6eiDvHoKhfJFZ2auyN8byXieDDJ96ViONg==
+  dependencies:
+    logform "^2.3.2"
+    readable-stream "^3.6.0"
+    triple-beam "^1.3.0"
+
+winston@^3.13.0:
+  version "3.13.0"
+  resolved "https://registry.yarnpkg.com/winston/-/winston-3.13.0.tgz#e76c0d722f78e04838158c61adc1287201de7ce3"
+  integrity sha512-rwidmA1w3SE4j0E5MuIufFhyJPBDG7Nu71RkZor1p2+qHvJSZ9GYDA81AyleQcZbh/+V6HjeBdfnTZJm9rSeQQ==
+  dependencies:
+    "@colors/colors" "^1.6.0"
+    "@dabh/diagnostics" "^2.0.2"
+    async "^3.2.3"
+    is-stream "^2.0.0"
+    logform "^2.4.0"
+    one-time "^1.0.0"
+    readable-stream "^3.4.0"
+    safe-stable-stringify "^2.3.1"
+    stack-trace "0.0.x"
+    triple-beam "^1.3.0"
+    winston-transport "^4.7.0"
+
 wrap-ansi@^7.0.0:
   version "7.0.0"
   resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43"

docker/.env.example

@@ -2,6 +2,8 @@ SERVER_PORT=3001
 STORAGE_DIR="/app/server/storage"
 UID='1000'
 GID='1000'
+# SIG_KEY='passphrase' # Please generate random string at least 32 chars long.
+# SIG_SALT='salt' # Please generate random string at least 32 chars long.
 # JWT_SECRET="my-random-string-for-seeding" # Only needed if AUTH_TOKEN is set. Please generate random string at least 12 chars long.
 
 ###########################################
@@ -92,6 +94,13 @@ GID='1000'
 # COHERE_API_KEY=
 # COHERE_MODEL_PREF='command-r'
 
+# LLM_PROVIDER='bedrock'
+# AWS_BEDROCK_LLM_ACCESS_KEY_ID=
+# AWS_BEDROCK_LLM_ACCESS_KEY=
+# AWS_BEDROCK_LLM_REGION=us-west-2
+# AWS_BEDROCK_LLM_MODEL_PREFERENCE=meta.llama3-1-8b-instruct-v1:0
+# AWS_BEDROCK_LLM_MODEL_TOKEN_LIMIT=8191
+
 ###########################################
 ######## Embedding API SElECTION ##########
 ###########################################
@@ -134,6 +143,12 @@ GID='1000'
 # LITE_LLM_BASE_PATH='http://127.0.0.1:4000'
 # LITE_LLM_API_KEY='sk-123abc'
 
+# EMBEDDING_ENGINE='generic-openai'
+# EMBEDDING_MODEL_PREF='text-embedding-ada-002'
+# EMBEDDING_MODEL_MAX_CHUNK_LENGTH=8192
+# EMBEDDING_BASE_PATH='http://127.0.0.1:4000'
+# GENERIC_OPEN_AI_EMBEDDING_API_KEY='sk-123abc'
+
 ###########################################
 ######## Vector Database Selection ########
 ###########################################
@@ -245,3 +260,6 @@ GID='1000'
 
 #------ Serply.io ----------- https://serply.io/
 # AGENT_SERPLY_API_KEY=
+
+#------ SearXNG ----------- https://github.com/searxng/searxng
+# AGENT_SEARXNG_API_URL=

docker/Dockerfile

@@ -1,5 +1,5 @@
 # Setup base image
-FROM ubuntu:jammy-20230916 AS base
+FROM ubuntu:jammy-20240627.1 AS base
 
 # Build arguments
 ARG ARG_UID=1000
@@ -27,7 +27,9 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get update && \
     apt-get install -yq --no-install-recommends nodejs && \
     curl -LO https://github.com/yarnpkg/yarn/releases/download/v1.22.19/yarn_1.22.19_all.deb \
         && dpkg -i yarn_1.22.19_all.deb \
-        && rm yarn_1.22.19_all.deb
+        && rm yarn_1.22.19_all.deb && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
 
 # Create a group and user with specific UID and GID
 RUN groupadd -g "$ARG_GID" anythingllm && \
@@ -58,7 +60,7 @@ ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true
 ENV CHROME_PATH=/app/chrome-linux/chrome
 ENV PUPPETEER_EXECUTABLE_PATH=/app/chrome-linux/chrome
 
-RUN echo "Done running arm64 specific installtion steps"
+RUN echo "Done running arm64 specific installation steps"
 
 #############################################
 
@@ -85,7 +87,9 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get update && \
     apt-get install -yq --no-install-recommends nodejs && \
     curl -LO https://github.com/yarnpkg/yarn/releases/download/v1.22.19/yarn_1.22.19_all.deb \
         && dpkg -i yarn_1.22.19_all.deb \
-        && rm yarn_1.22.19_all.deb
+        && rm yarn_1.22.19_all.deb && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
 
 # Create a group and user with specific UID and GID
 RUN groupadd -g "$ARG_GID" anythingllm && \
@@ -112,66 +116,63 @@ RUN echo "Running common build flow of AnythingLLM image for all architectures"
 USER anythingllm
 WORKDIR /app
 
-# Install frontend dependencies
-FROM build AS frontend-deps
-
-COPY ./frontend/package.json ./frontend/yarn.lock ./frontend/
+# Install & Build frontend layer
+FROM build AS frontend-build
+COPY --chown=anythingllm:anythingllm ./frontend /app/frontend/
 WORKDIR /app/frontend
 RUN yarn install --network-timeout 100000 && yarn cache clean
+RUN yarn build && \
+    cp -r dist /tmp/frontend-build && \
+    rm -rf * && \
+    cp -r /tmp/frontend-build dist && \
+    rm -rf /tmp/frontend-build
 WORKDIR /app
 
-# Install server dependencies
-FROM build AS server-deps
-COPY ./server/package.json ./server/yarn.lock ./server/
+# Install server layer & build node-llama-cpp
+# Also pull and build collector deps (chromium issues prevent bad bindings)
+FROM build AS backend-build
+COPY ./server /app/server/
 WORKDIR /app/server
 RUN yarn install --production --network-timeout 100000 && yarn cache clean
 WORKDIR /app
 
+# Install collector dependencies
+COPY ./collector/ ./collector/
+WORKDIR /app/collector
+ENV PUPPETEER_DOWNLOAD_BASE_URL=https://storage.googleapis.com/chrome-for-testing-public 
+RUN yarn install --production --network-timeout 100000 && yarn cache clean
+
 # Compile Llama.cpp bindings for node-llama-cpp for this operating system.
+# Creates appropriate bindings for the OS
 USER root
 WORKDIR /app/server
 RUN npx --no node-llama-cpp download
 WORKDIR /app
 USER anythingllm
 
-# Build the frontend
-FROM frontend-deps AS build-stage
-COPY ./frontend/ ./frontend/
-WORKDIR /app/frontend
-RUN yarn build && yarn cache clean
+# Since we are building from backend-build we just need to move built frontend into server/public
+FROM backend-build AS production-build
 WORKDIR /app
+COPY --chown=anythingllm:anythingllm --from=frontend-build /app/frontend/dist /app/server/public
+USER root
+RUN chown -R anythingllm:anythingllm /app/server && \
+    chown -R anythingllm:anythingllm /app/collector
+USER anythingllm
 
-# Setup the server
-FROM server-deps AS production-stage
-COPY --chown=anythingllm:anythingllm ./server/ ./server/
-
-# Copy built static frontend files to the server public directory
-COPY --chown=anythingllm:anythingllm --from=build-stage /app/frontend/dist ./server/public
-
-# Copy the collector
-COPY --chown=anythingllm:anythingllm ./collector/ ./collector/
-
-# Install collector dependencies
-WORKDIR /app/collector
-ENV PUPPETEER_DOWNLOAD_BASE_URL=https://storage.googleapis.com/chrome-for-testing-public 
-RUN yarn install --production --network-timeout 100000 && yarn cache clean
-
-# Migrate and Run Prisma against known schema
-WORKDIR /app/server
-RUN npx prisma generate --schema=./prisma/schema.prisma && \
-    npx prisma migrate deploy --schema=./prisma/schema.prisma
-WORKDIR /app
+# No longer needed? (deprecated)
+# WORKDIR /app/server
+# RUN npx prisma generate --schema=./prisma/schema.prisma && \
+#     npx prisma migrate deploy --schema=./prisma/schema.prisma
+# WORKDIR /app
 
 # Setup the environment
 ENV NODE_ENV=production
 ENV ANYTHING_LLM_RUNTIME=docker
 
-# Expose the server port
-EXPOSE 3001
-
 # Setup the healthcheck
 HEALTHCHECK --interval=1m --timeout=10s --start-period=1m \
   CMD /bin/bash /usr/local/bin/docker-healthcheck.sh || exit 1
 
 # Run the server
+# CMD ["sh", "-c", "tail -f /dev/null"] # For development: keep container open
 ENTRYPOINT ["/bin/bash", "/usr/local/bin/docker-entrypoint.sh"]

docker/HOW_TO_USE_DOCKER.md

@@ -89,6 +89,9 @@ mintplexlabs/anythingllm;
 <tr>
 <td> Docker Compose</td>
 <td>
+
+
+```yaml
 version: '3.8'
 services:
   anythingllm:
@@ -99,7 +102,7 @@ mintplexlabs/anythingllm;
     cap_add:
       - SYS_ADMIN
     environment:
-          # Adjust for your environemnt
+    # Adjust for your environment
       - STORAGE_DIR=/app/server/storage
       - JWT_SECRET="make this a large list of random numbers and letters 20+"
       - LLM_PROVIDER=ollama
@@ -127,6 +130,8 @@ mintplexlabs/anythingllm;
       type: none
       o: bind
       device: /path/on/local/disk
+```
+
   </td>
 </tr>
 </table>

docker/docker-compose.yml

@@ -1,5 +1,3 @@
-version: "3.9"
-
 name: anythingllm
 
 networks:
@@ -8,8 +6,7 @@ networks:
 
 services:
   anything-llm:
-    container_name: anything-llm
-    image: anything-llm:latest
+    container_name: anythingllm
     build:
       context: ../.
       dockerfile: ./docker/Dockerfile

docker/vex/CVE-2019-10790.vex.json

@@ -0,0 +1,22 @@
+{
+  "@context": "https://openvex.dev/ns/v0.2.0",
+  "@id": "https://openvex.dev/docs/public/vex-6750d79bb005487e11d10f81d0b3ac92c47e6e259292c6b2d02558f9f4bca52d",
+  "author": "tim@mintplexlabs.com",
+  "timestamp": "2024-07-22T13:49:12.883675-07:00",
+  "version": 1,
+  "statements": [
+    {
+      "vulnerability": {
+        "name": "CVE-2019-10790"
+      },
+      "timestamp": "2024-07-22T13:49:12.883678-07:00",
+      "products": [
+        {
+          "@id": "pkg:npm/taffydb@2.6.2"
+        }
+      ],
+      "status": "not_affected",
+      "justification": "vulnerable_code_not_in_execute_path"
+    }
+  ]
+}

docker/vex/CVE-2024-29415.vex.json

@@ -0,0 +1,22 @@
+{
+  "@context": "https://openvex.dev/ns/v0.2.0",
+  "@id": "https://openvex.dev/docs/public/vex-939548c125c5bfebd3fd91e64c1c53bffacbde06b3611b4474ea90fa58045004",
+  "author": "tim@mintplexlabs.com",
+  "timestamp": "2024-07-19T16:08:47.147169-07:00",
+  "version": 1,
+  "statements": [
+    {
+      "vulnerability": {
+        "name": "CVE-2024-29415"
+      },
+      "timestamp": "2024-07-19T16:08:47.147172-07:00",
+      "products": [
+        {
+          "@id": "pkg:npm/ip@2.0.0"
+        }
+      ],
+      "status": "not_affected",
+      "justification": "vulnerable_code_not_present"
+    }
+  ]
+}

docker/vex/CVE-2024-37890.vex.json

@@ -0,0 +1,22 @@
+{
+  "@context": "https://openvex.dev/ns/v0.2.0",
+  "@id": "https://openvex.dev/docs/public/vex-939548c125c5bfebd3fd91e64c1c53bffacbde06b3611b4474ea90fa58045004",
+  "author": "tim@mintplexlabs.com",
+  "timestamp": "2024-07-19T16:08:47.147169-07:00",
+  "version": 1,
+  "statements": [
+    {
+      "vulnerability": {
+        "name": "CVE-2024-37890"
+      },
+      "timestamp": "2024-07-19T16:08:47.147172-07:00",
+      "products": [
+        {
+          "@id": "pkg:npm/ws@8.14.2"
+        }
+      ],
+      "status": "not_affected",
+      "justification": "vulnerable_code_not_in_execute_path"
+    }
+  ]
+}

docker/vex/CVE-2024-4068.vex.json

@@ -0,0 +1,22 @@
+{
+  "@context": "https://openvex.dev/ns/v0.2.0",
+  "@id": "https://openvex.dev/docs/public/vex-939548c125c5bfebd3fd91e64c1c53bffacbde06b3611b4474ea90fa58045004",
+  "author": "tim@mintplexlabs.com",
+  "timestamp": "2024-07-19T16:08:47.147169-07:00",
+  "version": 1,
+  "statements": [
+    {
+      "vulnerability": {
+        "name": "CVE-2024-4068"
+      },
+      "timestamp": "2024-07-19T16:08:47.147172-07:00",
+      "products": [
+        {
+          "@id": "pkg:npm/braces@3.0.2"
+        }
+      ],
+      "status": "not_affected",
+      "justification": "vulnerable_code_not_present"
+    }
+  ]
+}

embed/README.md

@@ -63,7 +63,7 @@ REQUIRED data attributes:
 
 **Style Overrides**
 
-- `data-chat-icon` — The chat bubble icon show when chat is closed. Options are `plus`, `chatCircle`, `support`, `search2`, `search`, `magic`.
+- `data-chat-icon` — The chat bubble icon show when chat is closed. Options are `plus`, `chatBubble`, `support`, `search2`, `search`, `magic`.
 
 - `data-button-color` — The chat bubble background color shown when chat is closed. Value must be hex color code.
 
@@ -93,9 +93,13 @@ REQUIRED data attributes:
 
 - `data-text-size` - Set the text size of the chats in pixels.
 
+- `data-username` - A specific readable name or identifier for the client for your reference. Will be shown in AnythingLLM chat logs. If empty it will not be reported.
+
+- `data-default-messages` - A string of comma-separated messages you want to display to the user when the chat widget has no history. Example: `"How are you?, What is so interesting about this project?, Tell me a joke."`
+
 **Behavior Overrides**
 
-- `data-open-on-load` — Once loaded, open the chat as default. It can still be closed by the user.
+- `data-open-on-load` — Once loaded, open the chat as default. It can still be closed by the user. To enable set this attribute to `on`. All other values will be ignored.
 
 - `data-support-email` — Shows a support email that the user can used to draft an email via the "three dot" menu in the top right. Option will not appear if it is not set.
 

embed/package.json

@@ -6,9 +6,12 @@
   "scripts": {
     "dev": "nodemon -e js,jsx,css --watch src --exec \"yarn run dev:preview\"",
     "dev:preview": "yarn run dev:build && yarn serve . -p 3080 --no-clipboard",
-    "dev:build": "vite build && cat src/static/tailwind@3.4.1.js >> dist/anythingllm-chat-widget.js",
-    "build": "vite build && cat src/static/tailwind@3.4.1.js >> dist/anythingllm-chat-widget.js && npx terser --compress -o dist/anythingllm-chat-widget.min.js -- dist/anythingllm-chat-widget.js",
-    "build:publish": "yarn build && mkdir -p ../frontend/public/embed && cp -r dist/anythingllm-chat-widget.min.js ../frontend/public/embed/anythingllm-chat-widget.min.js",
+    "dev:build": "vite build && yarn styles",
+    "styles": "npx cleancss -o dist/anythingllm-chat-widget.min.css dist/style.css",
+    "build": "vite build && yarn styles && npx terser --compress -o dist/anythingllm-chat-widget.min.js -- dist/anythingllm-chat-widget.js",
+    "build:publish": "yarn build:publish:js && yarn build:publish:css",
+    "build:publish:js": "yarn build && mkdir -p ../frontend/public/embed && cp -r dist/anythingllm-chat-widget.min.js ../frontend/public/embed/anythingllm-chat-widget.min.js",
+    "build:publish:css": "cp -r dist/anythingllm-chat-widget.min.css ../frontend/public/embed/anythingllm-chat-widget.min.css",
     "lint": "yarn prettier --ignore-path ../.prettierignore --write ./src"
   },
   "dependencies": {
@@ -29,14 +32,18 @@
     "@types/react-dom": "^18.2.15",
     "@vitejs/plugin-react": "^4.2.0",
     "autoprefixer": "^10.4.14",
+    "clean-css": "^5.3.3",
+    "clean-css-cli": "^5.6.3",
     "eslint": "^8.53.0",
     "eslint-plugin-react": "^7.33.2",
     "eslint-plugin-react-hooks": "^4.6.0",
     "eslint-plugin-react-refresh": "^0.4.4",
     "globals": "^13.21.0",
     "nodemon": "^2.0.22",
+    "postcss": "^8.4.23",
     "prettier": "^3.0.3",
     "serve": "^14.2.1",
+    "tailwindcss": "3.4.1",
     "terser": "^5.27.0",
     "vite": "^5.0.0",
     "vite-plugin-singlefile": "^0.13.5"

embed/postcss.config.js

@@ -0,0 +1,10 @@
+import tailwind from 'tailwindcss'
+import autoprefixer from 'autoprefixer'
+import tailwindConfig from './tailwind.config.js'
+
+export default {
+  plugins: [
+    tailwind(tailwindConfig),
+    autoprefixer,
+  ],
+}

embed/src/App.jsx

@@ -20,29 +20,29 @@ export default function App() {
   if (!embedSettings.loaded) return null;
 
   const positionClasses = {
-    "bottom-left": "bottom-0 left-0 ml-4",
-    "bottom-right": "bottom-0 right-0 mr-4",
-    "top-left": "top-0 left-0 ml-4 mt-4",
-    "top-right": "top-0 right-0 mr-4 mt-4",
+    "bottom-left": "allm-bottom-0 allm-left-0 allm-ml-4",
+    "bottom-right": "allm-bottom-0 allm-right-0 allm-mr-4",
+    "top-left": "allm-top-0 allm-left-0 allm-ml-4 allm-mt-4",
+    "top-right": "allm-top-0 allm-right-0 allm-mr-4 allm-mt-4",
   };
 
   const position = embedSettings.position || "bottom-right";
-  const windowWidth = embedSettings.windowWidth
-    ? `max-w-[${embedSettings.windowWidth}]`
-    : "max-w-[400px]";
-  const windowHeight = embedSettings.windowHeight
-    ? `max-h-[${embedSettings.windowHeight}]`
-    : "max-h-[700px]";
+  const windowWidth = embedSettings.windowWidth ?? "400px";
+  const windowHeight = embedSettings.windowHeight ?? "700px";
 
   return (
     <>
       <Head />
       <div
         id="anything-llm-embed-chat-container"
-        className={`fixed inset-0 z-50 ${isChatOpen ? "block" : "hidden"}`}
+        className={`allm-fixed allm-inset-0 allm-z-50 ${isChatOpen ? "allm-block" : "allm-hidden"}`}
       >
         <div
-          className={`${windowHeight} ${windowWidth} h-full w-full bg-white fixed bottom-0 right-0 mb-4 md:mr-4 rounded-2xl border border-gray-300 shadow-[0_4px_14px_rgba(0,0,0,0.25)] ${positionClasses[position]}`}
+          style={{
+            maxWidth: windowWidth,
+            maxHeight: windowHeight,
+          }}
+          className={`allm-h-full allm-w-full allm-bg-white allm-fixed allm-bottom-0 allm-right-0 allm-mb-4 allm-md:mr-4 allm-rounded-2xl allm-border allm-border-gray-300 allm-shadow-[0_4px_14px_rgba(0,0,0,0.25)] ${positionClasses[position]}`}
           id="anything-llm-chat"
         >
           {isChatOpen && (
@@ -57,7 +57,7 @@ export default function App() {
       {!isChatOpen && (
         <div
           id="anything-llm-embed-chat-button-container"
-          className={`fixed bottom-0 ${positionClasses[position]} mb-4 z-50`}
+          className={`allm-fixed allm-bottom-0 ${positionClasses[position]} allm-mb-4 allm-z-50`}
         >
           <OpenButton
             settings={embedSettings}

embed/src/components/ChatWindow/ChatContainer/ChatHistory/HistoricalMessage/Actions/index.jsx

@@ -5,7 +5,7 @@ import { Tooltip } from "react-tooltip";
 
 const Actions = ({ message }) => {
   return (
-    <div className="flex justify-start items-center gap-x-4">
+    <div className="allm-flex allm-justify-start allm-items-center allm-gap-x-4">
       <CopyMessage message={message} />
       {/* Other actions to go here later. */}
     </div>
@@ -16,17 +16,17 @@ function CopyMessage({ message }) {
   const { copied, copyText } = useCopyText();
   return (
     <>
-      <div className="mt-3 relative">
+      <div className="allm-mt-3 allm-relative">
         <button
           data-tooltip-id="copy-assistant-text"
           data-tooltip-content="Copy"
-          className="text-zinc-300"
+          className="allm-border-none allm-text-zinc-300"
           onClick={() => copyText(message)}
         >
           {copied ? (
-            <Check size={18} className="mb-1" />
+            <Check size={18} className="allm-mb-1" />
           ) : (
-            <ClipboardText size={18} className="mb-1" />
+            <ClipboardText size={18} className="allm-mb-1" />
           )}
         </button>
       </div>
@@ -34,7 +34,7 @@ function CopyMessage({ message }) {
         id="copy-assistant-text"
         place="bottom"
         delayShow={300}
-        className="tooltip !text-xs"
+        className="allm-tooltip !allm-text-xs"
       />
     </>
   );

embed/src/components/ChatWindow/ChatContainer/ChatHistory/HistoricalMessage/index.jsx

@@ -14,14 +14,15 @@ const HistoricalMessage = forwardRef(
     ref
   ) => {
     const textSize = !!embedderSettings.settings.textSize
-      ? `text-[${embedderSettings.settings.textSize}px]`
-      : "text-sm";
+      ? `allm-text-[${embedderSettings.settings.textSize}px]`
+      : "allm-text-sm";
+    if (error) console.error(`ANYTHING_LLM_CHAT_WIDGET_ERROR: ${error}`);
 
     return (
       <div className="py-[5px]">
         {role === "assistant" && (
           <div
-            className={`text-[10px] font-medium text-gray-400 ml-[54px] mr-6 mb-2 text-left`}
+            className={`allm-text-[10px] allm-text-gray-400 allm-ml-[54px] allm-mr-6 allm-mb-2 allm-text-left allm-font-sans`}
           >
             {embedderSettings.settings.assistantName ||
               "Anything LLM Chat Assistant"}
@@ -30,42 +31,48 @@ const HistoricalMessage = forwardRef(
         <div
           key={uuid}
           ref={ref}
-          className={`flex items-start w-full h-fit ${
-            role === "user" ? "justify-end" : "justify-start"
+          className={`allm-flex allm-items-start allm-w-full allm-h-fit ${
+            role === "user" ? "allm-justify-end" : "allm-justify-start"
           }`}
         >
           {role === "assistant" && (
             <img
               src={embedderSettings.settings.assistantIcon || AnythingLLMIcon}
               alt="Anything LLM Icon"
-              className="w-9 h-9 flex-shrink-0 ml-2 mt-2"
+              className="allm-w-9 allm-h-9 allm-flex-shrink-0 allm-ml-2 allm-mt-2"
               id="anything-llm-icon"
             />
           )}
           <div
-            style={{ wordBreak: "break-word" }}
-            className={`py-[11px] px-4 flex flex-col ${
+            style={{
+              wordBreak: "break-word",
+              backgroundColor:
+                role === "user"
+                  ? embedderSettings.USER_STYLES.msgBg
+                  : embedderSettings.ASSISTANT_STYLES.msgBg,
+            }}
+            className={`allm-py-[11px] allm-px-4 allm-flex allm-flex-col allm-font-sans ${
               error
-                ? "bg-red-200 rounded-lg mr-[37px] ml-[9px]"
+                ? "allm-bg-red-200 allm-rounded-lg allm-mr-[37px] allm-ml-[9px]"
                 : role === "user"
-                  ? `${embedderSettings.USER_STYLES} anything-llm-user-message`
-                  : `${embedderSettings.ASSISTANT_STYLES} anything-llm-assistant-message`
-            } shadow-[0_4px_14px_rgba(0,0,0,0.25)]`}
+                  ? `${embedderSettings.USER_STYLES.base} allm-anything-llm-user-message`
+                  : `${embedderSettings.ASSISTANT_STYLES.base} allm-anything-llm-assistant-message`
+            } allm-shadow-[0_4px_14px_rgba(0,0,0,0.25)]`}
           >
-            <div className="flex">
+            <div className="allm-flex">
               {error ? (
-                <div className="p-2 rounded-lg bg-red-50 text-red-500">
-                  <span className={`inline-block `}>
-                    <Warning className="h-4 w-4 mb-1 inline-block" /> Could not
-                    respond to message.
+                <div className="allm-p-2 allm-rounded-lg allm-bg-red-50 allm-text-red-500">
+                  <span className={`allm-inline-block `}>
+                    <Warning className="allm-h-4 allm-w-4 allm-mb-1 allm-inline-block" />{" "}
+                    Could not respond to message.
                   </span>
-                  <p className="text-xs font-mono mt-2 border-l-2 border-red-500 pl-2 bg-red-300 p-2 rounded-sm">
-                    {error}
+                  <p className="allm-text-xs allm-font-mono allm-mt-2 allm-border-l-2 allm-border-red-500 allm-pl-2 allm-bg-red-300 allm-p-2 allm-rounded-sm">
+                    Server error
                   </p>
                 </div>
               ) : (
                 <span
-                  className={`whitespace-pre-line font-medium flex flex-col gap-y-1 ${textSize} leading-[20px]`}
+                  className={`allm-whitespace-pre-line allm-flex allm-flex-col allm-gap-y-1 ${textSize} allm-leading-[20px]`}
                   dangerouslySetInnerHTML={{
                     __html: DOMPurify.sanitize(renderMarkdown(message)),
                   }}
@@ -77,7 +84,7 @@ const HistoricalMessage = forwardRef(
 
         {sentAt && (
           <div
-            className={`text-[10px] font-medium text-gray-400 ml-[54px] mr-6 mt-2 ${role === "user" ? "text-right" : "text-left"}`}
+            className={`allm-font-sans allm-text-[10px] allm-text-gray-400 allm-ml-[54px] allm-mr-6 allm-mt-2 ${role === "user" ? "allm-text-right" : "allm-text-left"}`}
           >
             {formatDate(sentAt)}
           </div>

embed/src/components/ChatWindow/ChatContainer/ChatHistory/PromptReply/index.jsx

@@ -8,21 +8,27 @@ import { formatDate } from "@/utils/date";
 const PromptReply = forwardRef(
   ({ uuid, reply, pending, error, sources = [] }, ref) => {
     if (!reply && sources.length === 0 && !pending && !error) return null;
+    if (error) console.error(`ANYTHING_LLM_CHAT_WIDGET_ERROR: ${error}`);
 
     if (pending) {
       return (
-        <div className={`flex items-start w-full h-fit justify-start`}>
+        <div
+          className={`allm-flex allm-items-start allm-w-full allm-h-fit allm-justify-start`}
+        >
           <img
             src={embedderSettings.settings.assistantIcon || AnythingLLMIcon}
             alt="Anything LLM Icon"
-            className="w-9 h-9 flex-shrink-0 ml-2"
+            className="allm-w-9 allm-h-9 allm-flex-shrink-0 allm-ml-2"
           />
           <div
-            style={{ wordBreak: "break-word" }}
-            className={`py-[11px] px-4 flex flex-col ${embedderSettings.ASSISTANT_STYLES} shadow-[0_4px_14px_rgba(0,0,0,0.25)]`}
+            style={{
+              wordBreak: "break-word",
+              backgroundColor: embedderSettings.ASSISTANT_STYLES.msgBg,
+            }}
+            className={`allm-py-[11px] allm-px-4 allm-flex allm-flex-col ${embedderSettings.ASSISTANT_STYLES.base} allm-shadow-[0_4px_14px_rgba(0,0,0,0.25)]`}
           >
-            <div className="flex gap-x-5">
-              <div className="mx-4 my-1 dot-falling"></div>
+            <div className="allm-flex allm-gap-x-5">
+              <div className="allm-mx-4 allm-my-1 allm-dot-falling"></div>
             </div>
           </div>
         </div>
@@ -31,23 +37,25 @@ const PromptReply = forwardRef(
 
     if (error) {
       return (
-        <div className={`flex items-end w-full h-fit justify-start`}>
+        <div
+          className={`allm-flex allm-items-end allm-w-full allm-h-fit allm-justify-start`}
+        >
           <img
             src={embedderSettings.settings.assistantIcon || AnythingLLMIcon}
             alt="Anything LLM Icon"
-            className="w-9 h-9 flex-shrink-0 ml-2"
+            className="allm-w-9 allm-h-9 allm-flex-shrink-0 allm-ml-2"
           />
           <div
             style={{ wordBreak: "break-word" }}
-            className={`py-[11px] px-4 rounded-lg flex flex-col bg-red-200 shadow-[0_4px_14px_rgba(0,0,0,0.25)] mr-[37px] ml-[9px]`}
+            className={`allm-py-[11px] allm-px-4 allm-rounded-lg allm-flex allm-flex-col allm-bg-red-200 allm-shadow-[0_4px_14px_rgba(0,0,0,0.25)] allm-mr-[37px] allm-ml-[9px]`}
           >
-            <div className="flex gap-x-5">
+            <div className="allm-flex allm-gap-x-5">
               <span
-                className={`inline-block p-2 rounded-lg bg-red-50 text-red-500`}
+                className={`allm-inline-block allm-p-2 allm-rounded-lg allm-bg-red-50 allm-text-red-500`}
               >
-                <Warning className="h-4 w-4 mb-1 inline-block" /> Could not
-                respond to message.
-                <span className="text-xs">Reason: {error || "unknown"}</span>
+                <Warning className="allm-h-4 allm-w-4 allm-mb-1 allm-inline-block" />{" "}
+                Could not respond to message.
+                <span className="allm-text-xs">Server error</span>
               </span>
             </div>
           </div>
@@ -56,9 +64,9 @@ const PromptReply = forwardRef(
     }
 
     return (
-      <div className="py-[5px]">
+      <div className="allm-py-[5px]">
         <div
-          className={`text-[10px] font-medium text-gray-400 ml-[54px] mr-6 mb-2 text-left`}
+          className={`allm-text-[10px] allm-text-gray-400 allm-ml-[54px] allm-mr-6 allm-mb-2 allm-text-left allm-font-sans`}
         >
           {embedderSettings.settings.assistantName ||
             "Anything LLM Chat Assistant"}
@@ -66,29 +74,32 @@ const PromptReply = forwardRef(
         <div
           key={uuid}
           ref={ref}
-          className={`flex items-start w-full h-fit justify-start`}
+          className={`allm-flex allm-items-start allm-w-full allm-h-fit allm-justify-start`}
         >
           <img
             src={embedderSettings.settings.assistantIcon || AnythingLLMIcon}
             alt="Anything LLM Icon"
-            className="w-9 h-9 flex-shrink-0 ml-2"
+            className="allm-w-9 allm-h-9 allm-flex-shrink-0 allm-ml-2"
           />
           <div
-            style={{ wordBreak: "break-word" }}
-            className={`py-[11px] px-4 flex flex-col ${
-              error ? "bg-red-200" : embedderSettings.ASSISTANT_STYLES
-            } shadow-[0_4px_14px_rgba(0,0,0,0.25)]`}
+            style={{
+              wordBreak: "break-word",
+              backgroundColor: embedderSettings.ASSISTANT_STYLES.msgBg,
+            }}
+            className={`allm-py-[11px] allm-px-4 allm-flex allm-flex-col ${
+              error ? "allm-bg-red-200" : embedderSettings.ASSISTANT_STYLES.base
+            } allm-shadow-[0_4px_14px_rgba(0,0,0,0.25)]`}
           >
-            <div className="flex gap-x-5">
+            <div className="allm-flex allm-gap-x-5">
               <span
-                className={`reply whitespace-pre-line font-normal text-sm md:text-sm flex flex-col gap-y-1`}
+                className={`allm-font-sans allm-reply allm-whitespace-pre-line allm-font-normal allm-text-sm allm-md:text-sm allm-flex allm-flex-col allm-gap-y-1`}
                 dangerouslySetInnerHTML={{ __html: renderMarkdown(reply) }}
               />
             </div>
           </div>
         </div>
         <div
-          className={`text-[10px] font-medium text-gray-400 ml-[54px] mr-6 mt-2 text-left`}
+          className={`allm-text-[10px] allm-text-gray-400 allm-ml-[54px] allm-mr-6 allm-mt-2 allm-text-left allm-font-sans`}
         >
           {formatDate(Date.now() / 1000)}
         </div>

embed/src/components/ChatWindow/ChatContainer/ChatHistory/index.jsx

@@ -2,7 +2,9 @@ import HistoricalMessage from "./HistoricalMessage";
 import PromptReply from "./PromptReply";
 import { useEffect, useRef, useState } from "react";
 import { ArrowDown, CircleNotch } from "@phosphor-icons/react";
+import { embedderSettings } from "@/main";
 import debounce from "lodash.debounce";
+import { SEND_TEXT_EVENT } from "..";
 
 export default function ChatHistory({ settings = {}, history = [] }) {
   const replyRef = useRef(null);
@@ -46,11 +48,12 @@ export default function ChatHistory({ settings = {}, history = [] }) {
 
   if (history.length === 0) {
     return (
-      <div className="pb-[100px] pt-[5px] rounded-lg px-2 h-full mt-2 gap-y-2 overflow-y-scroll flex flex-col justify-start no-scroll">
-        <div className="flex h-full flex-col items-center justify-center">
-          <p className="text-slate-400 text-sm font-base py-4 text-center">
+      <div className="allm-pb-[100px] allm-pt-[5px] allm-rounded-lg allm-px-2 allm-h-full allm-mt-2 allm-gap-y-2 allm-overflow-y-scroll allm-flex allm-flex-col allm-justify-start allm-no-scroll">
+        <div className="allm-flex allm-h-full allm-flex-col allm-items-center allm-justify-center">
+          <p className="allm-text-slate-400 allm-text-sm allm-font-sans allm-py-4 allm-text-center">
             {settings?.greeting ?? "Send a chat to get started."}
           </p>
+          <SuggestedMessages settings={settings} />
         </div>
       </div>
     );
@@ -58,7 +61,7 @@ export default function ChatHistory({ settings = {}, history = [] }) {
 
   return (
     <div
-      className="pb-[30px] pt-[5px] rounded-lg px-2 h-full gap-y-2 overflow-y-scroll flex flex-col justify-start no-scroll md:max-h-[500px]"
+      className="allm-pb-[30px] allm-pt-[5px] allm-rounded-lg allm-px-2 allm-h-full allm-gap-y-2 allm-overflow-y-scroll allm-flex allm-flex-col allm-justify-start allm-no-scroll allm-md:max-h-[500px]"
       id="chat-history"
       ref={chatHistoryRef}
     >
@@ -97,12 +100,12 @@ export default function ChatHistory({ settings = {}, history = [] }) {
         );
       })}
       {!isAtBottom && (
-        <div className="fixed bottom-[10rem] right-[50px] z-50 cursor-pointer animate-pulse">
-          <div className="flex flex-col items-center">
-            <div className="p-1 rounded-full border border-white/10 bg-black/20 hover:bg-black/50">
+        <div className="allm-fixed allm-bottom-[10rem] allm-right-[50px] allm-z-50 allm-cursor-pointer allm-animate-pulse">
+          <div className="allm-flex allm-flex-col allm-items-center">
+            <div className="allm-p-1 allm-rounded-full allm-border allm-border-white/10 allm-bg-black/20 hover:allm-bg-black/50">
               <ArrowDown
                 weight="bold"
-                className="text-white/50 w-5 h-5"
+                className="allm-text-white/50 allm-w-5 allm-h-5"
                 onClick={scrollToBottom}
                 id="scroll-to-bottom-button"
                 aria-label="Scroll to bottom"
@@ -117,12 +120,44 @@ export default function ChatHistory({ settings = {}, history = [] }) {
 
 export function ChatHistoryLoading() {
   return (
-    <div className="h-full w-full relative">
-      <div className="h-full max-h-[82vh] pb-[100px] pt-[5px] bg-gray-100 rounded-lg px-2 h-full mt-2 gap-y-2 overflow-y-scroll flex flex-col justify-start no-scroll">
-        <div className="flex h-full flex-col items-center justify-center">
-          <CircleNotch size={14} className="text-slate-400 animate-spin" />
+    <div className="allm-h-full allm-w-full allm-relative">
+      <div className="allm-h-full allm-max-h-[82vh] allm-pb-[100px] allm-pt-[5px] allm-bg-gray-100 allm-rounded-lg allm-px-2 allm-h-full allm-mt-2 allm-gap-y-2 allm-overflow-y-scroll allm-flex allm-flex-col allm-justify-start allm-no-scroll">
+        <div className="allm-flex allm-h-full allm-flex-col allm-items-center allm-justify-center">
+          <CircleNotch
+            size={14}
+            className="allm-text-slate-400 allm-animate-spin"
+          />
         </div>
       </div>
     </div>
   );
 }
+
+function SuggestedMessages({ settings }) {
+  if (!settings?.defaultMessages?.length) return null;
+
+  return (
+    <div className="allm-flex allm-flex-col allm-gap-y-2 allm-w-[75%]">
+      {settings.defaultMessages.map((content, i) => (
+        <button
+          key={i}
+          style={{
+            opacity: 0,
+            wordBreak: "break-word",
+            backgroundColor: embedderSettings.USER_STYLES.msgBg,
+            fontSize: settings.textSize,
+          }}
+          type="button"
+          onClick={() => {
+            window.dispatchEvent(
+              new CustomEvent(SEND_TEXT_EVENT, { detail: { command: content } })
+            );
+          }}
+          className={`msg-suggestion allm-border-none hover:allm-shadow-[0_4px_14px_rgba(0,0,0,0.5)] allm-cursor-pointer allm-px-2 allm-py-2 allm-rounded-lg allm-text-white allm-w-full allm-shadow-[0_4px_14px_rgba(0,0,0,0.25)]`}
+        >
+          {content}
+        </button>
+      ))}
+    </div>
+  );
+}

embed/src/components/ChatWindow/ChatContainer/PromptInput/index.jsx

@@ -46,14 +46,17 @@ export default function PromptInput({
   };
 
   return (
-    <div className="w-full sticky bottom-0 z-10 flex justify-center items-center px-5 bg-white">
+    <div className="allm-w-full allm-sticky allm-bottom-0 allm-z-10 allm-flex allm-justify-center allm-items-center allm-bg-white">
       <form
         onSubmit={handleSubmit}
-        className="flex flex-col gap-y-1 rounded-t-lg w-full items-center justify-center"
+        className="allm-flex allm-flex-col allm-gap-y-1 allm-rounded-t-lg allm-w-full allm-items-center allm-justify-center"
+      >
+        <div className="allm-flex allm-items-center allm-w-full">
+          <div className="allm-bg-white allm-flex allm-flex-col allm-px-4 allm-overflow-hidden allm-w-full">
+            <div
+              style={{ border: "1.5px solid #22262833" }}
+              className="allm-flex allm-items-center allm-w-full allm-rounded-2xl"
             >
-        <div className="flex items-center w-full">
-          <div className="bg-white border-[1.5px] border-[#22262833]/20 rounded-2xl flex flex-col px-4 overflow-hidden w-full">
-            <div className="flex items-center w-full">
               <textarea
                 ref={textareaRef}
                 onKeyUp={adjustTextArea}
@@ -67,7 +70,7 @@ export default function PromptInput({
                   adjustTextArea(e);
                 }}
                 value={message}
-                className="cursor-text max-h-[100px] text-[14px] mx-2 py-2 w-full text-black bg-transparent placeholder:text-slate-800/60 resize-none active:outline-none focus:outline-none flex-grow"
+                className="allm-font-sans allm-border-none allm-cursor-text allm-max-h-[100px] allm-text-[14px] allm-mx-2 allm-py-2 allm-w-full allm-text-black allm-bg-transparent placeholder:allm-text-slate-800/60 allm-resize-none active:allm-outline-none focus:allm-outline-none allm-flex-grow"
                 placeholder={"Send a message"}
                 id="message-input"
               />
@@ -75,20 +78,20 @@ export default function PromptInput({
                 ref={formRef}
                 type="submit"
                 disabled={buttonDisabled}
-                className="inline-flex justify-center rounded-2xl cursor-pointer text-black group ml-4"
+                className="allm-bg-transparent allm-border-none allm-inline-flex allm-justify-center allm-rounded-2xl allm-cursor-pointer allm-text-black group"
                 id="send-message-button"
                 aria-label="Send message"
               >
                 {buttonDisabled ? (
-                  <CircleNotch className="w-4 h-4 animate-spin" />
+                  <CircleNotch className="allm-w-4 allm-h-4 allm-animate-spin" />
                 ) : (
                   <PaperPlaneRight
                     size={24}
-                    className="my-3 text-[#22262899]/60 group-hover:text-[#22262899]/90"
+                    className="allm-my-3 allm-text-[#22262899]/60 group-hover:allm-text-[#22262899]/90"
                     weight="fill"
                   />
                 )}
-                <span className="sr-only">Send message</span>
+                <span className="allm-sr-only">Send message</span>
               </button>
             </div>
           </div>

embed/src/components/ChatWindow/ChatContainer/index.jsx

@@ -3,6 +3,7 @@ import ChatHistory from "./ChatHistory";
 import PromptInput from "./PromptInput";
 import handleChat from "@/utils/chat";
 import ChatService from "@/models/chatService";
+export const SEND_TEXT_EVENT = "anythingllm-embed-send-prompt";
 
 export default function ChatContainer({
   sessionId,
@@ -45,6 +46,45 @@ export default function ChatContainer({
     setLoadingResponse(true);
   };
 
+  const sendCommand = (command, history = [], attachments = []) => {
+    if (!command || command === "") return false;
+
+    let prevChatHistory;
+    if (history.length > 0) {
+      // use pre-determined history chain.
+      prevChatHistory = [
+        ...history,
+        {
+          content: "",
+          role: "assistant",
+          pending: true,
+          userMessage: command,
+          attachments,
+          animate: true,
+        },
+      ];
+    } else {
+      prevChatHistory = [
+        ...chatHistory,
+        {
+          content: command,
+          role: "user",
+          attachments,
+        },
+        {
+          content: "",
+          role: "assistant",
+          pending: true,
+          userMessage: command,
+          animate: true,
+        },
+      ];
+    }
+
+    setChatHistory(prevChatHistory);
+    setLoadingResponse(true);
+  };
+
   useEffect(() => {
     async function fetchReply() {
       const promptMessage =
@@ -76,9 +116,21 @@ export default function ChatContainer({
     loadingResponse === true && fetchReply();
   }, [loadingResponse, chatHistory]);
 
+  const handleAutofillEvent = (event) => {
+    if (!event.detail.command) return;
+    sendCommand(event.detail.command, [], []);
+  };
+
+  useEffect(() => {
+    window.addEventListener(SEND_TEXT_EVENT, handleAutofillEvent);
+    return () => {
+      window.removeEventListener(SEND_TEXT_EVENT, handleAutofillEvent);
+    };
+  }, []);
+
   return (
-    <div className="h-full w-full flex flex-col">
-      <div className="flex-grow overflow-y-auto">
+    <div className="allm-h-full allm-w-full allm-flex allm-flex-col">
+      <div className="allm-flex-grow allm-overflow-y-auto">
         <ChatHistory settings={settings} history={chatHistory} />
       </div>
       <PromptInput

embed/src/components/ChatWindow/Header/index.jsx

@@ -45,23 +45,24 @@ export default function ChatWindowHeader({
 
   return (
     <div
-      className="flex items-center relative rounded-t-2xl bg-black/10"
+      style={{ borderBottom: "1px solid #E9E9E9" }}
+      className="allm-flex allm-items-center allm-relative allm-rounded-t-2xl"
       id="anything-llm-header"
     >
-      <div className="flex justify-center items-center w-full h-[76px]">
+      <div className="allm-flex allm-justify-center allm-items-center allm-w-full allm-h-[76px]">
         <img
           style={{ maxWidth: 48, maxHeight: 48 }}
           src={iconUrl ?? AnythingLLMIcon}
           alt={iconUrl ? "Brand" : "AnythingLLM Logo"}
         />
       </div>
-      <div className="absolute right-0 flex gap-x-1 items-center px-[22px]">
+      <div className="allm-absolute allm-right-0 allm-flex allm-gap-x-1 allm-items-center allm-px-[22px]">
         {settings.loaded && (
           <button
             ref={buttonRef}
             type="button"
             onClick={() => setShowOptions(!showingOptions)}
-            className="hover:bg-gray-100 rounded-sm text-slate-800"
+            className="allm-bg-transparent hover:allm-cursor-pointer allm-border-none hover:allm-bg-gray-100 allm-rounded-sm allm-text-slate-800/60"
             aria-label="Options"
           >
             <DotsThreeOutlineVertical size={20} weight="fill" />
@@ -70,7 +71,7 @@ export default function ChatWindowHeader({
         <button
           type="button"
           onClick={closeChat}
-          className="hover:bg-gray-100 rounded-sm text-slate-800"
+          className="allm-bg-transparent hover:allm-cursor-pointer allm-border-none hover:allm-bg-gray-100 allm-rounded-sm allm-text-slate-800/60"
           aria-label="Close"
         >
           <X size={20} weight="bold" />
@@ -92,14 +93,14 @@ function OptionsMenu({ settings, showing, resetChat, sessionId, menuRef }) {
   return (
     <div
       ref={menuRef}
-      className="absolute z-10 bg-white flex flex-col gap-y-1 rounded-xl shadow-lg border border-gray-300 top-[64px] right-[46px]"
+      className="allm-bg-white allm-absolute allm-z-10 allm-flex allm-flex-col allm-gap-y-1 allm-rounded-xl allm-shadow-lg allm-top-[64px] allm-right-[46px]"
     >
       <button
         onClick={resetChat}
-        className="flex items-center gap-x-2 hover:bg-gray-100 text-sm text-gray-700 py-2.5 px-4 rounded-xl"
+        className="hover:allm-cursor-pointer allm-bg-white allm-gap-x-[12px] hover:allm-bg-gray-100 allm-rounded-lg allm-border-none allm-flex allm-items-center allm-text-base allm-text-[#7A7D7E] allm-font-bold allm-px-4"
       >
         <ArrowCounterClockwise size={24} />
-        <p className="text-sm text-[#7A7D7E] font-bold">Reset Chat</p>
+        <p className="allm-text-[14px]">Reset Chat</p>
       </button>
       <ContactSupport email={settings.supportEmail} />
       <SessionID sessionId={sessionId} />
@@ -120,9 +121,9 @@ function SessionID({ sessionId }) {
 
   if (sessionIdCopied) {
     return (
-      <div className="flex items-center gap-x-2 hover:bg-gray-100 text-sm text-gray-700 py-2.5 px-4 rounded-xl">
+      <div className="hover:allm-cursor-pointer allm-bg-white allm-gap-x-[12px] hover:allm-bg-gray-100 allm-rounded-lg allm-border-none allm-flex allm-items-center allm-text-base allm-text-[#7A7D7E] allm-font-bold allm-px-4">
         <Check size={24} />
-        <p className="text-sm text-[#7A7D7E] font-bold">Copied!</p>
+        <p className="allm-text-[14px] allm-font-sans">Copied!</p>
       </div>
     );
   }
@@ -130,10 +131,10 @@ function SessionID({ sessionId }) {
   return (
     <button
       onClick={copySessionId}
-      className="flex items-center gap-x-2 hover:bg-gray-100 text-sm text-gray-700 py-2.5 px-4 rounded-xl"
+      className="hover:allm-cursor-pointer allm-bg-white allm-gap-x-[12px] hover:allm-bg-gray-100 allm-rounded-lg allm-border-none allm-flex allm-items-center allm-text-base allm-text-[#7A7D7E] allm-font-bold allm-px-4"
     >
       <Copy size={24} />
-      <p className="text-sm text-[#7A7D7E] font-bold">Session ID</p>
+      <p className="allm-text-[14px]">Session ID</p>
     </button>
   );
 }
@@ -145,10 +146,10 @@ function ContactSupport({ email = null }) {
   return (
     <a
       href={`mailto:${email}?Subject=${encodeURIComponent(subject)}`}
-      className="flex items-center gap-x-2 hover:bg-gray-100 text-sm text-gray-700 py-2.5 px-4 rounded-xl"
+      className="allm-no-underline hover:allm-underline hover:allm-cursor-pointer allm-bg-white allm-gap-x-[12px] hover:allm-bg-gray-100 allm-rounded-lg allm-border-none allm-flex allm-items-center allm-text-base allm-text-[#7A7D7E] allm-font-bold allm-px-4"
     >
       <Envelope size={24} />
-      <p className="text-sm text-[#7A7D7E] font-bold">Email Support</p>
+      <p className="allm-text-[14px] allm-font-sans">Email Support</p>
     </a>
   );
 }

embed/src/components/ChatWindow/index.jsx

@@ -14,7 +14,7 @@ export default function ChatWindow({ closeChat, settings, sessionId }) {
 
   if (loading) {
     return (
-      <div className="flex flex-col h-full">
+      <div className="allm-flex allm-flex-col allm-h-full">
         <ChatWindowHeader
           sessionId={sessionId}
           settings={settings}
@@ -23,7 +23,7 @@ export default function ChatWindow({ closeChat, settings, sessionId }) {
           setChatHistory={setChatHistory}
         />
         <ChatHistoryLoading />
-        <div className="pt-4 pb-2 h-fit gap-y-1">
+        <div className="allm-pt-4 allm-pb-2 allm-h-fit allm-gap-y-1">
           <SessionId />
           <Sponsor settings={settings} />
         </div>
@@ -34,7 +34,7 @@ export default function ChatWindow({ closeChat, settings, sessionId }) {
   setEventDelegatorForCodeSnippets();
 
   return (
-    <div className="flex flex-col h-full">
+    <div className="allm-flex allm-flex-col allm-h-full">
       <ChatWindowHeader
         sessionId={sessionId}
         settings={settings}
@@ -42,14 +42,14 @@ export default function ChatWindow({ closeChat, settings, sessionId }) {
         closeChat={closeChat}
         setChatHistory={setChatHistory}
       />
-      <div className="flex-grow overflow-y-auto">
+      <div className="allm-flex-grow allm-overflow-y-auto">
         <ChatContainer
           sessionId={sessionId}
           settings={settings}
           knownHistory={chatHistory}
         />
       </div>
-      <div className="mt-4 pb-4 h-fit gap-y-2 z-10">
+      <div className="allm-mt-4 allm-pb-4 allm-h-fit allm-gap-y-2 allm-z-10">
         <Sponsor settings={settings} />
         <ResetChat
           setChatHistory={setChatHistory}
@@ -76,13 +76,13 @@ function copyCodeSnippet(uuid) {
 
   window.navigator.clipboard.writeText(markdown);
 
-  target.classList.add("text-green-500");
+  target.classList.add("allm-text-green-500");
   const originalText = target.innerHTML;
   target.innerText = "Copied!";
   target.setAttribute("disabled", true);
 
   setTimeout(() => {
-    target.classList.remove("text-green-500");
+    target.classList.remove("allm-text-green-500");
     target.innerHTML = originalText;
     target.removeAttribute("disabled");
   }, 2500);

embed/src/components/Head.jsx

@@ -1,3 +1,5 @@
+import { embedderSettings } from "@/main";
+
 const hljsCss = `
 pre code.hljs{display:block;overflow-x:auto;padding:1em}code.hljs{padding:3px 5px}/*!
   Theme: GitHub Dark Dimmed
@@ -16,7 +18,7 @@ const customCss = `
    * Dot Falling
    * ==============================================
    */
-  .dot-falling {
+  .allm-dot-falling {
     position: relative;
     left: -9999px;
     width: 10px;
@@ -29,15 +31,15 @@ const customCss = `
     animation-delay: 0.1s;
   }
 
-  .dot-falling::before,
-  .dot-falling::after {
+  .allm-dot-falling::before,
+  .allm-dot-falling::after {
     content: "";
     display: inline-block;
     position: absolute;
     top: 0;
   }
 
-  .dot-falling::before {
+  .allm-dot-falling::before {
     width: 10px;
     height: 10px;
     border-radius: 5px;
@@ -47,7 +49,7 @@ const customCss = `
     animation-delay: 0s;
   }
 
-  .dot-falling::after {
+  .allm-dot-falling::after {
     width: 10px;
     height: 10px;
     border-radius: 5px;
@@ -101,50 +103,20 @@ const customCss = `
 
   #chat-history::-webkit-scrollbar,
   #chat-container::-webkit-scrollbar,
-  .no-scroll::-webkit-scrollbar {
+  .allm-no-scroll::-webkit-scrollbar {
     display: none !important;
   }
 
   /* Hide scrollbar for IE, Edge and Firefox */
   #chat-history,
   #chat-container,
-  .no-scroll {
+  .allm-no-scroll {
     -ms-overflow-style: none !important; /* IE and Edge */
     scrollbar-width: none !important; /* Firefox */
   }
 
-  .animate-slow-pulse {
-    transform: scale(1);
-    animation: subtlePulse 20s infinite;
-    will-change: transform;
-  }
-
-  @keyframes subtlePulse {
-    0% {
-      transform: scale(1);
-    }
-    50% {
-      transform: scale(1.1);
-    }
-    100% {
-      transform: scale(1);
-    }
-  }
-
-  @keyframes subtleShift {
-    0% {
-      background-position: 0% 50%;
-    }
-    50% {
-      background-position: 100% 50%;
-    }
-    100% {
-      background-position: 0% 50%;
-    }
-  }
-
-  .bg-black-900 {
-    background: #141414;
+  span.allm-whitespace-pre-line>p {
+    margin: 0px;
   }
 `;
 
@@ -153,6 +125,7 @@ export default function Head() {
     <head>
       <style>{hljsCss}</style>
       <style>{customCss}</style>
+      <link rel="stylesheet" href={embedderSettings.stylesSrc} />
     </head>
   );
 }

embed/src/components/OpenButton/index.jsx

@@ -23,9 +23,10 @@ export default function OpenButton({ settings, isOpen, toggleOpen }) {
     : CHAT_ICONS.plus;
   return (
     <button
+      style={{ backgroundColor: settings.buttonColor }}
       id="anything-llm-embed-chat-button"
       onClick={toggleOpen}
-      className={`flex items-center justify-center p-4 rounded-full bg-[${settings.buttonColor}] text-white text-2xl`}
+      className={`hover:allm-cursor-pointer allm-border-none allm-flex allm-items-center allm-justify-center allm-p-4 allm-rounded-full allm-text-white allm-text-2xl hover:allm-opacity-95`}
       aria-label="Toggle Menu"
     >
       <ChatIcon className="text-white" />

embed/src/components/ResetChat/index.jsx

@@ -7,9 +7,10 @@ export default function ResetChat({ setChatHistory, settings, sessionId }) {
   };
 
   return (
-    <div className="w-full flex justify-center">
+    <div className="allm-w-full allm-flex allm-justify-center">
       <button
-        className="text-sm text-[#7A7D7E] hover:text-[#7A7D7E]/80 hover:underline"
+        style={{ color: "#7A7D7E" }}
+        className="hover:allm-cursor-pointer allm-border-none allm-text-sm allm-bg-transparent hover:allm-opacity-80 hover:allm-underline"
         onClick={() => handleChatReset()}
       >
         Reset Chat

embed/src/components/SessionId/index.jsx

@@ -5,6 +5,8 @@ export default function SessionId() {
   if (!sessionId) return null;
 
   return (
-    <div className="text-xs text-gray-300 w-full text-center">{sessionId}</div>
+    <div className="allm-text-xs allm-text-gray-300 allm-w-full allm-text-center">
+      {sessionId}
+    </div>
   );
 }

embed/src/components/Sponsor/index.jsx

@@ -2,12 +2,13 @@ export default function Sponsor({ settings }) {
   if (!!settings.noSponsor) return null;
 
   return (
-    <div className="flex w-full items-center justify-center">
+    <div className="allm-flex allm-w-full allm-items-center allm-justify-center">
       <a
+        style={{ color: "#0119D9" }}
         href={settings.sponsorLink ?? "#"}
         target="_blank"
         rel="noreferrer"
-        className="text-xs text-[#0119D9] hover:text-[#0119D9]/80 hover:underline"
+        className="allm-text-xs allm-font-sans hover:allm-opacity-80 hover:allm-underline"
       >
         {settings.sponsorText}
       </a>

embed/src/hooks/useScriptAttributes.js

@@ -19,7 +19,7 @@ const DEFAULT_SETTINGS = {
   assistantBgColor: "#2563eb", // assistant text bubble color
   noSponsor: null, // Shows sponsor in footer of chat
   sponsorText: "Powered by AnythingLLM", // default sponsor text
-  sponsorLink: "https://useanything.com", // default sponsor link
+  sponsorLink: "https://anythingllm.com", // default sponsor link
   position: "bottom-right", // position of chat button/window
   assistantName: "AnythingLLM Chat Assistant", // default assistant name
   assistantIcon: null, // default assistant icon
@@ -30,6 +30,8 @@ const DEFAULT_SETTINGS = {
   // behaviors
   openOnLoad: "off", // or "on"
   supportEmail: null, // string of email for contact
+  username: null, // The display or readable name set on a script
+  defaultMessages: [], // list of strings for default messages.
 };
 
 export default function useGetScriptAttributes() {
@@ -51,7 +53,7 @@ export default function useGetScriptAttributes() {
 
       setSettings({
         ...DEFAULT_SETTINGS,
-        ...embedderSettings.settings,
+        ...parseAndValidateEmbedSettings(embedderSettings.settings),
         loaded: true,
       });
     }
@@ -60,3 +62,43 @@ export default function useGetScriptAttributes() {
 
   return settings;
 }
+
+const validations = {
+  _fallbacks: {
+    defaultMessages: [],
+  },
+
+  defaultMessages: function (value = null) {
+    if (typeof value !== "string") return this._fallbacks.defaultMessages;
+    try {
+      const list = value.split(",");
+      if (
+        !Array.isArray(list) ||
+        list.length === 0 ||
+        !list.every((v) => typeof v === "string" && v.length > 0)
+      )
+        throw new Error(
+          "Invalid default-messages attribute value. Must be array of strings"
+        );
+      return list.map((v) => v.trim());
+    } catch (e) {
+      console.error("AnythingLLMEmbed", e);
+      return this._fallbacks.defaultMessages;
+    }
+  },
+};
+
+function parseAndValidateEmbedSettings(settings = {}) {
+  const validated = {};
+  for (let [key, value] of Object.entries(settings)) {
+    if (!validations.hasOwnProperty(key)) {
+      validated[key] = value;
+      continue;
+    }
+
+    const validatedValue = validations[key](value);
+    validated[key] = validatedValue;
+  }
+
+  return validated;
+}

embed/src/index.css

@@ -0,0 +1,32 @@
+@tailwind base;
+@tailwind components;
+@tailwind utilities;
+
+.msg-suggestion {
+  animation-name: fadeIn;
+  animation-duration: 300ms;
+  animation-timing-function: linear;
+  animation-fill-mode: forwards;
+}
+
+@keyframes fadeIn {
+  0% {
+    opacity: 0%;
+  }
+
+  25% {
+    opacity: 25%;
+  }
+
+  50% {
+    opacity: 50%;
+  }
+
+  75% {
+    opacity: 75%;
+  }
+
+  100% {
+    opacity: 100%;
+  }
+}

embed/src/main.jsx

@@ -1,6 +1,8 @@
 import React from "react";
 import ReactDOM from "react-dom/client";
 import App from "./App.jsx";
+import "./index.css";
+import { parseStylesSrc } from "./utils/constants.js";
 const appElement = document.createElement("div");
 
 document.body.appendChild(appElement);
@@ -17,6 +19,13 @@ const scriptSettings = Object.assign(
 );
 export const embedderSettings = {
   settings: scriptSettings,
-  USER_STYLES: `bg-[${scriptSettings?.userBgColor ?? "#3DBEF5"}] text-white rounded-t-[18px] rounded-bl-[18px] rounded-br-[4px] mx-[20px]`,
-  ASSISTANT_STYLES: `bg-[${scriptSettings?.assistantBgColor ?? "#FFFFFF"}] text-[#222628] rounded-t-[18px] rounded-br-[18px] rounded-bl-[4px] mr-[37px] ml-[9px]`,
+  stylesSrc: parseStylesSrc(document?.currentScript?.src),
+  USER_STYLES: {
+    msgBg: scriptSettings?.userBgColor ?? "#3DBEF5",
+    base: `allm-text-white allm-rounded-t-[18px] allm-rounded-bl-[18px] allm-rounded-br-[4px] allm-mx-[20px]`,
+  },
+  ASSISTANT_STYLES: {
+    msgBg: scriptSettings?.assistantBgColor ?? "#FFFFFF",
+    base: `allm-text-[#222628] allm-rounded-t-[18px] allm-rounded-br-[18px] allm-rounded-bl-[4px] allm-mr-[37px] allm-ml-[9px]`,
+  },
 };

embed/src/models/chatService.js

@@ -32,7 +32,7 @@ const ChatService = {
       .catch(() => false);
   },
   streamChat: async function (sessionId, embedSettings, message, handleChat) {
-    const { baseApiUrl, embedId } = embedSettings;
+    const { baseApiUrl, embedId, username } = embedSettings;
     const overrides = {
       prompt: embedSettings?.prompt ?? null,
       model: embedSettings?.model ?? null,
@@ -45,6 +45,7 @@ const ChatService = {
       body: JSON.stringify({
         message,
         sessionId,
+        username,
         ...overrides,
       }),
       signal: ctrl.signal,

embed/src/utils/constants.js

@@ -1 +1,15 @@
 export const CHAT_UI_REOPEN = "___anythingllm-chat-widget-open___";
+export function parseStylesSrc(scriptSrc = null) {
+  try {
+    const _url = new URL(scriptSrc);
+    _url.pathname = _url.pathname
+      .replace("anythingllm-chat-widget.js", "anythingllm-chat-widget.min.css")
+      .replace(
+        "anythingllm-chat-widget.min.js",
+        "anythingllm-chat-widget.min.css"
+      );
+    return _url.toString();
+  } catch {
+    return "";
+  }
+}

embed/tailwind.config.js

@@ -0,0 +1,103 @@
+/** @type {import('tailwindcss').Config} */
+export default {
+  darkMode: 'false',
+  prefix: 'allm-',
+  corePlugins: {
+    preflight: false,
+  },
+  content: {
+    relative: true,
+    files: [
+      "./src/components/**/*.{js,jsx}",
+      "./src/hooks/**/*.js",
+      "./src/models/**/*.js",
+      "./src/pages/**/*.{js,jsx}",
+      "./src/utils/**/*.js",
+      "./src/*.jsx",
+      "./index.html",
+    ]
+  },
+  theme: {
+    extend: {
+      rotate: {
+        "270": "270deg",
+        "360": "360deg"
+      },
+      colors: {
+        "black-900": "#141414",
+        accent: "#3D4147",
+        "sidebar-button": "#31353A",
+        sidebar: "#25272C",
+        "historical-msg-system": "rgba(255, 255, 255, 0.05);",
+        "historical-msg-user": "#2C2F35",
+        outline: "#4E5153",
+        "primary-button": "#46C8FF",
+        secondary: "#2C2F36",
+        "dark-input": "#18181B",
+        "mobile-onboarding": "#2C2F35",
+        "dark-highlight": "#1C1E21",
+        "dark-text": "#222628",
+        description: "#D2D5DB",
+        "x-button": "#9CA3AF"
+      },
+      backgroundImage: {
+        "preference-gradient":
+          "linear-gradient(180deg, #5A5C63 0%, rgba(90, 92, 99, 0.28) 100%);",
+        "chat-msg-user-gradient":
+          "linear-gradient(180deg, #3D4147 0%, #2C2F35 100%);",
+        "selected-preference-gradient":
+          "linear-gradient(180deg, #313236 0%, rgba(63.40, 64.90, 70.13, 0) 100%);",
+        "main-gradient": "linear-gradient(180deg, #3D4147 0%, #2C2F35 100%)",
+        "modal-gradient": "linear-gradient(180deg, #3D4147 0%, #2C2F35 100%)",
+        "sidebar-gradient": "linear-gradient(90deg, #5B616A 0%, #3F434B 100%)",
+        "login-gradient": "linear-gradient(180deg, #3D4147 0%, #2C2F35 100%)",
+        "menu-item-gradient":
+          "linear-gradient(90deg, #3D4147 0%, #2C2F35 100%)",
+        "menu-item-selected-gradient":
+          "linear-gradient(90deg, #5B616A 0%, #3F434B 100%)",
+        "workspace-item-gradient":
+          "linear-gradient(90deg, #3D4147 0%, #2C2F35 100%)",
+        "workspace-item-selected-gradient":
+          "linear-gradient(90deg, #5B616A 0%, #3F434B 100%)",
+        "switch-selected": "linear-gradient(146deg, #5B616A 0%, #3F434B 100%)"
+      },
+      fontFamily: {
+        sans: [
+          "plus-jakarta-sans",
+          "ui-sans-serif",
+          "system-ui",
+          "-apple-system",
+          "BlinkMacSystemFont",
+          '"Segoe UI"',
+          "Roboto",
+          '"Helvetica Neue"',
+          "Arial",
+          '"Noto Sans"',
+          "sans-serif",
+          '"Apple Color Emoji"',
+          '"Segoe UI Emoji"',
+          '"Segoe UI Symbol"',
+          '"Noto Color Emoji"'
+        ]
+      },
+      animation: {
+        sweep: "sweep 0.5s ease-in-out"
+      },
+      keyframes: {
+        sweep: {
+          "0%": { transform: "scaleX(0)", transformOrigin: "bottom left" },
+          "100%": { transform: "scaleX(1)", transformOrigin: "bottom left" }
+        },
+        fadeIn: {
+          "0%": { opacity: 0 },
+          "100%": { opacity: 1 }
+        },
+        fadeOut: {
+          "0%": { opacity: 1 },
+          "100%": { opacity: 0 }
+        }
+      }
+    }
+  },
+  plugins: []
+}

embed/vite.config.js

@@ -1,6 +1,7 @@
 // vite.config.js
 import { defineConfig } from "vite"
 import { fileURLToPath, URL } from "url"
+import postcss from "./postcss.config.js"
 import react from "@vitejs/plugin-react"
 import image from "@rollup/plugin-image"
 
@@ -10,6 +11,9 @@ export default defineConfig({
     // In dev, we need to disable this, but in prod, we need to enable it
     "process.env.NODE_ENV": JSON.stringify("production")
   },
+  css: {
+    postcss
+  },
   resolve: {
     alias: [
       {

embed/yarn.lock

@@ -7,6 +7,11 @@
   resolved "https://registry.yarnpkg.com/@aashutoshrathi/word-wrap/-/word-wrap-1.2.6.tgz#bd9154aec9983f77b3a034ecaa015c2e4201f6cf"
   integrity sha512-1Yjs2SvM8TflER/OD3cOjhWWOZb58A2t7wpE2S9XfBYTiIl+XFhQG2bjy4Pu1I+EAlCNUzRDYDdFwFYUKvXcIA==
 
+"@alloc/quick-lru@^5.2.0":
+  version "5.2.0"
+  resolved "https://registry.yarnpkg.com/@alloc/quick-lru/-/quick-lru-5.2.0.tgz#7bf68b20c0a350f936915fcae06f58e32007ce30"
+  integrity sha512-UrcABB+4bUrFABwbluTIBErXwvbsU/V7TZWfmbgJfbkwiBuziS9gxdODUyuiecfdGQ85jglMW6juS3+z5TsKLw==
+
 "@ampproject/remapping@^2.2.0":
   version "2.2.1"
   resolved "https://registry.yarnpkg.com/@ampproject/remapping/-/remapping-2.2.1.tgz#99e8e11851128b8702cd57c33684f1d0f260b630"
@@ -379,6 +384,18 @@
   resolved "https://registry.yarnpkg.com/@humanwhocodes/object-schema/-/object-schema-2.0.2.tgz#d9fae00a2d5cb40f92cfe64b47ad749fbc38f917"
   integrity sha512-6EwiSjwWYP7pTckG6I5eyFANjPhmPjUX9JRLUSfNPC7FX7zK9gyZAfUEaECL6ALTpGX5AjnBq3C9XmVWPitNpw==
 
+"@isaacs/cliui@^8.0.2":
+  version "8.0.2"
+  resolved "https://registry.yarnpkg.com/@isaacs/cliui/-/cliui-8.0.2.tgz#b37667b7bc181c168782259bab42474fbf52b550"
+  integrity sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==
+  dependencies:
+    string-width "^5.1.2"
+    string-width-cjs "npm:string-width@^4.2.0"
+    strip-ansi "^7.0.1"
+    strip-ansi-cjs "npm:strip-ansi@^6.0.1"
+    wrap-ansi "^8.1.0"
+    wrap-ansi-cjs "npm:wrap-ansi@^7.0.0"
+
 "@jridgewell/gen-mapping@^0.3.0", "@jridgewell/gen-mapping@^0.3.2":
   version "0.3.3"
   resolved "https://registry.yarnpkg.com/@jridgewell/gen-mapping/-/gen-mapping-0.3.3.tgz#7e02e6eb5df901aaedb08514203b096614024098"
@@ -432,12 +449,12 @@
     "@nodelib/fs.stat" "2.0.5"
     run-parallel "^1.1.9"
 
-"@nodelib/fs.stat@2.0.5":
+"@nodelib/fs.stat@2.0.5", "@nodelib/fs.stat@^2.0.2":
   version "2.0.5"
   resolved "https://registry.yarnpkg.com/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz#5bd262af94e9d25bd1e71b05deed44876a222e8b"
   integrity sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==
 
-"@nodelib/fs.walk@^1.2.8":
+"@nodelib/fs.walk@^1.2.3", "@nodelib/fs.walk@^1.2.8":
   version "1.2.8"
   resolved "https://registry.yarnpkg.com/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz#e95737e8bb6746ddedf69c556953494f196fe69a"
   integrity sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==
@@ -450,6 +467,11 @@
   resolved "https://registry.yarnpkg.com/@phosphor-icons/react/-/react-2.0.15.tgz#4d8e28484d45649f53a6cd75db161cf8b8379e1d"
   integrity sha512-PQKNcRrfERlC8gJGNz0su0i9xVmeubXSNxucPcbCLDd9u0cwJVTEyYK87muul/svf0UXFdL2Vl6bbeOhT1Mwow==
 
+"@pkgjs/parseargs@^0.11.0":
+  version "0.11.0"
+  resolved "https://registry.yarnpkg.com/@pkgjs/parseargs/-/parseargs-0.11.0.tgz#a77ea742fab25775145434eb1d2328cf5013ac33"
+  integrity sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==
+
 "@rollup/plugin-image@^3.0.3":
   version "3.0.3"
   resolved "https://registry.yarnpkg.com/@rollup/plugin-image/-/plugin-image-3.0.3.tgz#025b557180bae20f2349ff5130ef2114169feaac"
@@ -684,7 +706,7 @@ ansi-styles@^3.2.1:
   dependencies:
     color-convert "^1.9.0"
 
-ansi-styles@^4.1.0:
+ansi-styles@^4.0.0, ansi-styles@^4.1.0:
   version "4.3.0"
   resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-4.3.0.tgz#edd803628ae71c04c85ae7a0906edad34b648937"
   integrity sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==
@@ -696,6 +718,11 @@ ansi-styles@^6.1.0:
   resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-6.2.1.tgz#0e62320cf99c21afff3b3012192546aacbfb05c5"
   integrity sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==
 
+any-promise@^1.0.0:
+  version "1.3.0"
+  resolved "https://registry.yarnpkg.com/any-promise/-/any-promise-1.3.0.tgz#abc6afeedcea52e809cdc0376aed3ce39635d17f"
+  integrity sha512-7UvmKalWRt1wgjL1RrGxoSJW/0QZFIegpeGvZG9kjp8vrRu55XTHbwnqq2GpXm9uLbcuhxm3IqX9OB4MZR1b2A==
+
 anymatch@~3.1.2:
   version "3.1.3"
   resolved "https://registry.yarnpkg.com/anymatch/-/anymatch-3.1.3.tgz#790c58b19ba1720a84205b57c618d5ad8524973e"
@@ -709,7 +736,7 @@ arch@^2.2.0:
   resolved "https://registry.yarnpkg.com/arch/-/arch-2.2.0.tgz#1bc47818f305764f23ab3306b0bfc086c5a29d11"
   integrity sha512-Of/R0wqp83cgHozfIYLbBMnej79U/SVGOOyuB3VVFv1NRM/PSFMK12x9KVtiYzJqmnU5WR2qp0Z5rHb7sWGnFQ==
 
-arg@5.0.2:
+arg@5.0.2, arg@^5.0.2:
   version "5.0.2"
   resolved "https://registry.yarnpkg.com/arg/-/arg-5.0.2.tgz#c81433cc427c92c4dcf4865142dbca6f15acd59c"
   integrity sha512-PYjyFOLKQ9y57JvQ6QLo8dAgNqswh8M1RMJYdQduT6xbWSgK36P/Z/v+p888pM69jMMfS8Xd8F6I1kQ/I9HUGg==
@@ -838,6 +865,13 @@ brace-expansion@^1.1.7:
     balanced-match "^1.0.0"
     concat-map "0.0.1"
 
+brace-expansion@^2.0.1:
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-2.0.1.tgz#1edc459e0f0c548486ecf9fc99f2221364b9a0ae"
+  integrity sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==
+  dependencies:
+    balanced-match "^1.0.0"
+
 braces@^3.0.2, braces@~3.0.2:
   version "3.0.2"
   resolved "https://registry.yarnpkg.com/braces/-/braces-3.0.2.tgz#3454e1a462ee8d599e236df336cd9ea4f8afe107"
@@ -845,6 +879,13 @@ braces@^3.0.2, braces@~3.0.2:
   dependencies:
     fill-range "^7.0.1"
 
+braces@^3.0.3:
+  version "3.0.3"
+  resolved "https://registry.yarnpkg.com/braces/-/braces-3.0.3.tgz#490332f40919452272d55a8480adc0c441358789"
+  integrity sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==
+  dependencies:
+    fill-range "^7.1.1"
+
 browserslist@^4.22.2:
   version "4.22.3"
   resolved "https://registry.yarnpkg.com/browserslist/-/browserslist-4.22.3.tgz#299d11b7e947a6b843981392721169e27d60c5a6"
@@ -879,6 +920,11 @@ callsites@^3.0.0:
   resolved "https://registry.yarnpkg.com/callsites/-/callsites-3.1.0.tgz#b3630abd8943432f54b3f0519238e33cd7df2f73"
   integrity sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==
 
+camelcase-css@^2.0.1:
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/camelcase-css/-/camelcase-css-2.0.1.tgz#ee978f6947914cc30c6b44741b6ed1df7f043fd5"
+  integrity sha512-QOSvevhslijgYwRx6Rv7zKdMF8lbRmx+uQGx2+vDc+KI/eBnsy9kit5aj23AgGu3pa4t9AgwbnXWqS+iOY+2aA==
+
 camelcase@^7.0.0:
   version "7.0.1"
   resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-7.0.1.tgz#f02e50af9fd7782bc8b88a3558c32fd3a388f048"
@@ -938,6 +984,38 @@ chokidar@^3.5.2:
   optionalDependencies:
     fsevents "~2.3.2"
 
+chokidar@^3.5.3:
+  version "3.6.0"
+  resolved "https://registry.yarnpkg.com/chokidar/-/chokidar-3.6.0.tgz#197c6cc669ef2a8dc5e7b4d97ee4e092c3eb0d5b"
+  integrity sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw==
+  dependencies:
+    anymatch "~3.1.2"
+    braces "~3.0.2"
+    glob-parent "~5.1.2"
+    is-binary-path "~2.1.0"
+    is-glob "~4.0.1"
+    normalize-path "~3.0.0"
+    readdirp "~3.6.0"
+  optionalDependencies:
+    fsevents "~2.3.2"
+
+clean-css-cli@^5.6.3:
+  version "5.6.3"
+  resolved "https://registry.yarnpkg.com/clean-css-cli/-/clean-css-cli-5.6.3.tgz#be2d65c9b17bba01a598d7eeb5ce68f4119f07b5"
+  integrity sha512-MUAta8pEqA/d2DKQwtZU5nm0Og8TCyAglOx3GlWwjhGdKBwY4kVF6E5M6LU/jmmuswv+HbYqG/dKKkq5p1dD0A==
+  dependencies:
+    chokidar "^3.5.2"
+    clean-css "^5.3.3"
+    commander "7.x"
+    glob "^7.1.6"
+
+clean-css@^5.3.3:
+  version "5.3.3"
+  resolved "https://registry.yarnpkg.com/clean-css/-/clean-css-5.3.3.tgz#b330653cd3bd6b75009cc25c714cae7b93351ccd"
+  integrity sha512-D5J+kHaVb/wKSFcyyV75uCn8fiY4sV38XJoe4CUyGQ+mOU/fMVYUdH1hJC+CJQ5uY3EnW27SbJYS4X8BiLrAFg==
+  dependencies:
+    source-map "~0.6.0"
+
 cli-boxes@^3.0.0:
   version "3.0.0"
   resolved "https://registry.yarnpkg.com/cli-boxes/-/cli-boxes-3.0.0.tgz#71a10c716feeba005e4504f36329ef0b17cf3145"
@@ -976,11 +1054,21 @@ color-name@~1.1.4:
   resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.4.tgz#c2a09a87acbde69543de6f63fa3995c826c536a2"
   integrity sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==
 
+commander@7.x:
+  version "7.2.0"
+  resolved "https://registry.yarnpkg.com/commander/-/commander-7.2.0.tgz#a36cb57d0b501ce108e4d20559a150a391d97ab7"
+  integrity sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw==
+
 commander@^2.20.0:
   version "2.20.3"
   resolved "https://registry.yarnpkg.com/commander/-/commander-2.20.3.tgz#fd485e84c03eb4881c20722ba48035e8531aeb33"
   integrity sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==
 
+commander@^4.0.0:
+  version "4.1.1"
+  resolved "https://registry.yarnpkg.com/commander/-/commander-4.1.1.tgz#9fd602bd936294e9e9ef46a3f4d6964044b18068"
+  integrity sha512-NOKm8xhkzAjzFx8B2v5OAHT+u5pRQc2UCa2Vq9jYL/31o2wi9mxBA7LIFs3sV5VSC49z6pEhfbMULvShKj26WA==
+
 compressible@~2.0.16:
   version "2.0.18"
   resolved "https://registry.yarnpkg.com/compressible/-/compressible-2.0.18.tgz#af53cca6b070d4c3c0750fbd77286a6d7cc46fba"
@@ -1016,7 +1104,7 @@ convert-source-map@^2.0.0:
   resolved "https://registry.yarnpkg.com/convert-source-map/-/convert-source-map-2.0.0.tgz#4b560f649fc4e918dd0ab75cf4961e8bc882d82a"
   integrity sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==
 
-cross-spawn@^7.0.2, cross-spawn@^7.0.3:
+cross-spawn@^7.0.0, cross-spawn@^7.0.2, cross-spawn@^7.0.3:
   version "7.0.3"
   resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.3.tgz#f73a85b9d5d41d045551c177e2882d4ac85728a6"
   integrity sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==
@@ -1025,6 +1113,11 @@ cross-spawn@^7.0.2, cross-spawn@^7.0.3:
     shebang-command "^2.0.0"
     which "^2.0.1"
 
+cssesc@^3.0.0:
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/cssesc/-/cssesc-3.0.0.tgz#37741919903b868565e1c09ea747445cd18983ee"
+  integrity sha512-/Tb/JcjK111nNScGob5MNtsntNM1aCNUDipB/TkwZFhyDrrE47SOx/18wF2bbjgc3ZzCSKW1T5nt5EbFoAz/Vg==
+
 csstype@^3.0.2:
   version "3.1.3"
   resolved "https://registry.yarnpkg.com/csstype/-/csstype-3.1.3.tgz#d80ff294d114fb0e6ac500fbf85b60137d7eff81"
@@ -1079,6 +1172,16 @@ define-properties@^1.1.3, define-properties@^1.2.0, define-properties@^1.2.1:
     has-property-descriptors "^1.0.0"
     object-keys "^1.1.1"
 
+didyoumean@^1.2.2:
+  version "1.2.2"
+  resolved "https://registry.yarnpkg.com/didyoumean/-/didyoumean-1.2.2.tgz#989346ffe9e839b4555ecf5666edea0d3e8ad037"
+  integrity sha512-gxtyfqMg7GKyhQmb056K7M3xszy/myH8w+B4RT+QXBQsvAOdc3XymqDDPHx1BgPgsdAA5SIifona89YtRATDzw==
+
+dlv@^1.1.3:
+  version "1.1.3"
+  resolved "https://registry.yarnpkg.com/dlv/-/dlv-1.1.3.tgz#5c198a8a11453596e751494d49874bc7732f2e79"
+  integrity sha512-+HlytyjlPKnIG8XuRG8WvmBP8xs8P71y+SKKS6ZXWoEgLuePxtDoUEiH7WkdePWrQ5JBpE6aoVqfZfJUQkjXwA==
+
 doctrine@^2.1.0:
   version "2.1.0"
   resolved "https://registry.yarnpkg.com/doctrine/-/doctrine-2.1.0.tgz#5cd01fc101621b42c4cd7f5d1a66243716d3f39d"
@@ -1404,6 +1507,17 @@ fast-deep-equal@^3.1.1, fast-deep-equal@^3.1.3:
   resolved "https://registry.yarnpkg.com/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz#3a7d56b559d6cbc3eb512325244e619a65c6c525"
   integrity sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==
 
+fast-glob@^3.3.0:
+  version "3.3.2"
+  resolved "https://registry.yarnpkg.com/fast-glob/-/fast-glob-3.3.2.tgz#a904501e57cfdd2ffcded45e99a54fef55e46129"
+  integrity sha512-oX2ruAFQwf/Orj8m737Y5adxDQO0LAB7/S5MnxCdTNDd4p6BsyIVsv9JQsATbTSq8KHRpLwIHbVlUNatxd+1Ow==
+  dependencies:
+    "@nodelib/fs.stat" "^2.0.2"
+    "@nodelib/fs.walk" "^1.2.3"
+    glob-parent "^5.1.2"
+    merge2 "^1.3.0"
+    micromatch "^4.0.4"
+
 fast-json-stable-stringify@^2.0.0:
   version "2.1.0"
   resolved "https://registry.yarnpkg.com/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz#874bf69c6f404c2b5d99c481341399fd55892633"
@@ -1442,6 +1556,13 @@ fill-range@^7.0.1:
   dependencies:
     to-regex-range "^5.0.1"
 
+fill-range@^7.1.1:
+  version "7.1.1"
+  resolved "https://registry.yarnpkg.com/fill-range/-/fill-range-7.1.1.tgz#44265d3cac07e3ea7dc247516380643754a05292"
+  integrity sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==
+  dependencies:
+    to-regex-range "^5.0.1"
+
 find-up@^5.0.0:
   version "5.0.0"
   resolved "https://registry.yarnpkg.com/find-up/-/find-up-5.0.0.tgz#4c92819ecb7083561e4f4a240a86be5198f536fc"
@@ -1471,6 +1592,14 @@ for-each@^0.3.3:
   dependencies:
     is-callable "^1.1.3"
 
+foreground-child@^3.1.0:
+  version "3.2.1"
+  resolved "https://registry.yarnpkg.com/foreground-child/-/foreground-child-3.2.1.tgz#767004ccf3a5b30df39bed90718bab43fe0a59f7"
+  integrity sha512-PXUUyLqrR2XCWICfv6ukppP96sdFwWbNEnfEMt7jNsISjMsvaLNinAHNDYyvkyU+SZG2BTSbT5NjG+vZslfGTA==
+  dependencies:
+    cross-spawn "^7.0.0"
+    signal-exit "^4.0.1"
+
 fraction.js@^4.3.7:
   version "4.3.7"
   resolved "https://registry.yarnpkg.com/fraction.js/-/fraction.js-4.3.7.tgz#06ca0085157e42fda7f9e726e79fefc4068840f7"
@@ -1534,6 +1663,13 @@ get-symbol-description@^1.0.0:
     call-bind "^1.0.2"
     get-intrinsic "^1.1.1"
 
+glob-parent@^5.1.2, glob-parent@~5.1.2:
+  version "5.1.2"
+  resolved "https://registry.yarnpkg.com/glob-parent/-/glob-parent-5.1.2.tgz#869832c58034fe68a4093c17dc15e8340d8401c4"
+  integrity sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==
+  dependencies:
+    is-glob "^4.0.1"
+
 glob-parent@^6.0.2:
   version "6.0.2"
   resolved "https://registry.yarnpkg.com/glob-parent/-/glob-parent-6.0.2.tgz#6d237d99083950c79290f24c7642a3de9a28f9e3"
@@ -1541,14 +1677,19 @@ glob-parent@^6.0.2:
   dependencies:
     is-glob "^4.0.3"
 
-glob-parent@~5.1.2:
-  version "5.1.2"
-  resolved "https://registry.yarnpkg.com/glob-parent/-/glob-parent-5.1.2.tgz#869832c58034fe68a4093c17dc15e8340d8401c4"
-  integrity sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==
+glob@^10.3.10:
+  version "10.4.2"
+  resolved "https://registry.yarnpkg.com/glob/-/glob-10.4.2.tgz#bed6b95dade5c1f80b4434daced233aee76160e5"
+  integrity sha512-GwMlUF6PkPo3Gk21UxkCohOv0PLcIXVtKyLlpEI28R/cO/4eNOdmLk3CMW1wROV/WR/EsZOWAfBbBOqYvs88/w==
   dependencies:
-    is-glob "^4.0.1"
+    foreground-child "^3.1.0"
+    jackspeak "^3.1.2"
+    minimatch "^9.0.4"
+    minipass "^7.1.2"
+    package-json-from-dist "^1.0.0"
+    path-scurry "^1.11.1"
 
-glob@^7.1.3:
+glob@^7.1.3, glob@^7.1.6:
   version "7.2.3"
   resolved "https://registry.yarnpkg.com/glob/-/glob-7.2.3.tgz#b8df0fb802bbfa8e89bd1d938b4e16578ed44f2b"
   integrity sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==
@@ -1921,6 +2062,20 @@ iterator.prototype@^1.1.2:
     reflect.getprototypeof "^1.0.4"
     set-function-name "^2.0.1"
 
+jackspeak@^3.1.2:
+  version "3.4.0"
+  resolved "https://registry.yarnpkg.com/jackspeak/-/jackspeak-3.4.0.tgz#a75763ff36ad778ede6a156d8ee8b124de445b4a"
+  integrity sha512-JVYhQnN59LVPFCEcVa2C3CrEKYacvjRfqIQl+h8oi91aLYQVWRYbxjPcv1bUiUy/kLmQaANrYfNMCO3kuEDHfw==
+  dependencies:
+    "@isaacs/cliui" "^8.0.2"
+  optionalDependencies:
+    "@pkgjs/parseargs" "^0.11.0"
+
+jiti@^1.19.1:
+  version "1.21.6"
+  resolved "https://registry.yarnpkg.com/jiti/-/jiti-1.21.6.tgz#6c7f7398dd4b3142767f9a168af2f317a428d268"
+  integrity sha512-2yTgeWTWzMWkHu6Jp9NKgePDaYHbntiwvYuuJLbbN9vl7DC9DvXKOB2BC3ZZ92D3cvV/aflH0osDfwpHepQ53w==
+
 "js-tokens@^3.0.0 || ^4.0.0", js-tokens@^4.0.0:
   version "4.0.0"
   resolved "https://registry.yarnpkg.com/js-tokens/-/js-tokens-4.0.0.tgz#19203fb59991df98e3a287050d4647cdeaf32499"
@@ -1988,6 +2143,21 @@ levn@^0.4.1:
     prelude-ls "^1.2.1"
     type-check "~0.4.0"
 
+lilconfig@^2.1.0:
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/lilconfig/-/lilconfig-2.1.0.tgz#78e23ac89ebb7e1bfbf25b18043de756548e7f52"
+  integrity sha512-utWOt/GHzuUxnLKxB6dk81RoOeoNeHgbrXiuGk4yyF5qlRz+iIVWu56E2fqGHFrXz0QNUhLB/8nKqvRH66JKGQ==
+
+lilconfig@^3.0.0:
+  version "3.1.2"
+  resolved "https://registry.yarnpkg.com/lilconfig/-/lilconfig-3.1.2.tgz#e4a7c3cb549e3a606c8dcc32e5ae1005e62c05cb"
+  integrity sha512-eop+wDAvpItUys0FWkHIKeC9ybYrTGbU41U5K7+bttZZeohvnY7M9dZ5kB21GNWiFT2q1OoPTvncPCgSOVO5ow==
+
+lines-and-columns@^1.1.6:
+  version "1.2.4"
+  resolved "https://registry.yarnpkg.com/lines-and-columns/-/lines-and-columns-1.2.4.tgz#eca284f75d2965079309dc0ad9255abb2ebc1632"
+  integrity sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==
+
 linkify-it@^4.0.1:
   version "4.0.1"
   resolved "https://registry.yarnpkg.com/linkify-it/-/linkify-it-4.0.1.tgz#01f1d5e508190d06669982ba31a7d9f56a5751ec"
@@ -2019,6 +2189,11 @@ loose-envify@^1.1.0, loose-envify@^1.4.0:
   dependencies:
     js-tokens "^3.0.0 || ^4.0.0"
 
+lru-cache@^10.2.0:
+  version "10.3.0"
+  resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-10.3.0.tgz#4a4aaf10c84658ab70f79a85a9a3f1e1fb11196b"
+  integrity sha512-CQl19J/g+Hbjbv4Y3mFNNXFEL/5t/KCg8POCuUqd4rMKjGG+j1ybER83hxV58zL+dFI1PTkt3GNFSHRt+d8qEQ==
+
 lru-cache@^5.1.1:
   version "5.1.1"
   resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-5.1.1.tgz#1da27e6710271947695daf6848e847f01d84b920"
@@ -2047,6 +2222,19 @@ merge-stream@^2.0.0:
   resolved "https://registry.yarnpkg.com/merge-stream/-/merge-stream-2.0.0.tgz#52823629a14dd00c9770fb6ad47dc6310f2c1f60"
   integrity sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==
 
+merge2@^1.3.0:
+  version "1.4.1"
+  resolved "https://registry.yarnpkg.com/merge2/-/merge2-1.4.1.tgz#4368892f885e907455a6fd7dc55c0c9d404990ae"
+  integrity sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==
+
+micromatch@^4.0.4:
+  version "4.0.7"
+  resolved "https://registry.yarnpkg.com/micromatch/-/micromatch-4.0.7.tgz#33e8190d9fe474a9895525f5618eee136d46c2e5"
+  integrity sha512-LPP/3KorzCwBxfeUuZmaR6bG2kdeHSbe0P2tY3FLRU4vYrjYz5hI4QZwV0njUx3jeuKe67YukQ1LSPZBKDqO/Q==
+  dependencies:
+    braces "^3.0.3"
+    picomatch "^2.3.1"
+
 micromatch@^4.0.5:
   version "4.0.5"
   resolved "https://registry.yarnpkg.com/micromatch/-/micromatch-4.0.5.tgz#bc8999a7cbbf77cdc89f132f6e467051b49090c6"
@@ -2096,11 +2284,23 @@ minimatch@3.1.2, minimatch@^3.0.5, minimatch@^3.1.1, minimatch@^3.1.2:
   dependencies:
     brace-expansion "^1.1.7"
 
+minimatch@^9.0.4:
+  version "9.0.5"
+  resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-9.0.5.tgz#d74f9dd6b57d83d8e98cfb82133b03978bc929e5"
+  integrity sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==
+  dependencies:
+    brace-expansion "^2.0.1"
+
 minimist@^1.2.0:
   version "1.2.8"
   resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.8.tgz#c1a464e7693302e082a075cee0c057741ac4772c"
   integrity sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==
 
+"minipass@^5.0.0 || ^6.0.2 || ^7.0.0", minipass@^7.1.2:
+  version "7.1.2"
+  resolved "https://registry.yarnpkg.com/minipass/-/minipass-7.1.2.tgz#93a9626ce5e5e66bd4db86849e7515e92340a707"
+  integrity sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==
+
 ms@2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8"
@@ -2116,6 +2316,15 @@ ms@^2.1.1:
   resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2"
   integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==
 
+mz@^2.7.0:
+  version "2.7.0"
+  resolved "https://registry.yarnpkg.com/mz/-/mz-2.7.0.tgz#95008057a56cafadc2bc63dde7f9ff6955948e32"
+  integrity sha512-z81GNO7nnYMEhrGh9LeymoE4+Yr0Wn5McHIZMK5cfQCl+NDX08sCZgUc9/6MHni9IWuFLm1Z3HTCXu2z9fN62Q==
+  dependencies:
+    any-promise "^1.0.0"
+    object-assign "^4.0.1"
+    thenify-all "^1.0.0"
+
 nanoid@^3.3.7:
   version "3.3.7"
   resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.3.7.tgz#d0c301a691bc8d54efa0a2226ccf3fe2fd656bd8"
@@ -2176,11 +2385,16 @@ npm-run-path@^4.0.1:
   dependencies:
     path-key "^3.0.0"
 
-object-assign@^4.1.1:
+object-assign@^4.0.1, object-assign@^4.1.1:
   version "4.1.1"
   resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.1.tgz#2109adc7965887cfc05cbbd442cac8bfbb360863"
   integrity sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==
 
+object-hash@^3.0.0:
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/object-hash/-/object-hash-3.0.0.tgz#73f97f753e7baffc0e2cc9d6e079079744ac82e9"
+  integrity sha512-RSn9F68PjH9HqtltsSnqYC1XXoWe9Bju5+213R98cNGttag9q9yAOTzdbsqvIa7aNm5WffBZFpWYr2aWrklWAw==
+
 object-inspect@^1.13.1, object-inspect@^1.9.0:
   version "1.13.1"
   resolved "https://registry.yarnpkg.com/object-inspect/-/object-inspect-1.13.1.tgz#b96c6109324ccfef6b12216a956ca4dc2ff94bc2"
@@ -2281,6 +2495,11 @@ p-locate@^5.0.0:
   dependencies:
     p-limit "^3.0.2"
 
+package-json-from-dist@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/package-json-from-dist/-/package-json-from-dist-1.0.0.tgz#e501cd3094b278495eb4258d4c9f6d5ac3019f00"
+  integrity sha512-dATvCeZN/8wQsGywez1mzHtTlP22H8OEfPrVMLNr4/eGa+ijtLn/6M5f0dY8UKNrC2O9UCU6SSoG3qRKnt7STw==
+
 parent-module@^1.0.0:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/parent-module/-/parent-module-1.0.1.tgz#691d2709e78c79fae3a156622452d00762caaaa2"
@@ -2313,6 +2532,14 @@ path-parse@^1.0.7:
   resolved "https://registry.yarnpkg.com/path-parse/-/path-parse-1.0.7.tgz#fbc114b60ca42b30d9daf5858e4bd68bbedb6735"
   integrity sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==
 
+path-scurry@^1.11.1:
+  version "1.11.1"
+  resolved "https://registry.yarnpkg.com/path-scurry/-/path-scurry-1.11.1.tgz#7960a668888594a0720b12a911d1a742ab9f11d2"
+  integrity sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA==
+  dependencies:
+    lru-cache "^10.2.0"
+    minipass "^5.0.0 || ^6.0.2 || ^7.0.0"
+
 path-to-regexp@2.2.1:
   version "2.2.1"
   resolved "https://registry.yarnpkg.com/path-to-regexp/-/path-to-regexp-2.2.1.tgz#90b617025a16381a879bc82a38d4e8bdeb2bcf45"
@@ -2323,16 +2550,79 @@ picocolors@^1.0.0:
   resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-1.0.0.tgz#cb5bdc74ff3f51892236eaf79d68bc44564ab81c"
   integrity sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ==
 
+picocolors@^1.0.1:
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-1.0.1.tgz#a8ad579b571952f0e5d25892de5445bcfe25aaa1"
+  integrity sha512-anP1Z8qwhkbmu7MFP5iTt+wQKXgwzf7zTyGlcdzabySa9vd0Xt392U0rVmz9poOaBj0uHJKyyo9/upk0HrEQew==
+
 picomatch@^2.0.4, picomatch@^2.2.1, picomatch@^2.3.1:
   version "2.3.1"
   resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.3.1.tgz#3ba3833733646d9d3e4995946c1365a67fb07a42"
   integrity sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==
 
-postcss-value-parser@^4.2.0:
+pify@^2.3.0:
+  version "2.3.0"
+  resolved "https://registry.yarnpkg.com/pify/-/pify-2.3.0.tgz#ed141a6ac043a849ea588498e7dca8b15330e90c"
+  integrity sha512-udgsAY+fTnvv7kI7aaxbqwWNb0AHiB0qBO89PZKPkoTmGOgdbrHDKD+0B2X4uTfJ/FT1R09r9gTsjUjNJotuog==
+
+pirates@^4.0.1:
+  version "4.0.6"
+  resolved "https://registry.yarnpkg.com/pirates/-/pirates-4.0.6.tgz#3018ae32ecfcff6c29ba2267cbf21166ac1f36b9"
+  integrity sha512-saLsH7WeYYPiD25LDuLRRY/i+6HaPYr6G1OUlN39otzkSTxKnubR9RTxS3/Kk50s1g2JTgFwWQDQyplC5/SHZg==
+
+postcss-import@^15.1.0:
+  version "15.1.0"
+  resolved "https://registry.yarnpkg.com/postcss-import/-/postcss-import-15.1.0.tgz#41c64ed8cc0e23735a9698b3249ffdbf704adc70"
+  integrity sha512-hpr+J05B2FVYUAXHeK1YyI267J/dDDhMU6B6civm8hSY1jYJnBXxzKDKDswzJmtLHryrjhnDjqqp/49t8FALew==
+  dependencies:
+    postcss-value-parser "^4.0.0"
+    read-cache "^1.0.0"
+    resolve "^1.1.7"
+
+postcss-js@^4.0.1:
+  version "4.0.1"
+  resolved "https://registry.yarnpkg.com/postcss-js/-/postcss-js-4.0.1.tgz#61598186f3703bab052f1c4f7d805f3991bee9d2"
+  integrity sha512-dDLF8pEO191hJMtlHFPRa8xsizHaM82MLfNkUHdUtVEV3tgTp5oj+8qbEqYM57SLfc74KSbw//4SeJma2LRVIw==
+  dependencies:
+    camelcase-css "^2.0.1"
+
+postcss-load-config@^4.0.1:
+  version "4.0.2"
+  resolved "https://registry.yarnpkg.com/postcss-load-config/-/postcss-load-config-4.0.2.tgz#7159dcf626118d33e299f485d6afe4aff7c4a3e3"
+  integrity sha512-bSVhyJGL00wMVoPUzAVAnbEoWyqRxkjv64tUl427SKnPrENtq6hJwUojroMz2VB+Q1edmi4IfrAPpami5VVgMQ==
+  dependencies:
+    lilconfig "^3.0.0"
+    yaml "^2.3.4"
+
+postcss-nested@^6.0.1:
+  version "6.0.1"
+  resolved "https://registry.yarnpkg.com/postcss-nested/-/postcss-nested-6.0.1.tgz#f83dc9846ca16d2f4fa864f16e9d9f7d0961662c"
+  integrity sha512-mEp4xPMi5bSWiMbsgoPfcP74lsWLHkQbZc3sY+jWYd65CUwXrUaTp0fmNpa01ZcETKlIgUdFN/MpS2xZtqL9dQ==
+  dependencies:
+    postcss-selector-parser "^6.0.11"
+
+postcss-selector-parser@^6.0.11:
+  version "6.1.0"
+  resolved "https://registry.yarnpkg.com/postcss-selector-parser/-/postcss-selector-parser-6.1.0.tgz#49694cb4e7c649299fea510a29fa6577104bcf53"
+  integrity sha512-UMz42UD0UY0EApS0ZL9o1XnLhSTtvvvLe5Dc2H2O56fvRZi+KulDyf5ctDhhtYJBGKStV2FL1fy6253cmLgqVQ==
+  dependencies:
+    cssesc "^3.0.0"
+    util-deprecate "^1.0.2"
+
+postcss-value-parser@^4.0.0, postcss-value-parser@^4.2.0:
   version "4.2.0"
   resolved "https://registry.yarnpkg.com/postcss-value-parser/-/postcss-value-parser-4.2.0.tgz#723c09920836ba6d3e5af019f92bc0971c02e514"
   integrity sha512-1NNCs6uurfkVbeXG4S8JFT9t19m45ICnif8zWLd5oPSZ50QnwMfK+H3jv408d4jw/7Bttv5axS5IiHoLaVNHeQ==
 
+postcss@^8.4.23:
+  version "8.4.39"
+  resolved "https://registry.yarnpkg.com/postcss/-/postcss-8.4.39.tgz#aa3c94998b61d3a9c259efa51db4b392e1bde0e3"
+  integrity sha512-0vzE+lAiG7hZl1/9I8yzKLx3aR9Xbof3fBHKunvMfOCYAtMhrsnccJY2iTURb9EZd5+pLuiNV9/c/GZJOHsgIw==
+  dependencies:
+    nanoid "^3.3.7"
+    picocolors "^1.0.1"
+    source-map-js "^1.2.0"
+
 postcss@^8.4.32:
   version "8.4.33"
   resolved "https://registry.yarnpkg.com/postcss/-/postcss-8.4.33.tgz#1378e859c9f69bf6f638b990a0212f43e2aaa742"
@@ -2421,6 +2711,13 @@ react@^18.2.0:
   dependencies:
     loose-envify "^1.1.0"
 
+read-cache@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/read-cache/-/read-cache-1.0.0.tgz#e664ef31161166c9751cdbe8dbcf86b5fb58f774"
+  integrity sha512-Owdv/Ft7IjOgm/i0xvNDZ1LrRANRfew4b2prF3OWMQLxLfu3bS8FVhCsrSCMK4lR56Y9ya+AThoTpDCTxCmpRA==
+  dependencies:
+    pify "^2.3.0"
+
 readdirp@~3.6.0:
   version "3.6.0"
   resolved "https://registry.yarnpkg.com/readdirp/-/readdirp-3.6.0.tgz#74a370bd857116e245b29cc97340cd431a02a6c7"
@@ -2474,6 +2771,15 @@ resolve-from@^4.0.0:
   resolved "https://registry.yarnpkg.com/resolve-from/-/resolve-from-4.0.0.tgz#4abcd852ad32dd7baabfe9b40e00a36db5f392e6"
   integrity sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==
 
+resolve@^1.1.7, resolve@^1.22.2:
+  version "1.22.8"
+  resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.22.8.tgz#b6c87a9f2aa06dfab52e3d70ac8cde321fa5a48d"
+  integrity sha512-oKWePCxqpd6FlLvGV1VU0x7bkPmmCNolxzjMf4NczoDnQcIWrAF+cPtZn5i6n+RfD2d9i0tzpKnG6Yk168yIyw==
+  dependencies:
+    is-core-module "^2.13.0"
+    path-parse "^1.0.7"
+    supports-preserve-symlinks-flag "^1.0.0"
+
 resolve@^2.0.0-next.4:
   version "2.0.0-next.5"
   resolved "https://registry.yarnpkg.com/resolve/-/resolve-2.0.0-next.5.tgz#6b0ec3107e671e52b68cd068ef327173b90dc03c"
@@ -2652,6 +2958,11 @@ signal-exit@^3.0.3:
   resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.7.tgz#a9a1767f8af84155114eaabd73f99273c8f59ad9"
   integrity sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==
 
+signal-exit@^4.0.1:
+  version "4.1.0"
+  resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-4.1.0.tgz#952188c1cbd546070e2dd20d0f41c0ae0530cb04"
+  integrity sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==
+
 simple-update-notifier@^1.0.7:
   version "1.1.0"
   resolved "https://registry.yarnpkg.com/simple-update-notifier/-/simple-update-notifier-1.1.0.tgz#67694c121de354af592b347cdba798463ed49c82"
@@ -2664,6 +2975,11 @@ source-map-js@^1.0.2:
   resolved "https://registry.yarnpkg.com/source-map-js/-/source-map-js-1.0.2.tgz#adbc361d9c62df380125e7f161f71c826f1e490c"
   integrity sha512-R0XvVJ9WusLiqTCEiGCmICCMplcCkIwwR11mOSD9CR5u+IXYdiseeEuXCVAjS54zqwkLcPNnmU4OeJ6tUrWhDw==
 
+source-map-js@^1.2.0:
+  version "1.2.0"
+  resolved "https://registry.yarnpkg.com/source-map-js/-/source-map-js-1.2.0.tgz#16b809c162517b5b8c3e7dcd315a2a5c2612b2af"
+  integrity sha512-itJW8lvSA0TXEphiRoawsCksnlf8SyvmFzIhltqAHluXd88pkCd+cXJVHTDwdCr0IzwptSm035IHQktUu1QUMg==
+
 source-map-support@~0.5.20:
   version "0.5.21"
   resolved "https://registry.yarnpkg.com/source-map-support/-/source-map-support-0.5.21.tgz#04fe7c7f9e1ed2d662233c28cb2b35b9f63f6e4f"
@@ -2672,12 +2988,12 @@ source-map-support@~0.5.20:
     buffer-from "^1.0.0"
     source-map "^0.6.0"
 
-source-map@^0.6.0:
+source-map@^0.6.0, source-map@~0.6.0:
   version "0.6.1"
   resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.6.1.tgz#74722af32e9614e9c287a8d0bbde48b5e2f1a263"
   integrity sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==
 
-string-width@^4.1.0:
+"string-width-cjs@npm:string-width@^4.2.0", string-width@^4.1.0:
   version "4.2.3"
   resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010"
   integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==
@@ -2737,7 +3053,7 @@ string.prototype.trimstart@^1.0.7:
     define-properties "^1.2.0"
     es-abstract "^1.22.1"
 
-strip-ansi@^6.0.1:
+"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.0, strip-ansi@^6.0.1:
   version "6.0.1"
   resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9"
   integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==
@@ -2766,6 +3082,19 @@ strip-json-comments@~2.0.1:
   resolved "https://registry.yarnpkg.com/strip-json-comments/-/strip-json-comments-2.0.1.tgz#3c531942e908c2697c0ec344858c286c7ca0a60a"
   integrity sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==
 
+sucrase@^3.32.0:
+  version "3.35.0"
+  resolved "https://registry.yarnpkg.com/sucrase/-/sucrase-3.35.0.tgz#57f17a3d7e19b36d8995f06679d121be914ae263"
+  integrity sha512-8EbVDiu9iN/nESwxeSxDKe0dunta1GOlHufmSSXxMD2z2/tMZpDMpvXQGsc+ajGo8y2uYUmixaSRUc/QPoQ0GA==
+  dependencies:
+    "@jridgewell/gen-mapping" "^0.3.2"
+    commander "^4.0.0"
+    glob "^10.3.10"
+    lines-and-columns "^1.1.6"
+    mz "^2.7.0"
+    pirates "^4.0.1"
+    ts-interface-checker "^0.1.9"
+
 supports-color@^5.3.0, supports-color@^5.5.0:
   version "5.5.0"
   resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-5.5.0.tgz#e2e69a44ac8772f78a1ec0b35b689df6530efc8f"
@@ -2785,6 +3114,34 @@ supports-preserve-symlinks-flag@^1.0.0:
   resolved "https://registry.yarnpkg.com/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz#6eda4bd344a3c94aea376d4cc31bc77311039e09"
   integrity sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==
 
+tailwindcss@3.4.1:
+  version "3.4.1"
+  resolved "https://registry.yarnpkg.com/tailwindcss/-/tailwindcss-3.4.1.tgz#f512ca5d1dd4c9503c7d3d28a968f1ad8f5c839d"
+  integrity sha512-qAYmXRfk3ENzuPBakNK0SRrUDipP8NQnEY6772uDhflcQz5EhRdD7JNZxyrFHVQNCwULPBn6FNPp9brpO7ctcA==
+  dependencies:
+    "@alloc/quick-lru" "^5.2.0"
+    arg "^5.0.2"
+    chokidar "^3.5.3"
+    didyoumean "^1.2.2"
+    dlv "^1.1.3"
+    fast-glob "^3.3.0"
+    glob-parent "^6.0.2"
+    is-glob "^4.0.3"
+    jiti "^1.19.1"
+    lilconfig "^2.1.0"
+    micromatch "^4.0.5"
+    normalize-path "^3.0.0"
+    object-hash "^3.0.0"
+    picocolors "^1.0.0"
+    postcss "^8.4.23"
+    postcss-import "^15.1.0"
+    postcss-js "^4.0.1"
+    postcss-load-config "^4.0.1"
+    postcss-nested "^6.0.1"
+    postcss-selector-parser "^6.0.11"
+    resolve "^1.22.2"
+    sucrase "^3.32.0"
+
 terser@^5.27.0:
   version "5.27.0"
   resolved "https://registry.yarnpkg.com/terser/-/terser-5.27.0.tgz#70108689d9ab25fef61c4e93e808e9fd092bf20c"
@@ -2800,6 +3157,20 @@ text-table@^0.2.0:
   resolved "https://registry.yarnpkg.com/text-table/-/text-table-0.2.0.tgz#7f5ee823ae805207c00af2df4a84ec3fcfa570b4"
   integrity sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw==
 
+thenify-all@^1.0.0:
+  version "1.6.0"
+  resolved "https://registry.yarnpkg.com/thenify-all/-/thenify-all-1.6.0.tgz#1a1918d402d8fc3f98fbf234db0bcc8cc10e9726"
+  integrity sha512-RNxQH/qI8/t3thXJDwcstUO4zeqo64+Uy/+sNVRBx4Xn2OX+OZ9oP+iJnNFqplFra2ZUVeKCSa2oVWi3T4uVmA==
+  dependencies:
+    thenify ">= 3.1.0 < 4"
+
+"thenify@>= 3.1.0 < 4":
+  version "3.3.1"
+  resolved "https://registry.yarnpkg.com/thenify/-/thenify-3.3.1.tgz#8932e686a4066038a016dd9e2ca46add9838a95f"
+  integrity sha512-RVZSIV5IG10Hk3enotrhvz0T9em6cyHBLkH/YAZuKqd8hRkKhSfCGIcP2KUY0EPxndzANBmNllzWPwak+bheSw==
+  dependencies:
+    any-promise "^1.0.0"
+
 to-fast-properties@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/to-fast-properties/-/to-fast-properties-2.0.0.tgz#dc5e698cbd079265bc73e0377681a4e4e83f616e"
@@ -2819,6 +3190,11 @@ touch@^3.1.0:
   dependencies:
     nopt "~1.0.10"
 
+ts-interface-checker@^0.1.9:
+  version "0.1.13"
+  resolved "https://registry.yarnpkg.com/ts-interface-checker/-/ts-interface-checker-0.1.13.tgz#784fd3d679722bc103b1b4b8030bcddb5db2a699"
+  integrity sha512-Y/arvbn+rrz3JCKl9C4kVNfTfSm2/mEp5FSz5EsZSANGPSlQrpRI5M4PKF+mJnE52jOO90PnPSc3Ur3bTQw0gA==
+
 type-check@^0.4.0, type-check@~0.4.0:
   version "0.4.0"
   resolved "https://registry.yarnpkg.com/type-check/-/type-check-0.4.0.tgz#07b8203bfa7056c0657050e3ccd2c37730bab8f1"
@@ -2918,6 +3294,11 @@ uri-js@^4.2.2:
   dependencies:
     punycode "^2.1.0"
 
+util-deprecate@^1.0.2:
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf"
+  integrity sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==
+
 uuid@^9.0.1:
   version "9.0.1"
   resolved "https://registry.yarnpkg.com/uuid/-/uuid-9.0.1.tgz#e188d4c8853cc722220392c424cd637f32293f30"
@@ -3010,7 +3391,16 @@ widest-line@^4.0.1:
   dependencies:
     string-width "^5.0.1"
 
-wrap-ansi@^8.0.1:
+"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0":
+  version "7.0.0"
+  resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43"
+  integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==
+  dependencies:
+    ansi-styles "^4.0.0"
+    string-width "^4.1.0"
+    strip-ansi "^6.0.0"
+
+wrap-ansi@^8.0.1, wrap-ansi@^8.1.0:
   version "8.1.0"
   resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-8.1.0.tgz#56dc22368ee570face1b49819975d9b9a5ead214"
   integrity sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==
@@ -3029,6 +3419,11 @@ yallist@^3.0.2:
   resolved "https://registry.yarnpkg.com/yallist/-/yallist-3.1.1.tgz#dbb7daf9bfd8bac9ab45ebf602b8cbad0d5d08fd"
   integrity sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==
 
+yaml@^2.3.4:
+  version "2.4.5"
+  resolved "https://registry.yarnpkg.com/yaml/-/yaml-2.4.5.tgz#60630b206dd6d84df97003d33fc1ddf6296cca5e"
+  integrity sha512-aBx2bnqDzVOyNKfsysjA2ms5ZlnjSAW2eG3/L5G/CSujfjLJTJsEw1bGw8kCf04KodQWk1pxlGnZ56CRxiawmg==
+
 yocto-queue@^0.1.0:
   version "0.1.0"
   resolved "https://registry.yarnpkg.com/yocto-queue/-/yocto-queue-0.1.0.tgz#0294eb3dee05028d31ee1a5fa2c556a6aaf10a1b"

frontend/index.html

@@ -12,7 +12,7 @@
 
     <!-- Facebook -->
     <meta property="og:type" content="website">
-  <meta property="og:url" content="https://useanything.com">
+    <meta property="og:url" content="https://anythingllm.com">
     <meta property="og:title" content="AnythingLLM | Your personal LLM trained on anything">
     <meta property="og:description" content="AnythingLLM | Your personal LLM trained on anything">
     <meta property="og:image"
@@ -20,7 +20,7 @@
 
     <!-- Twitter -->
     <meta property="twitter:card" content="summary_large_image">
-  <meta property="twitter:url" content="https://useanything.com">
+    <meta property="twitter:url" content="https://anythingllm.com">
     <meta property="twitter:title" content="AnythingLLM | Your personal LLM trained on anything">
     <meta property="twitter:description" content="AnythingLLM | Your personal LLM trained on anything">
     <meta property="twitter:image"

frontend/jsconfig.json

@@ -4,7 +4,9 @@
     "target": "esnext",
     "jsx": "react",
     "paths": {
-      "@/*": ["./src/*"]
+      "@/*": [
+        "./src/*"
+      ]
     }
   }
 }

frontend/package.json

@@ -6,27 +6,34 @@
   "scripts": {
     "start": "vite --open",
     "dev": "NODE_ENV=development vite --debug --host=0.0.0.0",
-    "build": "vite build",
+    "build": "vite build && node scripts/postbuild.js",
     "lint": "yarn prettier --ignore-path ../.prettierignore --write ./src",
     "preview": "vite preview"
   },
   "dependencies": {
     "@metamask/jazzicon": "^2.0.0",
     "@microsoft/fetch-event-source": "^2.0.1",
-    "@phosphor-icons/react": "^2.0.13",
+    "@mintplex-labs/piper-tts-web": "^1.0.4",
+    "@phosphor-icons/react": "^2.1.7",
     "@tremor/react": "^3.15.1",
     "dompurify": "^3.0.8",
     "file-saver": "^2.0.5",
     "he": "^1.2.0",
     "highlight.js": "^11.9.0",
+    "i18next": "^23.11.3",
+    "i18next-browser-languagedetector": "^7.2.1",
     "js-levenshtein": "^1.1.6",
     "lodash.debounce": "^4.0.8",
     "markdown-it": "^13.0.1",
+    "markdown-it-katex": "^2.0.3",
+    "moment": "^2.30.1",
+    "onnxruntime-web": "^1.18.0",
     "pluralize": "^8.0.0",
     "react": "^18.2.0",
     "react-device-detect": "^2.2.2",
     "react-dom": "^18.2.0",
     "react-dropzone": "^14.2.3",
+    "react-i18next": "^14.1.1",
     "react-loading-skeleton": "^3.1.0",
     "react-router-dom": "^6.3.0",
     "react-speech-recognition": "^3.10.0",

frontend/scripts/postbuild.js

@@ -0,0 +1,8 @@
+import { renameSync } from 'fs';
+import { fileURLToPath } from 'url';
+import path from 'path';
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+
+console.log(`Running frontend post build script...`)
+renameSync(path.resolve(__dirname, '../dist/index.html'), path.resolve(__dirname, '../dist/_index.html'));
+console.log(`index.html renamed to _index.html so SSR of the index page can be assumed.`);

frontend/src/App.jsx

@@ -1,5 +1,6 @@
 import React, { lazy, Suspense } from "react";
 import { Routes, Route } from "react-router-dom";
+import { I18nextProvider } from "react-i18next";
 import { ContextWrapper } from "@/AuthContext";
 import PrivateRoute, {
   AdminRoute,
@@ -9,9 +10,11 @@ import { ToastContainer } from "react-toastify";
 import "react-toastify/dist/ReactToastify.css";
 import Login from "@/pages/Login";
 import OnboardingFlow from "@/pages/OnboardingFlow";
+import i18n from "./i18n";
 
 import { PfpProvider } from "./PfpContext";
 import { LogoProvider } from "./LogoContext";
+import { FullScreenLoader } from "./components/Preloader";
 
 const Main = lazy(() => import("@/pages/Main"));
 const InvitePage = lazy(() => import("@/pages/Invite"));
@@ -21,6 +24,7 @@ const AdminInvites = lazy(() => import("@/pages/Admin/Invitations"));
 const AdminWorkspaces = lazy(() => import("@/pages/Admin/Workspaces"));
 const AdminSystem = lazy(() => import("@/pages/Admin/System"));
 const AdminLogs = lazy(() => import("@/pages/Admin/Logging"));
+const AdminAgents = lazy(() => import("@/pages/Admin/Agents"));
 const GeneralChats = lazy(() => import("@/pages/GeneralSettings/Chats"));
 const GeneralAppearance = lazy(
   () => import("@/pages/GeneralSettings/Appearance")
@@ -53,13 +57,21 @@ const EmbedChats = lazy(() => import("@/pages/GeneralSettings/EmbedChats"));
 const PrivacyAndData = lazy(
   () => import("@/pages/GeneralSettings/PrivacyAndData")
 );
+const ExperimentalFeatures = lazy(
+  () => import("@/pages/Admin/ExperimentalFeatures")
+);
+const LiveDocumentSyncManage = lazy(
+  () => import("@/pages/Admin/ExperimentalFeatures/Features/LiveSync/manage")
+);
+const FineTuningWalkthrough = lazy(() => import("@/pages/FineTuning"));
 
 export default function App() {
   return (
-    <Suspense fallback={<div />}>
+    <Suspense fallback={<FullScreenLoader />}>
       <ContextWrapper>
         <LogoProvider>
           <PfpProvider>
+            <I18nextProvider i18n={i18n}>
               <Routes>
                 <Route path="/" element={<PrivateRoute Component={Main} />} />
                 <Route path="/login" element={<Login />} />
@@ -94,7 +106,9 @@ export default function App() {
                 />
                 <Route
                   path="/settings/embedding-preference"
-                element={<AdminRoute Component={GeneralEmbeddingPreference} />}
+                  element={
+                    <AdminRoute Component={GeneralEmbeddingPreference} />
+                  }
                 />
                 <Route
                   path="/settings/text-splitter-preference"
@@ -106,6 +120,10 @@ export default function App() {
                   path="/settings/vector-database"
                   element={<AdminRoute Component={GeneralVectorDatabase} />}
                 />
+                <Route
+                  path="/settings/agents"
+                  element={<AdminRoute Component={AdminAgents} />}
+                />
                 <Route
                   path="/settings/event-logs"
                   element={<AdminRoute Component={AdminLogs} />}
@@ -131,6 +149,10 @@ export default function App() {
                   path="/settings/appearance"
                   element={<ManagerRoute Component={GeneralAppearance} />}
                 />
+                <Route
+                  path="/settings/beta-features"
+                  element={<AdminRoute Component={ExperimentalFeatures} />}
+                />
                 <Route
                   path="/settings/api-keys"
                   element={<AdminRoute Component={GeneralApiKeys} />}
@@ -158,8 +180,21 @@ export default function App() {
                 {/* Onboarding Flow */}
                 <Route path="/onboarding" element={<OnboardingFlow />} />
                 <Route path="/onboarding/:step" element={<OnboardingFlow />} />
+
+                {/* Experimental feature pages  */}
+                {/* Live Document Sync feature */}
+                <Route
+                  path="/settings/beta-features/live-document-sync/manage"
+                  element={<AdminRoute Component={LiveDocumentSyncManage} />}
+                />
+
+                <Route
+                  path="/fine-tuning"
+                  element={<AdminRoute Component={FineTuningWalkthrough} />}
+                />
               </Routes>
               <ToastContainer />
+            </I18nextProvider>
           </PfpProvider>
         </LogoProvider>
       </ContextWrapper>

frontend/src/components/ContextualSaveBar/index.jsx

@@ -0,0 +1,32 @@
+import { Warning } from "@phosphor-icons/react";
+
+export default function ContextualSaveBar({
+  showing = false,
+  onSave,
+  onCancel,
+}) {
+  if (!showing) return null;
+
+  return (
+    <div className="fixed top-0 left-0 right-0 h-14 bg-dark-input flex items-center justify-end px-4 z-[999]">
+      <div className="absolute ml-4 left-0 md:left-1/2 transform md:-translate-x-1/2 flex items-center gap-x-2">
+        <Warning size={18} className="text-white" />
+        <p className="text-white font-medium text-xs">Unsaved Changes</p>
+      </div>
+      <div className="flex items-center gap-x-2">
+        <button
+          className="border-none text-white font-medium text-sm px-[10px] py-[6px] rounded-md bg-white/5 hover:bg-white/10"
+          onClick={onCancel}
+        >
+          Cancel
+        </button>
+        <button
+          className="border-none text-dark-text font-medium text-sm px-[10px] py-[6px] rounded-md bg-primary-button hover:bg-[#3DB5E8]"
+          onClick={onSave}
+        >
+          Save
+        </button>
+      </div>
+    </div>
+  );
+}

frontend/src/components/DataConnectorOption/media/gitlab.svg

@@ -0,0 +1,7 @@
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 380 380">
+  <rect width="380" height="380" fill="#FFFFFF"/>
+  <path fill="#e24329" d="M282.83,170.73l-.27-.69-26.14-68.22a6.81,6.81,0,0,0-2.69-3.24,7,7,0,0,0-8,.43,7,7,0,0,0-2.32,3.52l-17.65,54H154.29l-17.65-54A6.86,6.86,0,0,0,134.32,99a7,7,0,0,0-8-.43,6.87,6.87,0,0,0-2.69,3.24L97.44,170l-.26.69a48.54,48.54,0,0,0,16.1,56.1l.09.07.24.17,39.82,29.82,19.7,14.91,12,9.06a8.07,8.07,0,0,0,9.76,0l12-9.06,19.7-14.91,40.06-30,.1-.08A48.56,48.56,0,0,0,282.83,170.73Z"/>
+  <path fill="#fc6d26" d="M282.83,170.73l-.27-.69a88.3,88.3,0,0,0-35.15,15.8L190,229.25c19.55,14.79,36.57,27.64,36.57,27.64l40.06-30,.1-.08A48.56,48.56,0,0,0,282.83,170.73Z"/>
+  <path fill="#fca326" d="M153.43,256.89l19.7,14.91,12,9.06a8.07,8.07,0,0,0,9.76,0l12-9.06,19.7-14.91S209.55,244,190,229.25C170.45,244,153.43,256.89,153.43,256.89Z"/>
+  <path fill="#fc6d26" d="M132.58,185.84A88.19,88.19,0,0,0,97.44,170l-.26.69a48.54,48.54,0,0,0,16.1,56.1l.09.07.24.17,39.82,29.82s17-12.85,36.57-27.64Z"/>
+</svg>

frontend/src/components/DataConnectorOption/media/index.js

@@ -1,10 +1,12 @@
 import Github from "./github.svg";
+import Gitlab from "./gitlab.svg";
 import YouTube from "./youtube.svg";
 import Link from "./link.svg";
 import Confluence from "./confluence.jpeg";
 
 const ConnectorImages = {
   github: Github,
+  gitlab: Gitlab,
   youtube: YouTube,
   websiteDepth: Link,
   confluence: Confluence,

frontend/src/components/DefaultChat/index.jsx

@@ -17,6 +17,7 @@ import UserIcon from "../UserIcon";
 import { userFromStorage } from "@/utils/request";
 import { AI_BACKGROUND_COLOR, USER_BACKGROUND_COLOR } from "@/utils/constants";
 import useUser from "@/hooks/useUser";
+import { useTranslation, Trans } from "react-i18next";
 
 export default function DefaultChatContainer() {
   const [mockMsgs, setMockMessages] = useState([]);
@@ -28,6 +29,7 @@ export default function DefaultChatContainer() {
     hideModal: hideNewWsModal,
   } = useNewWorkspaceModal();
   const popMsg = !window.localStorage.getItem("anythingllm_intro");
+  const { t } = useTranslation();
 
   useEffect(() => {
     const fetchData = async () => {
@@ -38,7 +40,7 @@ export default function DefaultChatContainer() {
   }, []);
 
   const MESSAGES = [
-    <React.Fragment>
+    <React.Fragment key="msg1">
       <div
         className={`flex justify-center items-end w-full ${AI_BACKGROUND_COLOR} md:mt-0 mt-[40px]`}
       >
@@ -51,18 +53,14 @@ export default function DefaultChatContainer() {
             <span
               className={`whitespace-pre-line text-white font-normal text-sm md:text-sm flex flex-col gap-y-1 mt-2`}
             >
-              Welcome to AnythingLLM, AnythingLLM is an open-source AI tool by
-              Mintplex Labs that turns anything into a trained chatbot you can
-              query and chat with. AnythingLLM is a BYOK (bring-your-own-keys)
-              software so there is no subscription, fee, or charges for this
-              software outside of the services you want to use with it.
+              {t("welcomeMessage.part1")}
             </span>
           </div>
         </div>
       </div>
     </React.Fragment>,
 
-    <React.Fragment>
+    <React.Fragment key="msg2">
       <div
         className={`flex justify-center items-end w-full ${AI_BACKGROUND_COLOR}`}
       >
@@ -75,17 +73,14 @@ export default function DefaultChatContainer() {
             <span
               className={`whitespace-pre-line text-white font-normal text-sm md:text-sm flex flex-col gap-y-1 mt-2`}
             >
-              AnythingLLM is the easiest way to put powerful AI products like
-              OpenAi, GPT-4, LangChain, PineconeDB, ChromaDB, and other services
-              together in a neat package with no fuss to increase your
-              productivity by 100x.
+              {t("welcomeMessage.part2")}
             </span>
           </div>
         </div>
       </div>
     </React.Fragment>,
 
-    <React.Fragment>
+    <React.Fragment key="msg3">
       <div
         className={`flex justify-center items-end w-full ${AI_BACKGROUND_COLOR}`}
       >
@@ -98,20 +93,16 @@ export default function DefaultChatContainer() {
               <span
                 className={`whitespace-pre-line text-white font-normal text-sm md:text-sm flex flex-col gap-y-1 mt-2`}
               >
-                AnythingLLM can run totally locally on your machine with little
-                overhead you wont even notice it's there! No GPU needed. Cloud
-                and on-premises installation is available as well.
-                <br />
-                The AI tooling ecosystem gets more powerful everyday.
-                AnythingLLM makes it easy to use.
+                {t("welcomeMessage.part3")}
               </span>
               <a
                 href={paths.github()}
                 target="_blank"
+                rel="noreferrer"
                 className="mt-5 w-fit transition-all duration-300 border border-slate-200 px-4 py-2 rounded-lg text-white text-sm items-center flex gap-x-2 hover:bg-slate-200 hover:text-slate-800 focus:ring-gray-800"
               >
                 <GitMerge className="h-4 w-4" />
-                <p>Create an issue on Github</p>
+                <p>{t("welcomeMessage.githubIssue")}</p>
               </a>
             </div>
           </div>
@@ -119,7 +110,7 @@ export default function DefaultChatContainer() {
       </div>
     </React.Fragment>,
 
-    <React.Fragment>
+    <React.Fragment key="msg4">
       <div
         className={`flex justify-center items-end w-full ${USER_BACKGROUND_COLOR}`}
       >
@@ -135,14 +126,14 @@ export default function DefaultChatContainer() {
             <span
               className={`whitespace-pre-line text-white font-normal text-sm md:text-sm flex flex-col gap-y-1 mt-2`}
             >
-              How do I get started?!
+              {t("welcomeMessage.user1")}
             </span>
           </div>
         </div>
       </div>
     </React.Fragment>,
 
-    <React.Fragment>
+    <React.Fragment key="msg5">
       <div
         className={`flex justify-center items-end w-full ${AI_BACKGROUND_COLOR}`}
       >
@@ -155,13 +146,7 @@ export default function DefaultChatContainer() {
               <span
                 className={`whitespace-pre-line text-white font-normal text-sm md:text-sm flex flex-col gap-y-1 mt-2`}
               >
-                It's simple. All collections are organized into buckets we call{" "}
-                "Workspaces". Workspaces are buckets of files, documents,
-                images, PDFs, and other files which will be transformed into
-                something LLM's can understand and use in conversation.
-                <br />
-                <br />
-                You can add and remove files at anytime.
+                {t("welcomeMessage.part4")}
               </span>
 
               {(!user || user?.role !== "default") && (
@@ -170,7 +155,7 @@ export default function DefaultChatContainer() {
                   className="mt-5 w-fit transition-all duration-300 border border-slate-200 px-4 py-2 rounded-lg text-white text-sm items-center flex gap-x-2 hover:bg-slate-200 hover:text-slate-800 focus:ring-gray-800"
                 >
                   <Plus className="h-4 w-4" />
-                  <p>Create your first workspace</p>
+                  <p>{t("welcomeMessage.createWorkspace")}</p>
                 </button>
               )}
             </div>
@@ -179,7 +164,7 @@ export default function DefaultChatContainer() {
       </div>
     </React.Fragment>,
 
-    <React.Fragment>
+    <React.Fragment key="msg6">
       <div
         className={`flex justify-center items-end w-full ${USER_BACKGROUND_COLOR}`}
       >
@@ -195,15 +180,14 @@ export default function DefaultChatContainer() {
             <span
               className={`whitespace-pre-line text-white font-normal text-sm md:text-sm flex flex-col gap-y-1 mt-2`}
             >
-              Is this like an AI dropbox or something? What about chatting? It
-              is a chatbot isn't it?
+              {t("welcomeMessage.user2")}
             </span>
           </div>
         </div>
       </div>
     </React.Fragment>,
 
-    <React.Fragment>
+    <React.Fragment key="msg7">
       <div
         className={`flex justify-center items-end w-full ${AI_BACKGROUND_COLOR}`}
       >
@@ -216,32 +200,20 @@ export default function DefaultChatContainer() {
             <span
               className={`whitespace-pre-line text-white font-normal text-sm md:text-sm flex flex-col gap-y-1 mt-2`}
             >
-              AnythingLLM is more than a smarter Dropbox.
-              <br />
-              <br />
-              AnythingLLM offers two ways of talking with your data:
-              <br />
-              <br />
-              <i>Query:</i> Your chats will return data or inferences found with
-              the documents in your workspace it has access to. Adding more
-              documents to the Workspace make it smarter!
-              <br />
-              <br />
-              <i>Conversational:</i> Your documents + your on-going chat history
-              both contribute to the LLM knowledge at the same time. Great for
-              appending real-time text-based info or corrections and
-              misunderstandings the LLM might have.
-              <br />
-              <br />
-              You can toggle between either mode{" "}
-              <i>in the middle of chatting!</i>
+              <Trans
+                i18nKey="welcomeMessage.part5"
+                components={{
+                  i: <i />,
+                  br: <br />,
+                }}
+              />
             </span>
           </div>
         </div>
       </div>
     </React.Fragment>,
 
-    <React.Fragment>
+    <React.Fragment key="msg8">
       <div
         className={`flex justify-center items-end w-full ${USER_BACKGROUND_COLOR}`}
       >
@@ -257,14 +229,14 @@ export default function DefaultChatContainer() {
             <span
               className={`whitespace-pre-line text-white font-normal text-sm md:text-sm flex flex-col gap-y-1 mt-2`}
             >
-              Wow, this sounds amazing, let me try it out already!
+              {t("welcomeMessage.user3")}
             </span>
           </div>
         </div>
       </div>
     </React.Fragment>,
 
-    <React.Fragment>
+    <React.Fragment key="msg9">
       <div
         className={`flex justify-center items-end w-full ${AI_BACKGROUND_COLOR}`}
       >
@@ -277,24 +249,25 @@ export default function DefaultChatContainer() {
               <span
                 className={`whitespace-pre-line text-white font-normal text-sm md:text-sm flex flex-col gap-y-1 mt-2`}
               >
-                Have Fun!
+                {t("welcomeMessage.part6")}
               </span>
 
               <div className="flex flex-col md:flex-row items-start md:items-center gap-1 md:gap-4">
                 <a
                   href={paths.github()}
                   target="_blank"
+                  rel="noreferrer"
                   className="mt-5 w-fit transition-all duration-300 border border-slate-200 px-4 py-2 rounded-lg text-white text-sm items-center flex gap-x-2 hover:bg-slate-200 hover:text-slate-800 focus:ring-gray-800"
                 >
                   <GithubLogo className="h-4 w-4" />
-                  <p>Star on GitHub</p>
+                  <p>{t("welcomeMessage.starOnGithub")}</p>
                 </a>
                 <a
                   href={paths.mailToMintplex()}
                   className="mt-5 w-fit transition-all duration-300 border border-slate-200 px-4 py-2 rounded-lg text-white text-sm items-center flex gap-x-2 hover:bg-slate-200 hover:text-slate-800 focus:ring-gray-800"
                 >
                   <EnvelopeSimple className="h-4 w-4" />
-                  <p>Contact Mintplex Labs</p>
+                  <p>{t("welcomeMessage.contact")}</p>
                 </a>
               </div>
             </div>

frontend/src/components/EditingChatBubble/index.jsx

@@ -1,5 +1,6 @@
 import React, { useState } from "react";
 import { X } from "@phosphor-icons/react";
+import { useTranslation } from "react-i18next";
 
 export default function EditingChatBubble({
   message,
@@ -11,11 +12,12 @@ export default function EditingChatBubble({
   const [isEditing, setIsEditing] = useState(false);
   const [tempMessage, setTempMessage] = useState(message[type]);
   const isUser = type === "user";
+  const { t } = useTranslation();
 
   return (
     <div>
       <p className={`text-xs text-[#D3D4D4] ${isUser ? "text-right" : ""}`}>
-        {isUser ? "User" : "AnythingLLM Chat Assistant"}
+        {isUser ? t("common.user") : t("appearance.message.assistant")}
       </p>
       <div
         className={`relative flex w-full mt-2 items-start ${

frontend/src/components/EmbeddingSelection/AzureAiOptions/index.jsx

@@ -1,15 +1,15 @@
 export default function AzureAiOptions({ settings }) {
   return (
     <div className="w-full flex flex-col gap-y-4">
-      <div className="w-full flex items-center gap-4">
+      <div className="w-full flex items-center gap-[36px] mt-1.5">
         <div className="flex flex-col w-60">
-          <label className="text-white text-sm font-semibold block mb-4">
+          <label className="text-white text-sm font-semibold block mb-3">
             Azure Service Endpoint
           </label>
           <input
             type="url"
             name="AzureOpenAiEndpoint"
-            className="bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:border-white block w-full p-2.5"
+            className="bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:outline-primary-button active:outline-primary-button outline-none block w-full p-2.5"
             placeholder="https://my-azure.openai.azure.com"
             defaultValue={settings?.AzureOpenAiEndpoint}
             required={true}
@@ -19,13 +19,13 @@ export default function AzureAiOptions({ settings }) {
         </div>
 
         <div className="flex flex-col w-60">
-          <label className="text-white text-sm font-semibold block mb-4">
+          <label className="text-white text-sm font-semibold block mb-3">
             API Key
           </label>
           <input
             type="password"
             name="AzureOpenAiKey"
-            className="bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:border-white block w-full p-2.5"
+            className="bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:outline-primary-button active:outline-primary-button outline-none block w-full p-2.5"
             placeholder="Azure OpenAI API Key"
             defaultValue={settings?.AzureOpenAiKey ? "*".repeat(20) : ""}
             required={true}
@@ -35,13 +35,13 @@ export default function AzureAiOptions({ settings }) {
         </div>
 
         <div className="flex flex-col w-60">
-          <label className="text-white text-sm font-semibold block mb-4">
+          <label className="text-white text-sm font-semibold block mb-3">
             Embedding Deployment Name
           </label>
           <input
             type="text"
             name="AzureOpenAiEmbeddingModelPref"
-            className="bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:border-white block w-full p-2.5"
+            className="bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:outline-primary-button active:outline-primary-button outline-none block w-full p-2.5"
             placeholder="Azure OpenAI embedding model deployment name"
             defaultValue={settings?.AzureOpenAiEmbeddingModelPref}
             required={true}

frontend/src/components/EmbeddingSelection/CohereOptions/index.jsx

@@ -1,15 +1,15 @@
 export default function CohereEmbeddingOptions({ settings }) {
   return (
     <div className="w-full flex flex-col gap-y-4">
-      <div className="w-full flex items-center gap-4">
+      <div className="w-full flex items-center gap-[36px] mt-1.5">
         <div className="flex flex-col w-60">
-          <label className="text-white text-sm font-semibold block mb-4">
+          <label className="text-white text-sm font-semibold block mb-3">
             API Key
           </label>
           <input
             type="password"
             name="CohereApiKey"
-            className="bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:border-white block w-full p-2.5"
+            className="bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:outline-primary-button active:outline-primary-button outline-none block w-full p-2.5"
             placeholder="Cohere API Key"
             defaultValue={settings?.CohereApiKey ? "*".repeat(20) : ""}
             required={true}
@@ -18,7 +18,7 @@ export default function CohereEmbeddingOptions({ settings }) {
           />
         </div>
         <div className="flex flex-col w-60">
-          <label className="text-white text-sm font-semibold block mb-4">
+          <label className="text-white text-sm font-semibold block mb-3">
             Model Preference
           </label>
           <select

frontend/src/components/EmbeddingSelection/EmbedderItem/index.jsx

@@ -29,7 +29,7 @@ export default function EmbedderItem({
         />
         <div className="flex flex-col">
           <div className="text-sm font-semibold text-white">{name}</div>
-          <div className="mt-1 text-xs text-[#D2D5DB]">{description}</div>
+          <div className="mt-1 text-xs text-description">{description}</div>
         </div>
       </div>
     </div>

frontend/src/components/EmbeddingSelection/GenericOpenAiOptions/index.jsx

@@ -0,0 +1,74 @@
+export default function GenericOpenAiEmbeddingOptions({ settings }) {
+  return (
+    <div className="w-full flex flex-col gap-y-7">
+      <div className="w-full flex items-center gap-[36px] mt-1.5 flex-wrap">
+        <div className="flex flex-col w-60">
+          <label className="text-white text-sm font-semibold block mb-3">
+            Base URL
+          </label>
+          <input
+            type="url"
+            name="EmbeddingBasePath"
+            className="bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:outline-primary-button active:outline-primary-button outline-none block w-full p-2.5"
+            placeholder="https://api.openai.com/v1"
+            defaultValue={settings?.EmbeddingBasePath}
+            required={true}
+            autoComplete="off"
+            spellCheck={false}
+          />
+        </div>
+        <div className="flex flex-col w-60">
+          <label className="text-white text-sm font-semibold block mb-3">
+            Embedding Model
+          </label>
+          <input
+            type="text"
+            name="EmbeddingModelPref"
+            className="bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:outline-primary-button active:outline-primary-button outline-none block w-full p-2.5"
+            placeholder="text-embedding-ada-002"
+            defaultValue={settings?.EmbeddingModelPref}
+            required={true}
+            autoComplete="off"
+            spellCheck={false}
+          />
+        </div>
+        <div className="flex flex-col w-60">
+          <label className="text-white text-sm font-semibold block mb-3">
+            Max embedding chunk length
+          </label>
+          <input
+            type="number"
+            name="EmbeddingModelMaxChunkLength"
+            className="bg-zinc-900 text-white placeholder-white/20 text-sm rounded-lg focus:outline-primary-button active:outline-primary-button outline-none block w-full p-2.5"
+            placeholder="8192"
+            min={1}
+            onScroll={(e) => e.target.blur()}
+            defaultValue={settings?.EmbeddingModelMaxChunkLength}
+            required={false}
+            autoComplete="off"
+          />
+        </div>
+      </div>
+      <div className="w-full flex items-center gap-[36px]">
+        <div className="flex flex-col w-60">
+          <div className="flex flex-col gap-y-1 mb-4">
+            <label className="text-white text-sm font-semibold flex items-center gap-x-2">
+              API Key <p className="!text-xs !italic !font-thin">optional</p>
+            </label>
+          </div>
+          <input
+            type="password"
+            name="GenericOpenAiEmbeddingApiKey"
+            className="bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:outline-primary-button active:outline-primary-button outline-none block w-full p-2.5"
+            placeholder="sk-mysecretkey"
+            defaultValue={
+              settings?.GenericOpenAiEmbeddingApiKey ? "*".repeat(20) : ""
+            }
+            autoComplete="off"
+            spellCheck={false}
+          />
+        </div>
+      </div>
+    </div>
+  );
+}

frontend/src/components/EmbeddingSelection/LMStudioOptions/index.jsx

@@ -1,48 +1,112 @@
 import React, { useEffect, useState } from "react";
 import System from "@/models/system";
+import PreLoader from "@/components/Preloader";
+import { LMSTUDIO_COMMON_URLS } from "@/utils/constants";
+import { CaretDown, CaretUp } from "@phosphor-icons/react";
+import useProviderEndpointAutoDiscovery from "@/hooks/useProviderEndpointAutoDiscovery";
 
 export default function LMStudioEmbeddingOptions({ settings }) {
-  const [basePathValue, setBasePathValue] = useState(
-    settings?.EmbeddingBasePath
+  const {
+    autoDetecting: loading,
+    basePath,
+    basePathValue,
+    showAdvancedControls,
+    setShowAdvancedControls,
+    handleAutoDetectClick,
+  } = useProviderEndpointAutoDiscovery({
+    provider: "lmstudio",
+    initialBasePath: settings?.EmbeddingBasePath,
+    ENDPOINTS: LMSTUDIO_COMMON_URLS,
+  });
+
+  const [maxChunkLength, setMaxChunkLength] = useState(
+    settings?.EmbeddingModelMaxChunkLength || 8192
   );
-  const [basePath, setBasePath] = useState(settings?.EmbeddingBasePath);
+
+  const handleMaxChunkLengthChange = (e) => {
+    setMaxChunkLength(Number(e.target.value));
+  };
 
   return (
-    <div className="w-full flex flex-col gap-y-4">
-      <div className="w-full flex items-center gap-4">
+    <div className="w-full flex flex-col gap-y-7">
+      <div className="w-full flex items-start gap-[36px] mt-1.5">
+        <LMStudioModelSelection settings={settings} basePath={basePath.value} />
         <div className="flex flex-col w-60">
-          <label className="text-white text-sm font-semibold block mb-4">
-            LMStudio Base URL
-          </label>
-          <input
-            type="url"
-            name="EmbeddingBasePath"
-            className="bg-zinc-900 text-white placeholder-white/20 text-sm rounded-lg focus:border-white block w-full p-2.5"
-            placeholder="http://localhost:1234/v1"
-            defaultValue={settings?.EmbeddingBasePath}
-            onChange={(e) => setBasePathValue(e.target.value)}
-            onBlur={() => setBasePath(basePathValue)}
-            required={true}
-            autoComplete="off"
-            spellCheck={false}
-          />
-        </div>
-        <LMStudioModelSelection settings={settings} basePath={basePath} />
-        <div className="flex flex-col w-60">
-          <label className="text-white text-sm font-semibold block mb-4">
-            Max embedding chunk length
+          <label className="text-white text-sm font-semibold block mb-2">
+            Max Embedding Chunk Length
           </label>
           <input
             type="number"
             name="EmbeddingModelMaxChunkLength"
-            className="bg-zinc-900 text-white placeholder-white/20 text-sm rounded-lg focus:border-white block w-full p-2.5"
+            className="bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:outline-primary-button active:outline-primary-button outline-none block w-full p-2.5"
             placeholder="8192"
             min={1}
+            value={maxChunkLength}
+            onChange={handleMaxChunkLengthChange}
             onScroll={(e) => e.target.blur()}
-            defaultValue={settings?.EmbeddingModelMaxChunkLength}
-            required={false}
+            required={true}
             autoComplete="off"
           />
+          <p className="text-xs leading-[18px] font-base text-white text-opacity-60 mt-2">
+            Maximum length of text chunks for embedding.
+          </p>
+        </div>
+      </div>
+      <div className="flex justify-start mt-4">
+        <button
+          onClick={(e) => {
+            e.preventDefault();
+            setShowAdvancedControls(!showAdvancedControls);
+          }}
+          className="text-white hover:text-white/70 flex items-center text-sm"
+        >
+          {showAdvancedControls ? "Hide" : "Show"} Manual Endpoint Input
+          {showAdvancedControls ? (
+            <CaretUp size={14} className="ml-1" />
+          ) : (
+            <CaretDown size={14} className="ml-1" />
+          )}
+        </button>
+      </div>
+
+      <div hidden={!showAdvancedControls}>
+        <div className="w-full flex items-start gap-4">
+          <div className="flex flex-col w-60">
+            <div className="flex justify-between items-center mb-2">
+              <label className="text-white text-sm font-semibold">
+                LM Studio Base URL
+              </label>
+              {loading ? (
+                <PreLoader size="6" />
+              ) : (
+                <>
+                  {!basePathValue.value && (
+                    <button
+                      onClick={handleAutoDetectClick}
+                      className="bg-primary-button text-xs font-medium px-2 py-1 rounded-lg hover:bg-secondary hover:text-white shadow-[0_4px_14px_rgba(0,0,0,0.25)]"
+                    >
+                      Auto-Detect
+                    </button>
+                  )}
+                </>
+              )}
+            </div>
+            <input
+              type="url"
+              name="EmbeddingBasePath"
+              className="bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:outline-primary-button active:outline-primary-button outline-none block w-full p-2.5"
+              placeholder="http://localhost:1234/v1"
+              value={basePathValue.value}
+              required={true}
+              autoComplete="off"
+              spellCheck={false}
+              onChange={basePath.onChange}
+              onBlur={basePath.onBlur}
+            />
+            <p className="text-xs leading-[18px] font-base text-white text-opacity-60 mt-2">
+              Enter the URL where LM Studio is running.
+            </p>
+          </div>
         </div>
       </div>
     </div>
@@ -55,14 +119,23 @@ function LMStudioModelSelection({ settings, basePath = null }) {
 
   useEffect(() => {
     async function findCustomModels() {
-      if (!basePath || !basePath.includes("/v1")) {
+      if (!basePath) {
         setCustomModels([]);
         setLoading(false);
         return;
       }
       setLoading(true);
-      const { models } = await System.customModels("lmstudio", null, basePath);
+      try {
+        const { models } = await System.customModels(
+          "lmstudio",
+          null,
+          basePath
+        );
         setCustomModels(models || []);
+      } catch (error) {
+        console.error("Failed to fetch custom models:", error);
+        setCustomModels([]);
+      }
       setLoading(false);
     }
     findCustomModels();
@@ -71,8 +144,8 @@ function LMStudioModelSelection({ settings, basePath = null }) {
   if (loading || customModels.length == 0) {
     return (
       <div className="flex flex-col w-60">
-        <label className="text-white text-sm font-semibold block mb-4">
-          Chat Model Selection
+        <label className="text-white text-sm font-semibold block mb-2">
+          LM Studio Embedding Model
         </label>
         <select
           name="EmbeddingModelPref"
@@ -80,19 +153,23 @@ function LMStudioModelSelection({ settings, basePath = null }) {
           className="bg-zinc-900 border-gray-500 text-white text-sm rounded-lg block w-full p-2.5"
         >
           <option disabled={true} selected={true}>
-            {basePath?.includes("/v1")
+            {!!basePath
               ? "--loading available models--"
-              : "-- waiting for URL --"}
+              : "Enter LM Studio URL first"}
           </option>
         </select>
+        <p className="text-xs leading-[18px] font-base text-white text-opacity-60 mt-2">
+          Select the LM Studio model for embeddings. Models will load after
+          entering a valid LM Studio URL.
+        </p>
       </div>
     );
   }
 
   return (
     <div className="flex flex-col w-60">
-      <label className="text-white text-sm font-semibold block mb-4">
-        Chat Model Selection
+      <label className="text-white text-sm font-semibold block mb-2">
+        LM Studio Embedding Model
       </label>
       <select
         name="EmbeddingModelPref"
@@ -115,6 +192,9 @@ function LMStudioModelSelection({ settings, basePath = null }) {
           </optgroup>
         )}
       </select>
+      <p className="text-xs leading-[18px] font-base text-white text-opacity-60 mt-2">
+        Choose the LM Studio model you want to use for generating embeddings.
+      </p>
     </div>
   );
 }

frontend/src/components/EmbeddingSelection/LiteLLMOptions/index.jsx

@@ -10,16 +10,16 @@ export default function LiteLLMOptions({ settings }) {
   const [apiKey, setApiKey] = useState(settings?.LiteLLMAPIKey);
 
   return (
-    <div className="w-full flex flex-col gap-y-4">
-      <div className="w-full flex items-center gap-4">
+    <div className="w-full flex flex-col gap-y-7">
+      <div className="w-full flex items-center gap-[36px] mt-1.5">
         <div className="flex flex-col w-60">
-          <label className="text-white text-sm font-semibold block mb-4">
+          <label className="text-white text-sm font-semibold block mb-3">
             Base URL
           </label>
           <input
             type="url"
             name="LiteLLMBasePath"
-            className="bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:border-white block w-full p-2.5"
+            className="bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:outline-primary-button active:outline-primary-button outline-none block w-full p-2.5"
             placeholder="http://127.0.0.1:4000"
             defaultValue={settings?.LiteLLMBasePath}
             required={true}
@@ -35,13 +35,13 @@ export default function LiteLLMOptions({ settings }) {
           apiKey={apiKey}
         />
         <div className="flex flex-col w-60">
-          <label className="text-white text-sm font-semibold block mb-4">
+          <label className="text-white text-sm font-semibold block mb-3">
             Max embedding chunk length
           </label>
           <input
             type="number"
             name="EmbeddingModelMaxChunkLength"
-            className="bg-zinc-900 text-white placeholder-white/20 text-sm rounded-lg focus:border-white block w-full p-2.5"
+            className="bg-zinc-900 text-white placeholder-white/20 text-sm rounded-lg focus:outline-primary-button active:outline-primary-button outline-none block w-full p-2.5"
             placeholder="8192"
             min={1}
             onScroll={(e) => e.target.blur()}
@@ -51,7 +51,7 @@ export default function LiteLLMOptions({ settings }) {
           />
         </div>
       </div>
-      <div className="w-full flex items-center gap-4">
+      <div className="w-full flex items-center gap-[36px]">
         <div className="flex flex-col w-60">
           <div className="flex flex-col gap-y-1 mb-4">
             <label className="text-white text-sm font-semibold flex items-center gap-x-2">
@@ -61,7 +61,7 @@ export default function LiteLLMOptions({ settings }) {
           <input
             type="password"
             name="LiteLLMAPIKey"
-            className="bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:border-white block w-full p-2.5"
+            className="bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:outline-primary-button active:outline-primary-button outline-none block w-full p-2.5"
             placeholder="sk-mysecretkey"
             defaultValue={settings?.LiteLLMAPIKey ? "*".repeat(20) : ""}
             autoComplete="off"
@@ -101,7 +101,7 @@ function LiteLLMModelSelection({ settings, basePath = null, apiKey = null }) {
   if (loading || customModels.length == 0) {
     return (
       <div className="flex flex-col w-60">
-        <label className="text-white text-sm font-semibold block mb-4">
+        <label className="text-white text-sm font-semibold block mb-3">
           Embedding Model Selection
         </label>
         <select
@@ -122,7 +122,7 @@ function LiteLLMModelSelection({ settings, basePath = null, apiKey = null }) {
   return (
     <div className="flex flex-col w-60">
       <div className="flex items-center">
-        <label className="text-white text-sm font-semibold block mb-4">
+        <label className="text-white text-sm font-semibold block mb-3">
           Embedding Model Selection
         </label>
         <EmbeddingModelTooltip />

frontend/src/components/EmbeddingSelection/LocalAiOptions/index.jsx

@@ -1,47 +1,42 @@
 import React, { useEffect, useState } from "react";
+import { CaretDown, CaretUp } from "@phosphor-icons/react";
 import System from "@/models/system";
+import PreLoader from "@/components/Preloader";
+import { LOCALAI_COMMON_URLS } from "@/utils/constants";
+import useProviderEndpointAutoDiscovery from "@/hooks/useProviderEndpointAutoDiscovery";
 
 export default function LocalAiOptions({ settings }) {
-  const [basePathValue, setBasePathValue] = useState(
-    settings?.EmbeddingBasePath
-  );
-  const [basePath, setBasePath] = useState(settings?.EmbeddingBasePath);
+  const {
+    autoDetecting: loading,
+    basePath,
+    basePathValue,
+    showAdvancedControls,
+    setShowAdvancedControls,
+    handleAutoDetectClick,
+  } = useProviderEndpointAutoDiscovery({
+    provider: "localai",
+    initialBasePath: settings?.EmbeddingBasePath,
+    ENDPOINTS: LOCALAI_COMMON_URLS,
+  });
   const [apiKeyValue, setApiKeyValue] = useState(settings?.LocalAiApiKey);
   const [apiKey, setApiKey] = useState(settings?.LocalAiApiKey);
 
   return (
-    <div className="w-full flex flex-col gap-y-4">
-      <div className="w-full flex items-center gap-4">
-        <div className="flex flex-col w-60">
-          <label className="text-white text-sm font-semibold block mb-4">
-            LocalAI Base URL
-          </label>
-          <input
-            type="url"
-            name="EmbeddingBasePath"
-            className="bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:border-white block w-full p-2.5"
-            placeholder="http://localhost:8080/v1"
-            defaultValue={settings?.EmbeddingBasePath}
-            onChange={(e) => setBasePathValue(e.target.value)}
-            onBlur={() => setBasePath(basePathValue)}
-            required={true}
-            autoComplete="off"
-            spellCheck={false}
-          />
-        </div>
+    <div className="w-full flex flex-col gap-y-7">
+      <div className="w-full flex items-center gap-[36px] mt-1.5">
         <LocalAIModelSelection
           settings={settings}
           apiKey={apiKey}
-          basePath={basePath}
+          basePath={basePath.value}
         />
         <div className="flex flex-col w-60">
-          <label className="text-white text-sm font-semibold block mb-4">
+          <label className="text-white text-sm font-semibold block mb-2">
             Max embedding chunk length
           </label>
           <input
             type="number"
             name="EmbeddingModelMaxChunkLength"
-            className="bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:border-white block w-full p-2.5"
+            className="bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:outline-primary-button active:outline-primary-button outline-none block w-full p-2.5"
             placeholder="1000"
             min={1}
             onScroll={(e) => e.target.blur()}
@@ -50,10 +45,8 @@ export default function LocalAiOptions({ settings }) {
             autoComplete="off"
           />
         </div>
-      </div>
-      <div className="w-full flex items-center gap-4">
         <div className="flex flex-col w-60">
-          <div className="flex flex-col gap-y-1 mb-4">
+          <div className="flex flex-col gap-y-1 mb-2">
             <label className="text-white text-sm font-semibold flex items-center gap-x-2">
               Local AI API Key{" "}
               <p className="!text-xs !italic !font-thin">optional</p>
@@ -62,7 +55,7 @@ export default function LocalAiOptions({ settings }) {
           <input
             type="password"
             name="LocalAiApiKey"
-            className="bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:border-white block w-full p-2.5"
+            className="bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:outline-primary-button active:outline-primary-button outline-none block w-full p-2.5"
             placeholder="sk-mysecretkey"
             defaultValue={settings?.LocalAiApiKey ? "*".repeat(20) : ""}
             autoComplete="off"
@@ -72,6 +65,59 @@ export default function LocalAiOptions({ settings }) {
           />
         </div>
       </div>
+      <div className="flex justify-start mt-4">
+        <button
+          onClick={(e) => {
+            e.preventDefault();
+            setShowAdvancedControls(!showAdvancedControls);
+          }}
+          className="text-white hover:text-white/70 flex items-center text-sm"
+        >
+          {showAdvancedControls ? "Hide" : "Show"} advanced settings
+          {showAdvancedControls ? (
+            <CaretUp size={14} className="ml-1" />
+          ) : (
+            <CaretDown size={14} className="ml-1" />
+          )}
+        </button>
+      </div>
+      <div hidden={!showAdvancedControls}>
+        <div className="w-full flex items-center gap-4">
+          <div className="flex flex-col w-60">
+            <div className="flex justify-between items-center mb-2">
+              <label className="text-white text-sm font-semibold">
+                LocalAI Base URL
+              </label>
+              {loading ? (
+                <PreLoader size="6" />
+              ) : (
+                <>
+                  {!basePathValue.value && (
+                    <button
+                      onClick={handleAutoDetectClick}
+                      className="bg-primary-button text-xs font-medium px-2 py-1 rounded-lg hover:bg-secondary hover:text-white shadow-[0_4px_14px_rgba(0,0,0,0.25)]"
+                    >
+                      Auto-Detect
+                    </button>
+                  )}
+                </>
+              )}
+            </div>
+            <input
+              type="url"
+              name="EmbeddingBasePath"
+              className="bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:outline-primary-button active:outline-primary-button outline-none block w-full p-2.5"
+              placeholder="http://localhost:8080/v1"
+              value={basePathValue.value}
+              required={true}
+              autoComplete="off"
+              spellCheck={false}
+              onChange={basePath.onChange}
+              onBlur={basePath.onBlur}
+            />
+          </div>
+        </div>
+      </div>
     </div>
   );
 }
@@ -102,7 +148,7 @@ function LocalAIModelSelection({ settings, apiKey = null, basePath = null }) {
   if (loading || customModels.length == 0) {
     return (
       <div className="flex flex-col w-60">
-        <label className="text-white text-sm font-semibold block mb-4">
+        <label className="text-white text-sm font-semibold block mb-2">
           Embedding Model Name
         </label>
         <select
@@ -122,7 +168,7 @@ function LocalAIModelSelection({ settings, apiKey = null, basePath = null }) {
 
   return (
     <div className="flex flex-col w-60">
-      <label className="text-white text-sm font-semibold block mb-4">
+      <label className="text-white text-sm font-semibold block mb-2">
         Embedding Model Name
       </label>
       <select

frontend/src/components/EmbeddingSelection/NativeEmbeddingOptions/index.jsx

@@ -1,9 +1,11 @@
+import { useTranslation } from "react-i18next";
+
 export default function NativeEmbeddingOptions() {
+  const { t } = useTranslation();
   return (
     <div className="w-full h-10 items-center flex">
       <p className="text-sm font-base text-white text-opacity-60">
-        There is no set up required when using AnythingLLM's native embedding
-        engine.
+        {t("embedding.provider.description")}
       </p>
     </div>
   );

frontend/src/components/EmbeddingSelection/OllamaOptions/index.jsx

@@ -1,55 +1,122 @@
 import React, { useEffect, useState } from "react";
 import System from "@/models/system";
+import PreLoader from "@/components/Preloader";
+import { OLLAMA_COMMON_URLS } from "@/utils/constants";
+import { CaretDown, CaretUp } from "@phosphor-icons/react";
+import useProviderEndpointAutoDiscovery from "@/hooks/useProviderEndpointAutoDiscovery";
 
 export default function OllamaEmbeddingOptions({ settings }) {
-  const [basePathValue, setBasePathValue] = useState(
-    settings?.EmbeddingBasePath
+  const {
+    autoDetecting: loading,
+    basePath,
+    basePathValue,
+    showAdvancedControls,
+    setShowAdvancedControls,
+    handleAutoDetectClick,
+  } = useProviderEndpointAutoDiscovery({
+    provider: "ollama",
+    initialBasePath: settings?.EmbeddingBasePath,
+    ENDPOINTS: OLLAMA_COMMON_URLS,
+  });
+
+  const [maxChunkLength, setMaxChunkLength] = useState(
+    settings?.EmbeddingModelMaxChunkLength || 8192
   );
-  const [basePath, setBasePath] = useState(settings?.EmbeddingBasePath);
+
+  const handleMaxChunkLengthChange = (e) => {
+    setMaxChunkLength(Number(e.target.value));
+  };
 
   return (
-    <div className="w-full flex flex-col gap-y-4">
-      <div className="w-full flex items-center gap-4">
-        <div className="flex flex-col w-60">
-          <label className="text-white text-sm font-semibold block mb-4">
-            Ollama Base URL
-          </label>
-          <input
-            type="url"
-            name="EmbeddingBasePath"
-            className="bg-zinc-900 text-white placeholder-white/20 text-sm rounded-lg focus:border-white block w-full p-2.5"
-            placeholder="http://127.0.0.1:11434"
-            defaultValue={settings?.EmbeddingBasePath}
-            onChange={(e) => setBasePathValue(e.target.value)}
-            onBlur={() => setBasePath(basePathValue)}
-            required={true}
-            autoComplete="off"
-            spellCheck={false}
+    <div className="w-full flex flex-col gap-y-7">
+      <div className="w-full flex items-start gap-[36px] mt-1.5">
+        <OllamaEmbeddingModelSelection
+          settings={settings}
+          basePath={basePath.value}
         />
-        </div>
-        <OllamaLLMModelSelection settings={settings} basePath={basePath} />
         <div className="flex flex-col w-60">
-          <label className="text-white text-sm font-semibold block mb-4">
-            Max embedding chunk length
+          <label className="text-white text-sm font-semibold block mb-2">
+            Max Embedding Chunk Length
           </label>
           <input
             type="number"
             name="EmbeddingModelMaxChunkLength"
-            className="bg-zinc-900 text-white placeholder-white/20 text-sm rounded-lg focus:border-white block w-full p-2.5"
+            className="bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:outline-primary-button active:outline-primary-button outline-none block w-full p-2.5"
             placeholder="8192"
             min={1}
+            value={maxChunkLength}
+            onChange={handleMaxChunkLengthChange}
             onScroll={(e) => e.target.blur()}
-            defaultValue={settings?.EmbeddingModelMaxChunkLength}
-            required={false}
+            required={true}
             autoComplete="off"
           />
+          <p className="text-xs leading-[18px] font-base text-white text-opacity-60 mt-2">
+            Maximum length of text chunks for embedding.
+          </p>
+        </div>
+      </div>
+      <div className="flex justify-start mt-4">
+        <button
+          onClick={(e) => {
+            e.preventDefault();
+            setShowAdvancedControls(!showAdvancedControls);
+          }}
+          className="text-white hover:text-white/70 flex items-center text-sm"
+        >
+          {showAdvancedControls ? "Hide" : "Show"} Manual Endpoint Input
+          {showAdvancedControls ? (
+            <CaretUp size={14} className="ml-1" />
+          ) : (
+            <CaretDown size={14} className="ml-1" />
+          )}
+        </button>
+      </div>
+
+      <div hidden={!showAdvancedControls}>
+        <div className="w-full flex items-start gap-4">
+          <div className="flex flex-col w-60">
+            <div className="flex justify-between items-center mb-2">
+              <label className="text-white text-sm font-semibold">
+                Ollama Base URL
+              </label>
+              {loading ? (
+                <PreLoader size="6" />
+              ) : (
+                <>
+                  {!basePathValue.value && (
+                    <button
+                      onClick={handleAutoDetectClick}
+                      className="bg-primary-button text-xs font-medium px-2 py-1 rounded-lg hover:bg-secondary hover:text-white shadow-[0_4px_14px_rgba(0,0,0,0.25)]"
+                    >
+                      Auto-Detect
+                    </button>
+                  )}
+                </>
+              )}
+            </div>
+            <input
+              type="url"
+              name="EmbeddingBasePath"
+              className="bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:outline-primary-button active:outline-primary-button outline-none block w-full p-2.5"
+              placeholder="http://127.0.0.1:11434"
+              value={basePathValue.value}
+              required={true}
+              autoComplete="off"
+              spellCheck={false}
+              onChange={basePath.onChange}
+              onBlur={basePath.onBlur}
+            />
+            <p className="text-xs leading-[18px] font-base text-white text-opacity-60 mt-2">
+              Enter the URL where Ollama is running.
+            </p>
+          </div>
         </div>
       </div>
     </div>
   );
 }
 
-function OllamaLLMModelSelection({ settings, basePath = null }) {
+function OllamaEmbeddingModelSelection({ settings, basePath = null }) {
   const [customModels, setCustomModels] = useState([]);
   const [loading, setLoading] = useState(true);
 
@@ -61,8 +128,13 @@ function OllamaLLMModelSelection({ settings, basePath = null }) {
         return;
       }
       setLoading(true);
+      try {
         const { models } = await System.customModels("ollama", null, basePath);
         setCustomModels(models || []);
+      } catch (error) {
+        console.error("Failed to fetch custom models:", error);
+        setCustomModels([]);
+      }
       setLoading(false);
     }
     findCustomModels();
@@ -71,33 +143,37 @@ function OllamaLLMModelSelection({ settings, basePath = null }) {
   if (loading || customModels.length == 0) {
     return (
       <div className="flex flex-col w-60">
-        <label className="text-white text-sm font-semibold block mb-4">
-          Embedding Model Selection
+        <label className="text-white text-sm font-semibold block mb-2">
+          Ollama Embedding Model
         </label>
         <select
           name="EmbeddingModelPref"
           disabled={true}
-          className="bg-zinc-900 border border-gray-500 text-white text-sm rounded-lg block w-full p-2.5"
+          className="bg-zinc-900 border-gray-500 text-white text-sm rounded-lg block w-full p-2.5"
         >
           <option disabled={true} selected={true}>
             {!!basePath
               ? "--loading available models--"
-              : "-- waiting for URL --"}
+              : "Enter Ollama URL first"}
           </option>
         </select>
+        <p className="text-xs leading-[18px] font-base text-white text-opacity-60 mt-2">
+          Select the Ollama model for embeddings. Models will load after
+          entering a valid Ollama URL.
+        </p>
       </div>
     );
   }
 
   return (
     <div className="flex flex-col w-60">
-      <label className="text-white text-sm font-semibold block mb-4">
-        Embedding Model Selection
+      <label className="text-white text-sm font-semibold block mb-2">
+        Ollama Embedding Model
       </label>
       <select
         name="EmbeddingModelPref"
         required={true}
-        className="bg-zinc-900 border border-gray-500 text-white text-sm rounded-lg block w-full p-2.5"
+        className="bg-zinc-900 border-gray-500 text-white text-sm rounded-lg block w-full p-2.5"
       >
         {customModels.length > 0 && (
           <optgroup label="Your loaded models">
@@ -115,6 +191,9 @@ function OllamaLLMModelSelection({ settings, basePath = null }) {
           </optgroup>
         )}
       </select>
+      <p className="text-xs leading-[18px] font-base text-white text-opacity-60 mt-2">
+        Choose the Ollama model you want to use for generating embeddings.
+      </p>
     </div>
   );
 }

frontend/src/components/EmbeddingSelection/OpenAiOptions/index.jsx

@@ -1,15 +1,15 @@
 export default function OpenAiOptions({ settings }) {
   return (
     <div className="w-full flex flex-col gap-y-4">
-      <div className="w-full flex items-center gap-4">
+      <div className="w-full flex items-center gap-[36px] mt-1.5">
         <div className="flex flex-col w-60">
-          <label className="text-white text-sm font-semibold block mb-4">
+          <label className="text-white text-sm font-semibold block mb-3">
             API Key
           </label>
           <input
             type="password"
             name="OpenAiKey"
-            className="bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:border-white block w-full p-2.5"
+            className="bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:outline-primary-button active:outline-primary-button outline-none block w-full p-2.5"
             placeholder="OpenAI API Key"
             defaultValue={settings?.OpenAiKey ? "*".repeat(20) : ""}
             required={true}
@@ -18,7 +18,7 @@ export default function OpenAiOptions({ settings }) {
           />
         </div>
         <div className="flex flex-col w-60">
-          <label className="text-white text-sm font-semibold block mb-4">
+          <label className="text-white text-sm font-semibold block mb-3">
             Model Preference
           </label>
           <select

frontend/src/components/EmbeddingSelection/VoyageAiOptions/index.jsx

@@ -1,15 +1,15 @@
 export default function VoyageAiOptions({ settings }) {
   return (
     <div className="w-full flex flex-col gap-y-4">
-      <div className="w-full flex items-center gap-4">
+      <div className="w-full flex items-center gap-[36px] mt-1.5">
         <div className="flex flex-col w-60">
-          <label className="text-white text-sm font-semibold block mb-4">
+          <label className="text-white text-sm font-semibold block mb-3">
             API Key
           </label>
           <input
             type="password"
             name="VoyageAiApiKey"
-            className="bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:border-white block w-full p-2.5"
+            className="bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:outline-primary-button active:outline-primary-button outline-none block w-full p-2.5"
             placeholder="Voyage AI API Key"
             defaultValue={settings?.VoyageAiApiKey ? "*".repeat(20) : ""}
             required={true}
@@ -18,7 +18,7 @@ export default function VoyageAiOptions({ settings }) {
           />
         </div>
         <div className="flex flex-col w-60">
-          <label className="text-white text-sm font-semibold block mb-4">
+          <label className="text-white text-sm font-semibold block mb-3">
             Model Preference
           </label>
           <select