Issue #543 support milvus vector db (#579)

* issue #543 support milvus vector db * migrate Milvus to use MilvusClient instead of ORM normalize env setup for docs/implementation feat: embedder model dimension added * update comments --------- Co-authored-by: timothycarambat <rambat1010@gmail.com>
2024-07-02 15:20:37 +02:00 · 2024-01-13 05:23:57 +08:00 · 2024-01-13 05:23:57 +08:00 · 6faa0efaa8
commit 6faa0efaa8
parent b2b2c2afe1
20 changed files with 759 additions and 5 deletions
--- a/README.md
+++ b/README.md
@ -87,6 +87,7 @@ Some cool features of AnythingLLM
 - [Chroma](https://trychroma.com)
 - [Weaviate](https://weaviate.io)
 - [QDrant](https://qdrant.tech)
+- [Milvus](https://milvus.io)

 ### Technical Overview

--- a/docker/.env.example
+++ b/docker/.env.example
@ -89,6 +89,12 @@ GID='1000'
 # QDRANT_ENDPOINT="http://localhost:6333"
 # QDRANT_API_KEY=

+# Enable all below if you are using vector database: Milvus.
+# VECTOR_DB="milvus"
+# MILVUS_ADDRESS="http://localhost:19530"
+# MILVUS_USERNAME=
+# MILVUS_PASSWORD=
+
 # CLOUD DEPLOYMENT VARIRABLES ONLY
 # AUTH_TOKEN="hunter2" # This is the password to your application if remote hosting.

--- a/frontend/src/components/VectorDBSelection/MilvusDBOptions/index.jsx
+++ b/frontend/src/components/VectorDBSelection/MilvusDBOptions/index.jsx
@ -0,0 +1,52 @@
+export default function MilvusDBOptions({ settings }) {
+  return (
+    <div className="w-full flex flex-col gap-y-4">
+      <div className="w-full flex items-center gap-4">
+        <div className="flex flex-col w-60">
+          <label className="text-white text-sm font-semibold block mb-4">
+            Milvus DB Address
+          </label>
+          <input
+            type="text"
+            name="MilvusAddress"
+            className="bg-zinc-900 text-white placeholder-white placeholder-opacity-60 text-sm rounded-lg focus:border-white block w-full p-2.5"
+            placeholder="http://localhost:19530"
+            defaultValue={settings?.MilvusAddress}
+            required={true}
+            autoComplete="off"
+            spellCheck={false}
+          />
+        </div>
+
+        <div className="flex flex-col w-60">
+          <label className="text-white text-sm font-semibold block mb-4">
+            Milvus Username
+          </label>
+          <input
+            type="text"
+            name="MilvusUsername"
+            className="bg-zinc-900 text-white placeholder-white placeholder-opacity-60 text-sm rounded-lg focus:border-white block w-full p-2.5"
+            placeholder="username"
+            defaultValue={settings?.MilvusUsername}
+            autoComplete="off"
+            spellCheck={false}
+          />
+        </div>
+        <div className="flex flex-col w-60">
+          <label className="text-white text-sm font-semibold block mb-4">
+            Milvus Password
+          </label>
+          <input
+            type="password"
+            name="MilvusPassword"
+            className="bg-zinc-900 text-white placeholder-white placeholder-opacity-60 text-sm rounded-lg focus:border-white block w-full p-2.5"
+            placeholder="password"
+            defaultValue={settings?.MilvusPassword ? "*".repeat(20) : ""}
+            autoComplete="off"
+            spellCheck={false}
+          />
+        </div>
+      </div>
+    </div>
+  );
+}
--- a/frontend/src/media/vectordbs/milvus.png
+++ b/frontend/src/media/vectordbs/milvus.png
--- a/frontend/src/pages/GeneralSettings/VectorDatabase/index.jsx
+++ b/frontend/src/pages/GeneralSettings/VectorDatabase/index.jsx
@ -8,6 +8,7 @@ import PineconeLogo from "@/media/vectordbs/pinecone.png";
 import LanceDbLogo from "@/media/vectordbs/lancedb.png";
 import WeaviateLogo from "@/media/vectordbs/weaviate.png";
 import QDrantLogo from "@/media/vectordbs/qdrant.png";
+import MilvusLogo from "@/media/vectordbs/milvus.png";
 import PreLoader from "@/components/Preloader";
 import ChangeWarningModal from "@/components/ChangeWarning";
 import { MagnifyingGlass } from "@phosphor-icons/react";
@ -17,6 +18,7 @@ import PineconeDBOptions from "@/components/VectorDBSelection/PineconeDBOptions"
 import QDrantDBOptions from "@/components/VectorDBSelection/QDrantDBOptions";
 import WeaviateDBOptions from "@/components/VectorDBSelection/WeaviateDBOptions";
 import VectorDBItem from "@/components/VectorDBSelection/VectorDBItem";
+import MilvusDBOptions from "@/components/VectorDBSelection/MilvusDBOptions";

 export default function GeneralVectorDatabase() {
  const [saving, setSaving] = useState(false);
@ -79,6 +81,13 @@ export default function GeneralVectorDatabase() {
      description:
        "Open source local and cloud hosted multi-modal vector database.",
    },
+    {
+      name: "Milvus",
+      value: "milvus",
+      logo: MilvusLogo,
+      options: <MilvusDBOptions settings={settings} />,
+      description: "Open-source, highly scalable, and blazing fast.",
+    },
  ];

  const updateVectorChoice = (selection) => {
--- a/frontend/src/pages/OnboardingFlow/Steps/DataHandling/index.jsx
+++ b/frontend/src/pages/OnboardingFlow/Steps/DataHandling/index.jsx
@ -14,6 +14,7 @@ import PineconeLogo from "@/media/vectordbs/pinecone.png";
 import LanceDbLogo from "@/media/vectordbs/lancedb.png";
 import WeaviateLogo from "@/media/vectordbs/weaviate.png";
 import QDrantLogo from "@/media/vectordbs/qdrant.png";
+import MilvusLogo from "@/media/vectordbs/milvus.png";
 import React, { useState, useEffect } from "react";
 import paths from "@/utils/paths";
 import { useNavigate } from "react-router-dom";
@ -123,6 +124,13 @@ const VECTOR_DB_PRIVACY = {
    ],
    logo: WeaviateLogo,
  },
+  milvus: {
+    name: "Milvus",
+    description: [
+      "Your vectors and document text are stored on your Milvus instance (cloud or self-hosted)",
+    ],
+    logo: MilvusLogo,
+  },
  lancedb: {
    name: "LanceDB",
    description: [
--- a/frontend/src/pages/OnboardingFlow/Steps/VectorDatabaseConnection/index.jsx
+++ b/frontend/src/pages/OnboardingFlow/Steps/VectorDatabaseConnection/index.jsx
@ -5,6 +5,7 @@ import PineconeLogo from "@/media/vectordbs/pinecone.png";
 import LanceDbLogo from "@/media/vectordbs/lancedb.png";
 import WeaviateLogo from "@/media/vectordbs/weaviate.png";
 import QDrantLogo from "@/media/vectordbs/qdrant.png";
+import MilvusLogo from "@/media/vectordbs/milvus.png";
 import System from "@/models/system";
 import paths from "@/utils/paths";
 import PineconeDBOptions from "@/components/VectorDBSelection/PineconeDBOptions";
@ -12,6 +13,7 @@ import ChromaDBOptions from "@/components/VectorDBSelection/ChromaDBOptions";
 import QDrantDBOptions from "@/components/VectorDBSelection/QDrantDBOptions";
 import WeaviateDBOptions from "@/components/VectorDBSelection/WeaviateDBOptions";
 import LanceDBOptions from "@/components/VectorDBSelection/LanceDBOptions";
+import MilvusOptions from "@/components/VectorDBSelection/MilvusDBOptions";
 import showToast from "@/utils/toast";
 import { useNavigate } from "react-router-dom";
 import VectorDBItem from "@/components/VectorDBSelection/VectorDBItem";
@ -81,6 +83,13 @@ export default function VectorDatabaseConnection({
      description:
        "Open source local and cloud hosted multi-modal vector database.",
    },
+    {
+      name: "Milvus",
+      value: "milvus",
+      logo: MilvusLogo,
+      options: <MilvusOptions settings={settings} />,
+      description: "Open-source, highly scalable, and blazing fast.",
+    },
  ];

  function handleForward() {
--- a/server/.env.example
+++ b/server/.env.example
@ -86,6 +86,11 @@ VECTOR_DB="lancedb"
 # QDRANT_ENDPOINT="http://localhost:6333"
 # QDRANT_API_KEY=

+# Enable all below if you are using vector database: Milvus.
+# VECTOR_DB="milvus"
+# MILVUS_ADDRESS="http://localhost:19530"
+# MILVUS_USERNAME=
+# MILVUS_PASSWORD=

 # CLOUD DEPLOYMENT VARIRABLES ONLY
 # AUTH_TOKEN="hunter2" # This is the password to your application if remote hosting.
--- a/server/models/systemSettings.js
+++ b/server/models/systemSettings.js
@ -56,6 +56,13 @@ const SystemSettings = {
            QdrantApiKey: process.env.QDRANT_API_KEY,
          }
        : {}),
+      ...(vectorDB === "milvus"
+        ? {
+            MilvusAddress: process.env.MILVUS_ADDRESS,
+            MilvusUsername: process.env.MILVUS_USERNAME,
+            MilvusPassword: !!process.env.MILVUS_PASSWORD,
+          }
+        : {}),
      LLMProvider: llmProvider,
      ...(llmProvider === "openai"
        ? {
--- a/server/package.json
+++ b/server/package.json
@ -28,6 +28,7 @@
    "@prisma/client": "5.3.0",
    "@qdrant/js-client-rest": "^1.4.0",
    "@xenova/transformers": "^2.10.0",
+    "@zilliz/milvus2-sdk-node": "^2.3.5",
    "archiver": "^5.3.1",
    "bcrypt": "^5.1.0",
    "body-parser": "^1.20.2",
@ -77,4 +78,4 @@
    "nodemon": "^2.0.22",
    "prettier": "^3.0.3"
  }
-}
+}
--- a/server/utils/EmbeddingEngines/azureOpenAi/index.js
+++ b/server/utils/EmbeddingEngines/azureOpenAi/index.js
@ -13,6 +13,7 @@ class AzureOpenAiEmbedder {
      new AzureKeyCredential(process.env.AZURE_OPENAI_KEY)
    );
    this.openai = openai;
+    this.dimensions = 1536;

    // Limit of how many strings we can process in a single pass to stay with resource or network limits
    // https://learn.microsoft.com/en-us/azure/ai-services/openai/faq#i-am-trying-to-use-embeddings-and-received-the-error--invalidrequesterror--too-many-inputs--the-max-number-of-inputs-is-1---how-do-i-fix-this-:~:text=consisting%20of%20up%20to%2016%20inputs%20per%20API%20request
--- a/server/utils/EmbeddingEngines/localAi/index.js
+++ b/server/utils/EmbeddingEngines/localAi/index.js
@ -16,6 +16,10 @@ class LocalAiEmbedder {
        : {}),
    });
    this.openai = new OpenAIApi(config);
+    // We don't know this for user's set model so for vectorDB integrations that requires dimensionality
+    // in schema, we will throw an error.
+    // Applies to QDrant and Milvus.
+    this.dimensions = null;

    // Limit of how many strings we can process in a single pass to stay with resource or network limits
    this.maxConcurrentChunks = 50;
--- a/server/utils/EmbeddingEngines/native/index.js
+++ b/server/utils/EmbeddingEngines/native/index.js
@ -12,6 +12,7 @@ class NativeEmbedder {
        : path.resolve(__dirname, `../../../storage/models`)
    );
    this.modelPath = path.resolve(this.cacheDir, "Xenova", "all-MiniLM-L6-v2");
+    this.dimensions = 384;

    // Limit of how many strings we can process in a single pass to stay with resource or network limits
    this.maxConcurrentChunks = 50;
--- a/server/utils/EmbeddingEngines/openAi/index.js
+++ b/server/utils/EmbeddingEngines/openAi/index.js
@ -9,6 +9,7 @@ class OpenAiEmbedder {
    });
    const openai = new OpenAIApi(config);
    this.openai = openai;
+    this.dimensions = 1536;

    // Limit of how many strings we can process in a single pass to stay with resource or network limits
    this.maxConcurrentChunks = 500;
--- a/server/utils/helpers/index.js
+++ b/server/utils/helpers/index.js
@ -16,6 +16,9 @@ function getVectorDbClass() {
    case "qdrant":
      const { QDrant } = require("../vectorDbProviders/qdrant");
      return QDrant;
+    case "milvus":
+      const { Milvus } = require("../vectorDbProviders/milvus");
+      return Milvus;
    default:
      throw new Error("ENV: No VECTOR_DB value found in environment!");
  }
--- a/server/utils/helpers/updateENV.js
+++ b/server/utils/helpers/updateENV.js
@ -170,6 +170,20 @@ const KEY_MAPPING = {
    checks: [],
  },

+  // Milvus Options
+  MilvusAddress: {
+    envKey: "MILVUS_ADDRESS",
+    checks: [isValidURL, validDockerizedUrl],
+  },
+  MilvusUsername: {
+    envKey: "MILVUS_USERNAME",
+    checks: [isNotEmpty],
+  },
+  MilvusPassword: {
+    envKey: "MILVUS_PASSWORD",
+    checks: [isNotEmpty],
+  },
+
  // Together Ai Options
  TogetherAiApiKey: {
    envKey: "TOGETHER_AI_API_KEY",
@ -279,7 +293,14 @@ function supportedEmbeddingModel(input = "") {
 }

 function supportedVectorDB(input = "") {
-  const supported = ["chroma", "pinecone", "lancedb", "weaviate", "qdrant"];
+  const supported = [
+    "chroma",
+    "pinecone",
+    "lancedb",
+    "weaviate",
+    "qdrant",
+    "milvus",
+  ];
  return supported.includes(input)
    ? null
    : `Invalid VectorDB type. Must be one of ${supported.join(", ")}.`;
--- a/server/utils/vectorDbProviders/milvus/MILVUS_SETUP.md
+++ b/server/utils/vectorDbProviders/milvus/MILVUS_SETUP.md
@ -0,0 +1,40 @@
+# How to setup a local (or remote) Milvus Vector Database
+
+[Official Milvus Docs](https://milvus.io/docs/example_code.md) for reference.
+
+### How to get started
+
+**Requirements**
+
+Choose one of the following
+
+- Cloud
+
+  - [Cloud account](https://cloud.zilliz.com/)
+
+- Local
+  - Docker
+  - `git` available in your CLI/terminal
+
+**Instructions**
+
+- Cloud
+
+  - Create a Cluster on your cloud account
+  - Get connect Public Endpoint and Token
+  - Set .env.development variable in server
+
+- Local
+  - Download yaml file `wget https://github.com/milvus-io/milvus/releases/download/v2.3.4/milvus-standalone-docker-compose.yml -O docker-compose.yml`
+  - Start Milvus `sudo docker compose up -d`
+  - Check the containers are up and running `sudo docker compose ps`
+  - Get port number and set .env.development variable in server
+
+eg: `server/.env.development`
+
+```
+VECTOR_DB="milvus"
+MILVUS_ADDRESS="http://localhost:19530"
+MILVUS_USERNAME=minioadmin # Whatever your username and password are
+MILVUS_PASSWORD=minioadmin
+```
--- a/server/utils/vectorDbProviders/milvus/index.js
+++ b/server/utils/vectorDbProviders/milvus/index.js
@ -0,0 +1,360 @@
+const {
+  DataType,
+  MetricType,
+  IndexType,
+  MilvusClient,
+} = require("@zilliz/milvus2-sdk-node");
+const { RecursiveCharacterTextSplitter } = require("langchain/text_splitter");
+const { v4: uuidv4 } = require("uuid");
+const { storeVectorResult, cachedVectorInformation } = require("../../files");
+const {
+  toChunks,
+  getLLMProvider,
+  getEmbeddingEngineSelection,
+} = require("../../helpers");
+
+const Milvus = {
+  name: "Milvus",
+  connect: async function () {
+    if (process.env.VECTOR_DB !== "milvus")
+      throw new Error("Milvus::Invalid ENV settings");
+
+    const client = new MilvusClient({
+      address: process.env.MILVUS_ADDRESS,
+      username: process.env.MILVUS_USERNAME,
+      password: process.env.MILVUS_PASSWORD,
+    });
+
+    const { isHealthy } = await client.checkHealth();
+    if (!isHealthy)
+      throw new Error(
+        "MilvusDB::Invalid Heartbeat received - is the instance online?"
+      );
+
+    return { client };
+  },
+  heartbeat: async function () {
+    await this.connect();
+    return { heartbeat: Number(new Date()) };
+  },
+  totalVectors: async function () {
+    const { client } = await this.connect();
+    const { collection_names } = await client.listCollections();
+    const total = collection_names.reduce(async (acc, collection_name) => {
+      const statistics = await client.getCollectionStatistics({
+        collection_name,
+      });
+      return Number(acc) + Number(statistics?.data?.row_count ?? 0);
+    }, 0);
+    return total;
+  },
+  namespaceCount: async function (_namespace = null) {
+    const { client } = await this.connect();
+    const statistics = await client.getCollectionStatistics({
+      collection_name: _namespace,
+    });
+    return Number(statistics?.data?.row_count ?? 0);
+  },
+  namespace: async function (client, namespace = null) {
+    if (!namespace) throw new Error("No namespace value provided.");
+    const collection = await client
+      .getCollectionStatistics({ collection_name: namespace })
+      .catch(() => null);
+    return collection;
+  },
+  hasNamespace: async function (namespace = null) {
+    if (!namespace) return false;
+    const { client } = await this.connect();
+    return await this.namespaceExists(client, namespace);
+  },
+  namespaceExists: async function (client, namespace = null) {
+    if (!namespace) throw new Error("No namespace value provided.");
+    const { value } = await client
+      .hasCollection({ collection_name: namespace })
+      .catch((e) => {
+        console.error("MilvusDB::namespaceExists", e.message);
+        return { value: false };
+      });
+    return value;
+  },
+  deleteVectorsInNamespace: async function (client, namespace = null) {
+    await client.dropCollection({ collection_name: namespace });
+    return true;
+  },
+  getOrCreateCollection: async function (client, namespace) {
+    const isExists = await this.namespaceExists(client, namespace);
+    if (!isExists) {
+      const embedder = getEmbeddingEngineSelection();
+      if (!embedder.dimensions)
+        throw new Error(
+          `Your embedder selection has unknown dimensions output. It should be defined when using ${this.name}. Open an issue on Github for support.`
+        );
+
+      await client.createCollection({
+        collection_name: namespace,
+        fields: [
+          {
+            name: "id",
+            description: "id",
+            data_type: DataType.VarChar,
+            max_length: 255,
+            is_primary_key: true,
+          },
+          {
+            name: "vector",
+            description: "vector",
+            data_type: DataType.FloatVector,
+            dim: embedder.dimensions,
+          },
+          {
+            name: "metadata",
+            decription: "metadata",
+            data_type: DataType.JSON,
+          },
+        ],
+      });
+      await client.createIndex({
+        collection_name: namespace,
+        field_name: "vector",
+        index_type: IndexType.AUTOINDEX,
+        metric_type: MetricType.COSINE,
+      });
+      await client.loadCollectionSync({
+        collection_name: namespace,
+      });
+    }
+  },
+  addDocumentToNamespace: async function (
+    namespace,
+    documentData = {},
+    fullFilePath = null
+  ) {
+    const { DocumentVectors } = require("../../../models/vectors");
+    try {
+      const { pageContent, docId, ...metadata } = documentData;
+      if (!pageContent || pageContent.length == 0) return false;
+
+      console.log("Adding new vectorized document into namespace", namespace);
+      const cacheResult = await cachedVectorInformation(fullFilePath);
+      if (cacheResult.exists) {
+        const { client } = await this.connect();
+        await this.getOrCreateCollection(client, namespace);
+
+        const { chunks } = cacheResult;
+        const documentVectors = [];
+
+        for (const chunk of chunks) {
+          // Before sending to Pinecone and saving the records to our db
+          // we need to assign the id of each chunk that is stored in the cached file.
+          const newChunks = chunk.map((chunk) => {
+            const id = uuidv4();
+            documentVectors.push({ docId, vectorId: id });
+            return { id, vector: chunk.values, metadata: chunk.metadata };
+          });
+          const insertResult = await client.insert({
+            collection_name: namespace,
+            data: newChunks,
+          });
+
+          if (insertResult?.status.error_code !== "Success") {
+            throw new Error(
+              `Error embedding into Milvus! Reason:${insertResult?.status.reason}`
+            );
+          }
+        }
+        await DocumentVectors.bulkInsert(documentVectors);
+        await client.flushSync({ collection_names: [namespace] });
+        return true;
+      }
+
+      const textSplitter = new RecursiveCharacterTextSplitter({
+        chunkSize:
+          getEmbeddingEngineSelection()?.embeddingMaxChunkLength || 1_000,
+        chunkOverlap: 20,
+      });
+      const textChunks = await textSplitter.splitText(pageContent);
+
+      console.log("Chunks created from document:", textChunks.length);
+      const LLMConnector = getLLMProvider();
+      const documentVectors = [];
+      const vectors = [];
+      const vectorValues = await LLMConnector.embedChunks(textChunks);
+
+      if (!!vectorValues && vectorValues.length > 0) {
+        for (const [i, vector] of vectorValues.entries()) {
+          const vectorRecord = {
+            id: uuidv4(),
+            values: vector,
+            // [DO NOT REMOVE]
+            // LangChain will be unable to find your text if you embed manually and dont include the `text` key.
+            metadata: { ...metadata, text: textChunks[i] },
+          };
+
+          vectors.push(vectorRecord);
+          documentVectors.push({ docId, vectorId: vectorRecord.id });
+        }
+      } else {
+        throw new Error(
+          "Could not embed document chunks! This document will not be recorded."
+        );
+      }
+
+      if (vectors.length > 0) {
+        const chunks = [];
+        const { client } = await this.connect();
+        await this.getOrCreateCollection(client, namespace);
+
+        console.log("Inserting vectorized chunks into Milvus.");
+        for (const chunk of toChunks(vectors, 100)) {
+          chunks.push(chunk);
+          const insertResult = await client.insert({
+            collection_name: namespace,
+            data: chunk.map((item) => ({
+              id: item.id,
+              vector: item.values,
+              metadata: chunk.metadata,
+            })),
+          });
+
+          if (insertResult?.status.error_code !== "Success") {
+            throw new Error(
+              `Error embedding into Milvus! Reason:${insertResult?.status.reason}`
+            );
+          }
+        }
+        await storeVectorResult(chunks, fullFilePath);
+        await client.flushSync({ collection_names: [namespace] });
+      }
+
+      await DocumentVectors.bulkInsert(documentVectors);
+      return true;
+    } catch (e) {
+      console.error(e);
+      console.error("addDocumentToNamespace", e.message);
+      return false;
+    }
+  },
+  deleteDocumentFromNamespace: async function (namespace, docId) {
+    const { DocumentVectors } = require("../../../models/vectors");
+    const { client } = await this.connect();
+    if (!(await this.namespaceExists(client, namespace))) return;
+    const knownDocuments = await DocumentVectors.where({ docId });
+    if (knownDocuments.length === 0) return;
+
+    const vectorIds = knownDocuments.map((doc) => doc.vectorId);
+    const queryIn = vectorIds.map((v) => `'${v}'`).join(",");
+    await client.deleteEntities({
+      collection_name: namespace,
+      expr: `id in [${queryIn}]`,
+    });
+
+    const indexes = knownDocuments.map((doc) => doc.id);
+    await DocumentVectors.deleteIds(indexes);
+
+    // Even after flushing Milvus can take some time to re-calc the count
+    // so all we can hope to do is flushSync so that the count can be correct
+    // on a later call.
+    await client.flushSync({ collection_names: [namespace] });
+    return true;
+  },
+  performSimilaritySearch: async function ({
+    namespace = null,
+    input = "",
+    LLMConnector = null,
+    similarityThreshold = 0.25,
+  }) {
+    if (!namespace || !input || !LLMConnector)
+      throw new Error("Invalid request to performSimilaritySearch.");
+
+    const { client } = await this.connect();
+    if (!(await this.namespaceExists(client, namespace))) {
+      return {
+        contextTexts: [],
+        sources: [],
+        message: "Invalid query - no documents found for workspace!",
+      };
+    }
+
+    const queryVector = await LLMConnector.embedTextInput(input);
+    const { contextTexts, sourceDocuments } = await this.similarityResponse(
+      client,
+      namespace,
+      queryVector,
+      similarityThreshold
+    );
+
+    const sources = sourceDocuments.map((metadata, i) => {
+      return { ...metadata, text: contextTexts[i] };
+    });
+    return {
+      contextTexts,
+      sources: this.curateSources(sources),
+      message: false,
+    };
+  },
+  similarityResponse: async function (
+    client,
+    namespace,
+    queryVector,
+    similarityThreshold = 0.25
+  ) {
+    const result = {
+      contextTexts: [],
+      sourceDocuments: [],
+      scores: [],
+    };
+    const response = await client.search({
+      collection_name: namespace,
+      vectors: queryVector,
+    });
+    response.results.forEach((match) => {
+      if (match.score < similarityThreshold) return;
+      result.contextTexts.push(match.metadata.text);
+      result.sourceDocuments.push(match);
+      result.scores.push(match.score);
+    });
+    return result;
+  },
+  "namespace-stats": async function (reqBody = {}) {
+    const { namespace = null } = reqBody;
+    if (!namespace) throw new Error("namespace required");
+    const { client } = await this.connect();
+    if (!(await this.namespaceExists(client, namespace)))
+      throw new Error("Namespace by that name does not exist.");
+    const stats = await this.namespace(client, namespace);
+    return stats
+      ? stats
+      : { message: "No stats were able to be fetched from DB for namespace" };
+  },
+  "delete-namespace": async function (reqBody = {}) {
+    const { namespace = null } = reqBody;
+    const { client } = await this.connect();
+    if (!(await this.namespaceExists(client, namespace)))
+      throw new Error("Namespace by that name does not exist.");
+
+    const statistics = await this.namespace(client, namespace);
+    await this.deleteVectorsInNamespace(client, namespace);
+    const vectorCount = Number(statistics?.data?.row_count ?? 0);
+    return {
+      message: `Namespace ${namespace} was deleted along with ${vectorCount} vectors.`,
+    };
+  },
+  curateSources: function (sources = []) {
+    const documents = [];
+    for (const source of sources) {
+      const { metadata = {} } = source;
+      if (Object.keys(metadata).length > 0) {
+        documents.push({
+          ...metadata,
+          ...(source.hasOwnProperty("pageContent")
+            ? { text: source.pageContent }
+            : {}),
+        });
+      }
+    }
+
+    return documents;
+  },
+};
+
+module.exports.Milvus = Milvus;
--- a/server/utils/vectorDbProviders/qdrant/index.js
+++ b/server/utils/vectorDbProviders/qdrant/index.js
@ -112,9 +112,15 @@ const QDrant = {
    if (await this.namespaceExists(client, namespace)) {
      return await client.getCollection(namespace);
    }
+
+    const embedder = getEmbeddingEngineSelection();
+    if (!embedder.dimensions)
+      throw new Error(
+        `Your embedder selection has unknown dimensions output. It should be defined when using ${this.name}. Open an issue on Github for support.`
+      );
    await client.createCollection(namespace, {
      vectors: {
-        size: 1536, //TODO: Fixed to OpenAI models - when other embeddings exist make variable.
+        size: embedder.dimensions,
        distance: "Cosine",
      },
    });
--- a/server/yarn.lock
+++ b/server/yarn.lock
@ -160,6 +160,20 @@
    "@azure/logger" "^1.0.3"
    tslib "^2.4.0"

+"@colors/colors@1.6.0", "@colors/colors@^1.6.0":
+  version "1.6.0"
+  resolved "https://registry.yarnpkg.com/@colors/colors/-/colors-1.6.0.tgz#ec6cd237440700bc23ca23087f513c75508958b0"
+  integrity sha512-Ir+AOibqzrIsL6ajt3Rz3LskB7OiMVHqltZmspbW/TJuTVuyOMirVqAkjfY6JISiLHgyNqicAC8AyHHGzNd/dA==
+
+"@dabh/diagnostics@^2.0.2":
+  version "2.0.3"
+  resolved "https://registry.yarnpkg.com/@dabh/diagnostics/-/diagnostics-2.0.3.tgz#7f7e97ee9a725dffc7808d93668cc984e1dc477a"
+  integrity sha512-hrlQOIi7hAfzsMqlGSFyVucrx38O+j6wiGOf//H2ecvIEqYN4ADBSS2iLMh5UFyDunCNniUIPk/q3riFv45xRA==
+  dependencies:
+    colorspace "1.1.x"
+    enabled "2.0.x"
+    kuler "^2.0.0"
+
 "@eslint-community/eslint-utils@^4.2.0":
  version "4.4.0"
  resolved "https://registry.yarnpkg.com/@eslint-community/eslint-utils/-/eslint-utils-4.4.0.tgz#a23514e8fb9af1269d5f7788aa556798d61c6b59"
@ -214,6 +228,35 @@
  resolved "https://registry.yarnpkg.com/@graphql-typed-document-node/core/-/core-3.2.0.tgz#5f3d96ec6b2354ad6d8a28bf216a1d97b5426861"
  integrity sha512-mB9oAsNCm9aM3/SOv4YtBMqZbYj10R7dkq8byBqxGY/ncFwhf2oQzMV+LCRlWoDSEBJ3COiR1yeDvMtsoOsuFQ==

+"@grpc/grpc-js@1.8.17":
+  version "1.8.17"
+  resolved "https://registry.yarnpkg.com/@grpc/grpc-js/-/grpc-js-1.8.17.tgz#a3a2f826fc033eae7d2f5ee41e0ab39cee948838"
+  integrity sha512-DGuSbtMFbaRsyffMf+VEkVu8HkSXEUfO3UyGJNtqxW9ABdtTIA+2UXAJpwbJS+xfQxuwqLUeELmL6FuZkOqPxw==
+  dependencies:
+    "@grpc/proto-loader" "^0.7.0"
+    "@types/node" ">=12.12.47"
+
+"@grpc/proto-loader@0.7.7":
+  version "0.7.7"
+  resolved "https://registry.yarnpkg.com/@grpc/proto-loader/-/proto-loader-0.7.7.tgz#d33677a77eea8407f7c66e2abd97589b60eb4b21"
+  integrity sha512-1TIeXOi8TuSCQprPItwoMymZXxWT0CPxUhkrkeCUH+D8U7QDwQ6b7SUz2MaLuWM2llT+J/TVFLmQI5KtML3BhQ==
+  dependencies:
+    "@types/long" "^4.0.1"
+    lodash.camelcase "^4.3.0"
+    long "^4.0.0"
+    protobufjs "^7.0.0"
+    yargs "^17.7.2"
+
+"@grpc/proto-loader@^0.7.0":
+  version "0.7.10"
+  resolved "https://registry.yarnpkg.com/@grpc/proto-loader/-/proto-loader-0.7.10.tgz#6bf26742b1b54d0a473067743da5d3189d06d720"
+  integrity sha512-CAqDfoaQ8ykFd9zqBDn4k6iWT9loLAlc2ETmDFS9JCD70gDcnA4L3AFEo2iV7KyAtAAHFW9ftq1Fz+Vsgq80RQ==
+  dependencies:
+    lodash.camelcase "^4.3.0"
+    long "^5.0.0"
+    protobufjs "^7.2.4"
+    yargs "^17.7.2"
+
 "@hapi/hoek@^9.0.0":
  version "9.3.0"
  resolved "https://registry.yarnpkg.com/@hapi/hoek/-/hoek-9.3.0.tgz#8368869dcb735be2e7f5cb7647de78e167a251fb"
@ -755,6 +798,13 @@
  resolved "https://registry.yarnpkg.com/@types/node/-/node-18.14.5.tgz#4a13a6445862159303fc38586598a9396fc408b3"
  integrity sha512-CRT4tMK/DHYhw1fcCEBwME9CSaZNclxfzVMe7GsO6ULSwsttbj70wSiX6rZdIjGblu93sTJxLdhNIT85KKI7Qw==

+"@types/node@>=12.12.47":
+  version "20.10.8"
+  resolved "https://registry.yarnpkg.com/@types/node/-/node-20.10.8.tgz#f1e223cbde9e25696661d167a5b93a9b2a5d57c7"
+  integrity sha512-f8nQs3cLxbAFc00vEU59yf9UyGUftkPaLGfvbVOIDdx2i1b8epBqj2aNGyP19fiyXWvlmZ7qC1XLjAzw/OKIeA==
+  dependencies:
+    undici-types "~5.26.4"
+
 "@types/node@>=13.7.0":
  version "20.10.3"
  resolved "https://registry.yarnpkg.com/@types/node/-/node-20.10.3.tgz#4900adcc7fc189d5af5bb41da8f543cea6962030"
@ -779,6 +829,11 @@
  resolved "https://registry.yarnpkg.com/@types/retry/-/retry-0.12.0.tgz#2b35eccfcee7d38cd72ad99232fbd58bffb3c84d"
  integrity sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA==

+"@types/triple-beam@^1.3.2":
+  version "1.3.5"
+  resolved "https://registry.yarnpkg.com/@types/triple-beam/-/triple-beam-1.3.5.tgz#74fef9ffbaa198eb8b588be029f38b00299caa2c"
+  integrity sha512-6WaYesThRMCl19iryMYP7/x2OVgCtbIVflDGFpWnb9irXI3UjYE4AzmYuiUKY1AJstGijoY+MgUszMgRxIYTYw==
+
 "@types/uuid@^9.0.1":
  version "9.0.7"
  resolved "https://registry.yarnpkg.com/@types/uuid/-/uuid-9.0.7.tgz#b14cebc75455eeeb160d5fe23c2fcc0c64f724d8"
@ -806,6 +861,18 @@
  optionalDependencies:
    onnxruntime-node "1.14.0"

+"@zilliz/milvus2-sdk-node@^2.3.5":
+  version "2.3.5"
+  resolved "https://registry.yarnpkg.com/@zilliz/milvus2-sdk-node/-/milvus2-sdk-node-2.3.5.tgz#6540bc03ebb99ab35f63e4eca7a1fd3ede2cf38c"
+  integrity sha512-bWbQnhvu+7jZXoqI+qySycwph3vloy0LDV54TBY4wRmu6HhMlqIqyIiI8sQNeSJFs8M1jHg1PlmhE/dvckA1bA==
+  dependencies:
+    "@grpc/grpc-js" "1.8.17"
+    "@grpc/proto-loader" "0.7.7"
+    dayjs "^1.11.7"
+    lru-cache "^9.1.2"
+    protobufjs "7.2.4"
+    winston "^3.9.0"
+
 abbrev@1:
  version "1.1.1"
  resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-1.1.1.tgz#f8f2c887ad10bf67f634f005b6987fed3179aac8"
@ -1487,7 +1554,7 @@ cmake-js@^7.2.1:
    which "^2.0.2"
    yargs "^17.6.0"

-color-convert@^1.9.0:
+color-convert@^1.9.0, color-convert@^1.9.3:
  version "1.9.3"
  resolved "https://registry.yarnpkg.com/color-convert/-/color-convert-1.9.3.tgz#bb71850690e1f136567de629d2d5471deda4c1e8"
  integrity sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==
@ -1511,7 +1578,7 @@ color-name@^1.0.0, color-name@~1.1.4:
  resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.4.tgz#c2a09a87acbde69543de6f63fa3995c826c536a2"
  integrity sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==

-color-string@^1.9.0:
+color-string@^1.6.0, color-string@^1.9.0:
  version "1.9.1"
  resolved "https://registry.yarnpkg.com/color-string/-/color-string-1.9.1.tgz#4467f9146f036f855b764dfb5bf8582bf342c7a4"
  integrity sha512-shrVawQFojnZv6xM40anx4CkoDP+fZsw/ZerEMsW/pyzsRbElpsL/DBVW7q3ExxwusdNXI3lXpuhEZkzs8p5Eg==
@ -1524,6 +1591,14 @@ color-support@^1.1.2, color-support@^1.1.3:
  resolved "https://registry.yarnpkg.com/color-support/-/color-support-1.1.3.tgz#93834379a1cc9a0c61f82f52f0d04322251bd5a2"
  integrity sha512-qiBjkpbMLO/HL68y+lh4q0/O1MZFj2RX6X/KmMa3+gJD3z+WwI1ZzDHysvqHGS3mP6mznPckpXmw1nI9cJjyRg==

+color@^3.1.3:
+  version "3.2.1"
+  resolved "https://registry.yarnpkg.com/color/-/color-3.2.1.tgz#3544dc198caf4490c3ecc9a790b54fe9ff45e164"
+  integrity sha512-aBl7dZI9ENN6fUGC7mWpMTPNHmWUSNan9tuWN6ahh5ZLNk9baLJOnSMlrQkHcrfFgz2/RigjUVAjdx36VcemKA==
+  dependencies:
+    color-convert "^1.9.3"
+    color-string "^1.6.0"
+
 color@^4.2.3:
  version "4.2.3"
  resolved "https://registry.yarnpkg.com/color/-/color-4.2.3.tgz#d781ecb5e57224ee43ea9627560107c0e0c6463a"
@ -1537,6 +1612,14 @@ colors@^1.4.0:
  resolved "https://registry.yarnpkg.com/colors/-/colors-1.4.0.tgz#c50491479d4c1bdaed2c9ced32cf7c7dc2360f78"
  integrity sha512-a+UqTh4kgZg/SlGvfbzDHpgRu7AAQOmmqRHJnxhRZICKFUT91brVhNNt58CMWU9PsBbv3PDCZUHbVxuDiH2mtA==

+colorspace@1.1.x:
+  version "1.1.4"
+  resolved "https://registry.yarnpkg.com/colorspace/-/colorspace-1.1.4.tgz#8d442d1186152f60453bf8070cd66eb364e59243"
+  integrity sha512-BgvKJiuVu1igBUF2kEjRCZXol6wiiGbY5ipL/oVPwm0BL9sIpMIzM8IK7vwuxIIzOXMV3Ey5w+vxhm0rR/TN8w==
+  dependencies:
+    color "^3.1.3"
+    text-hex "1.0.x"
+
 combined-stream@^1.0.8:
  version "1.0.8"
  resolved "https://registry.yarnpkg.com/combined-stream/-/combined-stream-1.0.8.tgz#c3d45a8b34fd730631a110a8a2520682b31d5a7f"
@ -1680,6 +1763,11 @@ crypt@0.0.2:
  resolved "https://registry.yarnpkg.com/crypt/-/crypt-0.0.2.tgz#88d7ff7ec0dfb86f713dc87bbb42d044d3e6c41b"
  integrity sha512-mCxBlsHFYh9C+HVpiEacem8FEBnMXgU9gy4zmNC+SXAZNB/1idgp/aulFJ4FgCi7GPEVbfyng092GqL2k2rmow==

+dayjs@^1.11.7:
+  version "1.11.10"
+  resolved "https://registry.yarnpkg.com/dayjs/-/dayjs-1.11.10.tgz#68acea85317a6e164457d6d6947564029a6a16a0"
+  integrity sha512-vjAczensTgRcqDERK0SR2XMwsF/tSvnvlv6VcF2GIhg6Sx4yOIt/irsr1RDJsKiIyBzJDpCoXiWWq28MqH2cnQ==
+
 debug@2.6.9:
  version "2.6.9"
  resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f"
@ -1835,6 +1923,11 @@ emoji-regex@^8.0.0:
  resolved "https://registry.yarnpkg.com/emoji-regex/-/emoji-regex-8.0.0.tgz#e818fd69ce5ccfcb404594f842963bf53164cc37"
  integrity sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==

+enabled@2.0.x:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/enabled/-/enabled-2.0.0.tgz#f9dd92ec2d6f4bbc0d5d1e64e21d61cd4665e7c2"
+  integrity sha512-AKrN98kuwOzMIdAizXGI86UFBoo26CL21UM763y1h/GMSJ4/OHU9k2YlsmBpyScFo/wbLzWQJBMCW4+IO3/+OQ==
+
 encode32@^1.1.0:
  version "1.1.0"
  resolved "https://registry.yarnpkg.com/encode32/-/encode32-1.1.0.tgz#0c54b45fb314ad5502e3c230cb95acdc5e5cd1dd"
@ -2254,6 +2347,11 @@ fd-slicer@~1.1.0:
  dependencies:
    pend "~1.2.0"

+fecha@^4.2.0:
+  version "4.2.3"
+  resolved "https://registry.yarnpkg.com/fecha/-/fecha-4.2.3.tgz#4d9ccdbc61e8629b259fdca67e65891448d569fd"
+  integrity sha512-OP2IUU6HeYKJi3i0z4A19kHMQoLVs4Hc+DPqqxI2h/DPZHTm/vjsfC6P0b4jCMy14XizLBqvndQ+UilD7707Jw==
+
 file-entry-cache@^6.0.1:
  version "6.0.1"
  resolved "https://registry.yarnpkg.com/file-entry-cache/-/file-entry-cache-6.0.1.tgz#211b2dd9659cb0394b073e7323ac3c933d522027"
@ -2339,6 +2437,11 @@ flow-remove-types@^2.217.1:
    pirates "^3.0.2"
    vlq "^0.2.1"

+fn.name@1.x.x:
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/fn.name/-/fn.name-1.1.0.tgz#26cad8017967aea8731bc42961d04a3d5988accc"
+  integrity sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw==
+
 follow-redirects@^1.14.8, follow-redirects@^1.14.9:
  version "1.15.2"
  resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.2.tgz#b460864144ba63f2681096f274c4e57026da2c13"
@ -3344,6 +3447,11 @@ keyv@^4.5.3:
  dependencies:
    json-buffer "3.0.1"

+kuler@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/kuler/-/kuler-2.0.0.tgz#e2c570a3800388fb44407e851531c1d670b061b3"
+  integrity sha512-Xq9nH7KlWZmXAtodXDDRE7vs6DU1gTU8zYDHDiWLSip45Egwq3plLHzPn27NgvzL2r1LMPC1vdqh98sQxtqj4A==
+
 ky@^0.33.1:
  version "0.33.3"
  resolved "https://registry.yarnpkg.com/ky/-/ky-0.33.3.tgz#bf1ad322a3f2c3428c13cfa4b3af95e6c4a2f543"
@ -3500,11 +3608,28 @@ log-symbols@^5.1.0:
    chalk "^5.0.0"
    is-unicode-supported "^1.1.0"

+logform@^2.3.2, logform@^2.4.0:
+  version "2.6.0"
+  resolved "https://registry.yarnpkg.com/logform/-/logform-2.6.0.tgz#8c82a983f05d6eaeb2d75e3decae7a768b2bf9b5"
+  integrity sha512-1ulHeNPp6k/LD8H91o7VYFBng5i1BDE7HoKxVbZiGFidS1Rj65qcywLxX+pVfAPoQJEjRdvKcusKwOupHCVOVQ==
+  dependencies:
+    "@colors/colors" "1.6.0"
+    "@types/triple-beam" "^1.3.2"
+    fecha "^4.2.0"
+    ms "^2.1.1"
+    safe-stable-stringify "^2.3.1"
+    triple-beam "^1.3.0"
+
 long@^4.0.0:
  version "4.0.0"
  resolved "https://registry.yarnpkg.com/long/-/long-4.0.0.tgz#9a7b71cfb7d361a194ea555241c92f7468d5bf28"
  integrity sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA==

+long@^5.0.0:
+  version "5.2.3"
+  resolved "https://registry.yarnpkg.com/long/-/long-5.2.3.tgz#a3ba97f3877cf1d778eccbcb048525ebb77499e1"
+  integrity sha512-lcHwpNoggQTObv5apGNCTdJrO69eHOZMi4BNC+rTLER8iHAqGrUVeLh/irVIM7zTw2bOXA8T6uNPeujwOLg/2Q==
+
 loose-envify@^1.4.0:
  version "1.4.0"
  resolved "https://registry.yarnpkg.com/loose-envify/-/loose-envify-1.4.0.tgz#71ee51fa7be4caec1a63839f7e682d8132d30caf"
@ -3524,6 +3649,11 @@ lru-cache@^6.0.0:
  dependencies:
    yallist "^4.0.0"

+lru-cache@^9.1.2:
+  version "9.1.2"
+  resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-9.1.2.tgz#255fdbc14b75589d6d0e73644ca167a8db506835"
+  integrity sha512-ERJq3FOzJTxBbFjZ7iDs+NiK4VI9Wz+RdrrAB8dio1oV+YvdPzUEE4QNiT2VD51DkIbCYRUUzCRkssXCHqSnKQ==
+
 make-dir@^3.1.0:
  version "3.1.0"
  resolved "https://registry.yarnpkg.com/make-dir/-/make-dir-3.1.0.tgz#415e967046b3a7f1d185277d84aa58203726a13f"
@ -4042,6 +4172,13 @@ once@^1.3.0, once@^1.3.1, once@^1.4.0:
  dependencies:
    wrappy "1"

+one-time@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/one-time/-/one-time-1.0.0.tgz#e06bc174aed214ed58edede573b433bbf827cb45"
+  integrity sha512-5DXOiRKwuSEcQ/l0kGCF6Q3jcADFv5tSmRaJck/OqkVFcOzutB134KRSfF0xDrL39MNnqxbHBbUUcjZIhTgb2g==
+  dependencies:
+    fn.name "1.x.x"
+
 onetime@^5.1.0:
  version "5.1.2"
  resolved "https://registry.yarnpkg.com/onetime/-/onetime-5.1.2.tgz#d0e96ebb56b07476df1dd9c4806e5237985ca45e"
@ -4334,6 +4471,24 @@ prop-types@^15.8.1:
    object-assign "^4.1.1"
    react-is "^16.13.1"

+protobufjs@7.2.4:
+  version "7.2.4"
+  resolved "https://registry.yarnpkg.com/protobufjs/-/protobufjs-7.2.4.tgz#3fc1ec0cdc89dd91aef9ba6037ba07408485c3ae"
+  integrity sha512-AT+RJgD2sH8phPmCf7OUZR8xGdcJRga4+1cOaXJ64hvcSkVhNcRHOwIxUatPH15+nj59WAGTDv3LSGZPEQbJaQ==
+  dependencies:
+    "@protobufjs/aspromise" "^1.1.2"
+    "@protobufjs/base64" "^1.1.2"
+    "@protobufjs/codegen" "^2.0.4"
+    "@protobufjs/eventemitter" "^1.1.0"
+    "@protobufjs/fetch" "^1.1.0"
+    "@protobufjs/float" "^1.0.2"
+    "@protobufjs/inquire" "^1.1.0"
+    "@protobufjs/path" "^1.1.2"
+    "@protobufjs/pool" "^1.1.0"
+    "@protobufjs/utf8" "^1.1.0"
+    "@types/node" ">=13.7.0"
+    long "^5.0.0"
+
 protobufjs@^6.8.8:
  version "6.11.4"
  resolved "https://registry.yarnpkg.com/protobufjs/-/protobufjs-6.11.4.tgz#29a412c38bf70d89e537b6d02d904a6f448173aa"
@ -4353,6 +4508,24 @@ protobufjs@^6.8.8:
    "@types/node" ">=13.7.0"
    long "^4.0.0"

+protobufjs@^7.0.0, protobufjs@^7.2.4:
+  version "7.2.5"
+  resolved "https://registry.yarnpkg.com/protobufjs/-/protobufjs-7.2.5.tgz#45d5c57387a6d29a17aab6846dcc283f9b8e7f2d"
+  integrity sha512-gGXRSXvxQ7UiPgfw8gevrfRWcTlSbOFg+p/N+JVJEK5VhueL2miT6qTymqAmjr1Q5WbOCyJbyrk6JfWKwlFn6A==
+  dependencies:
+    "@protobufjs/aspromise" "^1.1.2"
+    "@protobufjs/base64" "^1.1.2"
+    "@protobufjs/codegen" "^2.0.4"
+    "@protobufjs/eventemitter" "^1.1.0"
+    "@protobufjs/fetch" "^1.1.0"
+    "@protobufjs/float" "^1.0.2"
+    "@protobufjs/inquire" "^1.1.0"
+    "@protobufjs/path" "^1.1.2"
+    "@protobufjs/pool" "^1.1.0"
+    "@protobufjs/utf8" "^1.1.0"
+    "@types/node" ">=13.7.0"
+    long "^5.0.0"
+
 proxy-addr@~2.0.7:
  version "2.0.7"
  resolved "https://registry.yarnpkg.com/proxy-addr/-/proxy-addr-2.0.7.tgz#f19fe69ceab311eeb94b42e70e8c2070f9ba1025"
@ -4605,6 +4778,11 @@ safe-regex-test@^1.0.0:
    get-intrinsic "^1.1.3"
    is-regex "^1.1.4"

+safe-stable-stringify@^2.3.1:
+  version "2.4.3"
+  resolved "https://registry.yarnpkg.com/safe-stable-stringify/-/safe-stable-stringify-2.4.3.tgz#138c84b6f6edb3db5f8ef3ef7115b8f55ccbf886"
+  integrity sha512-e2bDA2WJT0wxseVd4lsDP4+3ONX6HpMXQa1ZhFQ7SU+GjvORCmShbCMltrtIDfkYhVHrOcPtj+KhmDBdPdZD1g==
+
 "safer-buffer@>= 2.1.2 < 3", "safer-buffer@>= 2.1.2 < 3.0.0":
  version "2.1.2"
  resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a"
@ -4835,6 +5013,11 @@ ssri@^8.0.0, ssri@^8.0.1:
  dependencies:
    minipass "^3.1.1"

+stack-trace@0.0.x:
+  version "0.0.10"
+  resolved "https://registry.yarnpkg.com/stack-trace/-/stack-trace-0.0.10.tgz#547c70b347e8d32b4e108ea1a2a159e5fdde19c0"
+  integrity sha512-KGzahc7puUKkzyMt+IqAep+TVNbKP+k2Lmwhub39m1AsTSkaDutx56aDCo+HLDzf/D26BIHTJWNiTG1KAJiQCg==
+
 statuses@2.0.1:
  version "2.0.1"
  resolved "https://registry.yarnpkg.com/statuses/-/statuses-2.0.1.tgz#55cb000ccf1d48728bd23c685a063998cf1a1b63"
@ -5078,6 +5261,11 @@ tar@^6.0.2, tar@^6.1.11, tar@^6.1.2:
    mkdirp "^1.0.3"
    yallist "^4.0.0"

+text-hex@1.0.x:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/text-hex/-/text-hex-1.0.0.tgz#69dc9c1b17446ee79a92bf5b884bb4b9127506f5"
+  integrity sha512-uuVGNWzgJ4yhRaNSiubPY7OjISw4sw4E5Uv0wbjp+OzcbmVU/rsT8ujgcXJhn9ypzsgr5vlzpPqP+MBBKcGvbg==
+
 text-table@^0.2.0:
  version "0.2.0"
  resolved "https://registry.yarnpkg.com/text-table/-/text-table-0.2.0.tgz#7f5ee823ae805207c00af2df4a84ec3fcfa570b4"
@ -5107,6 +5295,11 @@ tr46@~0.0.3:
  resolved "https://registry.yarnpkg.com/tr46/-/tr46-0.0.3.tgz#8184fd347dac9cdc185992f3a6622e14b9d9ab6a"
  integrity sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==

+triple-beam@^1.3.0:
+  version "1.4.1"
+  resolved "https://registry.yarnpkg.com/triple-beam/-/triple-beam-1.4.1.tgz#6fde70271dc6e5d73ca0c3b24e2d92afb7441984"
+  integrity sha512-aZbgViZrg1QNcG+LULa7nhZpJTZSLm/mXnHXnbAbjmN5aSa0y7V+wvv6+4WaBtpISJzThKy+PIPxc1Nq1EJ9mg==
+
 tslib@^2.2.0, tslib@^2.4.0:
  version "2.6.1"
  resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.6.1.tgz#fd8c9a0ff42590b25703c0acb3de3d3f4ede0410"
@ -5448,6 +5641,32 @@ wide-align@^1.1.2, wide-align@^1.1.5:
  dependencies:
    string-width "^1.0.2 || 2 || 3 || 4"

+winston-transport@^4.5.0:
+  version "4.6.0"
+  resolved "https://registry.yarnpkg.com/winston-transport/-/winston-transport-4.6.0.tgz#f1c1a665ad1b366df72199e27892721832a19e1b"
+  integrity sha512-wbBA9PbPAHxKiygo7ub7BYRiKxms0tpfU2ljtWzb3SjRjv5yl6Ozuy/TkXf00HTAt+Uylo3gSkNwzc4ME0wiIg==
+  dependencies:
+    logform "^2.3.2"
+    readable-stream "^3.6.0"
+    triple-beam "^1.3.0"
+
+winston@^3.9.0:
+  version "3.11.0"
+  resolved "https://registry.yarnpkg.com/winston/-/winston-3.11.0.tgz#2d50b0a695a2758bb1c95279f0a88e858163ed91"
+  integrity sha512-L3yR6/MzZAOl0DsysUXHVjOwv8mKZ71TrA/41EIduGpOOV5LQVodqN+QdQ6BS6PJ/RdIshZhq84P/fStEZkk7g==
+  dependencies:
+    "@colors/colors" "^1.6.0"
+    "@dabh/diagnostics" "^2.0.2"
+    async "^3.2.3"
+    is-stream "^2.0.0"
+    logform "^2.4.0"
+    one-time "^1.0.0"
+    readable-stream "^3.4.0"
+    safe-stable-stringify "^2.3.1"
+    stack-trace "0.0.x"
+    triple-beam "^1.3.0"
+    winston-transport "^4.5.0"
+
 wordwrapjs@^4.0.0:
  version "4.0.1"
  resolved "https://registry.yarnpkg.com/wordwrapjs/-/wordwrapjs-4.0.1.tgz#d9790bccfb110a0fc7836b5ebce0937b37a8b98f"