mirror of
https://github.com/Mintplex-Labs/anything-llm.git
synced 2024-11-11 01:10:11 +01:00
Compare commits
No commits in common. "8c86ad11b037725b9e890bac6eb0eb2461c8b43a" and "5baa01b737827730b16faf34bcf637ffdaa16209" have entirely different histories.
8c86ad11b0
...
5baa01b737
@ -4,7 +4,7 @@ import "react-loading-skeleton/dist/skeleton.css";
|
|||||||
import Workspace from "@/models/workspace";
|
import Workspace from "@/models/workspace";
|
||||||
import ManageWorkspace, {
|
import ManageWorkspace, {
|
||||||
useManageWorkspaceModal,
|
useManageWorkspaceModal,
|
||||||
} from "../../Modals/ManageWorkspace";
|
} from "../../Modals/MangeWorkspace";
|
||||||
import paths from "@/utils/paths";
|
import paths from "@/utils/paths";
|
||||||
import { useParams } from "react-router-dom";
|
import { useParams } from "react-router-dom";
|
||||||
import { GearSix, SquaresFour, UploadSimple } from "@phosphor-icons/react";
|
import { GearSix, SquaresFour, UploadSimple } from "@phosphor-icons/react";
|
||||||
|
@ -1,8 +1,8 @@
|
|||||||
import HistoricalMessage from "./HistoricalMessage";
|
import HistoricalMessage from "./HistoricalMessage";
|
||||||
import PromptReply from "./PromptReply";
|
import PromptReply from "./PromptReply";
|
||||||
import { useEffect, useRef, useState } from "react";
|
import { useEffect, useRef, useState } from "react";
|
||||||
import { useManageWorkspaceModal } from "../../../Modals/ManageWorkspace";
|
import { useManageWorkspaceModal } from "../../../Modals/MangeWorkspace";
|
||||||
import ManageWorkspace from "../../../Modals/ManageWorkspace";
|
import ManageWorkspace from "../../../Modals/MangeWorkspace";
|
||||||
import { ArrowDown } from "@phosphor-icons/react";
|
import { ArrowDown } from "@phosphor-icons/react";
|
||||||
import debounce from "lodash.debounce";
|
import debounce from "lodash.debounce";
|
||||||
import useUser from "@/hooks/useUser";
|
import useUser from "@/hooks/useUser";
|
||||||
|
@ -6,55 +6,9 @@ const { v4: uuidv4 } = require("uuid");
|
|||||||
const { toChunks, getEmbeddingEngineSelection } = require("../../helpers");
|
const { toChunks, getEmbeddingEngineSelection } = require("../../helpers");
|
||||||
const { parseAuthHeader } = require("../../http");
|
const { parseAuthHeader } = require("../../http");
|
||||||
const { sourceIdentifier } = require("../../chats");
|
const { sourceIdentifier } = require("../../chats");
|
||||||
const COLLECTION_REGEX = new RegExp(
|
|
||||||
/^(?!\d+\.\d+\.\d+\.\d+$)(?!.*\.\.)(?=^[a-zA-Z0-9][a-zA-Z0-9_-]{1,61}[a-zA-Z0-9]$).{3,63}$/
|
|
||||||
);
|
|
||||||
|
|
||||||
const Chroma = {
|
const Chroma = {
|
||||||
name: "Chroma",
|
name: "Chroma",
|
||||||
// Chroma DB has specific requirements for collection names:
|
|
||||||
// (1) Must contain 3-63 characters
|
|
||||||
// (2) Must start and end with an alphanumeric character
|
|
||||||
// (3) Can only contain alphanumeric characters, underscores, or hyphens
|
|
||||||
// (4) Cannot contain two consecutive periods (..)
|
|
||||||
// (5) Cannot be a valid IPv4 address
|
|
||||||
// We need to enforce these rules by normalizing the collection names
|
|
||||||
// before communicating with the Chroma DB.
|
|
||||||
normalize: function (inputString) {
|
|
||||||
if (COLLECTION_REGEX.test(inputString)) return inputString;
|
|
||||||
let normalized = inputString.replace(/[^a-zA-Z0-9_-]/g, "-");
|
|
||||||
|
|
||||||
// Replace consecutive periods with a single period (if any)
|
|
||||||
normalized = normalized.replace(/\.\.+/g, ".");
|
|
||||||
|
|
||||||
// Ensure the name doesn't start with a non-alphanumeric character
|
|
||||||
if (normalized[0] && !/^[a-zA-Z0-9]$/.test(normalized[0])) {
|
|
||||||
normalized = "anythingllm-" + normalized.slice(1);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Ensure the name doesn't end with a non-alphanumeric character
|
|
||||||
if (
|
|
||||||
normalized[normalized.length - 1] &&
|
|
||||||
!/^[a-zA-Z0-9]$/.test(normalized[normalized.length - 1])
|
|
||||||
) {
|
|
||||||
normalized = normalized.slice(0, -1);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Ensure the length is between 3 and 63 characters
|
|
||||||
if (normalized.length < 3) {
|
|
||||||
normalized = `anythingllm-${normalized}`;
|
|
||||||
} else if (normalized.length > 63) {
|
|
||||||
// Recheck the norm'd name if sliced since its ending can still be invalid.
|
|
||||||
normalized = this.normalize(normalized.slice(0, 63));
|
|
||||||
}
|
|
||||||
|
|
||||||
// Ensure the name is not an IPv4 address
|
|
||||||
if (/^\d+\.\d+\.\d+\.\d+$/.test(normalized)) {
|
|
||||||
normalized = "-" + normalized.slice(1);
|
|
||||||
}
|
|
||||||
|
|
||||||
return normalized;
|
|
||||||
},
|
|
||||||
connect: async function () {
|
connect: async function () {
|
||||||
if (process.env.VECTOR_DB !== "chroma")
|
if (process.env.VECTOR_DB !== "chroma")
|
||||||
throw new Error("Chroma::Invalid ENV settings");
|
throw new Error("Chroma::Invalid ENV settings");
|
||||||
@ -105,7 +59,7 @@ const Chroma = {
|
|||||||
},
|
},
|
||||||
namespaceCount: async function (_namespace = null) {
|
namespaceCount: async function (_namespace = null) {
|
||||||
const { client } = await this.connect();
|
const { client } = await this.connect();
|
||||||
const namespace = await this.namespace(client, this.normalize(_namespace));
|
const namespace = await this.namespace(client, _namespace);
|
||||||
return namespace?.vectorCount || 0;
|
return namespace?.vectorCount || 0;
|
||||||
},
|
},
|
||||||
similarityResponse: async function (
|
similarityResponse: async function (
|
||||||
@ -116,9 +70,7 @@ const Chroma = {
|
|||||||
topN = 4,
|
topN = 4,
|
||||||
filterIdentifiers = []
|
filterIdentifiers = []
|
||||||
) {
|
) {
|
||||||
const collection = await client.getCollection({
|
const collection = await client.getCollection({ name: namespace });
|
||||||
name: this.normalize(namespace),
|
|
||||||
});
|
|
||||||
const result = {
|
const result = {
|
||||||
contextTexts: [],
|
contextTexts: [],
|
||||||
sourceDocuments: [],
|
sourceDocuments: [],
|
||||||
@ -154,7 +106,7 @@ const Chroma = {
|
|||||||
namespace: async function (client, namespace = null) {
|
namespace: async function (client, namespace = null) {
|
||||||
if (!namespace) throw new Error("No namespace value provided.");
|
if (!namespace) throw new Error("No namespace value provided.");
|
||||||
const collection = await client
|
const collection = await client
|
||||||
.getCollection({ name: this.normalize(namespace) })
|
.getCollection({ name: namespace })
|
||||||
.catch(() => null);
|
.catch(() => null);
|
||||||
if (!collection) return null;
|
if (!collection) return null;
|
||||||
|
|
||||||
@ -166,12 +118,12 @@ const Chroma = {
|
|||||||
hasNamespace: async function (namespace = null) {
|
hasNamespace: async function (namespace = null) {
|
||||||
if (!namespace) return false;
|
if (!namespace) return false;
|
||||||
const { client } = await this.connect();
|
const { client } = await this.connect();
|
||||||
return await this.namespaceExists(client, this.normalize(namespace));
|
return await this.namespaceExists(client, namespace);
|
||||||
},
|
},
|
||||||
namespaceExists: async function (client, namespace = null) {
|
namespaceExists: async function (client, namespace = null) {
|
||||||
if (!namespace) throw new Error("No namespace value provided.");
|
if (!namespace) throw new Error("No namespace value provided.");
|
||||||
const collection = await client
|
const collection = await client
|
||||||
.getCollection({ name: this.normalize(namespace) })
|
.getCollection({ name: namespace })
|
||||||
.catch((e) => {
|
.catch((e) => {
|
||||||
console.error("ChromaDB::namespaceExists", e.message);
|
console.error("ChromaDB::namespaceExists", e.message);
|
||||||
return null;
|
return null;
|
||||||
@ -179,7 +131,7 @@ const Chroma = {
|
|||||||
return !!collection;
|
return !!collection;
|
||||||
},
|
},
|
||||||
deleteVectorsInNamespace: async function (client, namespace = null) {
|
deleteVectorsInNamespace: async function (client, namespace = null) {
|
||||||
await client.deleteCollection({ name: this.normalize(namespace) });
|
await client.deleteCollection({ name: namespace });
|
||||||
return true;
|
return true;
|
||||||
},
|
},
|
||||||
addDocumentToNamespace: async function (
|
addDocumentToNamespace: async function (
|
||||||
@ -197,7 +149,7 @@ const Chroma = {
|
|||||||
if (cacheResult.exists) {
|
if (cacheResult.exists) {
|
||||||
const { client } = await this.connect();
|
const { client } = await this.connect();
|
||||||
const collection = await client.getOrCreateCollection({
|
const collection = await client.getOrCreateCollection({
|
||||||
name: this.normalize(namespace),
|
name: namespace,
|
||||||
metadata: { "hnsw:space": "cosine" },
|
metadata: { "hnsw:space": "cosine" },
|
||||||
});
|
});
|
||||||
const { chunks } = cacheResult;
|
const { chunks } = cacheResult;
|
||||||
@ -293,7 +245,7 @@ const Chroma = {
|
|||||||
|
|
||||||
const { client } = await this.connect();
|
const { client } = await this.connect();
|
||||||
const collection = await client.getOrCreateCollection({
|
const collection = await client.getOrCreateCollection({
|
||||||
name: this.normalize(namespace),
|
name: namespace,
|
||||||
metadata: { "hnsw:space": "cosine" },
|
metadata: { "hnsw:space": "cosine" },
|
||||||
});
|
});
|
||||||
|
|
||||||
@ -322,7 +274,7 @@ const Chroma = {
|
|||||||
const { client } = await this.connect();
|
const { client } = await this.connect();
|
||||||
if (!(await this.namespaceExists(client, namespace))) return;
|
if (!(await this.namespaceExists(client, namespace))) return;
|
||||||
const collection = await client.getCollection({
|
const collection = await client.getCollection({
|
||||||
name: this.normalize(namespace),
|
name: namespace,
|
||||||
});
|
});
|
||||||
|
|
||||||
const knownDocuments = await DocumentVectors.where({ docId });
|
const knownDocuments = await DocumentVectors.where({ docId });
|
||||||
@ -347,7 +299,7 @@ const Chroma = {
|
|||||||
throw new Error("Invalid request to performSimilaritySearch.");
|
throw new Error("Invalid request to performSimilaritySearch.");
|
||||||
|
|
||||||
const { client } = await this.connect();
|
const { client } = await this.connect();
|
||||||
if (!(await this.namespaceExists(client, this.normalize(namespace)))) {
|
if (!(await this.namespaceExists(client, namespace))) {
|
||||||
return {
|
return {
|
||||||
contextTexts: [],
|
contextTexts: [],
|
||||||
sources: [],
|
sources: [],
|
||||||
@ -378,9 +330,9 @@ const Chroma = {
|
|||||||
const { namespace = null } = reqBody;
|
const { namespace = null } = reqBody;
|
||||||
if (!namespace) throw new Error("namespace required");
|
if (!namespace) throw new Error("namespace required");
|
||||||
const { client } = await this.connect();
|
const { client } = await this.connect();
|
||||||
if (!(await this.namespaceExists(client, this.normalize(namespace))))
|
if (!(await this.namespaceExists(client, namespace)))
|
||||||
throw new Error("Namespace by that name does not exist.");
|
throw new Error("Namespace by that name does not exist.");
|
||||||
const stats = await this.namespace(client, this.normalize(namespace));
|
const stats = await this.namespace(client, namespace);
|
||||||
return stats
|
return stats
|
||||||
? stats
|
? stats
|
||||||
: { message: "No stats were able to be fetched from DB for namespace" };
|
: { message: "No stats were able to be fetched from DB for namespace" };
|
||||||
@ -388,11 +340,11 @@ const Chroma = {
|
|||||||
"delete-namespace": async function (reqBody = {}) {
|
"delete-namespace": async function (reqBody = {}) {
|
||||||
const { namespace = null } = reqBody;
|
const { namespace = null } = reqBody;
|
||||||
const { client } = await this.connect();
|
const { client } = await this.connect();
|
||||||
if (!(await this.namespaceExists(client, this.normalize(namespace))))
|
if (!(await this.namespaceExists(client, namespace)))
|
||||||
throw new Error("Namespace by that name does not exist.");
|
throw new Error("Namespace by that name does not exist.");
|
||||||
|
|
||||||
const details = await this.namespace(client, this.normalize(namespace));
|
const details = await this.namespace(client, namespace);
|
||||||
await this.deleteVectorsInNamespace(client, this.normalize(namespace));
|
await this.deleteVectorsInNamespace(client, namespace);
|
||||||
return {
|
return {
|
||||||
message: `Namespace ${namespace} was deleted along with ${details?.vectorCount} vectors.`,
|
message: `Namespace ${namespace} was deleted along with ${details?.vectorCount} vectors.`,
|
||||||
};
|
};
|
||||||
|
Loading…
Reference in New Issue
Block a user