kamp
/
anything-llm
mirror of https://github.com/Mintplex-Labs/anything-llm.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[FEAT] support pinecone serverless (#639) 

* migrate pinecone package to latest version and migrate pinecone vectordb provider class

* remove pinecone environment name env variable and update docs to reflect removal & serverless support complete

* migrate query for pinecone db

* typo in log

---------

Co-authored-by: timothycarambat <rambat1010@gmail.com>
This commit is contained in:
Sean Hatfield 2024-01-22 16:41:20 -08:00 committed by GitHub
parent ad5a35b662
commit 2f3db0e63a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
11 changed files with 64 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
docker/.env.example
View File

@ -76,7 +76,6 @@ GID='1000'
# Enable all below if you are using vector database: Pinecone.
# VECTOR_DB="pinecone"
# PINECONE_ENVIRONMENT=
# PINECONE_API_KEY=
# PINECONE_INDEX=

17
frontend/src/components/VectorDBSelection/PineconeDBOptions/index.jsx
View File

@ -17,23 +17,6 @@ export default function PineconeDBOptions({ settings }) {
spellCheck={false}
/>
</div>
<div className="flex flex-col w-60">
<label className="text-white text-sm font-semibold block mb-4">
Pinecone Index Environment
</label>
<input
type="text"
name="PineConeEnvironment"
className="bg-zinc-900 text-white placeholder-white placeholder-opacity-60 text-sm rounded-lg focus:border-white block w-full p-2.5"
placeholder="us-gcp-west-1"
defaultValue={settings?.PineConeEnvironment}
required={true}
autoComplete="off"
spellCheck={false}
/>
</div>
<div className="flex flex-col w-60">
<label className="text-white text-sm font-semibold block mb-4">
Pinecone Index Name

1
server/.env.example
View File

@ -73,7 +73,6 @@ JWT_SECRET="my-random-string-for-seeding" # Please generate random string at lea
# Enable all below if you are using vector database: Pinecone.
# VECTOR_DB="pinecone"
# PINECONE_ENVIRONMENT=
# PINECONE_API_KEY=
# PINECONE_INDEX=

1
server/endpoints/api/system/index.js
View File

@ -40,7 +40,6 @@ function apiSystemEndpoints(app) {
example: {
"settings": {
"VectorDB": "pinecone",
"PineConeEnvironment": "us-west4-gcp-free",
"PineConeKey": true,
"PineConeIndex": "my-pinecone-index",
"LLMProvider": "azure",

1
server/models/systemSettings.js
View File

@ -32,7 +32,6 @@ const SystemSettings = {
LocalAiApiKey: !!process.env.LOCAL_AI_API_KEY,
...(vectorDB === "pinecone"
? {
PineConeEnvironment: process.env.PINECONE_ENVIRONMENT,
PineConeKey: !!process.env.PINECONE_API_KEY,
PineConeIndex: process.env.PINECONE_INDEX,
}

4
server/package.json
View File

@ -24,7 +24,7 @@
"@azure/openai": "1.0.0-beta.10",
"@google/generative-ai": "^0.1.3",
"@googleapis/youtube": "^9.0.0",
"@pinecone-database/pinecone": "^0.1.6",
"@pinecone-database/pinecone": "^2.0.1",
"@prisma/client": "5.3.0",
"@qdrant/js-client-rest": "^1.4.0",
"@xenova/transformers": "^2.14.0",
@ -77,4 +77,4 @@
"nodemon": "^2.0.22",
"prettier": "^3.0.3"
}
}
}

1
server/swagger/openapi.json
View File

@ -1941,7 +1941,6 @@
"example": {
"settings": {
"VectorDB": "pinecone",
"PineConeEnvironment": "us-west4-gcp-free",
"PineConeKey": true,
"PineConeIndex": "my-pinecone-index",
"LLMProvider": "azure",

5
server/utils/helpers/updateENV.js
View File

@ -171,11 +171,6 @@ const KEY_MAPPING = {
envKey: "QDRANT_API_KEY",
checks: [],
},
PineConeEnvironment: {
envKey: "PINECONE_ENVIRONMENT",
checks: [],
},
PineConeKey: {
envKey: "PINECONE_API_KEY",
checks: [],

1
server/utils/vectorDbProviders/pinecone/PINECONE_SETUP.md
View File

@ -19,7 +19,6 @@
```
VECTOR_DB="pinecone"
PINECONE_ENVIRONMENT=us-west4-gcp-free
PINECONE_API_KEY=sklive-123xyz
PINECONE_INDEX=my-primary-index # the value from the first instruction!
```

72
server/utils/vectorDbProviders/pinecone/index.js
View File

@ -1,4 +1,4 @@
const { PineconeClient } = require("@pinecone-database/pinecone");
const { Pinecone } = require("@pinecone-database/pinecone");
const { RecursiveCharacterTextSplitter } = require("langchain/text_splitter");
const { storeVectorResult, cachedVectorInformation } = require("../../files");
const { v4: uuidv4 } = require("uuid");
@ -8,37 +8,35 @@ const {
getEmbeddingEngineSelection,
} = require("../../helpers");
const Pinecone = {
const PineconeDB = {
name: "Pinecone",
connect: async function () {
if (process.env.VECTOR_DB !== "pinecone")
throw new Error("Pinecone::Invalid ENV settings");
const client = new PineconeClient();
await client.init({
const client = new Pinecone({
apiKey: process.env.PINECONE_API_KEY,
environment: process.env.PINECONE_ENVIRONMENT,
});
const pineconeIndex = client.Index(process.env.PINECONE_INDEX);
const { status } = await client.describeIndex({
indexName: process.env.PINECONE_INDEX,
});
if (!status.ready) throw new Error("Pinecode::Index not ready.");
const pineconeIndex = client.Index(process.env.PINECONE_INDEX);
const { status } = await client.describeIndex(process.env.PINECONE_INDEX);
if (!status.ready) throw new Error("Pinecone::Index not ready.");
return { client, pineconeIndex, indexName: process.env.PINECONE_INDEX };
},
totalVectors: async function () {
const { pineconeIndex } = await this.connect();
const { namespaces } = await pineconeIndex.describeIndexStats1();
const { namespaces } = await pineconeIndex.describeIndexStats();
return Object.values(namespaces).reduce(
(a, b) => a + (b?.vectorCount || 0),
(a, b) => a + (b?.recordCount || 0),
0
);
},
namespaceCount: async function (_namespace = null) {
const { pineconeIndex } = await this.connect();
const namespace = await this.namespace(pineconeIndex, _namespace);
return namespace?.vectorCount || 0;
return namespace?.recordCount || 0;
},
similarityResponse: async function (
index,
@ -52,13 +50,12 @@ const Pinecone = {
sourceDocuments: [],
scores: [],
};
const response = await index.query({
queryRequest: {
namespace,
vector: queryVector,
topK: topN,
includeMetadata: true,
},
const pineconeNamespace = index.namespace(namespace);
const response = await pineconeNamespace.query({
vector: queryVector,
topK: topN,
includeMetadata: true,
});
response.matches.forEach((match) => {
@ -70,10 +67,9 @@ const Pinecone = {
return result;
},
namespace: async function (index, namespace = null) {
if (!namespace) throw new Error("No namespace value provided.");
const { namespaces } = await index.describeIndexStats1();
const { namespaces } = await index.describeIndexStats();
return namespaces.hasOwnProperty(namespace) ? namespaces[namespace] : null;
},
hasNamespace: async function (namespace = null) {
@ -83,11 +79,12 @@ const Pinecone = {
},
namespaceExists: async function (index, namespace = null) {
if (!namespace) throw new Error("No namespace value provided.");
const { namespaces } = await index.describeIndexStats1();
const { namespaces } = await index.describeIndexStats();
return namespaces.hasOwnProperty(namespace);
},
deleteVectorsInNamespace: async function (index, namespace = null) {
await index.delete1({ namespace, deleteAll: true });
const pineconeNamespace = index.namespace(namespace);
await pineconeNamespace.deleteAll();
return true;
},
addDocumentToNamespace: async function (
@ -104,6 +101,7 @@ const Pinecone = {
const cacheResult = await cachedVectorInformation(fullFilePath);
if (cacheResult.exists) {
const { pineconeIndex } = await this.connect();
const pineconeNamespace = pineconeIndex.namespace(namespace);
const { chunks } = cacheResult;
const documentVectors = [];
@ -115,14 +113,7 @@ const Pinecone = {
documentVectors.push({ docId, vectorId: id });
return { ...chunk, id };
});
// Push chunks with new ids to pinecone.
await pineconeIndex.upsert({
upsertRequest: {
vectors: [...newChunks],
namespace,
},
});
await pineconeNamespace.upsert([...newChunks]);
}
await DocumentVectors.bulkInsert(documentVectors);
@ -170,15 +161,11 @@ const Pinecone = {
if (vectors.length > 0) {
const chunks = [];
const { pineconeIndex } = await this.connect();
const pineconeNamespace = pineconeIndex.namespace(namespace);
console.log("Inserting vectorized chunks into Pinecone.");
for (const chunk of toChunks(vectors, 100)) {
chunks.push(chunk);
await pineconeIndex.upsert({
upsertRequest: {
vectors: [...chunk],
namespace,
},
});
await pineconeNamespace.upsert([...chunk]);
}
await storeVectorResult(chunks, fullFilePath);
}
@ -199,11 +186,10 @@ const Pinecone = {
if (knownDocuments.length === 0) return;
const vectorIds = knownDocuments.map((doc) => doc.vectorId);
const pineconeNamespace = pineconeIndex.namespace(namespace);
for (const batchOfVectorIds of toChunks(vectorIds, 1000)) {
await pineconeIndex.delete1({
ids: batchOfVectorIds,
namespace,
});
await pineconeNamespace.deleteMany(batchOfVectorIds);
}
const indexes = knownDocuments.map((doc) => doc.id);
@ -285,4 +271,4 @@ const Pinecone = {
},
};
module.exports.Pinecone = Pinecone;
module.exports.Pinecone = PineconeDB;

38
server/yarn.lock
View File

@ -625,12 +625,15 @@
"@octokit/webhooks-types" "7.1.0"
aggregate-error "^3.1.0"
"@pinecone-database/pinecone@^0.1.6":
version "0.1.6"
resolved "https://registry.yarnpkg.com/@pinecone-database/pinecone/-/pinecone-0.1.6.tgz#13374ae9462c8eea0dc26683cafeddc4e7c0375f"
integrity sha512-tCnVc28udecthhgSBTdcMhYEW+xsR++AdZasp+ZE/AvUD1hOR2IR3edjk9m0sDxZyvXbno2KeqUbLIOZr7sCTw==
"@pinecone-database/pinecone@^2.0.1":
version "2.0.1"
resolved "https://registry.yarnpkg.com/@pinecone-database/pinecone/-/pinecone-2.0.1.tgz#1dce6e06e299dfe2c26490aacffe1103e316b8fc"
integrity sha512-a1ejzrqdSQ2yW+9QUi2TVlKwYUbrvGH+QH6POJhITyaOz9ANE+EhXqToC9af93Ctzq9n87+bOUvBvewLeW++Mw==
dependencies:
"@sinclair/typebox" "^0.29.0"
ajv "^8.12.0"
cross-fetch "^3.1.5"
encoding "^0.1.13"
"@pkgr/core@^0.1.0":
version "0.1.0"
@ -743,6 +746,11 @@
resolved "https://registry.yarnpkg.com/@sideway/pinpoint/-/pinpoint-2.0.0.tgz#cff8ffadc372ad29fd3f78277aeb29e632cc70df"
integrity sha512-RNiOoTPkptFtSVzQevY/yWtZwf/RxyVnPy/OcA9HBM3MlGDnBEYL5B41H0MTn0Uec8Hi+2qUtTfG2WWZBmMejQ==
"@sinclair/typebox@^0.29.0":
version "0.29.6"
resolved "https://registry.yarnpkg.com/@sinclair/typebox/-/typebox-0.29.6.tgz#4cd8372f9247372edd5fc5af44f67e2032c46e2f"
integrity sha512-aX5IFYWlMa7tQ8xZr3b2gtVReCvg7f3LEhjir/JAjX2bJCMVJA5tIPv30wTD4KDfcwMd7DDYY3hFDeGmOgtrZQ==
"@tootallnate/once@1":
version "1.1.2"
resolved "https://registry.yarnpkg.com/@tootallnate/once/-/once-1.1.2.tgz#ccb91445360179a04e7fe6aff78c00ffc1eeaf82"
@ -955,6 +963,16 @@ ajv@^6.12.4:
json-schema-traverse "^0.4.1"
uri-js "^4.2.2"
ajv@^8.12.0:
version "8.12.0"
resolved "https://registry.yarnpkg.com/ajv/-/ajv-8.12.0.tgz#d1a0527323e22f53562c567c00991577dfbe19d1"
integrity sha512-sRu1kpcO9yLtYxBKvqfTeh9KzZEwO3STyX1HT+4CaDzC6HpTGYhIhPIzj9XuKU7KYDwnaeh5hcOwjy1QuJzBPA==
dependencies:
fast-deep-equal "^3.1.1"
json-schema-traverse "^1.0.0"
require-from-string "^2.0.2"
uri-js "^4.2.2"
ansi-regex@^5.0.1:
version "5.0.1"
resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-5.0.1.tgz#082cb2c89c9fe8659a311a53bd6a4dc5301db304"
@ -1934,7 +1952,7 @@ encodeurl@~1.0.2:
resolved "https://registry.yarnpkg.com/encodeurl/-/encodeurl-1.0.2.tgz#ad3ff4c86ec2d029322f5a02c3a9a606c95b3f59"
integrity sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==
encoding@^0.1.12:
encoding@^0.1.12, encoding@^0.1.13:
version "0.1.13"
resolved "https://registry.yarnpkg.com/encoding/-/encoding-0.1.13.tgz#56574afdd791f54a8e9b2785c0582a2d26210fa9"
integrity sha512-ETBauow1T35Y/WZMkio9jiM0Z5xjHHmJ4XmjZOq1l/dXz3lr2sRn87nJy20RupqSh1F2m3HHPSp8ShIPQJrJ3A==
@ -3321,6 +3339,11 @@ json-schema-traverse@^0.4.1:
resolved "https://registry.yarnpkg.com/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz#69f6a87d9513ab8bb8fe63bdb0979c448e684660"
integrity sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==
json-schema-traverse@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz#ae7bcb3656ab77a73ba5c49bf654f38e6b6860e2"
integrity sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==
json-stable-stringify-without-jsonify@^1.0.1:
version "1.0.1"
resolved "https://registry.yarnpkg.com/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz#9db7b59496ad3f3cfef30a75142d2d930ad72651"
@ -4674,6 +4697,11 @@ require-directory@^2.1.1:
resolved "https://registry.yarnpkg.com/require-directory/-/require-directory-2.1.1.tgz#8c64ad5fd30dab1c976e2344ffe7f792a6a6df42"
integrity sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==
require-from-string@^2.0.2:
version "2.0.2"
resolved "https://registry.yarnpkg.com/require-from-string/-/require-from-string-2.0.2.tgz#89a7fdd938261267318eafe14f9c32e598c36909"
integrity sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==
resolve-from@^4.0.0:
version "4.0.0"
resolved "https://registry.yarnpkg.com/resolve-from/-/resolve-from-4.0.0.tgz#4abcd852ad32dd7baabfe9b40e00a36db5f392e6"