From a3f5a936e27eb74739de2d3a843a2f325ca98a1e Mon Sep 17 00:00:00 2001 From: Timothy Carambat Date: Mon, 26 Jun 2023 11:38:38 -0700 Subject: [PATCH] Enable debug mode of ENV at runtime (#111) * Enable debug mode of ENV at runtime Update Storage README for those with SQLite issues * add files --- aws/cloudformation/cf_template.template | 1 + docker/.env.example | 1 + frontend/src/components/Modals/Keys.jsx | 181 +++++++++++++++++++----- frontend/src/models/system.js | 12 ++ server/.env.example | 3 +- server/endpoints/system.js | 16 +++ server/storage/README.md | 24 ++++ server/utils/helpers/updateENV.js | 115 +++++++++++++++ 8 files changed, 320 insertions(+), 33 deletions(-) create mode 100644 server/storage/README.md create mode 100644 server/utils/helpers/updateENV.js diff --git a/aws/cloudformation/cf_template.template b/aws/cloudformation/cf_template.template index 405be8cd..b261a2d8 100644 --- a/aws/cloudformation/cf_template.template +++ b/aws/cloudformation/cf_template.template @@ -96,6 +96,7 @@ "!SUB::USER::CONTENT!", "UID=\"1000\"\n", "GID=\"1000\"\n", + "NO_DEBUG=\"true\"\n", "END\n", "cd ../frontend\n", "rm -rf .env.production\n", diff --git a/docker/.env.example b/docker/.env.example index b43c4c26..e44acd02 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -19,6 +19,7 @@ PINECONE_INDEX= # CLOUD DEPLOYMENT VARIRABLES ONLY # AUTH_TOKEN="hunter2" # This is the password to your application if remote hosting. # JWT_SECRET="my-random-string-for-seeding" # Only needed if AUTH_TOKEN is set. Please generate random string at least 12 chars long. +# NO_DEBUG="true" STORAGE_DIR="./server/storage" GOOGLE_APIS_KEY= UID='1000' diff --git a/frontend/src/components/Modals/Keys.jsx b/frontend/src/components/Modals/Keys.jsx index 7f472d86..272f5224 100644 --- a/frontend/src/components/Modals/Keys.jsx +++ b/frontend/src/components/Modals/Keys.jsx @@ -1,5 +1,5 @@ import React, { useState, useEffect } from "react"; -import { AlertCircle, X } from "react-feather"; +import { AlertCircle, Loader, X } from "react-feather"; import System from "../../models/system"; const noop = () => false; @@ -55,36 +55,48 @@ export default function KeysModal({ hideModal = noop }) {
{settings?.VectorDB === "pinecone" && ( <> )} @@ -92,8 +104,10 @@ export default function KeysModal({ hideModal = noop }) { <> )} @@ -115,47 +129,150 @@ export default function KeysModal({ hideModal = noop }) { ); } -function ShowKey({ name, value, valid }) { - if (!valid) { +function ShowKey({ name, env, value, valid, allowDebug = true }) { + const [isValid, setIsValid] = useState(valid); + const [debug, setDebug] = useState(false); + const [saving, setSaving] = useState(false); + const handleSubmit = async (e) => { + e.preventDefault(); + setSaving(true); + const data = {}; + const form = new FormData(e.target); + for (var [key, value] of form.entries()) data[key] = value; + const { newValues, error } = await System.updateSystem(data); + if (!!error) { + alert(error); + setSaving(false); + setIsValid(false); + return; + } + + setSaving(false); + setDebug(false); + setIsValid(true); + }; + + if (!isValid) { return ( -
+
+
+ + +
+

+ Need setup in .env file. +

+ {allowDebug && ( + <> + {debug ? ( +
+ {saving ? ( + <> + + + ) : ( + <> + + + + )} +
+ ) : ( + + )} + + )} +
+
+
+ ); + } + + return ( +
+
-

- Need setup in .env file. -

+ {allowDebug && ( +
+ {debug ? ( +
+ {saving ? ( + <> + + + ) : ( + <> + + + + )} +
+ ) : ( + + )} +
+ )}
- ); - } - - return ( -
- - -
+
); } diff --git a/frontend/src/models/system.js b/frontend/src/models/system.js index 41d6e355..1ce003d5 100644 --- a/frontend/src/models/system.js +++ b/frontend/src/models/system.js @@ -74,6 +74,18 @@ const System = { .then((res) => res?.types) .catch(() => null); }, + updateSystem: async (data) => { + return await fetch(`${API_BASE}/system/update-env`, { + method: "POST", + headers: baseHeaders(), + body: JSON.stringify(data), + }) + .then((res) => res.json()) + .catch((e) => { + console.error(e); + return { newValues: null, error: e.message }; + }); + }, }; export default System; diff --git a/server/.env.example b/server/.env.example index 3a7bf9c0..a74cec57 100644 --- a/server/.env.example +++ b/server/.env.example @@ -19,4 +19,5 @@ PINECONE_INDEX= # CLOUD DEPLOYMENT VARIRABLES ONLY # AUTH_TOKEN="hunter2" # This is the password to your application if remote hosting. # JWT_SECRET="my-random-string-for-seeding" # Only needed if AUTH_TOKEN is set. Please generate random string at least 12 chars long. -# STORAGE_DIR= # absolute filesystem path with no trailing slash \ No newline at end of file +# STORAGE_DIR= # absolute filesystem path with no trailing slash +# NO_DEBUG="true" \ No newline at end of file diff --git a/server/endpoints/system.js b/server/endpoints/system.js index ab472037..ba16b2e5 100644 --- a/server/endpoints/system.js +++ b/server/endpoints/system.js @@ -8,6 +8,7 @@ const { acceptedFileTypes, } = require("../utils/files/documentProcessor"); const { getVectorDbClass } = require("../utils/helpers"); +const { updateENV } = require("../utils/helpers/updateENV"); const { reqBody, makeJWT } = require("../utils/http"); function systemEndpoints(app) { @@ -26,10 +27,14 @@ function systemEndpoints(app) { try { const vectorDB = process.env.VECTOR_DB || "pinecone"; const results = { + CanDebug: !!!process.env.NO_DEBUG, RequiresAuth: !!process.env.AUTH_TOKEN, VectorDB: vectorDB, OpenAiKey: !!process.env.OPEN_AI_KEY, OpenAiModelPref: process.env.OPEN_MODEL_PREF || "gpt-3.5-turbo", + AuthToken: !!process.env.AUTH_TOKEN, + JWTSecret: !!process.env.JWT_SECRET, + StorageDir: process.env.STORAGE_DIR, ...(vectorDB === "pinecone" ? { PineConeEnvironment: process.env.PINECONE_ENVIRONMENT, @@ -123,6 +128,17 @@ function systemEndpoints(app) { response.sendStatus(500).end(); } }); + + app.post("/system/update-env", async (request, response) => { + try { + const body = reqBody(request); + const { newValues, error } = updateENV(body); + response.status(200).json({ newValues, error }); + } catch (e) { + console.log(e.message, e); + response.sendStatus(500).end(); + } + }); } module.exports = { systemEndpoints }; diff --git a/server/storage/README.md b/server/storage/README.md new file mode 100644 index 00000000..1282f4b3 --- /dev/null +++ b/server/storage/README.md @@ -0,0 +1,24 @@ +# AnythingLLM Storage + +This folder is for the local or disk storage of ready-to-embed documents, vector-cached embeddings, and the disk-storage of LanceDB and the local SQLite database. + +This folder should contain the following folders. +`documents` +`lancedb` (if using lancedb) +`vector-cache` +and a file named exactly `anythingllm.db` + + +### Common issues +**SQLITE_FILE_CANNOT_BE_OPENED** in the server log = The DB file does not exist probably because the node instance does not have the correct permissions to write a file to the disk. To solve this.. + +- Local dev + - Create a `anythingllm.db` empty file in this directory. Thats all. No need to reboot the server or anything. If your permissions are correct this should not ever occur since the server will create the file if it does not exist automatically. + +- Docker Instance + - Get your AnythingLLM docker container id with `docker ps -a`. The container must be running to execute the next commands. + - Run `docker container exec -u 0 -t mkdir -p /app/server/storage /app/server/storage/documents /app/server/storage/vector-cache /app/server/storage/lancedb` + - Run `docker container exec -u 0 -t touch /app/server/storage/anythingllm.db` + - Run `docker container exec -u 0 -t chown -R anythingllm:anythingllm /app/collector /app/server` + + - The above commands will create the appropriate folders inside of the docker container and will persist as long as you do not destroy the container and volume. This will also fix any ownership issues of folder files in the collector and the server. \ No newline at end of file diff --git a/server/utils/helpers/updateENV.js b/server/utils/helpers/updateENV.js new file mode 100644 index 00000000..4161aec1 --- /dev/null +++ b/server/utils/helpers/updateENV.js @@ -0,0 +1,115 @@ +const KEY_MAPPING = { + OpenAiKey: { + envKey: "OPEN_AI_KEY", + checks: [isNotEmpty, validOpenAIKey], + }, + OpenAiModelPref: { + envKey: "OPEN_MODEL_PREF", + checks: [isNotEmpty, validOpenAIModel], + }, + VectorDB: { + envKey: "VECTOR_DB", + checks: [isNotEmpty, supportedVectorDB], + }, + ChromaEndpoint: { + envKey: "CHROMA_ENDPOINT", + checks: [isValidURL, validChromaURL], + }, + PineConeEnvironment: { + envKey: "PINECONE_ENVIRONMENT", + checks: [], + }, + PineConeKey: { + envKey: "PINECONE_API_KEY", + checks: [], + }, + PineConeIndex: { + envKey: "PINECONE_INDEX", + checks: [], + }, + // Not supported yet. + // 'AuthToken': 'AUTH_TOKEN', + // 'JWTSecret': 'JWT_SECRET', + // 'StorageDir': 'STORAGE_DIR', +}; + +function isNotEmpty(input = "") { + return !input || input.length === 0 ? "Value cannot be empty" : null; +} + +function isValidURL(input = "") { + try { + new URL(input); + return null; + } catch (e) { + return "URL is not a valid URL."; + } +} + +function validOpenAIKey(input = "") { + return input.startsWith("sk-") ? null : "OpenAI Key must start with sk-"; +} + +function validOpenAIModel(input = "") { + const validModels = [ + "gpt-4", + "gpt-4-0613", + "gpt-4-32k", + "gpt-4-32k-0613", + "gpt-3.5-turbo", + "gpt-3.5-turbo-0613", + "gpt-3.5-turbo-16k", + "gpt-3.5-turbo-16k-0613", + ]; + return validModels.includes(input) + ? null + : `Invalid Model type. Must be one of ${validModels.join(", ")}.`; +} + +function supportedVectorDB(input = "") { + const supported = ["chroma", "pinecone", "lancedb"]; + return supported.includes(input) + ? null + : `Invalid VectorDB type. Must be one of ${supported.join(", ")}.`; +} + +function validChromaURL(input = "") { + return input.slice(-1) === "/" + ? `Chroma Instance URL should not end in a trailing slash.` + : null; +} + +// This will force update .env variables which for any which reason were not able to be parsed or +// read from an ENV file as this seems to be a complicating step for many so allowing people to write +// to the process will at least alleviate that issue. It does not perform comprehensive validity checks or sanity checks +// and is simply for debugging when the .env not found issue many come across. +function updateENV(newENVs = {}) { + let error = ""; + const validKeys = Object.keys(KEY_MAPPING); + const ENV_KEYS = Object.keys(newENVs).filter((key) => + validKeys.includes(key) + ); + const newValues = {}; + + ENV_KEYS.forEach((key) => { + const { envKey, checks } = KEY_MAPPING[key]; + const value = newENVs[key]; + const errors = checks + .map((validityCheck) => validityCheck(value)) + .filter((err) => typeof err === "string"); + + if (errors.length > 0) { + error += errors.join("\n"); + return; + } + + newValues[key] = value; + process.env[envKey] = value; + }); + + return { newValues, error: error?.length > 0 ? error : false }; +} + +module.exports = { + updateENV, +};