Enable debug mode of ENV at runtime (#111)

* Enable debug mode of ENV at runtime
Update Storage README for those with SQLite issues

* add files

aws/cloudformation/cf_template.template

@@ -96,6 +96,7 @@
                 "!SUB::USER::CONTENT!",
                 "UID=\"1000\"\n",
                 "GID=\"1000\"\n",
+                "NO_DEBUG=\"true\"\n",
                 "END\n",
                 "cd ../frontend\n",
                 "rm -rf .env.production\n",

docker/.env.example

@@ -19,6 +19,7 @@ PINECONE_INDEX=
 # CLOUD DEPLOYMENT VARIRABLES ONLY
 # AUTH_TOKEN="hunter2" # This is the password to your application if remote hosting.
 # JWT_SECRET="my-random-string-for-seeding" # Only needed if AUTH_TOKEN is set. Please generate random string at least 12 chars long.
+# NO_DEBUG="true"
 STORAGE_DIR="./server/storage"
 GOOGLE_APIS_KEY=
 UID='1000'

frontend/src/components/Modals/Keys.jsx

@@ -1,5 +1,5 @@
 import React, { useState, useEffect } from "react";
-import { AlertCircle, X } from "react-feather";
+import { AlertCircle, Loader, X } from "react-feather";
 import System from "../../models/system";
 
 const noop = () => false;
@@ -55,36 +55,48 @@ export default function KeysModal({ hideModal = noop }) {
                 </div>
                 <ShowKey
                   name="OpenAI API Key"
+                  env="OpenAiKey"
                   value={settings?.OpenAiKey ? "*".repeat(20) : ""}
                   valid={settings?.OpenAiKey}
+                  allowDebug={settings?.CanDebug}
                 />
                 <ShowKey
                   name="OpenAI Model for chats"
+                  env="OpenAiModelPref"
                   value={settings?.OpenAiModelPref}
                   valid={!!settings?.OpenAiModelPref}
+                  allowDebug={settings?.CanDebug}
                 />
                 <div className="h-[2px] w-full bg-gray-200 dark:bg-stone-600" />
                 <ShowKey
                   name="Vector DB Choice"
+                  env="VectorDB"
                   value={settings?.VectorDB}
                   valid={!!settings?.VectorDB}
+                  allowDebug={settings?.CanDebug}
                 />
                 {settings?.VectorDB === "pinecone" && (
                   <>
                     <ShowKey
                       name="Pinecone DB API Key"
+                      env="PineConeKey"
                       value={settings?.PineConeKey ? "*".repeat(20) : ""}
                       valid={!!settings?.PineConeKey}
+                      allowDebug={settings?.CanDebug}
                     />
                     <ShowKey
                       name="Pinecone DB Environment"
+                      env="PineConeEnvironment"
                       value={settings?.PineConeEnvironment}
                       valid={!!settings?.PineConeEnvironment}
+                      allowDebug={settings?.CanDebug}
                     />
                     <ShowKey
                       name="Pinecone DB Index"
+                      env="PineConeIndex"
                       value={settings?.PineConeIndex}
                       valid={!!settings?.PineConeIndex}
+                      allowDebug={settings?.CanDebug}
                     />
                   </>
                 )}
@@ -92,8 +104,10 @@ export default function KeysModal({ hideModal = noop }) {
                   <>
                     <ShowKey
                       name="Chroma Endpoint"
+                      env="ChromaEndpoint"
                       value={settings?.ChromaEndpoint}
                       valid={!!settings?.ChromaEndpoint}
+                      allowDebug={settings?.CanDebug}
                     />
                   </>
                 )}
@@ -115,9 +129,32 @@ export default function KeysModal({ hideModal = noop }) {
   );
 }
 
-function ShowKey({ name, value, valid }) {
+function ShowKey({ name, env, value, valid, allowDebug = true }) {
-  if (!valid) {
+  const [isValid, setIsValid] = useState(valid);
+  const [debug, setDebug] = useState(false);
+  const [saving, setSaving] = useState(false);
+  const handleSubmit = async (e) => {
+    e.preventDefault();
+    setSaving(true);
+    const data = {};
+    const form = new FormData(e.target);
+    for (var [key, value] of form.entries()) data[key] = value;
+    const { newValues, error } = await System.updateSystem(data);
+    if (!!error) {
+      alert(error);
+      setSaving(false);
+      setIsValid(false);
+      return;
+    }
+
+    setSaving(false);
+    setDebug(false);
+    setIsValid(true);
+  };
+
+  if (!isValid) {
     return (
+      <form onSubmit={handleSubmit}>
         <div>
           <label
             htmlFor="error"
@@ -128,19 +165,63 @@ function ShowKey({ name, value, valid }) {
           <input
             type="text"
             id="error"
-          disabled={true}
+            name={env}
+            disabled={!debug}
             className="bg-red-50 border border-red-500 text-red-900 placeholder-red-700 text-sm rounded-lg focus:ring-red-500 dark:bg-gray-700 focus:border-red-500 block w-full p-2.5 dark:text-red-500 dark:placeholder-red-500 dark:border-red-500"
             placeholder={name}
             defaultValue={value}
+            required={true}
+            autoComplete="off"
           />
+          <div className="flex items-center justify-between">
             <p className="mt-2 text-sm text-red-600 dark:text-red-500">
               Need setup in .env file.
             </p>
+            {allowDebug && (
+              <>
+                {debug ? (
+                  <div className="flex items-center gap-x-2 mt-2">
+                    {saving ? (
+                      <>
+                        <Loader className="animate-spin h-4 w-4 text-slate-300 dark:text-slate-500" />
+                      </>
+                    ) : (
+                      <>
+                        <button
+                          type="button"
+                          onClick={() => setDebug(false)}
+                          className="text-xs text-slate-300 dark:text-slate-500"
+                        >
+                          Cancel
+                        </button>
+                        <button
+                          type="submit"
+                          className="text-xs text-blue-300 dark:text-blue-500"
+                        >
+                          Save
+                        </button>
+                      </>
+                    )}
                   </div>
+                ) : (
+                  <button
+                    type="button"
+                    onClick={() => setDebug(true)}
+                    className="mt-2 text-xs text-slate-300 dark:text-slate-500"
+                  >
+                    Debug
+                  </button>
+                )}
+              </>
+            )}
+          </div>
+        </div>
+      </form>
     );
   }
 
   return (
+    <form onSubmit={handleSubmit}>
       <div className="mb-6">
         <label
           htmlFor="success"
@@ -151,11 +232,47 @@ function ShowKey({ name, value, valid }) {
         <input
           type="text"
           id="success"
-        disabled={true}
+          name={env}
+          disabled={!debug}
           className="border border-white text-green-900 dark:text-green-400 placeholder-green-700 dark:placeholder-green-500 text-sm rounded-lg focus:ring-green-500 focus:border-green-500 block w-full p-2.5 dark:bg-gray-700 dark:border-green-500"
           defaultValue={value}
+          required={true}
+          autoComplete="off"
         />
+        {allowDebug && (
+          <div className="flex items-center justify-end">
+            {debug ? (
+              <div className="flex items-center gap-x-2 mt-2">
+                {saving ? (
+                  <>
+                    <Loader className="animate-spin h-4 w-4 text-slate-300 dark:text-slate-500" />
+                  </>
+                ) : (
+                  <>
+                    <button
+                      onClick={() => setDebug(false)}
+                      className="text-xs text-slate-300 dark:text-slate-500"
+                    >
+                      Cancel
+                    </button>
+                    <button className="text-xs text-blue-300 dark:text-blue-500">
+                      Save
+                    </button>
+                  </>
+                )}
               </div>
+            ) : (
+              <button
+                onClick={() => setDebug(true)}
+                className="mt-2 text-xs text-slate-300 dark:text-slate-500"
+              >
+                Debug
+              </button>
+            )}
+          </div>
+        )}
+      </div>
+    </form>
   );
 }
 

frontend/src/models/system.js

@@ -74,6 +74,18 @@ const System = {
       .then((res) => res?.types)
       .catch(() => null);
   },
+  updateSystem: async (data) => {
+    return await fetch(`${API_BASE}/system/update-env`, {
+      method: "POST",
+      headers: baseHeaders(),
+      body: JSON.stringify(data),
+    })
+      .then((res) => res.json())
+      .catch((e) => {
+        console.error(e);
+        return { newValues: null, error: e.message };
+      });
+  },
 };
 
 export default System;

server/.env.example

@@ -20,3 +20,4 @@ PINECONE_INDEX=
 # AUTH_TOKEN="hunter2" # This is the password to your application if remote hosting.
 # JWT_SECRET="my-random-string-for-seeding" # Only needed if AUTH_TOKEN is set. Please generate random string at least 12 chars long.
 # STORAGE_DIR= # absolute filesystem path with no trailing slash
+# NO_DEBUG="true"

server/endpoints/system.js

@@ -8,6 +8,7 @@ const {
   acceptedFileTypes,
 } = require("../utils/files/documentProcessor");
 const { getVectorDbClass } = require("../utils/helpers");
+const { updateENV } = require("../utils/helpers/updateENV");
 const { reqBody, makeJWT } = require("../utils/http");
 
 function systemEndpoints(app) {
@@ -26,10 +27,14 @@ function systemEndpoints(app) {
     try {
       const vectorDB = process.env.VECTOR_DB || "pinecone";
       const results = {
+        CanDebug: !!!process.env.NO_DEBUG,
         RequiresAuth: !!process.env.AUTH_TOKEN,
         VectorDB: vectorDB,
         OpenAiKey: !!process.env.OPEN_AI_KEY,
         OpenAiModelPref: process.env.OPEN_MODEL_PREF || "gpt-3.5-turbo",
+        AuthToken: !!process.env.AUTH_TOKEN,
+        JWTSecret: !!process.env.JWT_SECRET,
+        StorageDir: process.env.STORAGE_DIR,
         ...(vectorDB === "pinecone"
           ? {
               PineConeEnvironment: process.env.PINECONE_ENVIRONMENT,
@@ -123,6 +128,17 @@ function systemEndpoints(app) {
       response.sendStatus(500).end();
     }
   });
+
+  app.post("/system/update-env", async (request, response) => {
+    try {
+      const body = reqBody(request);
+      const { newValues, error } = updateENV(body);
+      response.status(200).json({ newValues, error });
+    } catch (e) {
+      console.log(e.message, e);
+      response.sendStatus(500).end();
+    }
+  });
 }
 
 module.exports = { systemEndpoints };

server/storage/README.md

@@ -0,0 +1,24 @@
+# AnythingLLM Storage
+
+This folder is for the local or disk storage of ready-to-embed documents, vector-cached embeddings, and the disk-storage of LanceDB and the local SQLite database.
+
+This folder should contain the following folders.
+`documents`
+`lancedb` (if using lancedb)
+`vector-cache`
+and a file named exactly `anythingllm.db`
+
+
+### Common issues
+**SQLITE_FILE_CANNOT_BE_OPENED** in the server log = The DB file does not exist probably because the node instance does not have the correct permissions to write a file to the disk. To solve this..
+
+- Local dev
+  - Create a `anythingllm.db` empty file in this directory. Thats all. No need to reboot the server or anything. If your permissions are correct this should not ever occur since the server will create the file if it does not exist automatically.
+
+- Docker Instance
+  - Get your AnythingLLM docker container id with `docker ps -a`. The container must be running to execute the next commands.
+  - Run `docker container exec -u 0 -t <ANYTHINGLLM DOCKER CONTAINER ID> mkdir -p /app/server/storage /app/server/storage/documents /app/server/storage/vector-cache /app/server/storage/lancedb`
+  - Run `docker container exec -u 0 -t <ANYTHINGLLM DOCKER CONTAINER ID> touch /app/server/storage/anythingllm.db`
+  - Run `docker container exec -u 0 -t <ANYTHINGLLM DOCKER CONTAINER ID> chown -R anythingllm:anythingllm /app/collector /app/server`
+
+  - The above commands will create the appropriate folders inside of the docker container and will persist as long as you do not destroy the container and volume. This will also fix any ownership issues of folder files in the collector and the server.

server/utils/helpers/updateENV.js

@@ -0,0 +1,115 @@
+const KEY_MAPPING = {
+  OpenAiKey: {
+    envKey: "OPEN_AI_KEY",
+    checks: [isNotEmpty, validOpenAIKey],
+  },
+  OpenAiModelPref: {
+    envKey: "OPEN_MODEL_PREF",
+    checks: [isNotEmpty, validOpenAIModel],
+  },
+  VectorDB: {
+    envKey: "VECTOR_DB",
+    checks: [isNotEmpty, supportedVectorDB],
+  },
+  ChromaEndpoint: {
+    envKey: "CHROMA_ENDPOINT",
+    checks: [isValidURL, validChromaURL],
+  },
+  PineConeEnvironment: {
+    envKey: "PINECONE_ENVIRONMENT",
+    checks: [],
+  },
+  PineConeKey: {
+    envKey: "PINECONE_API_KEY",
+    checks: [],
+  },
+  PineConeIndex: {
+    envKey: "PINECONE_INDEX",
+    checks: [],
+  },
+  // Not supported yet.
+  // 'AuthToken': 'AUTH_TOKEN',
+  // 'JWTSecret': 'JWT_SECRET',
+  // 'StorageDir': 'STORAGE_DIR',
+};
+
+function isNotEmpty(input = "") {
+  return !input || input.length === 0 ? "Value cannot be empty" : null;
+}
+
+function isValidURL(input = "") {
+  try {
+    new URL(input);
+    return null;
+  } catch (e) {
+    return "URL is not a valid URL.";
+  }
+}
+
+function validOpenAIKey(input = "") {
+  return input.startsWith("sk-") ? null : "OpenAI Key must start with sk-";
+}
+
+function validOpenAIModel(input = "") {
+  const validModels = [
+    "gpt-4",
+    "gpt-4-0613",
+    "gpt-4-32k",
+    "gpt-4-32k-0613",
+    "gpt-3.5-turbo",
+    "gpt-3.5-turbo-0613",
+    "gpt-3.5-turbo-16k",
+    "gpt-3.5-turbo-16k-0613",
+  ];
+  return validModels.includes(input)
+    ? null
+    : `Invalid Model type. Must be one of ${validModels.join(", ")}.`;
+}
+
+function supportedVectorDB(input = "") {
+  const supported = ["chroma", "pinecone", "lancedb"];
+  return supported.includes(input)
+    ? null
+    : `Invalid VectorDB type. Must be one of ${supported.join(", ")}.`;
+}
+
+function validChromaURL(input = "") {
+  return input.slice(-1) === "/"
+    ? `Chroma Instance URL should not end in a trailing slash.`
+    : null;
+}
+
+// This will force update .env variables which for any which reason were not able to be parsed or
+// read from an ENV file as this seems to be a complicating step for many so allowing people to write
+// to the process will at least alleviate that issue. It does not perform comprehensive validity checks or sanity checks
+// and is simply for debugging when the .env not found issue many come across.
+function updateENV(newENVs = {}) {
+  let error = "";
+  const validKeys = Object.keys(KEY_MAPPING);
+  const ENV_KEYS = Object.keys(newENVs).filter((key) =>
+    validKeys.includes(key)
+  );
+  const newValues = {};
+
+  ENV_KEYS.forEach((key) => {
+    const { envKey, checks } = KEY_MAPPING[key];
+    const value = newENVs[key];
+    const errors = checks
+      .map((validityCheck) => validityCheck(value))
+      .filter((err) => typeof err === "string");
+
+    if (errors.length > 0) {
+      error += errors.join("\n");
+      return;
+    }
+
+    newValues[key] = value;
+    process.env[envKey] = value;
+  });
+
+  return { newValues, error: error?.length > 0 ? error : false };
+}
+
+module.exports = {
+  updateENV,
+};