From 92425782637175998ac9691fa7b55d5c998aae31 Mon Sep 17 00:00:00 2001 From: Timothy Carambat Date: Sun, 29 Oct 2023 11:03:41 -0700 Subject: [PATCH] Aws docker fixes (#309) * WIP aws fixes to latest changes; * change default in docker env * bump storage default for docker * Updates to docker deployment --- .../aws_build_from_source_no_credentials.json | 12 +- .../aws/cloudformation/cf_template.template | 255 ------------------ .../aws/cloudformation/generate.mjs | 70 ----- docker/.env.example | 4 +- docker/HOW_TO_USE_DOCKER.md | 11 +- docker/docker-entrypoint.sh | 6 +- 6 files changed, 15 insertions(+), 343 deletions(-) delete mode 100644 cloud-deployments/aws/cloudformation/cf_template.template delete mode 100644 cloud-deployments/aws/cloudformation/generate.mjs diff --git a/cloud-deployments/aws/cloudformation/aws_build_from_source_no_credentials.json b/cloud-deployments/aws/cloudformation/aws_build_from_source_no_credentials.json index 86971696..c96da542 100644 --- a/cloud-deployments/aws/cloudformation/aws_build_from_source_no_credentials.json +++ b/cloud-deployments/aws/cloudformation/aws_build_from_source_no_credentials.json @@ -91,17 +91,15 @@ "sudo systemctl start docker\n", "sudo yum install git -y\n", "git clone https://github.com/Mintplex-Labs/anything-llm.git /home/ec2-user/anything-llm\n", + "sudo touch /home/ec2-user/anything-llm/server/storage/anythingllm.db\n", "cd /home/ec2-user/anything-llm/docker\n", "cat >> .env << \"END\"\n", "SERVER_PORT=3001\n", "OPEN_AI_KEY=\n", "OPEN_MODEL_PREF='gpt-3.5-turbo'\n", "CACHE_VECTORS=\"true\"\n", - "VECTOR_DB=\"pinecone\"\n", - "PINECONE_ENVIRONMENT=\n", - "PINECONE_API_KEY=\n", - "PINECONE_INDEX=\n", - "STORAGE_DIR=\"./server/storage\"\n", + "VECTOR_DB=\"lancedb\"\n", + "STORAGE_DIR=\"/app/server/storage\"\n", "GOOGLE_APIS_KEY=\n", "UID=\"1000\"\n", "GID=\"1000\"\n", @@ -116,8 +114,6 @@ "echo \"Container ID: $(sudo docker ps --latest --quiet)\"\n", "sudo docker container exec -u 0 -t $(sudo docker ps --latest --quiet) mkdir -p /app/server/storage /app/server/storage/documents /app/server/storage/vector-cache /app/server/storage/lancedb\n", "echo \"Placeholder folders in storage created.\"\n", - "sudo docker container exec -u 0 -t $(sudo docker ps --latest --quiet) touch /app/server/storage/anythingllm.db\n", - "echo \"SQLite DB placeholder set.\"\n", "sudo docker container exec -u 0 -t $(sudo docker ps --latest --quiet) chown -R anythingllm:anythingllm /app/collector /app/server\n", "echo \"File permissions corrected.\"\n", "export ONLINE=$(curl -Is http://localhost:3001/api/ping | head -n 1|cut -d$' ' -f2)\n", @@ -260,4 +256,4 @@ } } } -} +} \ No newline at end of file diff --git a/cloud-deployments/aws/cloudformation/cf_template.template b/cloud-deployments/aws/cloudformation/cf_template.template deleted file mode 100644 index 7aeb28b1..00000000 --- a/cloud-deployments/aws/cloudformation/cf_template.template +++ /dev/null @@ -1,255 +0,0 @@ -{ - "AWSTemplateFormatVersion": "2010-09-09", - "Description": "Create a stack that runs AnythingLLM on a single instance", - "Parameters": { - "InstanceType": { - "Description": "EC2 instance type", - "Type": "String", - "Default": "t2.small" - }, - "InstanceVolume": { - "Description": "Storage size of disk on Instance in GB", - "Type": "Number", - "Default": 10, - "MinValue": 4 - } - }, - "Resources": { - "AnythingLLMInstance": { - "Type": "AWS::EC2::Instance", - "Properties": { - "ImageId": { - "Fn::FindInMap": [ - "Region2AMI", - { - "Ref": "AWS::Region" - }, - "AMI" - ] - }, - "InstanceType": { - "Ref": "InstanceType" - }, - "SecurityGroupIds": [ - { - "Ref": "AnythingLLMInstanceSecurityGroup" - } - ], - "BlockDeviceMappings": [ - { - "DeviceName": { - "Fn::FindInMap": [ - "Region2AMI", - { - "Ref": "AWS::Region" - }, - "RootDeviceName" - ] - }, - "Ebs": { - "VolumeSize": { - "Ref": "InstanceVolume" - } - } - } - ], - "UserData": { - "Fn::Base64": { - "Fn::Join": [ - "", - [ - "Content-Type: multipart/mixed; boundary=\"//\"\n", - "MIME-Version: 1.0\n", - "\n", - "--//\n", - "Content-Type: text/cloud-config; charset=\"us-ascii\"\n", - "MIME-Version: 1.0\n", - "Content-Transfer-Encoding: 7bit\n", - "Content-Disposition: attachment; filename=\"cloud-config.txt\"\n", - "\n", - "\n", - "#cloud-config\n", - "cloud_final_modules:\n", - "- [scripts-user, once-per-instance]\n", - "\n", - "\n", - "--//\n", - "Content-Type: text/x-shellscript; charset=\"us-ascii\"\n", - "MIME-Version: 1.0\n", - "Content-Transfer-Encoding: 7bit\n", - "Content-Disposition: attachment; filename=\"userdata.txt\"\n", - "\n", - "\n", - "#!/bin/bash\n", - "# check output of userdata script with sudo tail -f /var/log/cloud-init-output.log\n", - "sudo yum install docker -y\n", - "sudo usermod -a -G docker ec2-user\n", - "curl -L https://github.com/docker/compose/releases/latest/download/docker-compose-$(uname -s)-$(uname -m) -o /usr/local/bin/docker-compose\n", - "sudo chmod +x /usr/local/bin/docker-compose\n", - "sudo ln -s /usr/local/bin/docker-compose /usr/bin/docker-compose\n", - "sudo systemctl enable docker\n", - "sudo systemctl start docker\n", - "sudo yum install git -y\n", - "git clone https://github.com/Mintplex-Labs/anything-llm.git /home/ec2-user/anything-llm\n", - "cd /home/ec2-user/anything-llm/docker\n", - "cat >> .env << \"END\"\n", - "!SUB::USER::CONTENT!", - "UID=\"1000\"\n", - "GID=\"1000\"\n", - "NO_DEBUG=\"true\"\n", - "END\n", - "cd ../frontend\n", - "rm -rf .env.production\n", - "cat >> .env.production << \"END\"\n", - "GENERATE_SOURCEMAP=true\n", - "VITE_API_BASE=\"/api\"\n", - "END\n", - "sudo docker-compose -f /home/ec2-user/anything-llm/docker/docker-compose.yml up -d\n", - "echo \"Container ID: $(sudo docker ps --latest --quiet)\"\n", - "sudo docker container exec -u 0 -t $(sudo docker ps --latest --quiet) mkdir -p /app/server/storage /app/server/storage/documents /app/server/storage/vector-cache /app/server/storage/lancedb\n", - "echo \"Placeholder folders in storage created.\"\n", - "sudo docker container exec -u 0 -t $(sudo docker ps --latest --quiet) touch /app/server/storage/anythingllm.db\n", - "echo \"SQLite DB placeholder set.\"\n", - "sudo docker container exec -u 0 -t $(sudo docker ps --latest --quiet) chown -R anythingllm:anythingllm /app/collector /app/server\n", - "echo \"File permissions corrected.\"\n", - "export ONLINE=$(curl -Is http://localhost:3001/api/ping | head -n 1|cut -d$' ' -f2)\n", - "echo \"Health check: $ONLINE\"\n", - "if [ \"$ONLINE\" = 200 ] ; then echo \"Running migrations...\" && curl -Is http://localhost:3001/api/migrate | head -n 1|cut -d$' ' -f2; fi\n", - "echo \"Setup complete! AnythingLLM instance is now online!\"\n", - "\n", - "--//--\n" - ] - ] - } - } - } - }, - "AnythingLLMInstanceSecurityGroup": { - "Type": "AWS::EC2::SecurityGroup", - "Properties": { - "GroupDescription": "AnythingLLm Instance Security Group", - "SecurityGroupIngress": [ - { - "IpProtocol": "tcp", - "FromPort": "22", - "ToPort": "22", - "CidrIp": "0.0.0.0/0" - }, - { - "IpProtocol": "tcp", - "FromPort": "3001", - "ToPort": "3001", - "CidrIp": "0.0.0.0/0" - }, - { - "IpProtocol": "tcp", - "FromPort": "3001", - "ToPort": "3001", - "CidrIpv6": "::/0" - } - ] - } - } - }, - "Outputs": { - "ServerIp": { - "Description": "IP address of the AnythingLLM instance", - "Value": { - "Fn::GetAtt": [ - "AnythingLLMInstance", - "PublicIp" - ] - } - }, - "ServerURL": { - "Description": "URL of the AnythingLLM server", - "Value": { - "Fn::Join": [ - "", - [ - "http://", - { - "Fn::GetAtt": [ - "AnythingLLMInstance", - "PublicIp" - ] - }, - ":3001" - ] - ] - } - } - }, - "Mappings": { - "Region2AMI": { - "ap-south-1": { - "AMI": "ami-0e6329e222e662a52", - "RootDeviceName": "/dev/xvda" - }, - "eu-north-1": { - "AMI": "ami-08c308b1bb265e927", - "RootDeviceName": "/dev/xvda" - }, - "eu-west-3": { - "AMI": "ami-069d1ea6bc64443f0", - "RootDeviceName": "/dev/xvda" - }, - "eu-west-2": { - "AMI": "ami-06a566ca43e14780d", - "RootDeviceName": "/dev/xvda" - }, - "eu-west-1": { - "AMI": "ami-0a8dc52684ee2fee2", - "RootDeviceName": "/dev/xvda" - }, - "ap-northeast-3": { - "AMI": "ami-0c8a89b455fae8513", - "RootDeviceName": "/dev/xvda" - }, - "ap-northeast-2": { - "AMI": "ami-0ff56409a6e8ea2a0", - "RootDeviceName": "/dev/xvda" - }, - "ap-northeast-1": { - "AMI": "ami-0ab0bbbd329f565e6", - "RootDeviceName": "/dev/xvda" - }, - "ca-central-1": { - "AMI": "ami-033c256a10931f206", - "RootDeviceName": "/dev/xvda" - }, - "sa-east-1": { - "AMI": "ami-0dabf4dab6b183eef", - "RootDeviceName": "/dev/xvda" - }, - "ap-southeast-1": { - "AMI": "ami-0dc5785603ad4ff54", - "RootDeviceName": "/dev/xvda" - }, - "ap-southeast-2": { - "AMI": "ami-0c5d61202c3b9c33e", - "RootDeviceName": "/dev/xvda" - }, - "eu-central-1": { - "AMI": "ami-004359656ecac6a95", - "RootDeviceName": "/dev/xvda" - }, - "us-east-1": { - "AMI": "ami-0cff7528ff583bf9a", - "RootDeviceName": "/dev/xvda" - }, - "us-east-2": { - "AMI": "ami-02238ac43d6385ab3", - "RootDeviceName": "/dev/xvda" - }, - "us-west-1": { - "AMI": "ami-01163e76c844a2129", - "RootDeviceName": "/dev/xvda" - }, - "us-west-2": { - "AMI": "ami-0ceecbb0f30a902a6", - "RootDeviceName": "/dev/xvda" - } - } - } -} diff --git a/cloud-deployments/aws/cloudformation/generate.mjs b/cloud-deployments/aws/cloudformation/generate.mjs deleted file mode 100644 index 99ed03ca..00000000 --- a/cloud-deployments/aws/cloudformation/generate.mjs +++ /dev/null @@ -1,70 +0,0 @@ -// Note (tcarambat) This script should be executed from root via the `yarn generate::cloudformation` command only. -// This script will copy your current Docker .env settings being used into a slightly custom AWS CloudFormation template -// that you can upload and deploy on AWS in a single click! -// Recommended settings are already defined in the template but you can modify them as needed. -// AnythingLLM can run within the free tier services of AWS (t2.micro w/10GB of storage) -// -// This will deploy a fully public AnythingLLM so if you do not want anyone to access it please set the AUTH_TOKEN & JWT_SECRET envs -// before running this script. You can still run the collector scripts on AWS so no FTP or file uploads are required. -// Your documents and data do not leave your AWS instance when you host in the cloud this way. - -import fs from 'fs'; -import { fileURLToPath } from 'url'; -import path, { dirname } from 'path'; -import { exit } from 'process'; -const __dirname = dirname(fileURLToPath(import.meta.url)); -const REPLACEMENT_KEY = '!SUB::USER::CONTENT!' - -const envPath = path.resolve(__dirname, `../../../docker/.env`) -const envFileExists = fs.existsSync(envPath); - -const chalk = { - redBright: function (text) { - return `\x1b[31m${text}\x1b[0m` - }, - cyan: function (text) { - return `\x1b[36m${text}\x1b[0m` - }, - greenBright: function (text) { - return `\x1b[32m${text}\x1b[0m` - }, - blueBright: function (text) { - return `\x1b[34m${text}\x1b[0m` - } -} - -if (!envFileExists) { - console.log(chalk.redBright('[ABORT]'), 'You do not have an .env file in your ./docker/ folder. You need to create it first.'); - console.log('You can start by running', chalk.cyan('cp -n ./docker/.env.example ./docker/.env')) - exit(1); -} - -// Remove comments -// Remove UID,GID,etc -// Remove empty strings -// Split into array -const settings = fs.readFileSync(envPath, "utf8") - .replace(/^#.*\n?/gm, '') - .replace(/^UID.*\n?/gm, '') - .replace(/^GID.*\n?/gm, '') - .replace(/^CLOUD_BUILD.*\n?/gm, '') - .replace(/^\s*\n/gm, "") - .split('\n') - .filter((i) => !!i) - .map((i) => i + '\n') - -const templatePath = path.resolve(__dirname, `cf_template.template`); -const templateString = fs.readFileSync(templatePath, "utf8"); -const template = JSON.parse(templateString); - -const cmdIdx = template.Resources.AnythingLLMInstance.Properties.UserData['Fn::Base64']['Fn::Join'][1].findIndex((cmd) => cmd === REPLACEMENT_KEY) -template.Resources.AnythingLLMInstance.Properties.UserData['Fn::Base64']['Fn::Join'][1].splice(cmdIdx, 1, ...settings); - -const output = path.resolve(__dirname, `aws_cf_deploy_anything_llm.json`); -fs.writeFileSync(output, JSON.stringify(template, null, 2), "utf8"); - -console.log(chalk.greenBright('[SUCCESS]'), 'Deploy AnythingLLM on AWS CloudFormation using your template document.'); -console.log(chalk.greenBright('File Created:'), 'aws_cf_deploy_anything_llm.json in aws/cloudformation directory.'); -console.log(chalk.blueBright('[INFO]'), 'Refer to aws/cloudformation/DEPLOY.md for how to use this file.'); - -exit(); \ No newline at end of file diff --git a/docker/.env.example b/docker/.env.example index 508b20f4..b0182cc8 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -26,7 +26,7 @@ OPEN_MODEL_PREF='gpt-3.5-turbo' # CHROMA_API_KEY="sk-123abc" # Enable all below if you are using vector database: Pinecone. -VECTOR_DB="pinecone" +VECTOR_DB="lancedb" PINECONE_ENVIRONMENT= PINECONE_API_KEY= PINECONE_INDEX= @@ -47,7 +47,7 @@ PINECONE_INDEX= # CLOUD DEPLOYMENT VARIRABLES ONLY # AUTH_TOKEN="hunter2" # This is the password to your application if remote hosting. # NO_DEBUG="true" -STORAGE_DIR="./server/storage" +STORAGE_DIR="/app/server/storage" GOOGLE_APIS_KEY= UID='1000' GID='1000' diff --git a/docker/HOW_TO_USE_DOCKER.md b/docker/HOW_TO_USE_DOCKER.md index 12c94e72..2cc5a41e 100644 --- a/docker/HOW_TO_USE_DOCKER.md +++ b/docker/HOW_TO_USE_DOCKER.md @@ -5,11 +5,11 @@ Use the Dockerized version of AnythingLLM for a much faster and complete startup ## Requirements - Install [Docker](https://www.docker.com/) on your computer or machine. -## How to install +## How to install & run locally - `git clone` this repo and `cd anything-llm` to get to the root directory. -- `yarn setup` +- `touch server/storage/anythingllm.db` to create empty SQLite DB file. - `cd docker/` -- Edit `.env` file and update the variables +- `cp .env.example .env` - `docker-compose up -d --build` to build the image - this will take a few moments. Your docker host will show the image as online once the build process is completed. This will build the app to `http://localhost:3001`. @@ -17,7 +17,7 @@ Your docker host will show the image as online once the build process is complet ## How to use the user interface - To access the full application, visit `http://localhost:3001` in your browser. -## How to add files to my system +## How to add files to my system using the standalone scripts - Upload files from the UI in your Workspace settings - To run the collector scripts to grab external data (articles, URLs, etc.) @@ -27,9 +27,6 @@ Your docker host will show the image as online once the build process is complet - `docker exec -it --workdir=/app/collector anything-llm python watch.py` - Upload [compliant files](../collector/hotdir/__HOTDIR__.md) to `./collector/hotdir` and they will be processed and made available in the UI. -## How to update and rebuild the ENV? -- Update the `./docker/.env` and run `docker-compose up -d --build` to rebuild with new environments. - ## About UID and GID in the ENV - The UID and GID are set to 1000 by default. This is the default user in the Docker container and on most host operating systems. If there is a mismatch between your host user UID and GID and what is set in the `.env` file, you may experience permission issues. diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index eb9285a9..4c264e88 100755 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -1,5 +1,9 @@ #!/bin/bash -node /app/server/index.js & +{ cd /app/server/ &&\ + npx prisma generate --schema=./prisma/schema.prisma &&\ + npx prisma migrate deploy --schema=./prisma/schema.prisma &&\ + node /app/server/index.js +} & { FLASK_ENV=production FLASK_APP=wsgi.py cd collector && gunicorn --timeout 300 --workers 4 --bind 0.0.0.0:8888 wsgi:api; } & wait -n exit $? \ No newline at end of file