From 7aabfbcbf001289c14af8e0b903a51c9ea3d9733 Mon Sep 17 00:00:00 2001 From: paul mayero Date: Wed, 29 May 2024 14:08:07 +0000 Subject: [PATCH] Adding rclone as an option to fdroid deploy --- examples/config.yml | 59 ++++++++++++++-- fdroidserver/deploy.py | 152 ++++++++++++++++++++++++++++++++++++++++- tests/deploy.TestCase | 51 ++++++++++++++ 3 files changed, 255 insertions(+), 7 deletions(-) diff --git a/examples/config.yml b/examples/config.yml index 0a0a54f0..646726bb 100644 --- a/examples/config.yml +++ b/examples/config.yml @@ -267,10 +267,20 @@ # sync_from_local_copy_dir: true -# To upload the repo to an Amazon S3 bucket using `fdroid server -# update`. Warning, this deletes and recreates the whole fdroid/ -# directory each time. This prefers s3cmd, but can also use -# apache-libcloud. To customize how s3cmd interacts with the cloud +# To upload the repo to an Amazon S3 bucket using `fdroid deploy' +# . rclone, s3cmd and apache libcloud are the available options. +# If rclone and s3cmd are not installed, apache libcloud is used. +# To use apache libcloud, add the following options to this file +# (config.yml) +# +# awsbucket: myawsfdroid +# awsaccesskeyid: SEE0CHAITHEIMAUR2USA +# awssecretkey: {env: awssecretkey} +# +# In case s3cmd is installed and rclone is not installed, +# s3cmd will be the preferred sync option. +# It will delete and recreate the whole fdroid directory each time. +# To customize how s3cmd interacts with the cloud # provider, create a 's3cfg' file next to this file (config.yml), and # those settings will be used instead of any 'aws' variable below. # Secrets can be fetched from environment variables to ensure that @@ -279,6 +289,47 @@ # awsbucket: myawsfdroid # awsaccesskeyid: SEE0CHAITHEIMAUR2USA # awssecretkey: {env: awssecretkey} +# +# In case rclone is installed and s3cmd is not installed, +# rclone will be the preferred sync option. +# It will sync the local folders with remote folders without +# deleting anything in one go. +# To ensure success, install rclone as per +# the instructions at https://rclone.org/install/ and also configure for +# object storage services as detailed at https://rclone.org/s3/#configuration +# By default rclone uses the configuration file at ~/.config/rclone/rclone.conf +# To specify a custom configuration file, please add the full path to the +# configuration file as below +# +# path_to_custom_rclone_config: /home/mycomputer/somedir/example.conf +# +# This setting will ignore the default rclone config found at +# ~/.config/rclone/rclone.conf +# +# Please note that rclone_config can be assigned a string or list +# +# awsbucket: myawsfdroid +# rclone_config: aws-sample-config +# +# or +# +# awsbucket: myawsfdroid +# rclone_config: [aws-sample-config, rclone-supported-service-config] +# +# In case both rclone and s3cmd are installed, the preferred sync +# tool can be specified in this file (config.yml) +# if s3cmd is preferred, set it as below +# +# s3cmd: true +# +# if rclone is preferred, set it as below +# +# rclone: true +# +# Please note that only one can be set to true at any time +# Also, in the event that both s3cmd and rclone are installed +# and both are missing from the config.yml file, the preferred +# tool will be s3cmd. # If you want to force 'fdroid server' to use a non-standard serverwebroot. diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py index b4b4cd95..b120999f 100644 --- a/fdroidserver/deploy.py +++ b/fdroidserver/deploy.py @@ -28,6 +28,7 @@ import urllib import yaml from argparse import ArgumentParser import logging +from shlex import split import shutil from . import _ @@ -44,6 +45,7 @@ BINARY_TRANSPARENCY_DIR = 'binary_transparency' AUTO_S3CFG = '.fdroid-deploy-s3cfg' USER_S3CFG = 's3cfg' +USER_RCLONE_CONF = None REMOTE_HOSTNAME_REGEX = re.compile(r'\W*\w+\W+(\w+).*') INDEX_FILES = [ @@ -89,7 +91,7 @@ def _get_index_excludes(repo_section): return index_excludes -def update_awsbucket(repo_section): +def update_awsbucket(repo_section, verbose=False, quiet=False): """Upload the contents of the directory `repo_section` (including subdirectories) to the AWS S3 "bucket". The contents of that subdir of the @@ -101,8 +103,29 @@ def update_awsbucket(repo_section): f'''Syncing "{repo_section}" to Amazon S3 bucket "{config['awsbucket']}"''' ) - if common.set_command_in_config('s3cmd'): + if common.set_command_in_config('s3cmd') and common.set_command_in_config('rclone'): + logging.info( + 'Both rclone and s3cmd are installed. Checking config.yml for preference.' + ) + if config['s3cmd'] is not True and config['rclone'] is not True: + logging.warning( + 'No syncing tool set in config.yml!. Defaulting to using s3cmd' + ) + update_awsbucket_s3cmd(repo_section) + if config['s3cmd'] is True and config['rclone'] is True: + logging.warning( + 'Both syncing tools set in config.yml!. Defaulting to using s3cmd' + ) + update_awsbucket_s3cmd(repo_section) + if config['s3cmd'] is True and config['rclone'] is not True: + update_awsbucket_s3cmd(repo_section) + if config['rclone'] is True and config['s3cmd'] is not True: + update_remote_storage_with_rclone(repo_section, verbose, quiet) + + elif common.set_command_in_config('s3cmd'): update_awsbucket_s3cmd(repo_section) + elif common.set_command_in_config('rclone'): + update_remote_storage_with_rclone(repo_section, verbose, quiet) else: update_awsbucket_libcloud(repo_section) @@ -186,6 +209,129 @@ def update_awsbucket_s3cmd(repo_section): raise FDroidException() +def update_remote_storage_with_rclone(repo_section, verbose=False, quiet=False): + """ + Upload fdroid repo folder to remote storage using rclone sync. + + Rclone sync can send the files to any supported remote storage + service once without numerous polling. + If remote storage is s3 e.g aws s3, wasabi, filebase then path will be + bucket_name/fdroid/repo where bucket_name will be an s3 bucket + If remote storage is storage drive/sftp e.g google drive, rsync.net + the new path will be bucket_name/fdroid/repo where bucket_name + will be a folder + + Better than the s3cmd command as it does the syncing in one command + Check https://rclone.org/docs/#config-config-file (optional config file) + """ + logging.debug(_('Using rclone to sync with: {url}').format(url=config['awsbucket'])) + + if config.get('path_to_custom_rclone_config') is not None: + USER_RCLONE_CONF = config['path_to_custom_rclone_config'] + if os.path.exists(USER_RCLONE_CONF): + logging.info("'path_to_custom_rclone_config' found in config.yml") + logging.info( + _('Using "{path}" for syncing with remote storage.').format( + path=USER_RCLONE_CONF + ) + ) + configfilename = USER_RCLONE_CONF + else: + logging.info('Custom configuration not found.') + logging.info( + 'Using default configuration at {}'.format( + subprocess.check_output('rclone config file') + ) + ) + configfilename = None + else: + logging.warning("'path_to_custom_rclone_config' not found in config.yml") + logging.info('Custom configuration not found.') + logging.info( + 'Using default configuration at {}'.format( + subprocess.check_output('rclone config file') + ) + ) + configfilename = None + + upload_dir = 'fdroid/' + repo_section + + if not config.get('rclone_config') or not config.get('awsbucket'): + raise FDroidException( + _('To use rclone, rclone_config and awsbucket must be set in config.yml!') + ) + + if isinstance(config['rclone_config'], str): + rclone_sync_command = ( + 'rclone sync ' + + repo_section + + ' ' + + config['rclone_config'] + + ':' + + config['awsbucket'] + + '/' + + upload_dir + ) + + rclone_sync_command = split(rclone_sync_command) + + if verbose: + rclone_sync_command += ['--verbose'] + elif quiet: + rclone_sync_command += ['--quiet'] + + if configfilename: + rclone_sync_command += split('--config=' + configfilename) + + complete_remote_path = ( + config['rclone_config'] + ':' + config['awsbucket'] + '/' + upload_dir + ) + + logging.debug( + "rclone sync all files in " + repo_section + ' to ' + complete_remote_path + ) + + if subprocess.call(rclone_sync_command) != 0: + raise FDroidException() + + if isinstance(config['rclone_config'], list): + for remote_config in config['rclone_config']: + rclone_sync_command = ( + 'rclone sync ' + + repo_section + + ' ' + + remote_config + + ':' + + config['awsbucket'] + + '/' + + upload_dir + ) + + rclone_sync_command = split(rclone_sync_command) + + if verbose: + rclone_sync_command += ['--verbose'] + elif quiet: + rclone_sync_command += ['--quiet'] + + if configfilename: + rclone_sync_command += split('--config=' + configfilename) + + complete_remote_path = ( + remote_config + ':' + config['awsbucket'] + '/' + upload_dir + ) + + logging.debug( + "rclone sync all files in " + + repo_section + + ' to ' + + complete_remote_path + ) + + if subprocess.call(rclone_sync_command) != 0: + raise FDroidException() + + def update_awsbucket_libcloud(repo_section): """No summary. @@ -967,7 +1113,7 @@ def main(): # update_servergitmirrors will take care of multiple mirrors so don't need a foreach update_servergitmirrors(config['servergitmirrors'], repo_section) if config.get('awsbucket'): - update_awsbucket(repo_section) + update_awsbucket(repo_section, options.verbose, options.quiet) if config.get('androidobservatory'): upload_to_android_observatory(repo_section) if config.get('virustotal_apikey'): diff --git a/tests/deploy.TestCase b/tests/deploy.TestCase index 4a2365c9..7e5b123b 100755 --- a/tests/deploy.TestCase +++ b/tests/deploy.TestCase @@ -1,8 +1,10 @@ #!/usr/bin/env python3 +import configparser import inspect import logging import os +import shutil import sys import tempfile import unittest @@ -21,6 +23,11 @@ from fdroidserver.exception import FDroidException from testcommon import TmpCwd, mkdtemp, parse_args_for_test +class Options: + quiet = False + verbose = False + + class DeployTest(unittest.TestCase): '''fdroidserver/deploy.py''' @@ -31,6 +38,10 @@ class DeployTest(unittest.TestCase): self._td = mkdtemp() self.testdir = self._td.name + fdroidserver.deploy.options = mock.Mock() + fdroidserver.deploy.config = {} + fdroidserver.deploy.USER_RCLONE_CONF = False + def tearDown(self): self._td.cleanup() @@ -87,6 +98,44 @@ class DeployTest(unittest.TestCase): with self.assertRaises(SystemExit): fdroidserver.deploy.update_serverwebroots([{'url': 'ssh://nope'}], 'repo') + @unittest.skipUnless(shutil.which('rclone'), '/usr/bin/rclone') + def test_update_remote_storage_with_rclone(self): + os.chdir(self.testdir) + repo = Path('repo') + repo.mkdir(parents=True, exist_ok=True) + + fake_apk = repo / 'another_fake.apk' + with fake_apk.open('w') as fp: + fp.write('not an APK, but has the right filename') + + # write out rclone config for test use + rclone_config = configparser.ConfigParser() + rclone_config.add_section("test-local-config") + rclone_config.set("test-local-config", "type", "local") + + rclone_config_path = Path('rclone_config_path') + rclone_config_path.mkdir(parents=True, exist_ok=True) + rclone_file = rclone_config_path / 'rclone.conf' + with open(rclone_file, 'w') as configfile: + rclone_config.write(configfile) + + # setup parameters for this test run + fdroidserver.deploy.config['awsbucket'] = 'test_bucket_folder' + fdroidserver.deploy.config['rclone'] = True + fdroidserver.deploy.config['rclone_config'] = 'test-local-config' + fdroidserver.deploy.config['path_to_custom_rclone_config'] = str(rclone_file) + fdroidserver.deploy.options = Options + + # write out destination path + destination = Path('some_bucket_folder/fdroid') + destination.mkdir(parents=True, exist_ok=True) + dest_path = Path(destination) / fake_apk + self.assertFalse(dest_path.is_file()) + repo_section = str(repo) + # fdroidserver.deploy.USER_RCLONE_CONF = str(rclone_file) + fdroidserver.deploy.update_remote_storage_with_rclone(repo_section) + self.assertFalse(dest_path.is_file()) + def test_update_serverwebroot(self): """rsync works with file paths, so this test uses paths for the URLs""" os.chdir(self.testdir) @@ -100,6 +149,8 @@ class DeployTest(unittest.TestCase): dest_apk = url / fake_apk self.assertFalse(dest_apk.is_file()) + fdroidserver.deploy.options = mock.Mock() + fdroidserver.deploy.options.identity_file = None fdroidserver.deploy.update_serverwebroot({'url': str(url)}, 'repo') self.assertTrue(dest_apk.is_file())