Merge branch 'switching-s3cmd-for-rclone' into 'master'

Adding rclone as an option to fdroid deploy

Closes #1095

See merge request fdroid/fdroidserver!1095

examples/config.yml

@@ -267,10 +267,20 @@
 # sync_from_local_copy_dir: true
 
 
-# To upload the repo to an Amazon S3 bucket using `fdroid server
+# To upload the repo to an Amazon S3 bucket using `fdroid deploy'
-# update`.  Warning, this deletes and recreates the whole fdroid/
+# . rclone, s3cmd and apache libcloud are the available options.
-# directory each time. This prefers s3cmd, but can also use
+# If rclone and s3cmd are not installed, apache libcloud is used.
-# apache-libcloud.  To customize how s3cmd interacts with the cloud
+# To use apache libcloud, add the following options to this file
+# (config.yml)
+#
+# awsbucket: myawsfdroid
+# awsaccesskeyid: SEE0CHAITHEIMAUR2USA
+# awssecretkey: {env: awssecretkey}
+#
+# In case s3cmd is installed and rclone is not installed,
+# s3cmd will be the preferred sync option.
+# It will delete and recreate the whole fdroid directory each time.
+# To customize how s3cmd interacts with the cloud
 # provider, create a 's3cfg' file next to this file (config.yml), and
 # those settings will be used instead of any 'aws' variable below.
 # Secrets can be fetched from environment variables to ensure that
@@ -279,6 +289,47 @@
 # awsbucket: myawsfdroid
 # awsaccesskeyid: SEE0CHAITHEIMAUR2USA
 # awssecretkey: {env: awssecretkey}
+#
+# In case rclone is installed and s3cmd is not installed,
+# rclone will be the preferred sync option.
+# It will sync the local folders with remote folders without
+# deleting anything in one go.
+# To ensure success, install rclone as per
+# the instructions at https://rclone.org/install/ and also configure for
+# object storage services as detailed at https://rclone.org/s3/#configuration
+# By default rclone uses the configuration file at ~/.config/rclone/rclone.conf
+# To specify a custom configuration file, please add the full path to the
+# configuration file as below
+#
+# path_to_custom_rclone_config: /home/mycomputer/somedir/example.conf
+#
+# This setting will ignore the default rclone config found at
+# ~/.config/rclone/rclone.conf
+#
+# Please note that rclone_config can be assigned a string or list
+#
+# awsbucket: myawsfdroid
+# rclone_config: aws-sample-config
+#
+# or
+#
+# awsbucket: myawsfdroid
+# rclone_config: [aws-sample-config, rclone-supported-service-config]
+#
+# In case both rclone and s3cmd are installed, the preferred sync
+# tool can be specified in this file (config.yml)
+# if s3cmd is preferred, set it as below
+#
+# s3cmd: true
+#
+# if rclone is preferred, set it as below
+#
+# rclone: true
+#
+# Please note that only one can be set to true at any time
+# Also, in the event that both s3cmd and rclone are installed
+# and both are missing from the config.yml file, the preferred
+# tool will be s3cmd.
 
 
 # If you want to force 'fdroid server' to use a non-standard serverwebroot.

fdroidserver/deploy.py

@@ -28,6 +28,7 @@ import urllib
 import yaml
 from argparse import ArgumentParser
 import logging
+from shlex import split
 import shutil
 
 from . import _
@@ -44,6 +45,7 @@ BINARY_TRANSPARENCY_DIR = 'binary_transparency'
 
 AUTO_S3CFG = '.fdroid-deploy-s3cfg'
 USER_S3CFG = 's3cfg'
+USER_RCLONE_CONF = None
 REMOTE_HOSTNAME_REGEX = re.compile(r'\W*\w+\W+(\w+).*')
 
 INDEX_FILES = [
@@ -89,7 +91,7 @@ def _get_index_excludes(repo_section):
     return index_excludes
 
 
-def update_awsbucket(repo_section):
+def update_awsbucket(repo_section, verbose=False, quiet=False):
     """Upload the contents of the directory `repo_section` (including subdirectories) to the AWS S3 "bucket".
 
     The contents of that subdir of the
@@ -101,8 +103,29 @@ def update_awsbucket(repo_section):
         f'''Syncing "{repo_section}" to Amazon S3 bucket "{config['awsbucket']}"'''
     )
 
-    if common.set_command_in_config('s3cmd'):
+    if common.set_command_in_config('s3cmd') and common.set_command_in_config('rclone'):
+        logging.info(
+            'Both rclone and s3cmd are installed. Checking config.yml for preference.'
+        )
+        if config['s3cmd'] is not True and config['rclone'] is not True:
+            logging.warning(
+                'No syncing tool set in config.yml!. Defaulting to using s3cmd'
+            )
+            update_awsbucket_s3cmd(repo_section)
+        if config['s3cmd'] is True and config['rclone'] is True:
+            logging.warning(
+                'Both syncing tools set in config.yml!. Defaulting to using s3cmd'
+            )
+            update_awsbucket_s3cmd(repo_section)
+        if config['s3cmd'] is True and config['rclone'] is not True:
+            update_awsbucket_s3cmd(repo_section)
+        if config['rclone'] is True and config['s3cmd'] is not True:
+            update_remote_storage_with_rclone(repo_section, verbose, quiet)
+
+    elif common.set_command_in_config('s3cmd'):
         update_awsbucket_s3cmd(repo_section)
+    elif common.set_command_in_config('rclone'):
+        update_remote_storage_with_rclone(repo_section, verbose, quiet)
     else:
         update_awsbucket_libcloud(repo_section)
 
@@ -186,6 +209,129 @@ def update_awsbucket_s3cmd(repo_section):
         raise FDroidException()
 
 
+def update_remote_storage_with_rclone(repo_section, verbose=False, quiet=False):
+    """
+    Upload fdroid repo folder to remote storage using rclone sync.
+
+    Rclone sync can send the files to any supported remote storage
+    service once without numerous polling.
+    If remote storage is s3 e.g aws s3, wasabi, filebase then path will be
+    bucket_name/fdroid/repo where bucket_name will be an s3 bucket
+    If remote storage is storage drive/sftp e.g google drive, rsync.net
+    the new path will be bucket_name/fdroid/repo where bucket_name
+    will be a folder
+
+    Better than the s3cmd command as it does the syncing in one command
+    Check https://rclone.org/docs/#config-config-file (optional config file)
+    """
+    logging.debug(_('Using rclone to sync with: {url}').format(url=config['awsbucket']))
+
+    if config.get('path_to_custom_rclone_config') is not None:
+        USER_RCLONE_CONF = config['path_to_custom_rclone_config']
+        if os.path.exists(USER_RCLONE_CONF):
+            logging.info("'path_to_custom_rclone_config' found in config.yml")
+            logging.info(
+                _('Using "{path}" for syncing with remote storage.').format(
+                    path=USER_RCLONE_CONF
+                )
+            )
+            configfilename = USER_RCLONE_CONF
+        else:
+            logging.info('Custom configuration not found.')
+            logging.info(
+                'Using default configuration at {}'.format(
+                    subprocess.check_output('rclone config file')
+                )
+            )
+            configfilename = None
+    else:
+        logging.warning("'path_to_custom_rclone_config' not found in config.yml")
+        logging.info('Custom configuration not found.')
+        logging.info(
+            'Using default configuration at {}'.format(
+                subprocess.check_output('rclone config file')
+            )
+        )
+        configfilename = None
+
+    upload_dir = 'fdroid/' + repo_section
+
+    if not config.get('rclone_config') or not config.get('awsbucket'):
+        raise FDroidException(
+            _('To use rclone, rclone_config and awsbucket must be set in config.yml!')
+        )
+
+    if isinstance(config['rclone_config'], str):
+        rclone_sync_command = (
+            'rclone sync '
+            + repo_section
+            + ' '
+            + config['rclone_config']
+            + ':'
+            + config['awsbucket']
+            + '/'
+            + upload_dir
+        )
+
+        rclone_sync_command = split(rclone_sync_command)
+
+        if verbose:
+            rclone_sync_command += ['--verbose']
+        elif quiet:
+            rclone_sync_command += ['--quiet']
+
+        if configfilename:
+            rclone_sync_command += split('--config=' + configfilename)
+
+        complete_remote_path = (
+            config['rclone_config'] + ':' + config['awsbucket'] + '/' + upload_dir
+        )
+
+        logging.debug(
+            "rclone sync all files in " + repo_section + ' to ' + complete_remote_path
+        )
+
+        if subprocess.call(rclone_sync_command) != 0:
+            raise FDroidException()
+
+    if isinstance(config['rclone_config'], list):
+        for remote_config in config['rclone_config']:
+            rclone_sync_command = (
+                'rclone sync '
+                + repo_section
+                + ' '
+                + remote_config
+                + ':'
+                + config['awsbucket']
+                + '/'
+                + upload_dir
+            )
+
+            rclone_sync_command = split(rclone_sync_command)
+
+            if verbose:
+                rclone_sync_command += ['--verbose']
+            elif quiet:
+                rclone_sync_command += ['--quiet']
+
+            if configfilename:
+                rclone_sync_command += split('--config=' + configfilename)
+
+            complete_remote_path = (
+                remote_config + ':' + config['awsbucket'] + '/' + upload_dir
+            )
+
+            logging.debug(
+                "rclone sync all files in "
+                + repo_section
+                + ' to '
+                + complete_remote_path
+            )
+
+            if subprocess.call(rclone_sync_command) != 0:
+                raise FDroidException()
+
+
 def update_awsbucket_libcloud(repo_section):
     """No summary.
 
@@ -967,7 +1113,7 @@ def main():
             # update_servergitmirrors will take care of multiple mirrors so don't need a foreach
             update_servergitmirrors(config['servergitmirrors'], repo_section)
         if config.get('awsbucket'):
-            update_awsbucket(repo_section)
+            update_awsbucket(repo_section, options.verbose, options.quiet)
         if config.get('androidobservatory'):
             upload_to_android_observatory(repo_section)
         if config.get('virustotal_apikey'):

tests/deploy.TestCase

@@ -1,8 +1,10 @@
 #!/usr/bin/env python3
 
+import configparser
 import inspect
 import logging
 import os
+import shutil
 import sys
 import tempfile
 import unittest
@@ -21,6 +23,11 @@ from fdroidserver.exception import FDroidException
 from testcommon import TmpCwd, mkdtemp, parse_args_for_test
 
 
+class Options:
+    quiet = False
+    verbose = False
+
+
 class DeployTest(unittest.TestCase):
     '''fdroidserver/deploy.py'''
 
@@ -31,6 +38,10 @@ class DeployTest(unittest.TestCase):
         self._td = mkdtemp()
         self.testdir = self._td.name
 
+        fdroidserver.deploy.options = mock.Mock()
+        fdroidserver.deploy.config = {}
+        fdroidserver.deploy.USER_RCLONE_CONF = False
+
     def tearDown(self):
         self._td.cleanup()
 
@@ -87,6 +98,44 @@ class DeployTest(unittest.TestCase):
         with self.assertRaises(SystemExit):
             fdroidserver.deploy.update_serverwebroots([{'url': 'ssh://nope'}], 'repo')
 
+    @unittest.skipUnless(shutil.which('rclone'), '/usr/bin/rclone')
+    def test_update_remote_storage_with_rclone(self):
+        os.chdir(self.testdir)
+        repo = Path('repo')
+        repo.mkdir(parents=True, exist_ok=True)
+
+        fake_apk = repo / 'another_fake.apk'
+        with fake_apk.open('w') as fp:
+            fp.write('not an APK, but has the right filename')
+
+        # write out rclone config for test use
+        rclone_config = configparser.ConfigParser()
+        rclone_config.add_section("test-local-config")
+        rclone_config.set("test-local-config", "type", "local")
+
+        rclone_config_path = Path('rclone_config_path')
+        rclone_config_path.mkdir(parents=True, exist_ok=True)
+        rclone_file = rclone_config_path / 'rclone.conf'
+        with open(rclone_file, 'w') as configfile:
+            rclone_config.write(configfile)
+
+        # setup parameters for this test run
+        fdroidserver.deploy.config['awsbucket'] = 'test_bucket_folder'
+        fdroidserver.deploy.config['rclone'] = True
+        fdroidserver.deploy.config['rclone_config'] = 'test-local-config'
+        fdroidserver.deploy.config['path_to_custom_rclone_config'] = str(rclone_file)
+        fdroidserver.deploy.options = Options
+
+        # write out destination path
+        destination = Path('some_bucket_folder/fdroid')
+        destination.mkdir(parents=True, exist_ok=True)
+        dest_path = Path(destination) / fake_apk
+        self.assertFalse(dest_path.is_file())
+        repo_section = str(repo)
+        # fdroidserver.deploy.USER_RCLONE_CONF = str(rclone_file)
+        fdroidserver.deploy.update_remote_storage_with_rclone(repo_section)
+        self.assertFalse(dest_path.is_file())
+
     def test_update_serverwebroot(self):
         """rsync works with file paths, so this test uses paths for the URLs"""
         os.chdir(self.testdir)
@@ -100,6 +149,8 @@ class DeployTest(unittest.TestCase):
 
         dest_apk = url / fake_apk
         self.assertFalse(dest_apk.is_file())
+        fdroidserver.deploy.options = mock.Mock()
+        fdroidserver.deploy.options.identity_file = None
         fdroidserver.deploy.update_serverwebroot({'url': str(url)}, 'repo')
         self.assertTrue(dest_apk.is_file())