feat(deploy): add index-only mode to the rclone deployment method

fdroidserver/deploy.py

@@ -126,12 +126,12 @@ def update_awsbucket(repo_section, is_index_only=False, verbose=False, quiet=Fal
         if config['s3cmd'] is True and config['rclone'] is not True:
             update_awsbucket_s3cmd(repo_section, is_index_only)
         if config['rclone'] is True and config['s3cmd'] is not True:
-            update_remote_storage_with_rclone(repo_section, verbose, quiet)
+            update_remote_storage_with_rclone(repo_section, is_index_only, verbose, quiet)
 
     elif common.set_command_in_config('s3cmd'):
         update_awsbucket_s3cmd(repo_section, is_index_only)
     elif common.set_command_in_config('rclone'):
-        update_remote_storage_with_rclone(repo_section, verbose, quiet)
+        update_remote_storage_with_rclone(repo_section, is_index_only, verbose, quiet)
     else:
         update_awsbucket_libcloud(repo_section, is_index_only)
 
@@ -234,7 +234,7 @@ def update_awsbucket_s3cmd(repo_section, is_index_only=False):
             raise FDroidException()
 
 
-def update_remote_storage_with_rclone(repo_section, verbose=False, quiet=False):
+def update_remote_storage_with_rclone(repo_section, is_index_only=False, verbose=False, quiet=False):
     """
     Upload fdroid repo folder to remote storage using rclone sync.
 
@@ -286,46 +286,18 @@ def update_remote_storage_with_rclone(repo_section, verbose=False, quiet=False):
             _('To use rclone, rclone_config and awsbucket must be set in config.yml!')
         )
 
-    if isinstance(config['rclone_config'], str):
-        rclone_sync_command = (
-            'rclone sync '
-            + repo_section
-            + ' '
-            + config['rclone_config']
-            + ':'
-            + config['awsbucket']
-            + '/'
-            + upload_dir
-        )
+    if is_index_only:
+        sources = _get_index_file_paths(repo_section)
+    else:
+        sources = repo_section
 
-        rclone_sync_command = split(rclone_sync_command)
-
-        if verbose:
-            rclone_sync_command += ['--verbose']
-        elif quiet:
-            rclone_sync_command += ['--quiet']
-
-        if configfilename:
-            rclone_sync_command += split('--config=' + configfilename)
-
-        complete_remote_path = (
-            config['rclone_config'] + ':' + config['awsbucket'] + '/' + upload_dir
-        )
-
-        logging.debug(
-            "rclone sync all files in " + repo_section + ' to ' + complete_remote_path
-        )
-
-        if subprocess.call(rclone_sync_command) != 0:
-            raise FDroidException()
-
-    if isinstance(config['rclone_config'], list):
-        for remote_config in config['rclone_config']:
+    for source in sources:
+        if isinstance(config['rclone_config'], str):
             rclone_sync_command = (
                 'rclone sync '
-                + repo_section
+                + source
                 + ' '
-                + remote_config
+                + config['rclone_config']
                 + ':'
                 + config['awsbucket']
                 + '/'
@@ -343,19 +315,53 @@ def update_remote_storage_with_rclone(repo_section, verbose=False, quiet=False):
                 rclone_sync_command += split('--config=' + configfilename)
 
             complete_remote_path = (
-                remote_config + ':' + config['awsbucket'] + '/' + upload_dir
+                config['rclone_config'] + ':' + config['awsbucket'] + '/' + upload_dir
             )
 
             logging.debug(
-                "rclone sync all files in "
-                + repo_section
-                + ' to '
-                + complete_remote_path
+                "rclone sync all files in " + source + ' to ' + complete_remote_path
             )
 
             if subprocess.call(rclone_sync_command) != 0:
                 raise FDroidException()
 
+        if isinstance(config['rclone_config'], list):
+            for remote_config in config['rclone_config']:
+                rclone_sync_command = (
+                    'rclone sync '
+                    + source
+                    + ' '
+                    + remote_config
+                    + ':'
+                    + config['awsbucket']
+                    + '/'
+                    + upload_dir
+                )
+
+                rclone_sync_command = split(rclone_sync_command)
+
+                if verbose:
+                    rclone_sync_command += ['--verbose']
+                elif quiet:
+                    rclone_sync_command += ['--quiet']
+
+                if configfilename:
+                    rclone_sync_command += split('--config=' + configfilename)
+
+                complete_remote_path = (
+                    remote_config + ':' + config['awsbucket'] + '/' + upload_dir
+                )
+
+                logging.debug(
+                    "rclone sync all files in "
+                    + source
+                    + ' to '
+                    + complete_remote_path
+                )
+
+                if subprocess.call(rclone_sync_command) != 0:
+                    raise FDroidException()
+
 
 def update_awsbucket_libcloud(repo_section, is_index_only=False):
     """No summary.

tests/deploy.TestCase

@@ -113,6 +113,10 @@ class DeployTest(unittest.TestCase):
         fake_apk = repo / 'another_fake.apk'
         with fake_apk.open('w') as fp:
             fp.write('not an APK, but has the right filename')
+        fake_index = repo / fdroidserver.deploy.INDEX_FILES[0]
+        with fake_index.open('w') as fp:
+            fp.write('not an index, but has the right filename')
+
 
         # write out rclone config for test use
         rclone_config = configparser.ConfigParser()
@@ -135,12 +139,61 @@ class DeployTest(unittest.TestCase):
         # write out destination path
         destination = Path('some_bucket_folder/fdroid')
         destination.mkdir(parents=True, exist_ok=True)
-        dest_path = Path(destination) / fake_apk
-        self.assertFalse(dest_path.is_file())
+        dest_apk = Path(destination) / fake_apk
+        dest_index = Path(destination) / fake_index
+        self.assertFalse(dest_apk.is_file())
+        self.assertFalse(dest_index.is_file())
         repo_section = str(repo)
         # fdroidserver.deploy.USER_RCLONE_CONF = str(rclone_file)
         fdroidserver.deploy.update_remote_storage_with_rclone(repo_section)
-        self.assertFalse(dest_path.is_file())
+        self.assertTrue(dest_apk.is_file())
+        self.assertTrue(dest_index.is_file())
+
+    @unittest.skipUnless(shutil.which('rclone'), '/usr/bin/rclone')
+    def test_update_remote_storage_with_rclone_in_index_only_mode(self):
+        os.chdir(self.testdir)
+        repo = Path('repo')
+        repo.mkdir(parents=True, exist_ok=True)
+
+        fake_apk = repo / 'another_fake.apk'
+        with fake_apk.open('w') as fp:
+            fp.write('not an APK, but has the right filename')
+        fake_index = repo / fdroidserver.deploy.INDEX_FILES[0]
+        with fake_index.open('w') as fp:
+            fp.write('not an index, but has the right filename')
+
+        # write out rclone config for test use
+        rclone_config = configparser.ConfigParser()
+        rclone_config.add_section("test-local-config")
+        rclone_config.set("test-local-config", "type", "local")
+
+        rclone_config_path = Path('rclone_config_path')
+        rclone_config_path.mkdir(parents=True, exist_ok=True)
+        rclone_file = rclone_config_path / 'rclone.conf'
+        with open(rclone_file, 'w') as configfile:
+            rclone_config.write(configfile)
+
+        # setup parameters for this test run
+        fdroidserver.deploy.config['awsbucket'] = 'test_bucket_folder'
+        fdroidserver.deploy.config['rclone'] = True
+        fdroidserver.deploy.config['rclone_config'] = 'test-local-config'
+        fdroidserver.deploy.config['path_to_custom_rclone_config'] = str(rclone_file)
+        fdroidserver.deploy.options = Options
+
+        # write out destination path
+        destination = Path('some_bucket_folder/fdroid')
+        destination.mkdir(parents=True, exist_ok=True)
+        dest_apk = Path(destination) / fake_apk
+        dest_index = Path(destination) / fake_index
+        self.assertFalse(dest_apk.is_file())
+        self.assertFalse(dest_index.is_file())
+        repo_section = str(repo)
+        # fdroidserver.deploy.USER_RCLONE_CONF = str(rclone_file)
+        fdroidserver.deploy.update_remote_storage_with_rclone(
+            repo_section, is_index_only=True
+        )
+        self.assertFalse(dest_apk.is_file())
+        self.assertTrue(dest_index.is_file())
 
     def test_update_serverwebroot(self):
         """rsync works with file paths, so this test uses paths for the URLs"""
@@ -178,10 +231,9 @@ class DeployTest(unittest.TestCase):
         fake_apk = repo / 'fake.apk'
         with fake_apk.open('w') as fp:
             fp.write('not an APK, but has the right filename')
-        for i in fdroidserver.deploy.INDEX_FILES:
-            fake_index = repo / i
-            with fake_index.open('w') as fp:
-                fp.write('not an index, but has the right filename')
+        fake_index = repo / fdroidserver.deploy.INDEX_FILES[0]
+        with fake_index.open('w') as fp:
+            fp.write('not an index, but has the right filename')
         url = Path('url')
         url.mkdir()