feat(deploy): add index-only mode to the rclone deployment method

fdroidserver/deploy.py

@@ -126,12 +126,12 @@ def update_awsbucket(repo_section, is_index_only=False, verbose=False, quiet=Fal
         if config['s3cmd'] is True and config['rclone'] is not True:
             update_awsbucket_s3cmd(repo_section, is_index_only)
         if config['rclone'] is True and config['s3cmd'] is not True:
-            update_remote_storage_with_rclone(repo_section, verbose, quiet)
+            update_remote_storage_with_rclone(repo_section, is_index_only, verbose, quiet)
 
     elif common.set_command_in_config('s3cmd'):
         update_awsbucket_s3cmd(repo_section, is_index_only)
     elif common.set_command_in_config('rclone'):
-        update_remote_storage_with_rclone(repo_section, verbose, quiet)
+        update_remote_storage_with_rclone(repo_section, is_index_only, verbose, quiet)
     else:
         update_awsbucket_libcloud(repo_section, is_index_only)
 
@@ -234,7 +234,7 @@ def update_awsbucket_s3cmd(repo_section, is_index_only=False):
             raise FDroidException()
 
 
-def update_remote_storage_with_rclone(repo_section, verbose=False, quiet=False):
+def update_remote_storage_with_rclone(repo_section, is_index_only=False, verbose=False, quiet=False):
     """
     Upload fdroid repo folder to remote storage using rclone sync.
 
@@ -286,10 +286,16 @@ def update_remote_storage_with_rclone(repo_section, verbose=False, quiet=False):
             _('To use rclone, rclone_config and awsbucket must be set in config.yml!')
         )
 
+    if is_index_only:
+        sources = _get_index_file_paths(repo_section)
+    else:
+        sources = repo_section
+
+    for source in sources:
         if isinstance(config['rclone_config'], str):
             rclone_sync_command = (
                 'rclone sync '
-            + repo_section
+                + source
                 + ' '
                 + config['rclone_config']
                 + ':'
@@ -313,7 +319,7 @@ def update_remote_storage_with_rclone(repo_section, verbose=False, quiet=False):
             )
 
             logging.debug(
-            "rclone sync all files in " + repo_section + ' to ' + complete_remote_path
+                "rclone sync all files in " + source + ' to ' + complete_remote_path
             )
 
             if subprocess.call(rclone_sync_command) != 0:
@@ -323,7 +329,7 @@ def update_remote_storage_with_rclone(repo_section, verbose=False, quiet=False):
             for remote_config in config['rclone_config']:
                 rclone_sync_command = (
                     'rclone sync '
-                + repo_section
+                    + source
                     + ' '
                     + remote_config
                     + ':'
@@ -348,7 +354,7 @@ def update_remote_storage_with_rclone(repo_section, verbose=False, quiet=False):
 
                 logging.debug(
                     "rclone sync all files in "
-                + repo_section
+                    + source
                     + ' to '
                     + complete_remote_path
                 )

tests/deploy.TestCase

@@ -113,6 +113,10 @@ class DeployTest(unittest.TestCase):
         fake_apk = repo / 'another_fake.apk'
         with fake_apk.open('w') as fp:
             fp.write('not an APK, but has the right filename')
+        fake_index = repo / fdroidserver.deploy.INDEX_FILES[0]
+        with fake_index.open('w') as fp:
+            fp.write('not an index, but has the right filename')
+
 
         # write out rclone config for test use
         rclone_config = configparser.ConfigParser()
@@ -135,12 +139,61 @@ class DeployTest(unittest.TestCase):
         # write out destination path
         destination = Path('some_bucket_folder/fdroid')
         destination.mkdir(parents=True, exist_ok=True)
-        dest_path = Path(destination) / fake_apk
+        dest_apk = Path(destination) / fake_apk
-        self.assertFalse(dest_path.is_file())
+        dest_index = Path(destination) / fake_index
+        self.assertFalse(dest_apk.is_file())
+        self.assertFalse(dest_index.is_file())
         repo_section = str(repo)
         # fdroidserver.deploy.USER_RCLONE_CONF = str(rclone_file)
         fdroidserver.deploy.update_remote_storage_with_rclone(repo_section)
-        self.assertFalse(dest_path.is_file())
+        self.assertTrue(dest_apk.is_file())
+        self.assertTrue(dest_index.is_file())
+
+    @unittest.skipUnless(shutil.which('rclone'), '/usr/bin/rclone')
+    def test_update_remote_storage_with_rclone_in_index_only_mode(self):
+        os.chdir(self.testdir)
+        repo = Path('repo')
+        repo.mkdir(parents=True, exist_ok=True)
+
+        fake_apk = repo / 'another_fake.apk'
+        with fake_apk.open('w') as fp:
+            fp.write('not an APK, but has the right filename')
+        fake_index = repo / fdroidserver.deploy.INDEX_FILES[0]
+        with fake_index.open('w') as fp:
+            fp.write('not an index, but has the right filename')
+
+        # write out rclone config for test use
+        rclone_config = configparser.ConfigParser()
+        rclone_config.add_section("test-local-config")
+        rclone_config.set("test-local-config", "type", "local")
+
+        rclone_config_path = Path('rclone_config_path')
+        rclone_config_path.mkdir(parents=True, exist_ok=True)
+        rclone_file = rclone_config_path / 'rclone.conf'
+        with open(rclone_file, 'w') as configfile:
+            rclone_config.write(configfile)
+
+        # setup parameters for this test run
+        fdroidserver.deploy.config['awsbucket'] = 'test_bucket_folder'
+        fdroidserver.deploy.config['rclone'] = True
+        fdroidserver.deploy.config['rclone_config'] = 'test-local-config'
+        fdroidserver.deploy.config['path_to_custom_rclone_config'] = str(rclone_file)
+        fdroidserver.deploy.options = Options
+
+        # write out destination path
+        destination = Path('some_bucket_folder/fdroid')
+        destination.mkdir(parents=True, exist_ok=True)
+        dest_apk = Path(destination) / fake_apk
+        dest_index = Path(destination) / fake_index
+        self.assertFalse(dest_apk.is_file())
+        self.assertFalse(dest_index.is_file())
+        repo_section = str(repo)
+        # fdroidserver.deploy.USER_RCLONE_CONF = str(rclone_file)
+        fdroidserver.deploy.update_remote_storage_with_rclone(
+            repo_section, is_index_only=True
+        )
+        self.assertFalse(dest_apk.is_file())
+        self.assertTrue(dest_index.is_file())
 
     def test_update_serverwebroot(self):
         """rsync works with file paths, so this test uses paths for the URLs"""
@@ -178,8 +231,7 @@ class DeployTest(unittest.TestCase):
         fake_apk = repo / 'fake.apk'
         with fake_apk.open('w') as fp:
             fp.write('not an APK, but has the right filename')
-        for i in fdroidserver.deploy.INDEX_FILES:
+        fake_index = repo / fdroidserver.deploy.INDEX_FILES[0]
-            fake_index = repo / i
         with fake_index.open('w') as fp:
             fp.write('not an index, but has the right filename')
         url = Path('url')