From cb18d101976cce00166d08f5a11cd9ce97556a2c Mon Sep 17 00:00:00 2001 From: proletarius101 Date: Tue, 4 Jun 2024 21:15:21 +0800 Subject: [PATCH] feat(deploy): add index-only mode to the rclone deployment method --- fdroidserver/deploy.py | 94 ++++++++++++++++++++++-------------------- tests/deploy.TestCase | 66 +++++++++++++++++++++++++---- 2 files changed, 109 insertions(+), 51 deletions(-) diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py index 1279683e..afed4e30 100644 --- a/fdroidserver/deploy.py +++ b/fdroidserver/deploy.py @@ -126,12 +126,12 @@ def update_awsbucket(repo_section, is_index_only=False, verbose=False, quiet=Fal if config['s3cmd'] is True and config['rclone'] is not True: update_awsbucket_s3cmd(repo_section, is_index_only) if config['rclone'] is True and config['s3cmd'] is not True: - update_remote_storage_with_rclone(repo_section, verbose, quiet) + update_remote_storage_with_rclone(repo_section, is_index_only, verbose, quiet) elif common.set_command_in_config('s3cmd'): update_awsbucket_s3cmd(repo_section, is_index_only) elif common.set_command_in_config('rclone'): - update_remote_storage_with_rclone(repo_section, verbose, quiet) + update_remote_storage_with_rclone(repo_section, is_index_only, verbose, quiet) else: update_awsbucket_libcloud(repo_section, is_index_only) @@ -234,7 +234,7 @@ def update_awsbucket_s3cmd(repo_section, is_index_only=False): raise FDroidException() -def update_remote_storage_with_rclone(repo_section, verbose=False, quiet=False): +def update_remote_storage_with_rclone(repo_section, is_index_only=False, verbose=False, quiet=False): """ Upload fdroid repo folder to remote storage using rclone sync. @@ -286,46 +286,18 @@ def update_remote_storage_with_rclone(repo_section, verbose=False, quiet=False): _('To use rclone, rclone_config and awsbucket must be set in config.yml!') ) - if isinstance(config['rclone_config'], str): - rclone_sync_command = ( - 'rclone sync ' - + repo_section - + ' ' - + config['rclone_config'] - + ':' - + config['awsbucket'] - + '/' - + upload_dir - ) + if is_index_only: + sources = _get_index_file_paths(repo_section) + else: + sources = repo_section - rclone_sync_command = split(rclone_sync_command) - - if verbose: - rclone_sync_command += ['--verbose'] - elif quiet: - rclone_sync_command += ['--quiet'] - - if configfilename: - rclone_sync_command += split('--config=' + configfilename) - - complete_remote_path = ( - config['rclone_config'] + ':' + config['awsbucket'] + '/' + upload_dir - ) - - logging.debug( - "rclone sync all files in " + repo_section + ' to ' + complete_remote_path - ) - - if subprocess.call(rclone_sync_command) != 0: - raise FDroidException() - - if isinstance(config['rclone_config'], list): - for remote_config in config['rclone_config']: + for source in sources: + if isinstance(config['rclone_config'], str): rclone_sync_command = ( 'rclone sync ' - + repo_section + + source + ' ' - + remote_config + + config['rclone_config'] + ':' + config['awsbucket'] + '/' @@ -343,19 +315,53 @@ def update_remote_storage_with_rclone(repo_section, verbose=False, quiet=False): rclone_sync_command += split('--config=' + configfilename) complete_remote_path = ( - remote_config + ':' + config['awsbucket'] + '/' + upload_dir + config['rclone_config'] + ':' + config['awsbucket'] + '/' + upload_dir ) logging.debug( - "rclone sync all files in " - + repo_section - + ' to ' - + complete_remote_path + "rclone sync all files in " + source + ' to ' + complete_remote_path ) if subprocess.call(rclone_sync_command) != 0: raise FDroidException() + if isinstance(config['rclone_config'], list): + for remote_config in config['rclone_config']: + rclone_sync_command = ( + 'rclone sync ' + + source + + ' ' + + remote_config + + ':' + + config['awsbucket'] + + '/' + + upload_dir + ) + + rclone_sync_command = split(rclone_sync_command) + + if verbose: + rclone_sync_command += ['--verbose'] + elif quiet: + rclone_sync_command += ['--quiet'] + + if configfilename: + rclone_sync_command += split('--config=' + configfilename) + + complete_remote_path = ( + remote_config + ':' + config['awsbucket'] + '/' + upload_dir + ) + + logging.debug( + "rclone sync all files in " + + source + + ' to ' + + complete_remote_path + ) + + if subprocess.call(rclone_sync_command) != 0: + raise FDroidException() + def update_awsbucket_libcloud(repo_section, is_index_only=False): """No summary. diff --git a/tests/deploy.TestCase b/tests/deploy.TestCase index c5fd9b0d..57f15392 100755 --- a/tests/deploy.TestCase +++ b/tests/deploy.TestCase @@ -113,6 +113,10 @@ class DeployTest(unittest.TestCase): fake_apk = repo / 'another_fake.apk' with fake_apk.open('w') as fp: fp.write('not an APK, but has the right filename') + fake_index = repo / fdroidserver.deploy.INDEX_FILES[0] + with fake_index.open('w') as fp: + fp.write('not an index, but has the right filename') + # write out rclone config for test use rclone_config = configparser.ConfigParser() @@ -135,12 +139,61 @@ class DeployTest(unittest.TestCase): # write out destination path destination = Path('some_bucket_folder/fdroid') destination.mkdir(parents=True, exist_ok=True) - dest_path = Path(destination) / fake_apk - self.assertFalse(dest_path.is_file()) + dest_apk = Path(destination) / fake_apk + dest_index = Path(destination) / fake_index + self.assertFalse(dest_apk.is_file()) + self.assertFalse(dest_index.is_file()) repo_section = str(repo) # fdroidserver.deploy.USER_RCLONE_CONF = str(rclone_file) fdroidserver.deploy.update_remote_storage_with_rclone(repo_section) - self.assertFalse(dest_path.is_file()) + self.assertTrue(dest_apk.is_file()) + self.assertTrue(dest_index.is_file()) + + @unittest.skipUnless(shutil.which('rclone'), '/usr/bin/rclone') + def test_update_remote_storage_with_rclone_in_index_only_mode(self): + os.chdir(self.testdir) + repo = Path('repo') + repo.mkdir(parents=True, exist_ok=True) + + fake_apk = repo / 'another_fake.apk' + with fake_apk.open('w') as fp: + fp.write('not an APK, but has the right filename') + fake_index = repo / fdroidserver.deploy.INDEX_FILES[0] + with fake_index.open('w') as fp: + fp.write('not an index, but has the right filename') + + # write out rclone config for test use + rclone_config = configparser.ConfigParser() + rclone_config.add_section("test-local-config") + rclone_config.set("test-local-config", "type", "local") + + rclone_config_path = Path('rclone_config_path') + rclone_config_path.mkdir(parents=True, exist_ok=True) + rclone_file = rclone_config_path / 'rclone.conf' + with open(rclone_file, 'w') as configfile: + rclone_config.write(configfile) + + # setup parameters for this test run + fdroidserver.deploy.config['awsbucket'] = 'test_bucket_folder' + fdroidserver.deploy.config['rclone'] = True + fdroidserver.deploy.config['rclone_config'] = 'test-local-config' + fdroidserver.deploy.config['path_to_custom_rclone_config'] = str(rclone_file) + fdroidserver.deploy.options = Options + + # write out destination path + destination = Path('some_bucket_folder/fdroid') + destination.mkdir(parents=True, exist_ok=True) + dest_apk = Path(destination) / fake_apk + dest_index = Path(destination) / fake_index + self.assertFalse(dest_apk.is_file()) + self.assertFalse(dest_index.is_file()) + repo_section = str(repo) + # fdroidserver.deploy.USER_RCLONE_CONF = str(rclone_file) + fdroidserver.deploy.update_remote_storage_with_rclone( + repo_section, is_index_only=True + ) + self.assertFalse(dest_apk.is_file()) + self.assertTrue(dest_index.is_file()) def test_update_serverwebroot(self): """rsync works with file paths, so this test uses paths for the URLs""" @@ -178,10 +231,9 @@ class DeployTest(unittest.TestCase): fake_apk = repo / 'fake.apk' with fake_apk.open('w') as fp: fp.write('not an APK, but has the right filename') - for i in fdroidserver.deploy.INDEX_FILES: - fake_index = repo / i - with fake_index.open('w') as fp: - fp.write('not an index, but has the right filename') + fake_index = repo / fdroidserver.deploy.INDEX_FILES[0] + with fake_index.open('w') as fp: + fp.write('not an index, but has the right filename') url = Path('url') url.mkdir()