diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py index afed4e30..64b19406 100644 --- a/fdroidserver/deploy.py +++ b/fdroidserver/deploy.py @@ -25,10 +25,10 @@ import re import subprocess import time import urllib -from typing import Dict, Optional, List +from typing import Dict, List from git import Repo import yaml -from argparse import ArgumentParser, Namespace +from argparse import ArgumentParser import logging from shlex import split import shutil @@ -41,7 +41,6 @@ from . import index from .exception import FDroidException config = None -options: Optional[Namespace] = None start_timestamp = time.gmtime() GIT_BRANCH = 'master' @@ -126,7 +125,9 @@ def update_awsbucket(repo_section, is_index_only=False, verbose=False, quiet=Fal if config['s3cmd'] is True and config['rclone'] is not True: update_awsbucket_s3cmd(repo_section, is_index_only) if config['rclone'] is True and config['s3cmd'] is not True: - update_remote_storage_with_rclone(repo_section, is_index_only, verbose, quiet) + update_remote_storage_with_rclone( + repo_section, is_index_only, verbose, quiet + ) elif common.set_command_in_config('s3cmd'): update_awsbucket_s3cmd(repo_section, is_index_only) @@ -191,8 +192,11 @@ def update_awsbucket_s3cmd(repo_section, is_index_only=False): ) if is_index_only: - logging.debug(_('s3cmd syncs indexes from {path} to {url} and deletes removed') - .format(path=repo_section, url=s3url)) + logging.debug( + _('s3cmd syncs indexes from {path} to {url} and deletes removed').format( + path=repo_section, url=s3url + ) + ) sync_indexes_flags = [] sync_indexes_flags.extend(_get_index_includes(repo_section)) sync_indexes_flags.append('--delete-removed') @@ -201,7 +205,9 @@ def update_awsbucket_s3cmd(repo_section, is_index_only=False): sync_indexes_flags.append('--no-check-md5') else: sync_indexes_flags.append('--check-md5') - returncode = subprocess.call(s3cmd_sync + sync_indexes_flags + [repo_section, s3url]) + returncode = subprocess.call( + s3cmd_sync + sync_indexes_flags + [repo_section, s3url] + ) if returncode != 0: raise FDroidException() else: @@ -222,8 +228,11 @@ def update_awsbucket_s3cmd(repo_section, is_index_only=False): if returncode != 0: raise FDroidException() - logging.debug(_('s3cmd sync indexes {path} to {url} and delete') - .format(path=repo_section, url=s3url)) + logging.debug( + _('s3cmd sync indexes {path} to {url} and delete').format( + path=repo_section, url=s3url + ) + ) s3cmd_sync.append('--delete-removed') s3cmd_sync.append('--delete-after') if options.no_checksum: @@ -234,7 +243,9 @@ def update_awsbucket_s3cmd(repo_section, is_index_only=False): raise FDroidException() -def update_remote_storage_with_rclone(repo_section, is_index_only=False, verbose=False, quiet=False): +def update_remote_storage_with_rclone( + repo_section, is_index_only=False, verbose=False, quiet=False +): """ Upload fdroid repo folder to remote storage using rclone sync. @@ -353,10 +364,7 @@ def update_remote_storage_with_rclone(repo_section, is_index_only=False, verbose ) logging.debug( - "rclone sync all files in " - + source - + ' to ' - + complete_remote_path + "rclone sync all files in " + source + ' to ' + complete_remote_path ) if subprocess.call(rclone_sync_command) != 0: @@ -412,12 +420,22 @@ def update_awsbucket_libcloud(repo_section, is_index_only=False): objs[obj.name] = obj if is_index_only: - index_files = [f"{os.getcwd()}/{name}" for name in _get_index_file_paths(repo_section)] - files_to_upload = [os.path.join(root, name) for root, dirs, files in os.walk(os.path.join(os.getcwd(), repo_section)) for name in files] + index_files = [ + f"{os.getcwd()}/{name}" for name in _get_index_file_paths(repo_section) + ] + files_to_upload = [ + os.path.join(root, name) + for root, dirs, files in os.walk(os.path.join(os.getcwd(), repo_section)) + for name in files + ] files_to_upload = list(set(files_to_upload) & set(index_files)) else: - files_to_upload = [os.path.join(root, name) for root, dirs, files in os.walk(os.path.join(os.getcwd(), repo_section)) for name in files] + files_to_upload = [ + os.path.join(root, name) + for root, dirs, files in os.walk(os.path.join(os.getcwd(), repo_section)) + for name in files + ] for file_to_upload in files_to_upload: upload = False @@ -528,9 +546,15 @@ def update_serverwebroot(serverwebroot, repo_section): if subprocess.call(rsyncargs + [repo_section, url]) != 0: raise FDroidException() # upload "current version" symlinks if requested - if config and config.get('make_current_version_link') and repo_section == 'repo': + if ( + config + and config.get('make_current_version_link') + and repo_section == 'repo' + ): links_to_upload = [] - for f in glob.glob('*.apk') + glob.glob('*.apk.asc') + glob.glob('*.apk.sig'): + for f in ( + glob.glob('*.apk') + glob.glob('*.apk.asc') + glob.glob('*.apk.sig') + ): if os.path.islink(f): links_to_upload.append(f) if len(links_to_upload) > 0: @@ -708,35 +732,42 @@ def update_servergitmirrors(servergitmirrors, repo_section): # trailing slashes have a meaning in rsync which is not needed here, so # make sure both paths have exactly one trailing slash if is_index_only: - files_to_sync = [str(workspace_dir / repo_section / index_file) for index_file in INDEX_FILES] + files_to_sync = [ + str(workspace_dir / repo_section / index_file) + for index_file in INDEX_FILES + ] else: files_to_sync = [str(workspace_dir / repo_section).rstrip('/') + '/'] - common.local_rsync(common.get_options(), - files_to_sync, - git_repodir.rstrip('/') + '/') + common.local_rsync( + common.get_options(), files_to_sync, git_repodir.rstrip('/') + '/' + ) - upload_to_servergitmirror(mirror_config=d, - local_repo=repo, - enabled_remotes=enabled_remotes, - repo_section=repo_section, - is_index_only=is_index_only, - fdroid_dir=git_fdroiddir, - git_mirror_path=str(git_mirror_path), - ssh_cmd=ssh_cmd, - progress=progress) + upload_to_servergitmirror( + mirror_config=d, + local_repo=repo, + enabled_remotes=enabled_remotes, + repo_section=repo_section, + is_index_only=is_index_only, + fdroid_dir=git_fdroiddir, + git_mirror_path=str(git_mirror_path), + ssh_cmd=ssh_cmd, + progress=progress, + ) if progress: progressbar.done() -def upload_to_servergitmirror(mirror_config: Dict[str, str], - local_repo: Repo, - enabled_remotes: List[str], - repo_section: str, - is_index_only: bool, - fdroid_dir: str, - git_mirror_path: str, - ssh_cmd: str, - progress: git.RemoteProgress) -> None: +def upload_to_servergitmirror( + mirror_config: Dict[str, str], + local_repo: Repo, + enabled_remotes: List[str], + repo_section: str, + is_index_only: bool, + fdroid_dir: str, + git_mirror_path: str, + ssh_cmd: str, + progress: git.RemoteProgress, +) -> None: remote_branch_name = GIT_BRANCH local_branch_name = local_repo.active_branch.name @@ -754,7 +785,9 @@ def upload_to_servergitmirror(mirror_config: Dict[str, str], logging.info("git status:", local_repo.git.status()) if is_index_only: - local_repo.index.add(_get_index_file_paths(os.path.join('fdroid', repo_section))) + local_repo.index.add( + _get_index_file_paths(os.path.join('fdroid', repo_section)) + ) else: # sadly index.add don't allow the --all parameter logging.debug('Adding all files to git mirror') @@ -763,16 +796,13 @@ def upload_to_servergitmirror(mirror_config: Dict[str, str], logging.debug('Committing files into git mirror') local_repo.index.commit("fdroidserver git-mirror") - # only deploy to GitLab Artifacts if too big for GitLab Pages if common.get_dir_size(fdroid_dir) <= common.GITLAB_COM_PAGES_MAX_SIZE: gitlab_ci_job_name = 'pages' else: gitlab_ci_job_name = 'GitLab Artifacts' logging.warning( - _( - 'Skipping GitLab Pages mirror because the repo is too large (>%.2fGB)!' - ) + _('Skipping GitLab Pages mirror because the repo is too large (>%.2fGB)!') % (common.GITLAB_COM_PAGES_MAX_SIZE / 1000000000) ) @@ -803,19 +833,29 @@ def upload_to_servergitmirror(mirror_config: Dict[str, str], logging.info(_('Pushing to {url}').format(url=remote.url)) with local_repo.git.custom_environment(GIT_SSH_COMMAND=ssh_cmd): pushinfos = remote.push( - f"{local_branch_name}:{remote_branch_name}", force=True, set_upstream=True, progress=progress + f"{local_branch_name}:{remote_branch_name}", + force=True, + set_upstream=True, + progress=progress, ) for pushinfo in pushinfos: - if pushinfo.flags & (git.remote.PushInfo.ERROR - | git.remote.PushInfo.REJECTED - | git.remote.PushInfo.REMOTE_FAILURE - | git.remote.PushInfo.REMOTE_REJECTED): + if pushinfo.flags & ( + git.remote.PushInfo.ERROR + | git.remote.PushInfo.REJECTED + | git.remote.PushInfo.REMOTE_FAILURE + | git.remote.PushInfo.REMOTE_REJECTED + ): # Show potentially useful messages from git remote for line in progress.other_lines: if line.startswith('remote:'): logging.debug(line) - raise FDroidException(remote.url + ' push failed: ' + str(pushinfo.flags) - + ' ' + pushinfo.summary) + raise FDroidException( + remote.url + + ' push failed: ' + + str(pushinfo.flags) + + ' ' + + pushinfo.summary + ) else: logging.debug(remote.url + ': ' + pushinfo.summary) @@ -1083,14 +1123,6 @@ def main(): parser = ArgumentParser() common.setup_global_opts(parser) - parser.add_argument("-i", "--identity-file", default=None, - help=_("Specify an identity file to provide to SSH for rsyncing")) - parser.add_argument("--local-copy-dir", default=None, - help=_("Specify a local folder to sync the repo to")) - parser.add_argument("--no-checksum", action="store_true", default=False, - help=_("Don't use rsync checksums")) - parser.add_argument("--no-keep-git-mirror-archive", action="store_true", default=False, - help=_("If a git mirror gets to big, allow the archive to be deleted")) parser.add_argument( "-i", "--identity-file", @@ -1114,8 +1146,8 @@ def main(): default=False, help=_("If a git mirror gets to big, allow the archive to be deleted"), ) - options = parser.parse_args() - config = common.read_config(options) + options = common.parse_args(parser) + config = common.read_config() if config.get('nonstandardwebroot') is True: standardwebroot = False diff --git a/tests/deploy.TestCase b/tests/deploy.TestCase index 57f15392..860fa416 100755 --- a/tests/deploy.TestCase +++ b/tests/deploy.TestCase @@ -40,7 +40,7 @@ class DeployTest(unittest.TestCase): self._td = mkdtemp() self.testdir = self._td.name - fdroidserver.deploy.options = mock.Mock() + fdroidserver.common.options = mock.Mock() fdroidserver.deploy.config = {} fdroidserver.deploy.USER_RCLONE_CONF = False @@ -73,7 +73,7 @@ class DeployTest(unittest.TestCase): url1.mkdir(parents=True) # setup parameters for this test run - fdroidserver.deploy.options.identity_file = None + fdroidserver.common.options.identity_file = None fdroidserver.deploy.config['make_current_version_link'] = False dest_apk0 = url0 / fake_apk @@ -117,7 +117,6 @@ class DeployTest(unittest.TestCase): with fake_index.open('w') as fp: fp.write('not an index, but has the right filename') - # write out rclone config for test use rclone_config = configparser.ConfigParser() rclone_config.add_section("test-local-config") @@ -134,7 +133,7 @@ class DeployTest(unittest.TestCase): fdroidserver.deploy.config['rclone'] = True fdroidserver.deploy.config['rclone_config'] = 'test-local-config' fdroidserver.deploy.config['path_to_custom_rclone_config'] = str(rclone_file) - fdroidserver.deploy.options = Options + fdroidserver.common.options = Options # write out destination path destination = Path('some_bucket_folder/fdroid') @@ -178,7 +177,7 @@ class DeployTest(unittest.TestCase): fdroidserver.deploy.config['rclone'] = True fdroidserver.deploy.config['rclone_config'] = 'test-local-config' fdroidserver.deploy.config['path_to_custom_rclone_config'] = str(rclone_file) - fdroidserver.deploy.options = Options + fdroidserver.common.options = Options # write out destination path destination = Path('some_bucket_folder/fdroid') @@ -210,9 +209,9 @@ class DeployTest(unittest.TestCase): url.mkdir() # setup parameters for this test run - fdroidserver.deploy.options = mock.Mock() - fdroidserver.deploy.options.identity_file = None - fdroidserver.deploy.options.identity_file = None + fdroidserver.common.options = mock.Mock() + fdroidserver.common.options.identity_file = None + fdroidserver.common.options.identity_file = None fdroidserver.deploy.config['make_current_version_link'] = False dest_apk = Path(url) / fake_apk @@ -238,8 +237,8 @@ class DeployTest(unittest.TestCase): url.mkdir() # setup parameters for this test run - fdroidserver.deploy.options = mock.Mock() - fdroidserver.deploy.options.identity_file = None + fdroidserver.common.options = mock.Mock() + fdroidserver.common.options.identity_file = None fdroidserver.deploy.config['make_current_version_link'] = False dest_apk = Path(url) / fake_apk @@ -269,7 +268,7 @@ class DeployTest(unittest.TestCase): fdroidserver.common.options.identity_file = None fdroidserver.common.options.verbose = False fdroidserver.common.options.quiet = True - fdroidserver.deploy.options.index_only = False + fdroidserver.common.options.index_only = False fdroidserver.deploy.config = {'make_current_version_link': True} url = "example.com:/var/www/fdroid" repo_section = 'repo' @@ -358,12 +357,12 @@ class DeployTest(unittest.TestCase): def test_update_serverwebroot_make_cur_version_link_in_index_only_mode(self): # setup parameters for this test run - fdroidserver.deploy.options = mock.Mock() - fdroidserver.deploy.options.no_checksum = True - fdroidserver.deploy.options.identity_file = None - fdroidserver.deploy.options.verbose = False - fdroidserver.deploy.options.quiet = True - fdroidserver.deploy.options.identity_file = None + fdroidserver.common.options = mock.Mock() + fdroidserver.common.options.no_checksum = True + fdroidserver.common.options.identity_file = None + fdroidserver.common.options.verbose = False + fdroidserver.common.options.quiet = True + fdroidserver.common.options.identity_file = None fdroidserver.deploy.config['make_current_version_link'] = True url = "example.com:/var/www/fdroid" repo_section = 'repo' @@ -382,6 +381,7 @@ class DeployTest(unittest.TestCase): '--delete-after', '--safe-links', '--quiet', + 'repo/altstore-index.json', 'repo/entry.jar', 'repo/entry.json', 'repo/entry.json.asc', @@ -447,7 +447,7 @@ class DeployTest(unittest.TestCase): fdroidserver.common.options.verbose = True fdroidserver.common.options.quiet = False fdroidserver.common.options.identity_file = None - fdroidserver.deploy.options.index_only = False + fdroidserver.common.options.index_only = False fdroidserver.deploy.config = {'identity_file': './id_rsa'} url = "example.com:/var/www/fdroid" repo_section = 'archive' @@ -522,11 +522,11 @@ class DeployTest(unittest.TestCase): def test_update_serverwebroot_with_id_file_in_index_only_mode(self): # setup parameters for this test run - fdroidserver.deploy.options = mock.Mock() - fdroidserver.deploy.options.no_chcksum = False - fdroidserver.deploy.options.verbose = True - fdroidserver.deploy.options.quiet = False - fdroidserver.deploy.options.identity_file = None + fdroidserver.common.options = mock.Mock() + fdroidserver.common.options.no_chcksum = False + fdroidserver.common.options.verbose = True + fdroidserver.common.options.quiet = False + fdroidserver.common.options.identity_file = None fdroidserver.deploy.config['identity_file'] = './id_rsa' fdroidserver.deploy.config['make_current_version_link'] = False url = "example.com:/var/www/fdroid" @@ -549,6 +549,7 @@ class DeployTest(unittest.TestCase): '-e', 'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ' + fdroidserver.deploy.config['identity_file'], + 'archive/altstore-index.json', 'archive/entry.jar', 'archive/entry.json', 'archive/entry.json.asc', @@ -592,8 +593,8 @@ class DeployTest(unittest.TestCase): not os.getenv('VIRUSTOTAL_API_KEY'), 'VIRUSTOTAL_API_KEY is not set' ) def test_upload_to_virustotal(self): - fdroidserver.deploy.options = mock.Mock() - fdroidserver.deploy.options.verbose = True + fdroidserver.common.options = mock.Mock() + fdroidserver.common.options.verbose = True virustotal_apikey = os.getenv('VIRUSTOTAL_API_KEY') fdroidserver.deploy.upload_to_virustotal('repo', virustotal_apikey) @@ -611,10 +612,10 @@ class DeployTest(unittest.TestCase): def test_update_awsbucket_s3cmd(self): # setup parameters for this test run - fdroidserver.deploy.options = mock.Mock() - fdroidserver.deploy.options.no_checksum = True - fdroidserver.deploy.options.verbose = False - fdroidserver.deploy.options.quiet = True + fdroidserver.common.options = mock.Mock() + fdroidserver.common.options.no_checksum = True + fdroidserver.common.options.verbose = False + fdroidserver.common.options.quiet = True config = {} fdroidserver.common.fill_config_defaults(config) @@ -651,6 +652,8 @@ class DeployTest(unittest.TestCase): '--acl-public', '--quiet', '--exclude', + 'repo/altstore-index.json', + '--exclude', 'repo/entry.jar', '--exclude', 'repo/entry.json', @@ -686,6 +689,8 @@ class DeployTest(unittest.TestCase): '--acl-public', '--quiet', '--exclude', + 'repo/altstore-index.json', + '--exclude', 'repo/entry.jar', '--exclude', 'repo/entry.json', @@ -742,10 +747,10 @@ class DeployTest(unittest.TestCase): def test_update_awsbucket_s3cmd_in_index_only_mode(self): # setup parameters for this test run - fdroidserver.deploy.options = mock.Mock() - fdroidserver.deploy.options.no_checksum = True - fdroidserver.deploy.options.verbose = False - fdroidserver.deploy.options.quiet = True + fdroidserver.common.options = mock.Mock() + fdroidserver.common.options.no_checksum = True + fdroidserver.common.options.verbose = False + fdroidserver.common.options.quiet = True config = {} fdroidserver.common.fill_config_defaults(config) @@ -782,6 +787,8 @@ class DeployTest(unittest.TestCase): '--acl-public', '--quiet', '--include', + 'repo/altstore-index.json', + '--include', 'repo/entry.jar', '--include', 'repo/entry.json', @@ -820,7 +827,7 @@ class DeployTest(unittest.TestCase): os.symlink('repo/com.example.sym.apk.sig', 'Sym.apk.sig') with mock.patch('subprocess.call', side_effect=update_awsbucket_s3cmd_call): fdroidserver.deploy.update_awsbucket_s3cmd( - repo_section, index_only=True + repo_section, is_index_only=True ) self.assertEqual(call_iteration, 2, 'expected 2 invocations of subprocess.call') @@ -828,10 +835,10 @@ class DeployTest(unittest.TestCase): from libcloud.storage.base import Container # setup parameters for this test run - fdroidserver.deploy.options = mock.Mock() - fdroidserver.deploy.options.no_checksum = True - fdroidserver.deploy.options.verbose = False - fdroidserver.deploy.options.quiet = True + fdroidserver.common.options = mock.Mock() + fdroidserver.common.options.no_checksum = True + fdroidserver.common.options.verbose = False + fdroidserver.common.options.quiet = True config = {} fdroidserver.common.fill_config_defaults(config) @@ -872,7 +879,10 @@ class DeployTest(unittest.TestCase): container_name=fdroidserver.deploy.config["awsbucket"] ) mock_container.list_objects.assert_called_once_with() - files_to_upload = ['fdroid/repo/Sym.apk', 'fdroid/repo/entry.jar'] + files_to_upload = [ + 'fdroid/repo/Sym.apk', + f"fdroid/repo/{fdroidserver.deploy.INDEX_FILES[0]}", + ] calls = [ mock.call( iterator=mock.ANY, @@ -889,10 +899,10 @@ class DeployTest(unittest.TestCase): from libcloud.storage.base import Container # setup parameters for this test run - fdroidserver.deploy.options = mock.Mock() - fdroidserver.deploy.options.no_checksum = True - fdroidserver.deploy.options.verbose = False - fdroidserver.deploy.options.quiet = True + fdroidserver.common.options = mock.Mock() + fdroidserver.common.options.no_checksum = True + fdroidserver.common.options.verbose = False + fdroidserver.common.options.quiet = True config = {} fdroidserver.common.fill_config_defaults(config) @@ -927,7 +937,7 @@ class DeployTest(unittest.TestCase): mock_driver.get_container.return_value = mock_container mock_driver.upload_object_via_stream.return_value = None - fdroidserver.deploy.update_awsbucket_libcloud(repo_section, index_only=True) + fdroidserver.deploy.update_awsbucket_libcloud(repo_section, is_index_only=True) mock_driver.get_container.assert_called_once_with( container_name=fdroidserver.deploy.config["awsbucket"] @@ -951,11 +961,11 @@ class DeployTest(unittest.TestCase): def test_update_servergitmirrors(self): # setup parameters for this test run - fdroidserver.deploy.options = mock.Mock() - fdroidserver.deploy.options.identity_file = None - fdroidserver.deploy.options.no_keep_git_mirror_archive = False - fdroidserver.deploy.options.verbose = False - fdroidserver.deploy.options.quiet = True + fdroidserver.common.options = mock.Mock() + fdroidserver.common.options.identity_file = None + fdroidserver.common.options.no_keep_git_mirror_archive = False + fdroidserver.common.options.verbose = False + fdroidserver.common.options.quiet = True config = {} fdroidserver.common.fill_config_defaults(config) @@ -1002,11 +1012,11 @@ class DeployTest(unittest.TestCase): def test_update_servergitmirrors_in_index_only_mode(self): # setup parameters for this test run - fdroidserver.deploy.options = mock.Mock() - fdroidserver.deploy.options.identity_file = None - fdroidserver.deploy.options.no_keep_git_mirror_archive = False - fdroidserver.deploy.options.verbose = False - fdroidserver.deploy.options.quiet = True + fdroidserver.common.options = mock.Mock() + fdroidserver.common.options.identity_file = None + fdroidserver.common.options.no_keep_git_mirror_archive = False + fdroidserver.common.options.verbose = False + fdroidserver.common.options.quiet = True config = {} fdroidserver.common.fill_config_defaults(config) @@ -1062,12 +1072,12 @@ class DeployTest(unittest.TestCase): def test_upload_to_servergitmirror_in_index_only_mode(self): # setup parameters for this test run - fdroidserver.deploy.options = mock.Mock() - fdroidserver.deploy.options.identity_file = None - fdroidserver.deploy.options.no_keep_git_mirror_archive = False - fdroidserver.deploy.options.verbose = False - fdroidserver.deploy.options.quiet = True - fdroidserver.deploy.options.identity_file = None + fdroidserver.common.options = mock.Mock() + fdroidserver.common.options.identity_file = None + fdroidserver.common.options.no_keep_git_mirror_archive = False + fdroidserver.common.options.verbose = False + fdroidserver.common.options.quiet = True + fdroidserver.common.options.identity_file = None config = {} fdroidserver.common.fill_config_defaults(config)