From 4c15cea78a9c80c6c4ea6e4ae9cc222dbaff6226 Mon Sep 17 00:00:00 2001 From: proletarius101 Date: Mon, 26 Feb 2024 22:57:05 +0800 Subject: [PATCH] fix(deploy): remove the index-only mode from elsewhere except the config file To enable per-remote configuration --- completion/bash-completion | 2 +- fdroidserver/common.py | 3 +- fdroidserver/deploy.py | 128 ++++++++++++++++++------------------- tests/deploy.TestCase | 85 +++++------------------- 4 files changed, 80 insertions(+), 138 deletions(-) diff --git a/completion/bash-completion b/completion/bash-completion index aa68a220..940b188b 100644 --- a/completion/bash-completion +++ b/completion/bash-completion @@ -264,7 +264,7 @@ __complete_nightly() { __complete_deploy() { opts="-i -v -q" lopts="--identity-file --local-copy-dir --sync-from-local-copy-dir - --verbose --quiet --no-checksum --no-keep-git-mirror-archive --index-only" + --verbose --quiet --no-checksum --no-keep-git-mirror-archive" __complete_options } diff --git a/fdroidserver/common.py b/fdroidserver/common.py index ea4f6232..b8374109 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -4301,7 +4301,7 @@ def get_app_display_name(app): return app.get('AutoName') or app['id'] -def local_rsync(options, from_paths: List[str], todir: str, args: List[str] = []): +def local_rsync(options, from_paths: List[str], todir: str): """Rsync method for local to local copying of things. This is an rsync wrapper with all the settings for safe use within @@ -4318,7 +4318,6 @@ def local_rsync(options, from_paths: List[str], todir: str, args: List[str] = [] rsyncargs += ['--verbose'] if options.quiet: rsyncargs += ['--quiet'] - rsyncargs += args logging.debug(' '.join(rsyncargs + from_paths + [todir])) if subprocess.call(rsyncargs + from_paths + [todir]) != 0: raise FDroidException() diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py index 5d7b2797..cf6e6ab5 100644 --- a/fdroidserver/deploy.py +++ b/fdroidserver/deploy.py @@ -92,7 +92,7 @@ def _get_index_includes(base_dir): return index_includes -def update_awsbucket(repo_section, verbose=False, quiet=False): +def update_awsbucket(repo_section, index_only=False, verbose=False, quiet=False): """Upload the contents of the directory `repo_section` (including subdirectories) to the AWS S3 "bucket". The contents of that subdir of the @@ -112,26 +112,26 @@ def update_awsbucket(repo_section, verbose=False, quiet=False): logging.warning( 'No syncing tool set in config.yml!. Defaulting to using s3cmd' ) - update_awsbucket_s3cmd(repo_section) + update_awsbucket_s3cmd(repo_section, index_only) if config['s3cmd'] is True and config['rclone'] is True: logging.warning( 'Both syncing tools set in config.yml!. Defaulting to using s3cmd' ) - update_awsbucket_s3cmd(repo_section) + update_awsbucket_s3cmd(repo_section, index_only) if config['s3cmd'] is True and config['rclone'] is not True: - update_awsbucket_s3cmd(repo_section) + update_awsbucket_s3cmd(repo_section, index_only) if config['rclone'] is True and config['s3cmd'] is not True: update_remote_storage_with_rclone(repo_section, verbose, quiet) elif common.set_command_in_config('s3cmd'): - update_awsbucket_s3cmd(repo_section) + update_awsbucket_s3cmd(repo_section, index_only) elif common.set_command_in_config('rclone'): update_remote_storage_with_rclone(repo_section, verbose, quiet) else: - update_awsbucket_libcloud(repo_section) + update_awsbucket_libcloud(repo_section, index_only) -def update_awsbucket_s3cmd(repo_section): +def update_awsbucket_s3cmd(repo_section, index_only=False): """Upload using the CLI tool s3cmd, which provides rsync-like sync. The upload is done in multiple passes to reduce the chance of @@ -185,7 +185,7 @@ def update_awsbucket_s3cmd(repo_section): ) ) - if options.index_only: + if index_only: logging.debug(_('s3cmd syncs indexes from {path} to {url} and deletes removed') .format(path=repo_section, url=s3url)) sync_indexes_flags = [] @@ -352,7 +352,7 @@ def update_remote_storage_with_rclone(repo_section, verbose=False, quiet=False): raise FDroidException() -def update_awsbucket_libcloud(repo_section): +def update_awsbucket_libcloud(repo_section, index_only=False): """No summary. Upload the contents of the directory `repo_section` (including @@ -400,7 +400,7 @@ def update_awsbucket_libcloud(repo_section): if obj.name.startswith(upload_dir + '/'): objs[obj.name] = obj - if options.index_only: + if index_only: index_files = [f"{os.getcwd()}/{name}" for name in _get_index_file_paths(repo_section)] files_to_upload = [os.path.join(root, name) for root, dirs, files in os.walk(os.path.join(os.getcwd(), repo_section)) for name in files] files_to_upload = list(set(files_to_upload) & set(index_files)) @@ -502,8 +502,9 @@ def update_serverwebroot(serverwebroot, repo_section): 'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ' + config['identity_file'], ] url = serverwebroot['url'] + index_only = serverwebroot.get('index_only', False) logging.info('rsyncing ' + repo_section + ' to ' + url) - if options.index_only: + if index_only: rsyncargs += _get_index_file_paths(repo_section) rsyncargs += [f'{url}/{repo_section}/'] logging.info(rsyncargs) @@ -563,8 +564,7 @@ def sync_from_localcopy(repo_section, local_copy_dir): push to all the servers that are configured. """ - if options.index_only: - raise FDroidException(_('The index only mode cannot be used when syncing to the local copy filesystem')) + logging.info('Syncing from local_copy_dir to this repo.') # trailing slashes have a meaning in rsync which is not needed here, so # make sure both paths have exactly one trailing slash @@ -588,8 +588,6 @@ def update_localcopy(repo_section, local_copy_dir): drive. """ - if options.index_only: - raise FDroidException(_('The index only mode cannot be used when syncing to the local copy filesystem')) # local_copy_dir is guaranteed to have a trailing slash in main() below common.local_rsync(common.get_options(), [repo_section], local_copy_dir) @@ -662,17 +660,11 @@ def update_servergitmirrors(servergitmirrors, repo_section): archive_path = os.path.join(git_mirror_path, 'fdroid', 'archive') shutil.rmtree(archive_path, ignore_errors=True) - if options.index_only: - # rsync is very particular about trailing slashes - common.local_rsync(common.get_options(), - _get_index_file_paths(repo_section), - git_repodir.rstrip('/') + '/') - else: - # trailing slashes have a meaning in rsync which is not needed here, so - # make sure both paths have exactly one trailing slash - common.local_rsync(common.get_options(), - [repo_section.rstrip('/') + '/'], - git_repodir.rstrip('/') + '/') + # trailing slashes have a meaning in rsync which is not needed here, so + # make sure both paths have exactly one trailing slash + common.local_rsync(common.get_options(), + [repo_section.rstrip('/') + '/'], + git_repodir.rstrip('/') + '/') # use custom SSH command if identity_file specified ssh_cmd = 'ssh -oBatchMode=yes' @@ -681,28 +673,6 @@ def update_servergitmirrors(servergitmirrors, repo_section): elif 'identity_file' in config: ssh_cmd += ' -oIdentitiesOnly=yes -i "%s"' % config['identity_file'] - repo = git.Repo.init(git_mirror_path, initial_branch=GIT_BRANCH) - - enabled_remotes = [] - for d in servergitmirrors: - remote_url = d['url'] - name = REMOTE_HOSTNAME_REGEX.sub(r'\1', remote_url) - enabled_remotes.append(name) - r = git.remote.Remote(repo, name) - if r in repo.remotes: - r = repo.remote(name) - if 'set_url' in dir(r): # force remote URL if using GitPython 2.x - r.set_url(remote_url) - else: - repo.create_remote(name, remote_url) - logging.info('Mirroring to: ' + remote_url) - - # sadly index.add don't allow the --all parameter - logging.debug('Adding all files to git mirror') - repo.git.add(all=True) - logging.debug('Committing all files into git mirror') - repo.index.commit("fdroidserver git-mirror") - if options.verbose: progressbar = progress.Bar() @@ -715,23 +685,49 @@ def update_servergitmirrors(servergitmirrors, repo_section): else: progress = None - # only deploy to GitLab Artifacts if too big for GitLab Pages - if common.get_dir_size(git_fdroiddir) <= common.GITLAB_COM_PAGES_MAX_SIZE: - gitlab_ci_job_name = 'pages' - else: - gitlab_ci_job_name = 'GitLab Artifacts' - logging.warning( - _( - 'Skipping GitLab Pages mirror because the repo is too large (>%.2fGB)!' - ) - % (common.GITLAB_COM_PAGES_MAX_SIZE / 1000000000) - ) + repo = git.Repo.init(git_mirror_path, initial_branch=GIT_BRANCH) + initial_commit_ref = repo.head.ref - # push for every remote. This will overwrite the git history - for remote in repo.remotes: - if remote.name not in enabled_remotes: - repo.delete_remote(remote) - continue + enabled_remotes = [] + for d in servergitmirrors: + index_only = d.get('index_only', False) + remote_url = d['url'] + name = REMOTE_HOSTNAME_REGEX.sub(r'\1', remote_url) + enabled_remotes.append(name) + r = git.remote.Remote(repo, name) + if r in repo.remotes: + r = repo.remote(name) + if 'set_url' in dir(r): # force remote URL if using GitPython 2.x + r.set_url(remote_url) + else: + repo.create_remote(name, remote_url) + logging.info('Mirroring to: ' + remote_url) + + if index_only: + logging.debug('Adding index files to git mirror') + repo.index.add(_get_index_file_paths(repo_section)) + else: + # sadly index.add don't allow the --all parameter + logging.debug('Adding all files to git mirror') + repo.git.add(all=True) + + logging.debug('Committing files into git mirror') + repo.index.commit("fdroidserver git-mirror") + + # only deploy to GitLab Artifacts if too big for GitLab Pages + if common.get_dir_size(git_fdroiddir) <= common.GITLAB_COM_PAGES_MAX_SIZE: + gitlab_ci_job_name = 'pages' + else: + gitlab_ci_job_name = 'GitLab Artifacts' + logging.warning( + _( + 'Skipping GitLab Pages mirror because the repo is too large (>%.2fGB)!' + ) + % (common.GITLAB_COM_PAGES_MAX_SIZE / 1000000000) + ) + + # push. This will overwrite the git history + remote = repo.remote(name) if remote.name == 'gitlab': logging.debug('Writing .gitlab-ci.yml to deploy to GitLab Pages') with open(os.path.join(git_mirror_path, ".gitlab-ci.yml"), "wt") as fp: @@ -751,7 +747,7 @@ def update_servergitmirrors(servergitmirrors, repo_section): default_flow_style=False, ) - repo.git.add(all=True) + repo.index.add(['.gitlab-ci.yml']) repo.index.commit("fdroidserver git-mirror: Deploy to GitLab Pages") logging.debug(_('Pushing to {url}').format(url=remote.url)) @@ -780,6 +776,8 @@ def update_servergitmirrors(servergitmirrors, repo_section): else: logging.debug(remote.url + ': ' + pushinfo.summary) + repo.head.reset(initial_commit_ref, index=True, working_tree=True) + if progress: progressbar.done() diff --git a/tests/deploy.TestCase b/tests/deploy.TestCase index dee28340..8465bd10 100755 --- a/tests/deploy.TestCase +++ b/tests/deploy.TestCase @@ -10,6 +10,7 @@ import tempfile import unittest from pathlib import Path from unittest import mock +from fdroidserver import index localmodule = os.path.realpath( os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..') @@ -72,7 +73,6 @@ class DeployTest(unittest.TestCase): # setup parameters for this test run fdroidserver.deploy.options.identity_file = None - fdroidserver.deploy.options.index_only = False fdroidserver.deploy.config['make_current_version_link'] = False dest_apk0 = url0 / fake_apk @@ -159,7 +159,6 @@ class DeployTest(unittest.TestCase): fdroidserver.deploy.options = mock.Mock() fdroidserver.deploy.options.identity_file = None fdroidserver.deploy.options.identity_file = None - fdroidserver.deploy.options.index_only = False fdroidserver.deploy.config['make_current_version_link'] = False dest_apk = Path(url) / fake_apk @@ -187,7 +186,6 @@ class DeployTest(unittest.TestCase): # setup parameters for this test run fdroidserver.deploy.options.identity_file = None - fdroidserver.deploy.options.index_only = True fdroidserver.deploy.config['make_current_version_link'] = False dest_apk = Path(url) / fake_apk @@ -195,7 +193,9 @@ class DeployTest(unittest.TestCase): self.assertFalse(dest_apk.is_file()) self.assertFalse(dest_index.is_file()) - fdroidserver.deploy.update_serverwebroot({'url': str(url)}, 'repo') + fdroidserver.deploy.update_serverwebroot( + {'url': str(url), 'index_only': True}, 'repo' + ) self.assertFalse(dest_apk.is_file()) self.assertTrue(dest_index.is_file()) @@ -310,7 +310,6 @@ class DeployTest(unittest.TestCase): fdroidserver.deploy.options.verbose = False fdroidserver.deploy.options.quiet = True fdroidserver.deploy.options.identity_file = None - fdroidserver.deploy.options.index_only = True fdroidserver.deploy.config['make_current_version_link'] = True url = "example.com:/var/www/fdroid" repo_section = 'repo' @@ -370,34 +369,6 @@ class DeployTest(unittest.TestCase): 'example.com:/var/www/fdroid', ], ) - # elif call_iteration == 1: - # self.assertListEqual( - # cmd, - # [ - # 'rsync', - # '--archive', - # '--delete-after', - # '--safe-links', - # '--quiet', - # 'repo', - # serverwebroot, - # ], - # ) - # elif call_iteration == 2: - # self.assertListEqual( - # cmd, - # [ - # 'rsync', - # '--archive', - # '--delete-after', - # '--safe-links', - # '--quiet', - # 'Sym.apk', - # 'Sym.apk.asc', - # 'Sym.apk.sig', - # 'example.com:/var/www/fdroid', - # ], - # ) else: self.fail('unexpected subprocess.call invocation') call_iteration += 1 @@ -409,7 +380,9 @@ class DeployTest(unittest.TestCase): os.symlink('repo/com.example.sym.apk.asc', 'Sym.apk.asc') os.symlink('repo/com.example.sym.apk.sig', 'Sym.apk.sig') with mock.patch('subprocess.call', side_effect=update_server_webroot_call): - fdroidserver.deploy.update_serverwebroot({'url': url}, repo_section) + fdroidserver.deploy.update_serverwebroot( + {'url': url, 'index_only': True}, repo_section + ) self.assertEqual(call_iteration, 1, 'expected 1 invocations of subprocess.call') def test_update_serverwebroot_with_id_file(self): @@ -499,7 +472,6 @@ class DeployTest(unittest.TestCase): fdroidserver.deploy.options.verbose = True fdroidserver.deploy.options.quiet = False fdroidserver.deploy.options.identity_file = None - fdroidserver.deploy.options.index_only = True fdroidserver.deploy.config['identity_file'] = './id_rsa' fdroidserver.deploy.config['make_current_version_link'] = False url = "example.com:/var/www/fdroid" @@ -550,29 +522,15 @@ class DeployTest(unittest.TestCase): "example.com:/var/www/fdroid/archive/", ], ) - # elif call_iteration == 1: - # self.assertListEqual( - # cmd, - # [ - # 'rsync', - # '--archive', - # '--delete-after', - # '--safe-links', - # '--verbose', - # '-e', - # 'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ' - # + fdroidserver.deploy.config['identity_file'], - # 'archive', - # serverwebroot, - # ], - # ) else: self.fail('unexpected subprocess.call invocation') call_iteration += 1 return 0 with mock.patch('subprocess.call', side_effect=update_server_webroot_call): - fdroidserver.deploy.update_serverwebroot({'url': url}, repo_section) + fdroidserver.deploy.update_serverwebroot( + {'url': url, 'index_only': True}, repo_section + ) self.assertEqual(call_iteration, 1, 'expected 1 invocations of subprocess.call') @unittest.skipIf( @@ -601,7 +559,6 @@ class DeployTest(unittest.TestCase): fdroidserver.deploy.options.no_checksum = True fdroidserver.deploy.options.verbose = False fdroidserver.deploy.options.quiet = True - fdroidserver.deploy.options.index_only = False config = {} fdroidserver.common.fill_config_defaults(config) @@ -733,7 +690,6 @@ class DeployTest(unittest.TestCase): fdroidserver.deploy.options.no_checksum = True fdroidserver.deploy.options.verbose = False fdroidserver.deploy.options.quiet = True - fdroidserver.deploy.options.index_only = True config = {} fdroidserver.common.fill_config_defaults(config) @@ -807,7 +763,9 @@ class DeployTest(unittest.TestCase): os.symlink('repo/com.example.sym.apk.asc', 'Sym.apk.asc') os.symlink('repo/com.example.sym.apk.sig', 'Sym.apk.sig') with mock.patch('subprocess.call', side_effect=update_awsbucket_s3cmd_call): - fdroidserver.deploy.update_awsbucket_s3cmd(repo_section) + fdroidserver.deploy.update_awsbucket_s3cmd( + repo_section, index_only=True + ) self.assertEqual(call_iteration, 2, 'expected 2 invocations of subprocess.call') def test_update_awsbucket_libcloud(self): @@ -818,7 +776,6 @@ class DeployTest(unittest.TestCase): fdroidserver.deploy.options.no_checksum = True fdroidserver.deploy.options.verbose = False fdroidserver.deploy.options.quiet = True - fdroidserver.deploy.options.index_only = False config = {} fdroidserver.common.fill_config_defaults(config) @@ -880,7 +837,6 @@ class DeployTest(unittest.TestCase): fdroidserver.deploy.options.no_checksum = True fdroidserver.deploy.options.verbose = False fdroidserver.deploy.options.quiet = True - fdroidserver.deploy.options.index_only = True config = {} fdroidserver.common.fill_config_defaults(config) @@ -915,7 +871,7 @@ class DeployTest(unittest.TestCase): mock_driver.get_container.return_value = mock_container mock_driver.upload_object_via_stream.return_value = None - fdroidserver.deploy.update_awsbucket_libcloud(repo_section) + fdroidserver.deploy.update_awsbucket_libcloud(repo_section, index_only=True) mock_driver.get_container.assert_called_once_with( container_name=fdroidserver.deploy.config["awsbucket"] @@ -944,7 +900,6 @@ class DeployTest(unittest.TestCase): fdroidserver.deploy.options.no_keep_git_mirror_archive = False fdroidserver.deploy.options.verbose = False fdroidserver.deploy.options.quiet = True - fdroidserver.deploy.options.index_only = False config = {} fdroidserver.common.fill_config_defaults(config) @@ -1001,7 +956,6 @@ class DeployTest(unittest.TestCase): fdroidserver.deploy.options.no_keep_git_mirror_archive = False fdroidserver.deploy.options.verbose = False fdroidserver.deploy.options.quiet = True - fdroidserver.deploy.options.index_only = True config = {} fdroidserver.common.fill_config_defaults(config) @@ -1038,16 +992,7 @@ class DeployTest(unittest.TestCase): '--delete', '--chmod=Da+rx,Fa-x,a+r,u+w', '--quiet', - 'repo/entry.jar', - 'repo/entry.json', - 'repo/entry.json.asc', - 'repo/index-v1.jar', - 'repo/index-v1.json', - 'repo/index-v1.json.asc', - 'repo/index-v2.json', - 'repo/index-v2.json.asc', - 'repo/index.jar', - 'repo/index.xml', + 'repo/', "git-mirror/fdroid/repo/", ], )