diff --git a/completion/bash-completion b/completion/bash-completion index aa68a220..940b188b 100644 --- a/completion/bash-completion +++ b/completion/bash-completion @@ -264,7 +264,7 @@ __complete_nightly() { __complete_deploy() { opts="-i -v -q" lopts="--identity-file --local-copy-dir --sync-from-local-copy-dir - --verbose --quiet --no-checksum --no-keep-git-mirror-archive --index-only" + --verbose --quiet --no-checksum --no-keep-git-mirror-archive" __complete_options } diff --git a/fdroidserver/common.py b/fdroidserver/common.py index c3c9b3eb..e91b608e 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -3997,7 +3997,7 @@ def get_app_display_name(app): return app.get('AutoName') or app['id'] -def local_rsync(options, from_paths: List[str], todir: str, args: List[str] = []): +def local_rsync(options, from_paths: List[str], todir: str): """Rsync method for local to local copying of things. This is an rsync wrapper with all the settings for safe use within @@ -4014,7 +4014,6 @@ def local_rsync(options, from_paths: List[str], todir: str, args: List[str] = [] rsyncargs += ['--verbose'] if options.quiet: rsyncargs += ['--quiet'] - rsyncargs += args logging.debug(' '.join(rsyncargs + from_paths + [todir])) if subprocess.call(rsyncargs + from_paths + [todir]) != 0: raise FDroidException() diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py index 1bb8cc81..4f04b3e7 100644 --- a/fdroidserver/deploy.py +++ b/fdroidserver/deploy.py @@ -90,7 +90,7 @@ def _get_index_includes(base_dir): return index_includes -def update_awsbucket(repo_section): +def update_awsbucket(repo_section, index_only=False): """Upload the contents of the directory `repo_section` (including subdirectories) to the AWS S3 "bucket". The contents of that subdir of the @@ -102,12 +102,12 @@ def update_awsbucket(repo_section): + config['awsbucket'] + '"') if common.set_command_in_config('s3cmd'): - update_awsbucket_s3cmd(repo_section) + update_awsbucket_s3cmd(repo_section, index_only) else: - update_awsbucket_libcloud(repo_section) + update_awsbucket_libcloud(repo_section, index_only) -def update_awsbucket_s3cmd(repo_section): +def update_awsbucket_s3cmd(repo_section, index_only=False): """Upload using the CLI tool s3cmd, which provides rsync-like sync. The upload is done in multiple passes to reduce the chance of @@ -151,7 +151,7 @@ def update_awsbucket_s3cmd(repo_section): s3url = s3bucketurl + '/fdroid/' - if options.index_only: + if index_only: logging.debug(_('s3cmd syncs indexes from {path} to {url} and deletes removed') .format(path=repo_section, url=s3url)) sync_indexes_flags = [] @@ -195,7 +195,7 @@ def update_awsbucket_s3cmd(repo_section): raise FDroidException() -def update_awsbucket_libcloud(repo_section): +def update_awsbucket_libcloud(repo_section, index_only=False): """No summary. Upload the contents of the directory `repo_section` (including @@ -238,7 +238,7 @@ def update_awsbucket_libcloud(repo_section): if obj.name.startswith(upload_dir + '/'): objs[obj.name] = obj - if options.index_only: + if index_only: index_files = [f"{os.getcwd()}/{name}" for name in _get_index_file_paths(repo_section)] files_to_upload = [os.path.join(root, name) for root, dirs, files in os.walk(os.path.join(os.getcwd(), repo_section)) for name in files] files_to_upload = list(set(files_to_upload) & set(index_files)) @@ -331,8 +331,9 @@ def update_serverwebroot(serverwebroot, repo_section): elif config and config.get('identity_file'): rsyncargs += ['-e', 'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ' + config['identity_file']] url = serverwebroot['url'] + index_only = serverwebroot.get('index_only', False) logging.info('rsyncing ' + repo_section + ' to ' + url) - if options.index_only: + if index_only: rsyncargs += _get_index_file_paths(repo_section) rsyncargs += [f'{url}/{repo_section}/'] logging.info(rsyncargs) @@ -393,8 +394,7 @@ def sync_from_localcopy(repo_section, local_copy_dir): push to all the servers that are configured. """ - if options.index_only: - raise FDroidException(_('The index only mode cannot be used when syncing to the local copy filesystem')) + logging.info('Syncing from local_copy_dir to this repo.') # trailing slashes have a meaning in rsync which is not needed here, so # make sure both paths have exactly one trailing slash @@ -416,8 +416,6 @@ def update_localcopy(repo_section, local_copy_dir): drive. """ - if options.index_only: - raise FDroidException(_('The index only mode cannot be used when syncing to the local copy filesystem')) # local_copy_dir is guaranteed to have a trailing slash in main() below common.local_rsync(options, [repo_section], local_copy_dir) @@ -480,17 +478,11 @@ def update_servergitmirrors(servergitmirrors, repo_section): archive_path = os.path.join(git_mirror_path, 'fdroid', 'archive') shutil.rmtree(archive_path, ignore_errors=True) - if options.index_only: - # rsync is very particular about trailing slashes - common.local_rsync(options, - _get_index_file_paths(repo_section), - git_repodir.rstrip('/') + '/') - else: - # trailing slashes have a meaning in rsync which is not needed here, so - # make sure both paths have exactly one trailing slash - common.local_rsync(options, - [repo_section.rstrip('/') + '/'], - git_repodir.rstrip('/') + '/') + # trailing slashes have a meaning in rsync which is not needed here, so + # make sure both paths have exactly one trailing slash + common.local_rsync(options, + [repo_section.rstrip('/') + '/'], + git_repodir.rstrip('/') + '/') # use custom SSH command if identity_file specified ssh_cmd = 'ssh -oBatchMode=yes' @@ -499,28 +491,6 @@ def update_servergitmirrors(servergitmirrors, repo_section): elif 'identity_file' in config: ssh_cmd += ' -oIdentitiesOnly=yes -i "%s"' % config['identity_file'] - repo = git.Repo.init(git_mirror_path, initial_branch=GIT_BRANCH) - - enabled_remotes = [] - for d in servergitmirrors: - remote_url = d['url'] - name = REMOTE_HOSTNAME_REGEX.sub(r'\1', remote_url) - enabled_remotes.append(name) - r = git.remote.Remote(repo, name) - if r in repo.remotes: - r = repo.remote(name) - if 'set_url' in dir(r): # force remote URL if using GitPython 2.x - r.set_url(remote_url) - else: - repo.create_remote(name, remote_url) - logging.info('Mirroring to: ' + remote_url) - - # sadly index.add don't allow the --all parameter - logging.debug('Adding all files to git mirror') - repo.git.add(all=True) - logging.debug('Committing all files into git mirror') - repo.index.commit("fdroidserver git-mirror") - if options.verbose: progressbar = progress.Bar() @@ -532,23 +502,49 @@ def update_servergitmirrors(servergitmirrors, repo_section): else: progress = None - # only deploy to GitLab Artifacts if too big for GitLab Pages - if common.get_dir_size(git_fdroiddir) <= common.GITLAB_COM_PAGES_MAX_SIZE: - gitlab_ci_job_name = 'pages' - else: - gitlab_ci_job_name = 'GitLab Artifacts' - logging.warning( - _( - 'Skipping GitLab Pages mirror because the repo is too large (>%.2fGB)!' - ) - % (common.GITLAB_COM_PAGES_MAX_SIZE / 1000000000) - ) + repo = git.Repo.init(git_mirror_path, initial_branch=GIT_BRANCH) + initial_commit_ref = repo.head.ref - # push for every remote. This will overwrite the git history - for remote in repo.remotes: - if remote.name not in enabled_remotes: - repo.delete_remote(remote) - continue + enabled_remotes = [] + for d in servergitmirrors: + index_only = d.get('index_only', False) + remote_url = d['url'] + name = REMOTE_HOSTNAME_REGEX.sub(r'\1', remote_url) + enabled_remotes.append(name) + r = git.remote.Remote(repo, name) + if r in repo.remotes: + r = repo.remote(name) + if 'set_url' in dir(r): # force remote URL if using GitPython 2.x + r.set_url(remote_url) + else: + repo.create_remote(name, remote_url) + logging.info('Mirroring to: ' + remote_url) + + if index_only: + logging.debug('Adding index files to git mirror') + repo.index.add(_get_index_file_paths(repo_section)) + else: + # sadly index.add don't allow the --all parameter + logging.debug('Adding all files to git mirror') + repo.git.add(all=True) + + logging.debug('Committing files into git mirror') + repo.index.commit("fdroidserver git-mirror") + + # only deploy to GitLab Artifacts if too big for GitLab Pages + if common.get_dir_size(git_fdroiddir) <= common.GITLAB_COM_PAGES_MAX_SIZE: + gitlab_ci_job_name = 'pages' + else: + gitlab_ci_job_name = 'GitLab Artifacts' + logging.warning( + _( + 'Skipping GitLab Pages mirror because the repo is too large (>%.2fGB)!' + ) + % (common.GITLAB_COM_PAGES_MAX_SIZE / 1000000000) + ) + + # push. This will overwrite the git history + remote = repo.remote(name) if remote.name == 'gitlab': logging.debug('Writing .gitlab-ci.yml to deploy to GitLab Pages') with open(os.path.join(git_mirror_path, ".gitlab-ci.yml"), "wt") as fp: @@ -568,7 +564,7 @@ def update_servergitmirrors(servergitmirrors, repo_section): default_flow_style=False, ) - repo.git.add(all=True) + repo.index.add(['.gitlab-ci.yml']) repo.index.commit("fdroidserver git-mirror: Deploy to GitLab Pages") logging.debug(_('Pushing to {url}').format(url=remote.url)) @@ -590,6 +586,8 @@ def update_servergitmirrors(servergitmirrors, repo_section): else: logging.debug(remote.url + ': ' + pushinfo.summary) + repo.head.reset(initial_commit_ref, index=True, working_tree=True) + if progress: progressbar.done() diff --git a/tests/deploy.TestCase b/tests/deploy.TestCase index ae436ec8..ef7a1911 100755 --- a/tests/deploy.TestCase +++ b/tests/deploy.TestCase @@ -9,6 +9,7 @@ import tempfile import unittest from pathlib import Path from unittest import mock +from fdroidserver import index localmodule = os.path.realpath( os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..') @@ -62,7 +63,6 @@ class DeployTest(unittest.TestCase): # setup parameters for this test run fdroidserver.deploy.options.identity_file = None - fdroidserver.deploy.options.index_only = False fdroidserver.deploy.config['make_current_version_link'] = False dest_apk0 = url0 / fake_apk @@ -109,7 +109,6 @@ class DeployTest(unittest.TestCase): # setup parameters for this test run fdroidserver.deploy.options.identity_file = None - fdroidserver.deploy.options.index_only = False fdroidserver.deploy.config['make_current_version_link'] = False dest_apk = Path(url) / fake_apk @@ -137,7 +136,6 @@ class DeployTest(unittest.TestCase): # setup parameters for this test run fdroidserver.deploy.options.identity_file = None - fdroidserver.deploy.options.index_only = True fdroidserver.deploy.config['make_current_version_link'] = False dest_apk = Path(url) / fake_apk @@ -145,7 +143,9 @@ class DeployTest(unittest.TestCase): self.assertFalse(dest_apk.is_file()) self.assertFalse(dest_index.is_file()) - fdroidserver.deploy.update_serverwebroot({'url': str(url)}, 'repo') + fdroidserver.deploy.update_serverwebroot( + {'url': str(url), 'index_only': True}, 'repo' + ) self.assertFalse(dest_apk.is_file()) self.assertTrue(dest_index.is_file()) @@ -164,7 +164,6 @@ class DeployTest(unittest.TestCase): fdroidserver.deploy.options.verbose = False fdroidserver.deploy.options.quiet = True fdroidserver.deploy.options.identity_file = None - fdroidserver.deploy.options.index_only = False fdroidserver.deploy.config['make_current_version_link'] = True url = "example.com:/var/www/fdroid" repo_section = 'repo' @@ -257,7 +256,6 @@ class DeployTest(unittest.TestCase): fdroidserver.deploy.options.verbose = False fdroidserver.deploy.options.quiet = True fdroidserver.deploy.options.identity_file = None - fdroidserver.deploy.options.index_only = True fdroidserver.deploy.config['make_current_version_link'] = True url = "example.com:/var/www/fdroid" repo_section = 'repo' @@ -317,34 +315,6 @@ class DeployTest(unittest.TestCase): 'example.com:/var/www/fdroid', ], ) - # elif call_iteration == 1: - # self.assertListEqual( - # cmd, - # [ - # 'rsync', - # '--archive', - # '--delete-after', - # '--safe-links', - # '--quiet', - # 'repo', - # serverwebroot, - # ], - # ) - # elif call_iteration == 2: - # self.assertListEqual( - # cmd, - # [ - # 'rsync', - # '--archive', - # '--delete-after', - # '--safe-links', - # '--quiet', - # 'Sym.apk', - # 'Sym.apk.asc', - # 'Sym.apk.sig', - # 'example.com:/var/www/fdroid', - # ], - # ) else: self.fail('unexpected subprocess.call invocation') call_iteration += 1 @@ -356,7 +326,9 @@ class DeployTest(unittest.TestCase): os.symlink('repo/com.example.sym.apk.asc', 'Sym.apk.asc') os.symlink('repo/com.example.sym.apk.sig', 'Sym.apk.sig') with mock.patch('subprocess.call', side_effect=update_server_webroot_call): - fdroidserver.deploy.update_serverwebroot({'url': url}, repo_section) + fdroidserver.deploy.update_serverwebroot( + {'url': url, 'index_only': True}, repo_section + ) self.assertEqual(call_iteration, 1, 'expected 1 invocations of subprocess.call') def test_update_serverwebroot_with_id_file(self): @@ -367,7 +339,6 @@ class DeployTest(unittest.TestCase): fdroidserver.deploy.options.verbose = True fdroidserver.deploy.options.quiet = False fdroidserver.deploy.options.identity_file = None - fdroidserver.deploy.options.index_only = False fdroidserver.deploy.config['identity_file'] = './id_rsa' fdroidserver.deploy.config['make_current_version_link'] = False url = "example.com:/var/www/fdroid" @@ -445,7 +416,6 @@ class DeployTest(unittest.TestCase): fdroidserver.deploy.options.verbose = True fdroidserver.deploy.options.quiet = False fdroidserver.deploy.options.identity_file = None - fdroidserver.deploy.options.index_only = True fdroidserver.deploy.config['identity_file'] = './id_rsa' fdroidserver.deploy.config['make_current_version_link'] = False url = "example.com:/var/www/fdroid" @@ -496,29 +466,15 @@ class DeployTest(unittest.TestCase): "example.com:/var/www/fdroid/archive/", ], ) - # elif call_iteration == 1: - # self.assertListEqual( - # cmd, - # [ - # 'rsync', - # '--archive', - # '--delete-after', - # '--safe-links', - # '--verbose', - # '-e', - # 'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ' - # + fdroidserver.deploy.config['identity_file'], - # 'archive', - # serverwebroot, - # ], - # ) else: self.fail('unexpected subprocess.call invocation') call_iteration += 1 return 0 with mock.patch('subprocess.call', side_effect=update_server_webroot_call): - fdroidserver.deploy.update_serverwebroot({'url': url}, repo_section) + fdroidserver.deploy.update_serverwebroot( + {'url': url, 'index_only': True}, repo_section + ) self.assertEqual(call_iteration, 1, 'expected 1 invocations of subprocess.call') @unittest.skipIf( @@ -547,7 +503,6 @@ class DeployTest(unittest.TestCase): fdroidserver.deploy.options.no_checksum = True fdroidserver.deploy.options.verbose = False fdroidserver.deploy.options.quiet = True - fdroidserver.deploy.options.index_only = False config = {} fdroidserver.common.fill_config_defaults(config) @@ -679,7 +634,6 @@ class DeployTest(unittest.TestCase): fdroidserver.deploy.options.no_checksum = True fdroidserver.deploy.options.verbose = False fdroidserver.deploy.options.quiet = True - fdroidserver.deploy.options.index_only = True config = {} fdroidserver.common.fill_config_defaults(config) @@ -753,7 +707,9 @@ class DeployTest(unittest.TestCase): os.symlink('repo/com.example.sym.apk.asc', 'Sym.apk.asc') os.symlink('repo/com.example.sym.apk.sig', 'Sym.apk.sig') with mock.patch('subprocess.call', side_effect=update_awsbucket_s3cmd_call): - fdroidserver.deploy.update_awsbucket_s3cmd(repo_section) + fdroidserver.deploy.update_awsbucket_s3cmd( + repo_section, index_only=True + ) self.assertEqual(call_iteration, 2, 'expected 2 invocations of subprocess.call') def test_update_awsbucket_libcloud(self): @@ -764,7 +720,6 @@ class DeployTest(unittest.TestCase): fdroidserver.deploy.options.no_checksum = True fdroidserver.deploy.options.verbose = False fdroidserver.deploy.options.quiet = True - fdroidserver.deploy.options.index_only = False config = {} fdroidserver.common.fill_config_defaults(config) @@ -826,7 +781,6 @@ class DeployTest(unittest.TestCase): fdroidserver.deploy.options.no_checksum = True fdroidserver.deploy.options.verbose = False fdroidserver.deploy.options.quiet = True - fdroidserver.deploy.options.index_only = True config = {} fdroidserver.common.fill_config_defaults(config) @@ -861,7 +815,7 @@ class DeployTest(unittest.TestCase): mock_driver.get_container.return_value = mock_container mock_driver.upload_object_via_stream.return_value = None - fdroidserver.deploy.update_awsbucket_libcloud(repo_section) + fdroidserver.deploy.update_awsbucket_libcloud(repo_section, index_only=True) mock_driver.get_container.assert_called_once_with( container_name=fdroidserver.deploy.config["awsbucket"] @@ -890,7 +844,6 @@ class DeployTest(unittest.TestCase): fdroidserver.deploy.options.no_keep_git_mirror_archive = False fdroidserver.deploy.options.verbose = False fdroidserver.deploy.options.quiet = True - fdroidserver.deploy.options.index_only = False config = {} fdroidserver.common.fill_config_defaults(config) @@ -947,7 +900,6 @@ class DeployTest(unittest.TestCase): fdroidserver.deploy.options.no_keep_git_mirror_archive = False fdroidserver.deploy.options.verbose = False fdroidserver.deploy.options.quiet = True - fdroidserver.deploy.options.index_only = True config = {} fdroidserver.common.fill_config_defaults(config) @@ -984,16 +936,7 @@ class DeployTest(unittest.TestCase): '--delete', '--chmod=Da+rx,Fa-x,a+r,u+w', '--quiet', - 'repo/entry.jar', - 'repo/entry.json', - 'repo/entry.json.asc', - 'repo/index-v1.jar', - 'repo/index-v1.json', - 'repo/index-v1.json.asc', - 'repo/index-v2.json', - 'repo/index-v2.json.asc', - 'repo/index.jar', - 'repo/index.xml', + 'repo/', "git-mirror/fdroid/repo/", ], )