diff --git a/examples/config.yml b/examples/config.yml index 646726bb..0337e6f0 100644 --- a/examples/config.yml +++ b/examples/config.yml @@ -183,7 +183,7 @@ # # serverwebroot: # - url: 'me@b.az:/srv/fdroid' -# indexOnly: true +# index_only: true # When running fdroid processes on a remote server, it is possible to @@ -209,7 +209,7 @@ # servergitmirrors: # - url: https://github.com/user/repo # - url: https://gitlab.com/user/repo -# indexOnly: true +# index_only: true # Most git hosting services have hard size limits for each git repo. # `fdroid deploy` will delete the git history when the git mirror repo diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 5a546386..b8374109 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -29,6 +29,7 @@ # libraries here as they will become a requirement for all commands. import difflib +from typing import List import git import glob import io @@ -4300,7 +4301,7 @@ def get_app_display_name(app): return app.get('AutoName') or app['id'] -def local_rsync(options, fromdir, todir): +def local_rsync(options, from_paths: List[str], todir: str): """Rsync method for local to local copying of things. This is an rsync wrapper with all the settings for safe use within @@ -4317,8 +4318,8 @@ def local_rsync(options, fromdir, todir): rsyncargs += ['--verbose'] if options.quiet: rsyncargs += ['--quiet'] - logging.debug(' '.join(rsyncargs + [fromdir, todir])) - if subprocess.call(rsyncargs + [fromdir, todir]) != 0: + logging.debug(' '.join(rsyncargs + from_paths + [todir])) + if subprocess.call(rsyncargs + from_paths + [todir]) != 0: raise FDroidException() diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py index b120999f..1e598673 100644 --- a/fdroidserver/deploy.py +++ b/fdroidserver/deploy.py @@ -25,11 +25,15 @@ import re import subprocess import time import urllib +from typing import Dict, List +from git import Repo import yaml from argparse import ArgumentParser import logging from shlex import split import shutil +import git +from pathlib import Path from . import _ from . import common @@ -49,6 +53,7 @@ USER_RCLONE_CONF = None REMOTE_HOSTNAME_REGEX = re.compile(r'\W*\w+\W+(\w+).*') INDEX_FILES = [ + "altstore-index.json", "entry.jar", "entry.json", "entry.json.asc", @@ -62,28 +67,19 @@ INDEX_FILES = [ ] -def _get_index_excludes(repo_section): +def _get_index_file_paths(base_dir): """Return the list of files to be synced last, since they finalize the deploy. The process of pushing all the new packages to the various services can take a while. So the index files should be updated last. That ensures that the package files are available when the client learns about them from the new index files. - """ - indexes = [ - os.path.join(repo_section, 'altstore-index.json'), - os.path.join(repo_section, 'entry.jar'), - os.path.join(repo_section, 'entry.json'), - os.path.join(repo_section, 'entry.json.asc'), - os.path.join(repo_section, 'index-v1.jar'), - os.path.join(repo_section, 'index-v1.json'), - os.path.join(repo_section, 'index-v1.json.asc'), - os.path.join(repo_section, 'index-v2.json'), - os.path.join(repo_section, 'index-v2.json.asc'), - os.path.join(repo_section, 'index.jar'), - os.path.join(repo_section, 'index.xml'), - ] + return [os.path.join(base_dir, filename) for filename in INDEX_FILES] + + +def _get_index_excludes(base_dir): + indexes = _get_index_file_paths(base_dir) index_excludes = [] for f in indexes: index_excludes.append('--exclude') @@ -91,7 +87,25 @@ def _get_index_excludes(repo_section): return index_excludes -def update_awsbucket(repo_section, verbose=False, quiet=False): +def _get_index_includes(base_dir): + indexes = _get_index_file_paths(base_dir) + index_includes = [] + for f in indexes: + index_includes.append('--include') + index_includes.append(f) + return index_includes + + +def _remove_missing_files(files: List[str]) -> List[str]: + """Remove files that are missing from the file system.""" + existing = [] + for f in files: + if os.path.exists(f): + existing.append(f) + return existing + + +def update_awsbucket(repo_section, is_index_only=False, verbose=False, quiet=False): """Upload the contents of the directory `repo_section` (including subdirectories) to the AWS S3 "bucket". The contents of that subdir of the @@ -111,26 +125,28 @@ def update_awsbucket(repo_section, verbose=False, quiet=False): logging.warning( 'No syncing tool set in config.yml!. Defaulting to using s3cmd' ) - update_awsbucket_s3cmd(repo_section) + update_awsbucket_s3cmd(repo_section, is_index_only) if config['s3cmd'] is True and config['rclone'] is True: logging.warning( 'Both syncing tools set in config.yml!. Defaulting to using s3cmd' ) - update_awsbucket_s3cmd(repo_section) + update_awsbucket_s3cmd(repo_section, is_index_only) if config['s3cmd'] is True and config['rclone'] is not True: - update_awsbucket_s3cmd(repo_section) + update_awsbucket_s3cmd(repo_section, is_index_only) if config['rclone'] is True and config['s3cmd'] is not True: - update_remote_storage_with_rclone(repo_section, verbose, quiet) + update_remote_storage_with_rclone( + repo_section, is_index_only, verbose, quiet + ) elif common.set_command_in_config('s3cmd'): - update_awsbucket_s3cmd(repo_section) + update_awsbucket_s3cmd(repo_section, is_index_only) elif common.set_command_in_config('rclone'): - update_remote_storage_with_rclone(repo_section, verbose, quiet) + update_remote_storage_with_rclone(repo_section, is_index_only, verbose, quiet) else: - update_awsbucket_libcloud(repo_section) + update_awsbucket_libcloud(repo_section, is_index_only) -def update_awsbucket_s3cmd(repo_section): +def update_awsbucket_s3cmd(repo_section, is_index_only=False): """Upload using the CLI tool s3cmd, which provides rsync-like sync. The upload is done in multiple passes to reduce the chance of @@ -177,39 +193,68 @@ def update_awsbucket_s3cmd(repo_section): s3cmd_sync += ['--quiet'] s3url = s3bucketurl + '/fdroid/' - logging.debug('s3cmd sync new files in ' + repo_section + ' to ' + s3url) - logging.debug(_('Running first pass with MD5 checking disabled')) - excludes = _get_index_excludes(repo_section) - returncode = subprocess.call( - s3cmd_sync - + excludes - + ['--no-check-md5', '--skip-existing', repo_section, s3url] - ) - if returncode != 0: - raise FDroidException() - logging.debug('s3cmd sync all files in ' + repo_section + ' to ' + s3url) - returncode = subprocess.call( - s3cmd_sync + excludes + ['--no-check-md5', repo_section, s3url] - ) - if returncode != 0: - raise FDroidException() logging.debug( _('s3cmd sync indexes {path} to {url} and delete').format( path=repo_section, url=s3url ) ) - s3cmd_sync.append('--delete-removed') - s3cmd_sync.append('--delete-after') - if options.no_checksum: - s3cmd_sync.append('--no-check-md5') + + if is_index_only: + logging.debug( + _('s3cmd syncs indexes from {path} to {url} and deletes removed').format( + path=repo_section, url=s3url + ) + ) + sync_indexes_flags = [] + sync_indexes_flags.extend(_get_index_includes(repo_section)) + sync_indexes_flags.append('--delete-removed') + sync_indexes_flags.append('--delete-after') + if options.no_checksum: + sync_indexes_flags.append('--no-check-md5') + else: + sync_indexes_flags.append('--check-md5') + returncode = subprocess.call( + s3cmd_sync + sync_indexes_flags + [repo_section, s3url] + ) + if returncode != 0: + raise FDroidException() else: - s3cmd_sync.append('--check-md5') - if subprocess.call(s3cmd_sync + [repo_section, s3url]) != 0: - raise FDroidException() + logging.debug('s3cmd sync new files in ' + repo_section + ' to ' + s3url) + logging.debug(_('Running first pass with MD5 checking disabled')) + excludes = _get_index_excludes(repo_section) + returncode = subprocess.call( + s3cmd_sync + + excludes + + ['--no-check-md5', '--skip-existing', repo_section, s3url] + ) + if returncode != 0: + raise FDroidException() + logging.debug('s3cmd sync all files in ' + repo_section + ' to ' + s3url) + returncode = subprocess.call( + s3cmd_sync + excludes + ['--no-check-md5', repo_section, s3url] + ) + if returncode != 0: + raise FDroidException() + + logging.debug( + _('s3cmd sync indexes {path} to {url} and delete').format( + path=repo_section, url=s3url + ) + ) + s3cmd_sync.append('--delete-removed') + s3cmd_sync.append('--delete-after') + if options.no_checksum: + s3cmd_sync.append('--no-check-md5') + else: + s3cmd_sync.append('--check-md5') + if subprocess.call(s3cmd_sync + [repo_section, s3url]) != 0: + raise FDroidException() -def update_remote_storage_with_rclone(repo_section, verbose=False, quiet=False): +def update_remote_storage_with_rclone( + repo_section, is_index_only=False, verbose=False, quiet=False +): """ Upload fdroid repo folder to remote storage using rclone sync. @@ -261,46 +306,19 @@ def update_remote_storage_with_rclone(repo_section, verbose=False, quiet=False): _('To use rclone, rclone_config and awsbucket must be set in config.yml!') ) - if isinstance(config['rclone_config'], str): - rclone_sync_command = ( - 'rclone sync ' - + repo_section - + ' ' - + config['rclone_config'] - + ':' - + config['awsbucket'] - + '/' - + upload_dir - ) + if is_index_only: + sources = _get_index_file_paths(repo_section) + sources = _remove_missing_files(sources) + else: + sources = [repo_section] - rclone_sync_command = split(rclone_sync_command) - - if verbose: - rclone_sync_command += ['--verbose'] - elif quiet: - rclone_sync_command += ['--quiet'] - - if configfilename: - rclone_sync_command += split('--config=' + configfilename) - - complete_remote_path = ( - config['rclone_config'] + ':' + config['awsbucket'] + '/' + upload_dir - ) - - logging.debug( - "rclone sync all files in " + repo_section + ' to ' + complete_remote_path - ) - - if subprocess.call(rclone_sync_command) != 0: - raise FDroidException() - - if isinstance(config['rclone_config'], list): - for remote_config in config['rclone_config']: + for source in sources: + if isinstance(config['rclone_config'], str): rclone_sync_command = ( 'rclone sync ' - + repo_section + + source + ' ' - + remote_config + + config['rclone_config'] + ':' + config['awsbucket'] + '/' @@ -318,21 +336,52 @@ def update_remote_storage_with_rclone(repo_section, verbose=False, quiet=False): rclone_sync_command += split('--config=' + configfilename) complete_remote_path = ( - remote_config + ':' + config['awsbucket'] + '/' + upload_dir + config['rclone_config'] + ':' + config['awsbucket'] + '/' + upload_dir ) logging.debug( - "rclone sync all files in " - + repo_section - + ' to ' - + complete_remote_path + "rclone sync all files in " + source + ' to ' + complete_remote_path ) if subprocess.call(rclone_sync_command) != 0: raise FDroidException() + if isinstance(config['rclone_config'], list): + for remote_config in config['rclone_config']: + rclone_sync_command = ( + 'rclone sync ' + + source + + ' ' + + remote_config + + ':' + + config['awsbucket'] + + '/' + + upload_dir + ) -def update_awsbucket_libcloud(repo_section): + rclone_sync_command = split(rclone_sync_command) + + if verbose: + rclone_sync_command += ['--verbose'] + elif quiet: + rclone_sync_command += ['--quiet'] + + if configfilename: + rclone_sync_command += split('--config=' + configfilename) + + complete_remote_path = ( + remote_config + ':' + config['awsbucket'] + '/' + upload_dir + ) + + logging.debug( + "rclone sync all files in " + source + ' to ' + complete_remote_path + ) + + if subprocess.call(rclone_sync_command) != 0: + raise FDroidException() + + +def update_awsbucket_libcloud(repo_section, is_index_only=False): """No summary. Upload the contents of the directory `repo_section` (including @@ -380,49 +429,66 @@ def update_awsbucket_libcloud(repo_section): if obj.name.startswith(upload_dir + '/'): objs[obj.name] = obj - for root, dirs, files in os.walk(os.path.join(os.getcwd(), repo_section)): - for name in files: - upload = False - file_to_upload = os.path.join(root, name) - object_name = 'fdroid/' + os.path.relpath(file_to_upload, os.getcwd()) - if object_name not in objs: + if is_index_only: + index_files = [ + f"{os.getcwd()}/{name}" for name in _get_index_file_paths(repo_section) + ] + files_to_upload = [ + os.path.join(root, name) + for root, dirs, files in os.walk(os.path.join(os.getcwd(), repo_section)) + for name in files + ] + files_to_upload = list(set(files_to_upload) & set(index_files)) + files_to_upload = _remove_missing_files(files_to_upload) + + else: + files_to_upload = [ + os.path.join(root, name) + for root, dirs, files in os.walk(os.path.join(os.getcwd(), repo_section)) + for name in files + ] + + for file_to_upload in files_to_upload: + upload = False + object_name = 'fdroid/' + os.path.relpath(file_to_upload, os.getcwd()) + if object_name not in objs: + upload = True + else: + obj = objs.pop(object_name) + if obj.size != os.path.getsize(file_to_upload): upload = True else: - obj = objs.pop(object_name) - if obj.size != os.path.getsize(file_to_upload): + # if the sizes match, then compare by MD5 + md5 = hashlib.md5() # nosec AWS uses MD5 + with open(file_to_upload, 'rb') as f: + while True: + data = f.read(8192) + if not data: + break + md5.update(data) + if obj.hash != md5.hexdigest(): + s3url = 's3://' + awsbucket + '/' + obj.name + logging.info(' deleting ' + s3url) + if not driver.delete_object(obj): + logging.warning('Could not delete ' + s3url) upload = True - else: - # if the sizes match, then compare by MD5 - md5 = hashlib.md5() # nosec AWS uses MD5 - with open(file_to_upload, 'rb') as f: - while True: - data = f.read(8192) - if not data: - break - md5.update(data) - if obj.hash != md5.hexdigest(): - s3url = 's3://' + awsbucket + '/' + obj.name - logging.info(' deleting ' + s3url) - if not driver.delete_object(obj): - logging.warning('Could not delete ' + s3url) - upload = True - if upload: - logging.debug(' uploading "' + file_to_upload + '"...') - extra = {'acl': 'public-read'} - if file_to_upload.endswith('.sig'): - extra['content_type'] = 'application/pgp-signature' - elif file_to_upload.endswith('.asc'): - extra['content_type'] = 'application/pgp-signature' - path = os.path.relpath(file_to_upload) - logging.info(f' uploading {path} to s3://{awsbucket}/{object_name}') - with open(file_to_upload, 'rb') as iterator: - obj = driver.upload_object_via_stream( - iterator=iterator, - container=container, - object_name=object_name, - extra=extra, - ) + if upload: + logging.debug(' uploading "' + file_to_upload + '"...') + extra = {'acl': 'public-read'} + if file_to_upload.endswith('.sig'): + extra['content_type'] = 'application/pgp-signature' + elif file_to_upload.endswith('.asc'): + extra['content_type'] = 'application/pgp-signature' + path = os.path.relpath(file_to_upload) + logging.info(f' uploading {path} to s3://{awsbucket}/{object_name}') + with open(file_to_upload, 'rb') as iterator: + obj = driver.upload_object_via_stream( + iterator=iterator, + container=container, + object_name=object_name, + extra=extra, + ) # delete the remnants in the bucket, they do not exist locally while objs: object_name, obj = objs.popitem() @@ -476,21 +542,38 @@ def update_serverwebroot(serverwebroot, repo_section): 'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ' + config['identity_file'], ] url = serverwebroot['url'] + is_index_only = serverwebroot.get('index_only', False) logging.info('rsyncing ' + repo_section + ' to ' + url) - excludes = _get_index_excludes(repo_section) - if subprocess.call(rsyncargs + excludes + [repo_section, url]) != 0: - raise FDroidException() - if subprocess.call(rsyncargs + [repo_section, url]) != 0: - raise FDroidException() - # upload "current version" symlinks if requested - if config and config.get('make_current_version_link') and repo_section == 'repo': - links_to_upload = [] - for f in glob.glob('*.apk') + glob.glob('*.apk.asc') + glob.glob('*.apk.sig'): - if os.path.islink(f): - links_to_upload.append(f) - if len(links_to_upload) > 0: - if subprocess.call(rsyncargs + links_to_upload + [url]) != 0: - raise FDroidException() + if is_index_only: + files_to_upload = _get_index_file_paths(repo_section) + files_to_upload = _remove_missing_files(files_to_upload) + + rsyncargs += files_to_upload + rsyncargs += [f'{url}/{repo_section}/'] + logging.info(rsyncargs) + if subprocess.call(rsyncargs) != 0: + raise FDroidException() + else: + excludes = _get_index_excludes(repo_section) + if subprocess.call(rsyncargs + excludes + [repo_section, url]) != 0: + raise FDroidException() + if subprocess.call(rsyncargs + [repo_section, url]) != 0: + raise FDroidException() + # upload "current version" symlinks if requested + if ( + config + and config.get('make_current_version_link') + and repo_section == 'repo' + ): + links_to_upload = [] + for f in ( + glob.glob('*.apk') + glob.glob('*.apk.asc') + glob.glob('*.apk.sig') + ): + if os.path.islink(f): + links_to_upload.append(f) + if len(links_to_upload) > 0: + if subprocess.call(rsyncargs + links_to_upload + [url]) != 0: + raise FDroidException() def update_serverwebroots(serverwebroots, repo_section, standardwebroot=True): @@ -531,11 +614,12 @@ def sync_from_localcopy(repo_section, local_copy_dir): """ logging.info('Syncing from local_copy_dir to this repo.') + # trailing slashes have a meaning in rsync which is not needed here, so # make sure both paths have exactly one trailing slash common.local_rsync( common.get_options(), - os.path.join(local_copy_dir, repo_section).rstrip('/') + '/', + [os.path.join(local_copy_dir, repo_section).rstrip('/') + '/'], repo_section.rstrip('/') + '/', ) @@ -554,7 +638,7 @@ def update_localcopy(repo_section, local_copy_dir): """ # local_copy_dir is guaranteed to have a trailing slash in main() below - common.local_rsync(common.get_options(), repo_section, local_copy_dir) + common.local_rsync(common.get_options(), [repo_section], local_copy_dir) offline_copy = os.path.join(os.getcwd(), BINARY_TRANSPARENCY_DIR) if os.path.isdir(os.path.join(offline_copy, '.git')): @@ -584,7 +668,6 @@ def update_servergitmirrors(servergitmirrors, repo_section): transparency log. """ - import git from clint.textui import progress if config.get('local_copy_dir') and not config.get('sync_from_local_copy_dir'): @@ -594,10 +677,11 @@ def update_servergitmirrors(servergitmirrors, repo_section): return options = common.get_options() + workspace_dir = Path(os.getcwd()) # right now we support only 'repo' git-mirroring if repo_section == 'repo': - git_mirror_path = 'git-mirror' + git_mirror_path = workspace_dir / 'git-mirror' dotgit = os.path.join(git_mirror_path, '.git') git_fdroiddir = os.path.join(git_mirror_path, 'fdroid') git_repodir = os.path.join(git_fdroiddir, repo_section) @@ -624,11 +708,6 @@ def update_servergitmirrors(servergitmirrors, repo_section): archive_path = os.path.join(git_mirror_path, 'fdroid', 'archive') shutil.rmtree(archive_path, ignore_errors=True) - # rsync is very particular about trailing slashes - common.local_rsync( - options, repo_section.rstrip('/') + '/', git_repodir.rstrip('/') + '/' - ) - # use custom SSH command if identity_file specified ssh_cmd = 'ssh -oBatchMode=yes' if options.identity_file is not None: @@ -636,28 +715,6 @@ def update_servergitmirrors(servergitmirrors, repo_section): elif 'identity_file' in config: ssh_cmd += ' -oIdentitiesOnly=yes -i "%s"' % config['identity_file'] - repo = git.Repo.init(git_mirror_path, initial_branch=GIT_BRANCH) - - enabled_remotes = [] - for d in servergitmirrors: - remote_url = d['url'] - name = REMOTE_HOSTNAME_REGEX.sub(r'\1', remote_url) - enabled_remotes.append(name) - r = git.remote.Remote(repo, name) - if r in repo.remotes: - r = repo.remote(name) - if 'set_url' in dir(r): # force remote URL if using GitPython 2.x - r.set_url(remote_url) - else: - repo.create_remote(name, remote_url) - logging.info('Mirroring to: ' + remote_url) - - # sadly index.add don't allow the --all parameter - logging.debug('Adding all files to git mirror') - repo.git.add(all=True) - logging.debug('Committing all files into git mirror') - repo.index.commit("fdroidserver git-mirror") - if options.verbose: progressbar = progress.Bar() @@ -670,75 +727,152 @@ def update_servergitmirrors(servergitmirrors, repo_section): else: progress = None - # only deploy to GitLab Artifacts if too big for GitLab Pages - if common.get_dir_size(git_fdroiddir) <= common.GITLAB_COM_PAGES_MAX_SIZE: - gitlab_ci_job_name = 'pages' - else: - gitlab_ci_job_name = 'GitLab Artifacts' - logging.warning( - _( - 'Skipping GitLab Pages mirror because the repo is too large (>%.2fGB)!' - ) - % (common.GITLAB_COM_PAGES_MAX_SIZE / 1000000000) + repo = git.Repo.init(git_mirror_path, initial_branch=GIT_BRANCH) + + enabled_remotes = [] + for d in servergitmirrors: + is_index_only = d.get('index_only', False) + + # Use a separate branch for the index only mode as it needs a different set of files to commit + if is_index_only: + local_branch_name = 'index_only' + else: + local_branch_name = 'full' + if local_branch_name in repo.heads: + repo.git.switch(local_branch_name) + else: + repo.git.switch('--orphan', local_branch_name) + + # trailing slashes have a meaning in rsync which is not needed here, so + # make sure both paths have exactly one trailing slash + if is_index_only: + files_to_sync = _get_index_file_paths(str(workspace_dir / repo_section)) + files_to_sync = _remove_missing_files(files_to_sync) + else: + files_to_sync = [str(workspace_dir / repo_section).rstrip('/') + '/'] + common.local_rsync( + common.get_options(), files_to_sync, git_repodir.rstrip('/') + '/' ) - # push for every remote. This will overwrite the git history - for remote in repo.remotes: - if remote.name not in enabled_remotes: - repo.delete_remote(remote) - continue - if remote.name == 'gitlab': - logging.debug('Writing .gitlab-ci.yml to deploy to GitLab Pages') - with open(os.path.join(git_mirror_path, ".gitlab-ci.yml"), "wt") as fp: - yaml.dump( - { - gitlab_ci_job_name: { - 'script': [ - 'mkdir .public', - 'cp -r * .public/', - 'mv .public public', - ], - 'artifacts': {'paths': ['public']}, - 'variables': {'GIT_DEPTH': 1}, - } - }, - fp, - default_flow_style=False, - ) - - repo.git.add(all=True) - repo.index.commit("fdroidserver git-mirror: Deploy to GitLab Pages") - - logging.debug(_('Pushing to {url}').format(url=remote.url)) - with repo.git.custom_environment(GIT_SSH_COMMAND=ssh_cmd): - pushinfos = remote.push( - GIT_BRANCH, force=True, set_upstream=True, progress=progress - ) - for pushinfo in pushinfos: - if pushinfo.flags & ( - git.remote.PushInfo.ERROR - | git.remote.PushInfo.REJECTED - | git.remote.PushInfo.REMOTE_FAILURE - | git.remote.PushInfo.REMOTE_REJECTED - ): - # Show potentially useful messages from git remote - for line in progress.other_lines: - if line.startswith('remote:'): - logging.debug(line) - raise FDroidException( - '{url} push failed: {flags} {summary}'.format( - url=remote.url, - flags=pushinfo.flags, - summary=pushinfo.summary, - ) - ) - else: - logging.debug(remote.url + ': ' + pushinfo.summary) - + upload_to_servergitmirror( + mirror_config=d, + local_repo=repo, + enabled_remotes=enabled_remotes, + repo_section=repo_section, + is_index_only=is_index_only, + fdroid_dir=git_fdroiddir, + git_mirror_path=str(git_mirror_path), + ssh_cmd=ssh_cmd, + progress=progress, + ) if progress: progressbar.done() +def upload_to_servergitmirror( + mirror_config: Dict[str, str], + local_repo: Repo, + enabled_remotes: List[str], + repo_section: str, + is_index_only: bool, + fdroid_dir: str, + git_mirror_path: str, + ssh_cmd: str, + progress: git.RemoteProgress, +) -> None: + remote_branch_name = GIT_BRANCH + local_branch_name = local_repo.active_branch.name + + remote_url = mirror_config['url'] + name = REMOTE_HOSTNAME_REGEX.sub(r'\1', remote_url) + enabled_remotes.append(name) + r = git.remote.Remote(local_repo, name) + if r in local_repo.remotes: + r = local_repo.remote(name) + if 'set_url' in dir(r): # force remote URL if using GitPython 2.x + r.set_url(remote_url) + else: + local_repo.create_remote(name, remote_url) + logging.info('Mirroring to: ' + remote_url) + + if is_index_only: + files_to_upload = _get_index_file_paths( + os.path.join(local_repo.working_tree_dir, 'fdroid', repo_section) + ) + files_to_upload = _remove_missing_files(files_to_upload) + local_repo.index.add(files_to_upload) + else: + # sadly index.add don't allow the --all parameter + logging.debug('Adding all files to git mirror') + local_repo.git.add(all=True) + + logging.debug('Committing files into git mirror') + local_repo.index.commit("fdroidserver git-mirror") + + # only deploy to GitLab Artifacts if too big for GitLab Pages + if common.get_dir_size(fdroid_dir) <= common.GITLAB_COM_PAGES_MAX_SIZE: + gitlab_ci_job_name = 'pages' + else: + gitlab_ci_job_name = 'GitLab Artifacts' + logging.warning( + _('Skipping GitLab Pages mirror because the repo is too large (>%.2fGB)!') + % (common.GITLAB_COM_PAGES_MAX_SIZE / 1000000000) + ) + + # push. This will overwrite the git history + remote = local_repo.remote(name) + if remote.name == 'gitlab': + logging.debug('Writing .gitlab-ci.yml to deploy to GitLab Pages') + with open(os.path.join(git_mirror_path, ".gitlab-ci.yml"), "wt") as fp: + yaml.dump( + { + gitlab_ci_job_name: { + 'script': [ + 'mkdir .public', + 'cp -r * .public/', + 'mv .public public', + ], + 'artifacts': {'paths': ['public']}, + 'variables': {'GIT_DEPTH': 1}, + } + }, + fp, + default_flow_style=False, + ) + + local_repo.index.add(['.gitlab-ci.yml']) + local_repo.index.commit("fdroidserver git-mirror: Deploy to GitLab Pages") + + logging.debug(_('Pushing to {url}').format(url=remote.url)) + with local_repo.git.custom_environment(GIT_SSH_COMMAND=ssh_cmd): + pushinfos = remote.push( + f"{local_branch_name}:{remote_branch_name}", + force=True, + set_upstream=True, + progress=progress, + ) + for pushinfo in pushinfos: + if pushinfo.flags & ( + git.remote.PushInfo.ERROR + | git.remote.PushInfo.REJECTED + | git.remote.PushInfo.REMOTE_FAILURE + | git.remote.PushInfo.REMOTE_REJECTED + ): + # Show potentially useful messages from git remote + for line in progress.other_lines: + if line.startswith('remote:'): + logging.debug(line) + raise FDroidException( + remote.url + + ' push failed: ' + + str(pushinfo.flags) + + ' ' + + pushinfo.summary + ) + else: + logging.debug(remote.url + ': ' + pushinfo.summary) + + def upload_to_android_observatory(repo_section): import requests @@ -955,8 +1089,6 @@ def push_binary_transparency(git_repo_path, git_remote): drive. """ - import git - logging.info(_('Pushing binary transparency log to {url}').format(url=git_remote)) if os.path.isdir(os.path.dirname(git_remote)): diff --git a/fdroidserver/nightly.py b/fdroidserver/nightly.py index 074a9eee..9c2c7175 100644 --- a/fdroidserver/nightly.py +++ b/fdroidserver/nightly.py @@ -378,11 +378,11 @@ Last updated: {date}'''.format(repo_git_base=repo_git_base, os.chdir(repo_basedir) if os.path.isdir(git_mirror_repodir): - common.local_rsync(options, git_mirror_repodir + '/', 'repo/') + common.local_rsync(options, [git_mirror_repodir + '/'], 'repo/') if os.path.isdir(git_mirror_metadatadir): - common.local_rsync(options, git_mirror_metadatadir + '/', 'metadata/') + common.local_rsync(options, [git_mirror_metadatadir + '/'], 'metadata/') if os.path.isdir(git_mirror_statsdir): - common.local_rsync(options, git_mirror_statsdir + '/', 'stats/') + common.local_rsync(options, [git_mirror_statsdir + '/'], 'stats/') ssh_private_key_file = _ssh_key_from_debug_keystore() # this is needed for GitPython to find the SSH key @@ -484,9 +484,9 @@ Last updated: {date}'''.format(repo_git_base=repo_git_base, cwd=repo_basedir, ) common.local_rsync( - options, repo_basedir + '/metadata/', git_mirror_metadatadir + '/' + options, [repo_basedir + '/metadata/'], git_mirror_metadatadir + '/' ) - common.local_rsync(options, repo_basedir + '/stats/', git_mirror_statsdir + '/') + common.local_rsync(options, [repo_basedir + '/stats/'], git_mirror_statsdir + '/') mirror_git_repo.git.add(all=True) mirror_git_repo.index.commit("update app metadata") diff --git a/tests/deploy.TestCase b/tests/deploy.TestCase index 7e5b123b..00a62888 100755 --- a/tests/deploy.TestCase +++ b/tests/deploy.TestCase @@ -11,6 +11,8 @@ import unittest from pathlib import Path from unittest import mock +import git + localmodule = os.path.realpath( os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..') ) @@ -38,7 +40,7 @@ class DeployTest(unittest.TestCase): self._td = mkdtemp() self.testdir = self._td.name - fdroidserver.deploy.options = mock.Mock() + fdroidserver.common.options = mock.Mock() fdroidserver.deploy.config = {} fdroidserver.deploy.USER_RCLONE_CONF = False @@ -70,6 +72,10 @@ class DeployTest(unittest.TestCase): url1 = Path('url1/fdroid') url1.mkdir(parents=True) + # setup parameters for this test run + fdroidserver.common.options.identity_file = None + fdroidserver.deploy.config['make_current_version_link'] = False + dest_apk0 = url0 / fake_apk dest_apk1 = url1 / fake_apk self.assertFalse(dest_apk0.is_file()) @@ -107,6 +113,9 @@ class DeployTest(unittest.TestCase): fake_apk = repo / 'another_fake.apk' with fake_apk.open('w') as fp: fp.write('not an APK, but has the right filename') + fake_index = repo / fdroidserver.deploy.INDEX_FILES[0] + with fake_index.open('w') as fp: + fp.write('not an index, but has the right filename') # write out rclone config for test use rclone_config = configparser.ConfigParser() @@ -124,17 +133,66 @@ class DeployTest(unittest.TestCase): fdroidserver.deploy.config['rclone'] = True fdroidserver.deploy.config['rclone_config'] = 'test-local-config' fdroidserver.deploy.config['path_to_custom_rclone_config'] = str(rclone_file) - fdroidserver.deploy.options = Options + fdroidserver.common.options = Options # write out destination path - destination = Path('some_bucket_folder/fdroid') + destination = Path('test_bucket_folder/fdroid') destination.mkdir(parents=True, exist_ok=True) - dest_path = Path(destination) / fake_apk - self.assertFalse(dest_path.is_file()) + dest_apk = Path(destination) / fake_apk + dest_index = Path(destination) / fake_index + self.assertFalse(dest_apk.is_file()) + self.assertFalse(dest_index.is_file()) repo_section = str(repo) # fdroidserver.deploy.USER_RCLONE_CONF = str(rclone_file) fdroidserver.deploy.update_remote_storage_with_rclone(repo_section) - self.assertFalse(dest_path.is_file()) + self.assertTrue(dest_apk.is_file()) + self.assertTrue(dest_index.is_file()) + + @unittest.skipUnless(shutil.which('rclone'), '/usr/bin/rclone') + def test_update_remote_storage_with_rclone_in_index_only_mode(self): + os.chdir(self.testdir) + repo = Path('repo') + repo.mkdir(parents=True, exist_ok=True) + + fake_apk = repo / 'another_fake.apk' + with fake_apk.open('w') as fp: + fp.write('not an APK, but has the right filename') + fake_index = repo / fdroidserver.deploy.INDEX_FILES[0] + with fake_index.open('w') as fp: + fp.write('not an index, but has the right filename') + + # write out rclone config for test use + rclone_config = configparser.ConfigParser() + rclone_config.add_section("test-local-config") + rclone_config.set("test-local-config", "type", "local") + + rclone_config_path = Path('rclone_config_path') + rclone_config_path.mkdir(parents=True, exist_ok=True) + rclone_file = rclone_config_path / 'rclone.conf' + with open(rclone_file, 'w') as configfile: + rclone_config.write(configfile) + + # setup parameters for this test run + fdroidserver.deploy.config['awsbucket'] = 'test_bucket_folder' + fdroidserver.deploy.config['rclone'] = True + fdroidserver.deploy.config['rclone_config'] = 'test-local-config' + fdroidserver.deploy.config['path_to_custom_rclone_config'] = str(rclone_file) + fdroidserver.common.options = Options + + # write out destination path + destination = Path('test_bucket_folder/fdroid') + destination.mkdir(parents=True, exist_ok=True) + dest_apk = Path(destination) / fake_apk + dest_index = Path(destination) / fake_index + self.assertFalse(dest_apk.is_file()) + self.assertFalse(dest_index.is_file()) + repo_section = str(repo) + # fdroidserver.deploy.USER_RCLONE_CONF = str(rclone_file) + fdroidserver.deploy.update_remote_storage_with_rclone( + repo_section, is_index_only=True + ) + self.assertFalse(dest_apk.is_file()) + self.assertTrue(dest_index.is_file()) def test_update_serverwebroot(self): """rsync works with file paths, so this test uses paths for the URLs""" @@ -144,15 +202,55 @@ class DeployTest(unittest.TestCase): fake_apk = repo / 'fake.apk' with fake_apk.open('w') as fp: fp.write('not an APK, but has the right filename') + fake_index = repo / fdroidserver.deploy.INDEX_FILES[0] + with fake_index.open('w') as fp: + fp.write('not an index, but has the right filename') url = Path('url') url.mkdir() - dest_apk = url / fake_apk + # setup parameters for this test run + fdroidserver.common.options = mock.Mock() + fdroidserver.common.options.identity_file = None + fdroidserver.common.options.identity_file = None + fdroidserver.deploy.config['make_current_version_link'] = False + + dest_apk = Path(url) / fake_apk + dest_index = Path(url) / fake_index self.assertFalse(dest_apk.is_file()) - fdroidserver.deploy.options = mock.Mock() - fdroidserver.deploy.options.identity_file = None + self.assertFalse(dest_index.is_file()) + fdroidserver.deploy.update_serverwebroot({'url': str(url)}, 'repo') self.assertTrue(dest_apk.is_file()) + self.assertTrue(dest_index.is_file()) + + def test_update_serverwebroot_in_index_only_mode(self): + os.chdir(self.testdir) + repo = Path('repo') + repo.mkdir() + fake_apk = repo / 'fake.apk' + with fake_apk.open('w') as fp: + fp.write('not an APK, but has the right filename') + fake_index = repo / fdroidserver.deploy.INDEX_FILES[0] + with fake_index.open('w') as fp: + fp.write('not an index, but has the right filename') + url = Path('url') + url.mkdir() + + # setup parameters for this test run + fdroidserver.common.options = mock.Mock() + fdroidserver.common.options.identity_file = None + fdroidserver.deploy.config['make_current_version_link'] = False + + dest_apk = Path(url) / fake_apk + dest_index = Path(url) / fake_index + self.assertFalse(dest_apk.is_file()) + self.assertFalse(dest_index.is_file()) + + fdroidserver.deploy.update_serverwebroot( + {'url': str(url), 'index_only': True}, 'repo' + ) + self.assertFalse(dest_apk.is_file()) + self.assertTrue(dest_index.is_file()) @mock.patch.dict(os.environ, clear=True) def test_update_serverwebroot_no_rsync_error(self): @@ -170,6 +268,7 @@ class DeployTest(unittest.TestCase): fdroidserver.common.options.identity_file = None fdroidserver.common.options.verbose = False fdroidserver.common.options.quiet = True + fdroidserver.common.options.index_only = False fdroidserver.deploy.config = {'make_current_version_link': True} url = "example.com:/var/www/fdroid" repo_section = 'repo' @@ -256,6 +355,97 @@ class DeployTest(unittest.TestCase): fdroidserver.deploy.update_serverwebroot({'url': url}, repo_section) self.assertEqual(call_iteration, 3, 'expected 3 invocations of subprocess.call') + def test_update_serverwebroot_make_cur_version_link_in_index_only_mode(self): + # setup parameters for this test run + fdroidserver.common.options = mock.Mock() + fdroidserver.common.options.no_checksum = True + fdroidserver.common.options.identity_file = None + fdroidserver.common.options.verbose = False + fdroidserver.common.options.quiet = True + fdroidserver.common.options.identity_file = None + fdroidserver.deploy.config['make_current_version_link'] = True + url = "example.com:/var/www/fdroid" + repo_section = 'repo' + + # setup function for asserting subprocess.call invocations + call_iteration = 0 + + def update_server_webroot_call(cmd): + nonlocal call_iteration + if call_iteration == 0: + self.assertListEqual( + cmd, + [ + 'rsync', + '--archive', + '--delete-after', + '--safe-links', + '--quiet', + 'repo/altstore-index.json', + 'repo/entry.jar', + 'repo/entry.json', + 'repo/entry.json.asc', + 'repo/index-v1.jar', + 'repo/index-v1.json', + 'repo/index-v1.json.asc', + 'repo/index-v2.json', + 'repo/index-v2.json.asc', + 'repo/index.jar', + 'repo/index.xml', + 'example.com:/var/www/fdroid/repo/', + ], + ) + elif call_iteration == 1: + self.assertListEqual( + cmd, + [ + 'rsync', + '--archive', + '--delete-after', + '--safe-links', + '--quiet', + 'repo', + url, + ], + ) + elif call_iteration == 2: + self.assertListEqual( + cmd, + [ + 'rsync', + '--archive', + '--delete-after', + '--safe-links', + '--quiet', + 'Sym.apk', + 'Sym.apk.asc', + 'Sym.apk.sig', + 'example.com:/var/www/fdroid', + ], + ) + else: + self.fail('unexpected subprocess.call invocation') + call_iteration += 1 + return 0 + + with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir): + os.mkdir(repo_section) + os.symlink('repo/com.example.sym.apk', 'Sym.apk') + os.symlink('repo/com.example.sym.apk.asc', 'Sym.apk.asc') + os.symlink('repo/com.example.sym.apk.sig', 'Sym.apk.sig') + + fake_files = fdroidserver.deploy.INDEX_FILES + for filename in fake_files: + fake_file = Path(repo_section) / filename + with fake_file.open('w') as fp: + fp.write('not a real one, but has the right filename') + + with mock.patch('subprocess.call', side_effect=update_server_webroot_call): + fdroidserver.deploy.update_serverwebroot( + {'url': url, 'index_only': True}, repo_section + ) + self.assertEqual(call_iteration, 1, 'expected 1 invocations of subprocess.call') + def test_update_serverwebroot_with_id_file(self): # setup parameters for this test run fdroidserver.common.options = mock.Mock() @@ -264,6 +454,7 @@ class DeployTest(unittest.TestCase): fdroidserver.common.options.verbose = True fdroidserver.common.options.quiet = False fdroidserver.common.options.identity_file = None + fdroidserver.common.options.index_only = False fdroidserver.deploy.config = {'identity_file': './id_rsa'} url = "example.com:/var/www/fdroid" repo_section = 'archive' @@ -336,10 +527,88 @@ class DeployTest(unittest.TestCase): fdroidserver.deploy.update_serverwebroot({'url': url}, repo_section) self.assertEqual(call_iteration, 2, 'expected 2 invocations of subprocess.call') + def test_update_serverwebroot_with_id_file_in_index_only_mode(self): + # setup parameters for this test run + fdroidserver.common.options = mock.Mock() + fdroidserver.common.options.no_chcksum = False + fdroidserver.common.options.verbose = True + fdroidserver.common.options.quiet = False + fdroidserver.common.options.identity_file = None + fdroidserver.deploy.config['identity_file'] = './id_rsa' + fdroidserver.deploy.config['make_current_version_link'] = False + url = "example.com:/var/www/fdroid" + repo_section = 'archive' + + # setup function for asserting subprocess.call invocations + call_iteration = 0 + + def update_server_webroot_call(cmd): + nonlocal call_iteration + if call_iteration == 0: + self.assertListEqual( + cmd, + [ + 'rsync', + '--archive', + '--delete-after', + '--safe-links', + '--verbose', + '-e', + 'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ' + + fdroidserver.deploy.config['identity_file'], + 'archive/altstore-index.json', + 'archive/entry.jar', + 'archive/entry.json', + 'archive/entry.json.asc', + 'archive/index-v1.jar', + 'archive/index-v1.json', + 'archive/index-v1.json.asc', + 'archive/index-v2.json', + 'archive/index-v2.json.asc', + 'archive/index.jar', + 'archive/index.xml', + "example.com:/var/www/fdroid/archive/", + ], + ) + elif call_iteration == 1: + self.assertListEqual( + cmd, + [ + 'rsync', + '--archive', + '--delete-after', + '--safe-links', + '--verbose', + '-e', + 'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ' + + fdroidserver.deploy.config['identity_file'], + "example.com:/var/www/fdroid/archive/", + ], + ) + else: + self.fail('unexpected subprocess.call invocation') + call_iteration += 1 + return 0 + + with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir): + with mock.patch('subprocess.call', side_effect=update_server_webroot_call): + os.mkdir(repo_section) + fake_files = fdroidserver.deploy.INDEX_FILES + for filename in fake_files: + fake_file = Path(repo_section) / filename + with fake_file.open('w') as fp: + fp.write('not a real one, but has the right filename') + + fdroidserver.deploy.update_serverwebroot( + {'url': url, 'index_only': True}, repo_section + ) + self.assertEqual(call_iteration, 1, 'expected 1 invocations of subprocess.call') + @unittest.skipIf( not os.getenv('VIRUSTOTAL_API_KEY'), 'VIRUSTOTAL_API_KEY is not set' ) def test_upload_to_virustotal(self): + fdroidserver.common.options = mock.Mock() fdroidserver.common.options.verbose = True virustotal_apikey = os.getenv('VIRUSTOTAL_API_KEY') fdroidserver.deploy.upload_to_virustotal('repo', virustotal_apikey) @@ -356,25 +625,245 @@ class DeployTest(unittest.TestCase): name, fdroidserver.deploy.REMOTE_HOSTNAME_REGEX.sub(r'\1', remote_url) ) - def test_update_servergitmirrors(self): + def test_update_awsbucket_s3cmd(self): # setup parameters for this test run fdroidserver.common.options = mock.Mock() - fdroidserver.common.options.identity_file = None - fdroidserver.common.options.no_keep_git_mirror_archive = False + fdroidserver.common.options.no_checksum = True fdroidserver.common.options.verbose = False fdroidserver.common.options.quiet = True - fdroidserver.common.options.index_only = False config = {} fdroidserver.common.fill_config_defaults(config) fdroidserver.deploy.config = config - fdroidserver.deploy.config["servergitmirrors"] = [] + fdroidserver.deploy.config["awsbucket"] = "bucket" + fdroidserver.deploy.config["awsaccesskeyid"] = "accesskeyid" + fdroidserver.deploy.config["awssecretkey"] = "secretkey" + fdroidserver.deploy.config["s3cmd"] = "s3cmd" repo_section = 'repo' # setup function for asserting subprocess.call invocations - update_servergitmirrors_call_iteration = 0 - remote_push_call_iteration = 0 + call_iteration = 0 + + def update_awsbucket_s3cmd_call(cmd): + nonlocal call_iteration + if call_iteration == 0: + self.assertListEqual( + cmd, + [ + 's3cmd', + f"--config={fdroidserver.deploy.AUTO_S3CFG}", + 'info', + f"s3://{fdroidserver.deploy.config['awsbucket']}", + ], + ) + elif call_iteration == 1: + self.assertListEqual( + cmd, + [ + 's3cmd', + f"--config={fdroidserver.deploy.AUTO_S3CFG}", + 'sync', + '--acl-public', + '--quiet', + '--exclude', + 'repo/altstore-index.json', + '--exclude', + 'repo/entry.jar', + '--exclude', + 'repo/entry.json', + '--exclude', + 'repo/entry.json.asc', + '--exclude', + 'repo/index-v1.jar', + '--exclude', + 'repo/index-v1.json', + '--exclude', + 'repo/index-v1.json.asc', + '--exclude', + 'repo/index-v2.json', + '--exclude', + 'repo/index-v2.json.asc', + '--exclude', + 'repo/index.jar', + '--exclude', + 'repo/index.xml', + '--no-check-md5', + '--skip-existing', + repo_section, + f"s3://{fdroidserver.deploy.config['awsbucket']}/fdroid/", + ], + ) + elif call_iteration == 2: + self.assertListEqual( + cmd, + [ + 's3cmd', + f"--config={fdroidserver.deploy.AUTO_S3CFG}", + 'sync', + '--acl-public', + '--quiet', + '--exclude', + 'repo/altstore-index.json', + '--exclude', + 'repo/entry.jar', + '--exclude', + 'repo/entry.json', + '--exclude', + 'repo/entry.json.asc', + '--exclude', + 'repo/index-v1.jar', + '--exclude', + 'repo/index-v1.json', + '--exclude', + 'repo/index-v1.json.asc', + '--exclude', + 'repo/index-v2.json', + '--exclude', + 'repo/index-v2.json.asc', + '--exclude', + 'repo/index.jar', + '--exclude', + 'repo/index.xml', + '--no-check-md5', + repo_section, + f"s3://{fdroidserver.deploy.config['awsbucket']}/fdroid/", + ], + ) + elif call_iteration == 3: + self.assertListEqual( + cmd, + [ + 's3cmd', + f"--config={fdroidserver.deploy.AUTO_S3CFG}", + 'sync', + '--acl-public', + '--quiet', + '--delete-removed', + '--delete-after', + '--no-check-md5', + repo_section, + f"s3://{fdroidserver.deploy.config['awsbucket']}/fdroid/", + ], + ) + else: + self.fail('unexpected subprocess.call invocation') + call_iteration += 1 + return 0 + + with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir): + os.mkdir('repo') + os.symlink('repo/com.example.sym.apk', 'Sym.apk') + os.symlink('repo/com.example.sym.apk.asc', 'Sym.apk.asc') + os.symlink('repo/com.example.sym.apk.sig', 'Sym.apk.sig') + with mock.patch('subprocess.call', side_effect=update_awsbucket_s3cmd_call): + fdroidserver.deploy.update_awsbucket_s3cmd(repo_section) + self.assertEqual(call_iteration, 4, 'expected 4 invocations of subprocess.call') + + def test_update_awsbucket_s3cmd_in_index_only_mode(self): + # setup parameters for this test run + fdroidserver.common.options = mock.Mock() + fdroidserver.common.options.no_checksum = True + fdroidserver.common.options.verbose = False + fdroidserver.common.options.quiet = True + + config = {} + fdroidserver.common.fill_config_defaults(config) + fdroidserver.deploy.config = config + fdroidserver.deploy.config["awsbucket"] = "bucket" + fdroidserver.deploy.config["awsaccesskeyid"] = "accesskeyid" + fdroidserver.deploy.config["awssecretkey"] = "secretkey" + fdroidserver.deploy.config["s3cmd"] = "s3cmd" + + repo_section = 'repo' + + # setup function for asserting subprocess.call invocations + call_iteration = 0 + + def update_awsbucket_s3cmd_call(cmd): + nonlocal call_iteration + if call_iteration == 0: + self.assertListEqual( + cmd, + [ + 's3cmd', + f"--config={fdroidserver.deploy.AUTO_S3CFG}", + 'info', + f"s3://{fdroidserver.deploy.config['awsbucket']}", + ], + ) + elif call_iteration == 1: + self.assertListEqual( + cmd, + [ + 's3cmd', + f"--config={fdroidserver.deploy.AUTO_S3CFG}", + 'sync', + '--acl-public', + '--quiet', + '--include', + 'repo/altstore-index.json', + '--include', + 'repo/entry.jar', + '--include', + 'repo/entry.json', + '--include', + 'repo/entry.json.asc', + '--include', + 'repo/index-v1.jar', + '--include', + 'repo/index-v1.json', + '--include', + 'repo/index-v1.json.asc', + '--include', + 'repo/index-v2.json', + '--include', + 'repo/index-v2.json.asc', + '--include', + 'repo/index.jar', + '--include', + 'repo/index.xml', + '--delete-removed', + '--delete-after', + '--no-check-md5', + repo_section, + f"s3://{fdroidserver.deploy.config['awsbucket']}/fdroid/", + ], + ) + else: + self.fail('unexpected subprocess.call invocation') + call_iteration += 1 + return 0 + + with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir): + os.mkdir('repo') + os.symlink('repo/com.example.sym.apk', 'Sym.apk') + os.symlink('repo/com.example.sym.apk.asc', 'Sym.apk.asc') + os.symlink('repo/com.example.sym.apk.sig', 'Sym.apk.sig') + with mock.patch('subprocess.call', side_effect=update_awsbucket_s3cmd_call): + fdroidserver.deploy.update_awsbucket_s3cmd( + repo_section, is_index_only=True + ) + self.assertEqual(call_iteration, 2, 'expected 2 invocations of subprocess.call') + + def test_update_awsbucket_libcloud(self): + from libcloud.storage.base import Container + + # setup parameters for this test run + fdroidserver.common.options = mock.Mock() + fdroidserver.common.options.no_checksum = True + fdroidserver.common.options.verbose = False + fdroidserver.common.options.quiet = True + + config = {} + fdroidserver.common.fill_config_defaults(config) + fdroidserver.deploy.config = config + fdroidserver.deploy.config["awsbucket"] = "bucket" + fdroidserver.deploy.config["awsaccesskeyid"] = "accesskeyid" + fdroidserver.deploy.config["awssecretkey"] = "secretkey" + fdroidserver.deploy.config["s3cmd"] = "s3cmd" + + repo_section = 'repo' os.chdir(self.testdir) repo = Path('repo') @@ -386,56 +875,292 @@ class DeployTest(unittest.TestCase): with fake_index.open('w') as fp: fp.write('not an index, but has the right filename') - def update_servergitmirrors_call(cmd): - nonlocal update_servergitmirrors_call_iteration - if update_servergitmirrors_call_iteration == 0: - self.assertListEqual( - cmd, - [ - 'rsync', - '--recursive', - '--safe-links', - '--times', - '--perms', - '--one-file-system', - '--delete', - '--chmod=Da+rx,Fa-x,a+r,u+w', - '--quiet', - 'repo/', - "git-mirror/fdroid/repo/", - ], - ) - else: - self.fail('unexpected subprocess.call invocation') - update_servergitmirrors_call_iteration += 1 - return 0 + with mock.patch( + 'libcloud.storage.drivers.s3.S3StorageDriver' + ) as mock_driver_class: + mock_driver = mock_driver_class.return_value + mock_container = mock.MagicMock(spec=Container) + mock_container.list_objects.return_value = [ + mock.MagicMock(name='Sym.apk'), + mock.MagicMock(name=fdroidserver.deploy.INDEX_FILES[0]), + ] - def remote_push_call(ref, force=False, set_upstream=False, **_args): - nonlocal remote_push_call_iteration - if remote_push_call_iteration == 0: - self.assertEqual([ref, force, set_upstream], ['master', True, True]) - else: - self.fail('unexpected git.Remote.push invocation') - remote_push_call_iteration += 1 - return [] + mock_driver.get_container.return_value = mock_container + mock_driver.upload_object_via_stream.return_value = None - with mock.patch('subprocess.call', side_effect=update_servergitmirrors_call): - with mock.patch( - 'git.Remote.push', side_effect=remote_push_call - ) as mock_remote_push: - mock_remote_push.return_value = [] - fdroidserver.deploy.update_servergitmirrors( - [{'url': 'https://github.com/user/repo'}], repo_section + fdroidserver.deploy.update_awsbucket_libcloud(repo_section) + + mock_driver.get_container.assert_called_once_with( + container_name=fdroidserver.deploy.config["awsbucket"] + ) + mock_container.list_objects.assert_called_once_with() + files_to_upload = [ + 'fdroid/repo/Sym.apk', + f"fdroid/repo/{fdroidserver.deploy.INDEX_FILES[0]}", + ] + calls = [ + mock.call( + iterator=mock.ANY, + container=mock_container, + object_name=file, + extra={'acl': 'public-read'}, ) - self.assertEqual( - update_servergitmirrors_call_iteration, - 1, - 'expected 1 invocations of subprocess.call', + for file in files_to_upload + ] + mock_driver.upload_object_via_stream.assert_has_calls(calls, any_order=True) + assert mock_driver.upload_object_via_stream.call_count == 2 + + def test_update_awsbucket_libcloud_in_index_only_mode(self): + from libcloud.storage.base import Container + + # setup parameters for this test run + fdroidserver.common.options = mock.Mock() + fdroidserver.common.options.no_checksum = True + fdroidserver.common.options.verbose = False + fdroidserver.common.options.quiet = True + + config = {} + fdroidserver.common.fill_config_defaults(config) + fdroidserver.deploy.config = config + fdroidserver.deploy.config["awsbucket"] = "bucket" + fdroidserver.deploy.config["awsaccesskeyid"] = "accesskeyid" + fdroidserver.deploy.config["awssecretkey"] = "secretkey" + fdroidserver.deploy.config["s3cmd"] = "s3cmd" + + repo_section = 'repo' + + os.chdir(self.testdir) + repo = Path('repo') + repo.mkdir(parents=True) + fake_apk = repo / 'Sym.apk' + with fake_apk.open('w') as fp: + fp.write('not an APK, but has the right filename') + fake_index = repo / fdroidserver.deploy.INDEX_FILES[0] + with fake_index.open('w') as fp: + fp.write('not an index, but has the right filename') + + with mock.patch( + 'libcloud.storage.drivers.s3.S3StorageDriver' + ) as mock_driver_class: + mock_driver = mock_driver_class.return_value + mock_container = mock.MagicMock(spec=Container) + mock_container.list_objects.return_value = [ + mock.MagicMock(name='Sym.apk'), + mock.MagicMock(name=fdroidserver.deploy.INDEX_FILES[0]), + ] + + mock_driver.get_container.return_value = mock_container + mock_driver.upload_object_via_stream.return_value = None + + fdroidserver.deploy.update_awsbucket_libcloud( + repo_section, is_index_only=True + ) + + mock_driver.get_container.assert_called_once_with( + container_name=fdroidserver.deploy.config["awsbucket"] + ) + mock_container.list_objects.assert_called_once_with() + files_to_upload = [f"fdroid/repo/{fdroidserver.deploy.INDEX_FILES[0]}"] + calls = [ + mock.call( + iterator=mock.ANY, + container=mock_container, + object_name=file, + extra={'acl': 'public-read'}, + ) + for file in files_to_upload + ] + mock_driver.upload_object_via_stream.assert_has_calls( + calls, + any_order=False, + ) + assert mock_driver.upload_object_via_stream.call_count == 1 + + def test_update_servergitmirrors(self): + # setup parameters for this test run + fdroidserver.common.options = mock.Mock() + fdroidserver.common.options.identity_file = None + fdroidserver.common.options.no_keep_git_mirror_archive = False + fdroidserver.common.options.verbose = False + fdroidserver.common.options.quiet = True + + config = {} + fdroidserver.common.fill_config_defaults(config) + fdroidserver.deploy.config = config + + os.chdir(self.testdir) + + repo_section = 'repo' + initial_branch = fdroidserver.deploy.GIT_BRANCH + + remote_repo = Path(self.testdir) / 'remote' + remote_repo.mkdir(parents=True) + remote_git_repo = git.Repo.init( + remote_repo, initial_branch=initial_branch, bare=True ) - self.assertEqual( - remote_push_call_iteration, 1, 'expected 1 invocations of git.Remote.push' + fdroidserver.deploy.config["servergitmirrors"] = [{"url": str(remote_repo)}] + + os.chdir(self.testdir) + repo = Path('repo') + repo.mkdir(parents=True) + fake_apk = 'Sym.apk' + fake_files = fdroidserver.deploy.INDEX_FILES + [fake_apk] + for filename in fake_files: + fake_file = repo / filename + with fake_file.open('w') as fp: + fp.write('not a real one, but has the right filename') + + fdroidserver.deploy.update_servergitmirrors( + fdroidserver.deploy.config["servergitmirrors"], repo_section ) + verify_repo = remote_git_repo.clone( + Path(self.testdir) / 'verify', + ) + + for filename in fake_files: + remote_file = f"fdroid/{repo_section}/{filename}" + + self.assertIsNotNone(verify_repo.working_tree_dir) + if verify_repo.working_tree_dir is not None: + self.assertTrue( + (Path(verify_repo.working_tree_dir) / remote_file).exists() + ) + + def test_update_servergitmirrors_in_index_only_mode(self): + # setup parameters for this test run + fdroidserver.common.options = mock.Mock() + fdroidserver.common.options.identity_file = None + fdroidserver.common.options.no_keep_git_mirror_archive = False + fdroidserver.common.options.verbose = False + fdroidserver.common.options.quiet = True + + config = {} + fdroidserver.common.fill_config_defaults(config) + fdroidserver.deploy.config = config + + os.chdir(self.testdir) + + repo_section = 'repo' + initial_branch = fdroidserver.deploy.GIT_BRANCH + + remote_repo = Path(self.testdir) / 'remote' + remote_repo.mkdir(parents=True) + remote_git_repo = git.Repo.init( + remote_repo, initial_branch=initial_branch, bare=True + ) + fdroidserver.deploy.config["servergitmirrors"] = [ + {"url": str(remote_repo), "index_only": True} + ] + + os.chdir(self.testdir) + repo = Path('repo') + repo.mkdir(parents=True) + fake_apk = 'Sym.apk' + fake_files = fdroidserver.deploy.INDEX_FILES + [fake_apk] + for filename in fake_files: + fake_file = repo / filename + with fake_file.open('w') as fp: + fp.write('not a real one, but has the right filename') + + fdroidserver.deploy.update_servergitmirrors( + fdroidserver.deploy.config["servergitmirrors"], repo_section + ) + + verify_repo = remote_git_repo.clone( + Path(self.testdir) / 'verify', + ) + + for filename in fdroidserver.deploy.INDEX_FILES: + remote_file = f"fdroid/{repo_section}/{filename}" + + self.assertIsNotNone(verify_repo.working_tree_dir) + if verify_repo.working_tree_dir is not None: + self.assertTrue( + (Path(verify_repo.working_tree_dir) / remote_file).exists() + ) + + # Should not have the APK file + remote_file = f"fdroid/{repo_section}/{fake_apk}" + if verify_repo.working_tree_dir is not None: + self.assertFalse( + (Path(verify_repo.working_tree_dir) / remote_file).exists() + ) + + def test_upload_to_servergitmirror_in_index_only_mode(self): + # setup parameters for this test run + fdroidserver.common.options = mock.Mock() + fdroidserver.common.options.identity_file = None + fdroidserver.common.options.no_keep_git_mirror_archive = False + fdroidserver.common.options.verbose = False + fdroidserver.common.options.quiet = True + fdroidserver.common.options.identity_file = None + + config = {} + fdroidserver.common.fill_config_defaults(config) + fdroidserver.deploy.config = config + + repo_section = 'repo' + initial_branch = fdroidserver.deploy.GIT_BRANCH + + os.chdir(self.testdir) + + local_git_repo_path = Path(self.testdir) / 'local' + local_git_repo = git.Repo.init( + local_git_repo_path, initial_branch=initial_branch + ) + + fdroid_dir = local_git_repo_path / 'fdroid' + repo_dir = fdroid_dir / repo_section + repo_dir.mkdir(parents=True) + fake_apk = 'Sym.apk' + fake_files = fdroidserver.deploy.INDEX_FILES + [fake_apk] + for filename in fake_files: + fake_file = repo_dir / filename + with fake_file.open('w') as fp: + fp.write('not a real one, but has the right filename') + + # The remote repo must be a bare repo to allow being pushed to + remote_git_repo_dir = Path(self.testdir) / 'remote' + remote_git_repo = git.Repo.init( + remote_git_repo_dir, initial_branch=initial_branch, bare=True + ) + + mirror_config = {"url": str(remote_git_repo_dir), "index_only": True} + enabled_remotes = [] + ssh_cmd = 'ssh -oBatchMode=yes' + fdroidserver.deploy.upload_to_servergitmirror( + mirror_config=mirror_config, + local_repo=local_git_repo, + enabled_remotes=enabled_remotes, + repo_section=repo_section, + is_index_only=mirror_config['index_only'], + fdroid_dir=str(fdroid_dir), + git_mirror_path=str(local_git_repo_path), + ssh_cmd=ssh_cmd, + progress=git.RemoteProgress(), + ) + + verify_repo = remote_git_repo.clone( + Path(self.testdir) / 'verify', + ) + + for filename in fdroidserver.deploy.INDEX_FILES: + remote_file = f"fdroid/{repo_section}/{filename}" + + self.assertIsNotNone(verify_repo.working_tree_dir) + if verify_repo.working_tree_dir is not None: + self.assertTrue( + (Path(verify_repo.working_tree_dir) / remote_file).exists() + ) + + # Should not have the APK file + remote_file = f"fdroid/{repo_section}/{fake_apk}" + if verify_repo.working_tree_dir is not None: + self.assertFalse( + (Path(verify_repo.working_tree_dir) / remote_file).exists() + ) + if __name__ == "__main__": os.chdir(os.path.dirname(__file__))