Merge branch 'index-only-deploy-mode' into 'master'

Add index only deployment mode

Closes #1181

See merge request fdroid/fdroidserver!1420

examples/config.yml

@@ -183,7 +183,7 @@
 #
 # serverwebroot:
 #   - url: 'me@b.az:/srv/fdroid'
-#     indexOnly: true
+#     index_only: true
 
 
 # When running fdroid processes on a remote server, it is possible to
@@ -209,7 +209,7 @@
 # servergitmirrors:
 #   - url: https://github.com/user/repo
 #   - url: https://gitlab.com/user/repo
-#     indexOnly: true
+#     index_only: true
 
 # Most git hosting services have hard size limits for each git repo.
 # `fdroid deploy` will delete the git history when the git mirror repo

fdroidserver/common.py

@@ -29,6 +29,7 @@
 # libraries here as they will become a requirement for all commands.
 
 import difflib
+from typing import List
 import git
 import glob
 import io
@@ -4037,7 +4038,7 @@ def get_app_display_name(app):
     return app.get('AutoName') or app['id']
 
 
-def local_rsync(options, fromdir, todir):
+def local_rsync(options, from_paths: List[str], todir: str):
     """Rsync method for local to local copying of things.
 
     This is an rsync wrapper with all the settings for safe use within
@@ -4054,8 +4055,8 @@ def local_rsync(options, fromdir, todir):
         rsyncargs += ['--verbose']
     if options.quiet:
         rsyncargs += ['--quiet']
-    logging.debug(' '.join(rsyncargs + [fromdir, todir]))
-    if subprocess.call(rsyncargs + [fromdir, todir]) != 0:
+    logging.debug(' '.join(rsyncargs + from_paths + [todir]))
+    if subprocess.call(rsyncargs + from_paths + [todir]) != 0:
         raise FDroidException()
 
 

fdroidserver/deploy.py

@@ -25,8 +25,9 @@ import re
 import subprocess
 import time
 import urllib
+from typing import Optional
 import yaml
-from argparse import ArgumentParser
+from argparse import ArgumentParser, Namespace
 import logging
 import shutil
 
@@ -36,7 +37,7 @@ from . import index
 from .exception import FDroidException
 
 config = None
-options = None
+options: Optional[Namespace] = None
 start_timestamp = time.gmtime()
 
 GIT_BRANCH = 'master'
@@ -61,27 +62,19 @@ INDEX_FILES = [
 ]
 
 
-def _get_index_excludes(repo_section):
+def _get_index_file_paths(base_dir):
     """Return the list of files to be synced last, since they finalize the deploy.
 
     The process of pushing all the new packages to the various
     services can take a while.  So the index files should be updated
     last.  That ensures that the package files are available when the
     client learns about them from the new index files.
-
     """
-    indexes = [
-        os.path.join(repo_section, 'entry.jar'),
-        os.path.join(repo_section, 'entry.json'),
-        os.path.join(repo_section, 'entry.json.asc'),
-        os.path.join(repo_section, 'index-v1.jar'),
-        os.path.join(repo_section, 'index-v1.json'),
-        os.path.join(repo_section, 'index-v1.json.asc'),
-        os.path.join(repo_section, 'index-v2.json'),
-        os.path.join(repo_section, 'index-v2.json.asc'),
-        os.path.join(repo_section, 'index.jar'),
-        os.path.join(repo_section, 'index.xml'),
-    ]
+    return [os.path.join(base_dir, filename) for filename in INDEX_FILES]
+
+
+def _get_index_excludes(base_dir):
+    indexes = _get_index_file_paths(base_dir)
     index_excludes = []
     for f in indexes:
         index_excludes.append('--exclude')
@@ -89,7 +82,16 @@ def _get_index_excludes(repo_section):
     return index_excludes
 
 
-def update_awsbucket(repo_section):
+def _get_index_includes(base_dir):
+    indexes = _get_index_file_paths(base_dir)
+    index_includes = []
+    for f in indexes:
+        index_includes.append('--include')
+        index_includes.append(f)
+    return index_includes
+
+
+def update_awsbucket(repo_section, index_only=False):
     """Upload the contents of the directory `repo_section` (including subdirectories) to the AWS S3 "bucket".
 
     The contents of that subdir of the
@@ -101,12 +103,12 @@ def update_awsbucket(repo_section):
                   + config['awsbucket'] + '"')
 
     if common.set_command_in_config('s3cmd'):
-        update_awsbucket_s3cmd(repo_section)
+        update_awsbucket_s3cmd(repo_section, index_only)
     else:
-        update_awsbucket_libcloud(repo_section)
+        update_awsbucket_libcloud(repo_section, index_only)
 
 
-def update_awsbucket_s3cmd(repo_section):
+def update_awsbucket_s3cmd(repo_section, index_only=False):
     """Upload using the CLI tool s3cmd, which provides rsync-like sync.
 
     The upload is done in multiple passes to reduce the chance of
@@ -149,6 +151,22 @@ def update_awsbucket_s3cmd(repo_section):
         s3cmd_sync += ['--quiet']
 
     s3url = s3bucketurl + '/fdroid/'
+
+    if index_only:
+        logging.debug(_('s3cmd syncs indexes from {path} to {url} and deletes removed')
+                      .format(path=repo_section, url=s3url))
+        sync_indexes_flags = []
+        sync_indexes_flags.extend(_get_index_includes(repo_section))
+        sync_indexes_flags.append('--delete-removed')
+        sync_indexes_flags.append('--delete-after')
+        if options.no_checksum:
+            sync_indexes_flags.append('--no-check-md5')
+        else:
+            sync_indexes_flags.append('--check-md5')
+        returncode = subprocess.call(s3cmd_sync + sync_indexes_flags + [repo_section, s3url])
+        if returncode != 0:
+            raise FDroidException()
+    else:
         logging.debug('s3cmd sync new files in ' + repo_section + ' to ' + s3url)
         logging.debug(_('Running first pass with MD5 checking disabled'))
         excludes = _get_index_excludes(repo_section)
@@ -178,7 +196,7 @@ def update_awsbucket_s3cmd(repo_section):
             raise FDroidException()
 
 
-def update_awsbucket_libcloud(repo_section):
+def update_awsbucket_libcloud(repo_section, index_only=False):
     """No summary.
 
     Upload the contents of the directory `repo_section` (including
@@ -221,10 +239,16 @@ def update_awsbucket_libcloud(repo_section):
         if obj.name.startswith(upload_dir + '/'):
             objs[obj.name] = obj
 
-    for root, dirs, files in os.walk(os.path.join(os.getcwd(), repo_section)):
-        for name in files:
+    if index_only:
+        index_files = [f"{os.getcwd()}/{name}" for name in _get_index_file_paths(repo_section)]
+        files_to_upload = [os.path.join(root, name) for root, dirs, files in os.walk(os.path.join(os.getcwd(), repo_section)) for name in files]
+        files_to_upload = list(set(files_to_upload) & set(index_files))
+
+    else:
+        files_to_upload = [os.path.join(root, name) for root, dirs, files in os.walk(os.path.join(os.getcwd(), repo_section)) for name in files]
+
+    for file_to_upload in files_to_upload:
         upload = False
-            file_to_upload = os.path.join(root, name)
         object_name = 'fdroid/' + os.path.relpath(file_to_upload, os.getcwd())
         if object_name not in objs:
             upload = True
@@ -308,14 +332,22 @@ def update_serverwebroot(serverwebroot, repo_section):
     elif config and config.get('identity_file'):
         rsyncargs += ['-e', 'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ' + config['identity_file']]
     url = serverwebroot['url']
+    index_only = serverwebroot.get('index_only', False)
     logging.info('rsyncing ' + repo_section + ' to ' + url)
+    if index_only:
+        rsyncargs += _get_index_file_paths(repo_section)
+        rsyncargs += [f'{url}/{repo_section}/']
+        logging.info(rsyncargs)
+        if subprocess.call(rsyncargs) != 0:
+            raise FDroidException()
+    else:
         excludes = _get_index_excludes(repo_section)
         if subprocess.call(rsyncargs + excludes + [repo_section, url]) != 0:
             raise FDroidException()
         if subprocess.call(rsyncargs + [repo_section, url]) != 0:
             raise FDroidException()
         # upload "current version" symlinks if requested
-    if config and config.get('make_current_version_link') and repo_section == 'repo':
+        if config['make_current_version_link'] and repo_section == 'repo':
             links_to_upload = []
             for f in glob.glob('*.apk') \
                     + glob.glob('*.apk.asc') + glob.glob('*.apk.sig'):
@@ -364,10 +396,11 @@ def sync_from_localcopy(repo_section, local_copy_dir):
 
     """
     logging.info('Syncing from local_copy_dir to this repo.')
+
     # trailing slashes have a meaning in rsync which is not needed here, so
     # make sure both paths have exactly one trailing slash
     common.local_rsync(options,
-                       os.path.join(local_copy_dir, repo_section).rstrip('/') + '/',
+                       [os.path.join(local_copy_dir, repo_section).rstrip('/') + '/'],
                        repo_section.rstrip('/') + '/')
 
     offline_copy = os.path.join(local_copy_dir, BINARY_TRANSPARENCY_DIR)
@@ -385,7 +418,7 @@ def update_localcopy(repo_section, local_copy_dir):
 
     """
     # local_copy_dir is guaranteed to have a trailing slash in main() below
-    common.local_rsync(options, repo_section, local_copy_dir)
+    common.local_rsync(options, [repo_section], local_copy_dir)
 
     offline_copy = os.path.join(os.getcwd(), BINARY_TRANSPARENCY_DIR)
     if os.path.isdir(os.path.join(offline_copy, '.git')):
@@ -445,9 +478,10 @@ def update_servergitmirrors(servergitmirrors, repo_section):
             archive_path = os.path.join(git_mirror_path, 'fdroid', 'archive')
             shutil.rmtree(archive_path, ignore_errors=True)
 
-        # rsync is very particular about trailing slashes
+        # trailing slashes have a meaning in rsync which is not needed here, so
+        # make sure both paths have exactly one trailing slash
         common.local_rsync(options,
-                           repo_section.rstrip('/') + '/',
+                           [repo_section.rstrip('/') + '/'],
                            git_repodir.rstrip('/') + '/')
 
         # use custom SSH command if identity_file specified
@@ -457,10 +491,44 @@ def update_servergitmirrors(servergitmirrors, repo_section):
         elif 'identity_file' in config:
             ssh_cmd += ' -oIdentitiesOnly=yes -i "%s"' % config['identity_file']
 
+        if options.verbose:
+            progressbar = progress.Bar()
+
+            class MyProgressPrinter(git.RemoteProgress):
+                def update(self, op_code, current, maximum=None, message=None):
+                    if isinstance(maximum, float):
+                        progressbar.show(current, maximum)
+            progress = MyProgressPrinter()
+        else:
+            progress = None
+
         repo = git.Repo.init(git_mirror_path, initial_branch=GIT_BRANCH)
 
+        # An initial commit of the git tree is required be for other operations
+        initial_branch_ref = repo.head.ref
+        repo.index.commit('Initial commit')
+
         enabled_remotes = []
         for d in servergitmirrors:
+            # Test
+            print(f"d: {d}")
+            print("files:")
+            print(glob.glob('.' + '/**/*', recursive=True))
+
+            index_only = d.get('index_only', False)
+
+            # Use a separate branch for the index only mode as it needs a different set of files to commit
+            if index_only:
+                branch_name = 'index_only'
+            else:
+                branch_name = 'full'
+            if not branch_name in repo.heads:
+                repo.create_head(branch_name, initial_branch_ref)
+            repo.head.reference = repo.heads[branch_name]
+
+            # test
+            print(repo.git.status())
+
             remote_url = d['url']
             name = REMOTE_HOSTNAME_REGEX.sub(r'\1', remote_url)
             enabled_remotes.append(name)
@@ -473,22 +541,24 @@ def update_servergitmirrors(servergitmirrors, repo_section):
                 repo.create_remote(name, remote_url)
             logging.info('Mirroring to: ' + remote_url)
 
+            if index_only:
+                # test
+                print(glob.glob('.' + '/**/*', recursive=True))
+
+                logging.debug('Adding index files to git mirror')
+                repo.index.add(_get_index_file_paths(os.path.join('fdroid', repo_section)))
+            else:
                 # sadly index.add don't allow the --all parameter
                 logging.debug('Adding all files to git mirror')
                 repo.git.add(all=True)
-        logging.debug('Committing all files into git mirror')
+
+            logging.debug('Committing files into git mirror')
             repo.index.commit("fdroidserver git-mirror")
 
-        if options.verbose:
-            progressbar = progress.Bar()
-
-            class MyProgressPrinter(git.RemoteProgress):
-                def update(self, op_code, current, maximum=None, message=None):
-                    if isinstance(maximum, float):
-                        progressbar.show(current, maximum)
-            progress = MyProgressPrinter()
-        else:
-            progress = None
+            # Test
+            print(f"In index-only: {index_only} mode")
+            print(repo.git.status())
+            print(repo.head.log())
 
             # only deploy to GitLab Artifacts if too big for GitLab Pages
             if common.get_dir_size(git_fdroiddir) <= common.GITLAB_COM_PAGES_MAX_SIZE:
@@ -502,11 +572,8 @@ def update_servergitmirrors(servergitmirrors, repo_section):
                     % (common.GITLAB_COM_PAGES_MAX_SIZE / 1000000000)
                 )
 
-        # push for every remote. This will overwrite the git history
-        for remote in repo.remotes:
-            if remote.name not in enabled_remotes:
-                repo.delete_remote(remote)
-                continue
+            # push. This will overwrite the git history
+            remote = repo.remote(name)
             if remote.name == 'gitlab':
                 logging.debug('Writing .gitlab-ci.yml to deploy to GitLab Pages')
                 with open(os.path.join(git_mirror_path, ".gitlab-ci.yml"), "wt") as fp:
@@ -526,7 +593,7 @@ def update_servergitmirrors(servergitmirrors, repo_section):
                         default_flow_style=False,
                     )
 
-                repo.git.add(all=True)
+                repo.index.add(['.gitlab-ci.yml'])
                 repo.index.commit("fdroidserver git-mirror: Deploy to GitLab Pages")
 
             logging.debug(_('Pushing to {url}').format(url=remote.url))
@@ -548,6 +615,10 @@ def update_servergitmirrors(servergitmirrors, repo_section):
                     else:
                         logging.debug(remote.url + ': ' + pushinfo.summary)
 
+            # Switch to the initial branch and unstage all files
+            repo.head.reference = initial_branch_ref
+            repo.head.reset(index=True, working_tree=False)
+            repo.delete_head(repo.branches[branch_name], force=True)
         if progress:
             progressbar.done()
 
@@ -782,6 +853,12 @@ def main():
                         help=_("Don't use rsync checksums"))
     parser.add_argument("--no-keep-git-mirror-archive", action="store_true", default=False,
                         help=_("If a git mirror gets to big, allow the archive to be deleted"))
+    parser.add_argument(
+        "--index-only",
+        action="store_true",
+        default=False,
+        help="Only deploy the index files entry.* and index-v*.json",
+    )
     options = parser.parse_args()
     config = common.read_config(options)
 

fdroidserver/nightly.py

@@ -324,11 +324,11 @@ Last updated: {date}'''.format(repo_git_base=repo_git_base,
 
         os.chdir(repo_basedir)
         if os.path.isdir(git_mirror_repodir):
-            common.local_rsync(options, git_mirror_repodir + '/', 'repo/')
+            common.local_rsync(options, [git_mirror_repodir + '/'], 'repo/')
         if os.path.isdir(git_mirror_metadatadir):
-            common.local_rsync(options, git_mirror_metadatadir + '/', 'metadata/')
+            common.local_rsync(options, [git_mirror_metadatadir + '/'], 'metadata/')
         if os.path.isdir(git_mirror_statsdir):
-            common.local_rsync(options, git_mirror_statsdir + '/', 'stats/')
+            common.local_rsync(options, [git_mirror_statsdir + '/'], 'stats/')
 
         ssh_private_key_file = _ssh_key_from_debug_keystore()
         # this is needed for GitPython to find the SSH key
@@ -430,9 +430,9 @@ Last updated: {date}'''.format(repo_git_base=repo_git_base,
             cwd=repo_basedir,
         )
         common.local_rsync(
-            options, repo_basedir + '/metadata/', git_mirror_metadatadir + '/'
+            options, [repo_basedir + '/metadata/'], git_mirror_metadatadir + '/'
         )
-        common.local_rsync(options, repo_basedir + '/stats/', git_mirror_statsdir + '/')
+        common.local_rsync(options, [repo_basedir + '/stats/'], git_mirror_statsdir + '/')
         mirror_git_repo.git.add(all=True)
         mirror_git_repo.index.commit("update app metadata")
 

tests/deploy.TestCase

@@ -60,6 +60,10 @@ class DeployTest(unittest.TestCase):
         url1 = Path('url1/fdroid')
         url1.mkdir(parents=True)
 
+        # setup parameters for this test run
+        fdroidserver.deploy.options.identity_file = None
+        fdroidserver.deploy.config['make_current_version_link'] = False
+
         dest_apk0 = url0 / fake_apk
         dest_apk1 = url1 / fake_apk
         self.assertFalse(dest_apk0.is_file())
@@ -96,13 +100,53 @@ class DeployTest(unittest.TestCase):
         fake_apk = repo / 'fake.apk'
         with fake_apk.open('w') as fp:
             fp.write('not an APK, but has the right filename')
+        fake_index = repo / fdroidserver.deploy.INDEX_FILES[0]
+        with fake_index.open('w') as fp:
+            fp.write('not an index, but has the right filename')
         url = Path('url')
         url.mkdir()
 
-        dest_apk = url / fake_apk
+        # setup parameters for this test run
+        fdroidserver.deploy.options.identity_file = None
+        fdroidserver.deploy.config['make_current_version_link'] = False
+
+        dest_apk = Path(url) / fake_apk
+        dest_index = Path(url) / fake_index
         self.assertFalse(dest_apk.is_file())
+        self.assertFalse(dest_index.is_file())
+
         fdroidserver.deploy.update_serverwebroot({'url': str(url)}, 'repo')
         self.assertTrue(dest_apk.is_file())
+        self.assertTrue(dest_index.is_file())
+
+    def test_update_serverwebroot_in_index_only_mode(self):
+        os.chdir(self.testdir)
+        repo = Path('repo')
+        repo.mkdir()
+        fake_apk = repo / 'fake.apk'
+        with fake_apk.open('w') as fp:
+            fp.write('not an APK, but has the right filename')
+        for i in fdroidserver.deploy.INDEX_FILES:
+            fake_index = repo / i
+            with fake_index.open('w') as fp:
+                fp.write('not an index, but has the right filename')
+        url = Path('url')
+        url.mkdir()
+
+        # setup parameters for this test run
+        fdroidserver.deploy.options.identity_file = None
+        fdroidserver.deploy.config['make_current_version_link'] = False
+
+        dest_apk = Path(url) / fake_apk
+        dest_index = Path(url) / fake_index
+        self.assertFalse(dest_apk.is_file())
+        self.assertFalse(dest_index.is_file())
+
+        fdroidserver.deploy.update_serverwebroot(
+            {'url': str(url), 'index_only': True}, 'repo'
+        )
+        self.assertFalse(dest_apk.is_file())
+        self.assertTrue(dest_index.is_file())
 
     @mock.patch.dict(os.environ, clear=True)
     def test_update_serverwebroot_no_rsync_error(self):
@@ -118,7 +162,8 @@ class DeployTest(unittest.TestCase):
         fdroidserver.deploy.options.identity_file = None
         fdroidserver.deploy.options.verbose = False
         fdroidserver.deploy.options.quiet = True
-        fdroidserver.deploy.config = {'make_current_version_link': True}
+        fdroidserver.deploy.options.identity_file = None
+        fdroidserver.deploy.config['make_current_version_link'] = True
         url = "example.com:/var/www/fdroid"
         repo_section = 'repo'
 
@@ -202,6 +247,89 @@ class DeployTest(unittest.TestCase):
                 fdroidserver.deploy.update_serverwebroot({'url': url}, repo_section)
         self.assertEqual(call_iteration, 3, 'expected 3 invocations of subprocess.call')
 
+    def test_update_serverwebroot_make_cur_version_link_in_index_only_mode(self):
+        # setup parameters for this test run
+        fdroidserver.deploy.options = mock.Mock()
+        fdroidserver.deploy.options.no_checksum = True
+        fdroidserver.deploy.options.identity_file = None
+        fdroidserver.deploy.options.verbose = False
+        fdroidserver.deploy.options.quiet = True
+        fdroidserver.deploy.options.identity_file = None
+        fdroidserver.deploy.config['make_current_version_link'] = True
+        url = "example.com:/var/www/fdroid"
+        repo_section = 'repo'
+
+        # setup function for asserting subprocess.call invocations
+        call_iteration = 0
+
+        def update_server_webroot_call(cmd):
+            nonlocal call_iteration
+            if call_iteration == 0:
+                self.assertListEqual(
+                    cmd,
+                    [
+                        'rsync',
+                        '--archive',
+                        '--delete-after',
+                        '--safe-links',
+                        '--quiet',
+                        'repo/entry.jar',
+                        'repo/entry.json',
+                        'repo/entry.json.asc',
+                        'repo/index-v1.jar',
+                        'repo/index-v1.json',
+                        'repo/index-v1.json.asc',
+                        'repo/index-v2.json',
+                        'repo/index-v2.json.asc',
+                        'repo/index.jar',
+                        'repo/index.xml',
+                        'example.com:/var/www/fdroid/repo/',
+                    ],
+                )
+            elif call_iteration == 1:
+                self.assertListEqual(
+                    cmd,
+                    [
+                        'rsync',
+                        '--archive',
+                        '--delete-after',
+                        '--safe-links',
+                        '--quiet',
+                        'repo',
+                        url,
+                    ],
+                )
+            elif call_iteration == 2:
+                self.assertListEqual(
+                    cmd,
+                    [
+                        'rsync',
+                        '--archive',
+                        '--delete-after',
+                        '--safe-links',
+                        '--quiet',
+                        'Sym.apk',
+                        'Sym.apk.asc',
+                        'Sym.apk.sig',
+                        'example.com:/var/www/fdroid',
+                    ],
+                )
+            else:
+                self.fail('unexpected subprocess.call invocation')
+            call_iteration += 1
+            return 0
+
+        with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
+            os.mkdir('repo')
+            os.symlink('repo/com.example.sym.apk', 'Sym.apk')
+            os.symlink('repo/com.example.sym.apk.asc', 'Sym.apk.asc')
+            os.symlink('repo/com.example.sym.apk.sig', 'Sym.apk.sig')
+            with mock.patch('subprocess.call', side_effect=update_server_webroot_call):
+                fdroidserver.deploy.update_serverwebroot(
+                    {'url': url, 'index_only': True}, repo_section
+                )
+        self.assertEqual(call_iteration, 1, 'expected 1 invocations of subprocess.call')
+
     def test_update_serverwebroot_with_id_file(self):
         # setup parameters for this test run
         fdroidserver.deploy.options = mock.Mock()
@@ -210,7 +338,8 @@ class DeployTest(unittest.TestCase):
         fdroidserver.deploy.options.verbose = True
         fdroidserver.deploy.options.quiet = False
         fdroidserver.deploy.options.identity_file = None
-        fdroidserver.deploy.config = {'identity_file': './id_rsa'}
+        fdroidserver.deploy.config['identity_file'] = './id_rsa'
+        fdroidserver.deploy.config['make_current_version_link'] = False
         url = "example.com:/var/www/fdroid"
         repo_section = 'archive'
 
@@ -280,6 +409,73 @@ class DeployTest(unittest.TestCase):
             fdroidserver.deploy.update_serverwebroot({'url': url}, repo_section)
         self.assertEqual(call_iteration, 2, 'expected 2 invocations of subprocess.call')
 
+    def test_update_serverwebroot_with_id_file_in_index_only_mode(self):
+        # setup parameters for this test run
+        fdroidserver.deploy.options.no_chcksum = False
+        fdroidserver.deploy.options.verbose = True
+        fdroidserver.deploy.options.quiet = False
+        fdroidserver.deploy.options.identity_file = None
+        fdroidserver.deploy.config['identity_file'] = './id_rsa'
+        fdroidserver.deploy.config['make_current_version_link'] = False
+        url = "example.com:/var/www/fdroid"
+        repo_section = 'archive'
+
+        # setup function for asserting subprocess.call invocations
+        call_iteration = 0
+
+        def update_server_webroot_call(cmd):
+            nonlocal call_iteration
+            if call_iteration == 0:
+                self.assertListEqual(
+                    cmd,
+                    [
+                        'rsync',
+                        '--archive',
+                        '--delete-after',
+                        '--safe-links',
+                        '--verbose',
+                        '-e',
+                        'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i '
+                        + fdroidserver.deploy.config['identity_file'],
+                        'archive/entry.jar',
+                        'archive/entry.json',
+                        'archive/entry.json.asc',
+                        'archive/index-v1.jar',
+                        'archive/index-v1.json',
+                        'archive/index-v1.json.asc',
+                        'archive/index-v2.json',
+                        'archive/index-v2.json.asc',
+                        'archive/index.jar',
+                        'archive/index.xml',
+                        "example.com:/var/www/fdroid/archive/",
+                    ],
+                )
+            elif call_iteration == 1:
+                self.assertListEqual(
+                    cmd,
+                    [
+                        'rsync',
+                        '--archive',
+                        '--delete-after',
+                        '--safe-links',
+                        '--verbose',
+                        '-e',
+                        'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i '
+                        + fdroidserver.deploy.config['identity_file'],
+                        "example.com:/var/www/fdroid/archive/",
+                    ],
+                )
+            else:
+                self.fail('unexpected subprocess.call invocation')
+            call_iteration += 1
+            return 0
+
+        with mock.patch('subprocess.call', side_effect=update_server_webroot_call):
+            fdroidserver.deploy.update_serverwebroot(
+                {'url': url, 'index_only': True}, repo_section
+            )
+        self.assertEqual(call_iteration, 1, 'expected 1 invocations of subprocess.call')
+
     @unittest.skipIf(
         not os.getenv('VIRUSTOTAL_API_KEY'), 'VIRUSTOTAL_API_KEY is not set'
     )
@@ -300,14 +496,352 @@ class DeployTest(unittest.TestCase):
                 name, fdroidserver.deploy.REMOTE_HOSTNAME_REGEX.sub(r'\1', remote_url)
             )
 
-    def test_update_servergitmirrors(self):
+    def test_update_awsbucket_s3cmd(self):
         # setup parameters for this test run
         fdroidserver.deploy.options = mock.Mock()
+        fdroidserver.deploy.options.no_checksum = True
+        fdroidserver.deploy.options.verbose = False
+        fdroidserver.deploy.options.quiet = True
+
+        config = {}
+        fdroidserver.common.fill_config_defaults(config)
+        fdroidserver.deploy.config = config
+        fdroidserver.deploy.config["awsbucket"] = "bucket"
+        fdroidserver.deploy.config["awsaccesskeyid"] = "accesskeyid"
+        fdroidserver.deploy.config["awssecretkey"] = "secretkey"
+        fdroidserver.deploy.config["s3cmd"] = "s3cmd"
+
+        repo_section = 'repo'
+
+        # setup function for asserting subprocess.call invocations
+        call_iteration = 0
+
+        def update_awsbucket_s3cmd_call(cmd):
+            nonlocal call_iteration
+            if call_iteration == 0:
+                self.assertListEqual(
+                    cmd,
+                    [
+                        's3cmd',
+                        f"--config={fdroidserver.deploy.AUTO_S3CFG}",
+                        'info',
+                        f"s3://{fdroidserver.deploy.config['awsbucket']}",
+                    ],
+                )
+            elif call_iteration == 1:
+                self.assertListEqual(
+                    cmd,
+                    [
+                        's3cmd',
+                        f"--config={fdroidserver.deploy.AUTO_S3CFG}",
+                        'sync',
+                        '--acl-public',
+                        '--quiet',
+                        '--exclude',
+                        'repo/entry.jar',
+                        '--exclude',
+                        'repo/entry.json',
+                        '--exclude',
+                        'repo/entry.json.asc',
+                        '--exclude',
+                        'repo/index-v1.jar',
+                        '--exclude',
+                        'repo/index-v1.json',
+                        '--exclude',
+                        'repo/index-v1.json.asc',
+                        '--exclude',
+                        'repo/index-v2.json',
+                        '--exclude',
+                        'repo/index-v2.json.asc',
+                        '--exclude',
+                        'repo/index.jar',
+                        '--exclude',
+                        'repo/index.xml',
+                        '--no-check-md5',
+                        '--skip-existing',
+                        repo_section,
+                        f"s3://{fdroidserver.deploy.config['awsbucket']}/fdroid/",
+                    ],
+                )
+            elif call_iteration == 2:
+                self.assertListEqual(
+                    cmd,
+                    [
+                        's3cmd',
+                        f"--config={fdroidserver.deploy.AUTO_S3CFG}",
+                        'sync',
+                        '--acl-public',
+                        '--quiet',
+                        '--exclude',
+                        'repo/entry.jar',
+                        '--exclude',
+                        'repo/entry.json',
+                        '--exclude',
+                        'repo/entry.json.asc',
+                        '--exclude',
+                        'repo/index-v1.jar',
+                        '--exclude',
+                        'repo/index-v1.json',
+                        '--exclude',
+                        'repo/index-v1.json.asc',
+                        '--exclude',
+                        'repo/index-v2.json',
+                        '--exclude',
+                        'repo/index-v2.json.asc',
+                        '--exclude',
+                        'repo/index.jar',
+                        '--exclude',
+                        'repo/index.xml',
+                        '--no-check-md5',
+                        repo_section,
+                        f"s3://{fdroidserver.deploy.config['awsbucket']}/fdroid/",
+                    ],
+                )
+            elif call_iteration == 3:
+                self.assertListEqual(
+                    cmd,
+                    [
+                        's3cmd',
+                        f"--config={fdroidserver.deploy.AUTO_S3CFG}",
+                        'sync',
+                        '--acl-public',
+                        '--quiet',
+                        '--delete-removed',
+                        '--delete-after',
+                        '--no-check-md5',
+                        repo_section,
+                        f"s3://{fdroidserver.deploy.config['awsbucket']}/fdroid/",
+                    ],
+                )
+            else:
+                self.fail('unexpected subprocess.call invocation')
+            call_iteration += 1
+            return 0
+
+        with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
+            os.mkdir('repo')
+            os.symlink('repo/com.example.sym.apk', 'Sym.apk')
+            os.symlink('repo/com.example.sym.apk.asc', 'Sym.apk.asc')
+            os.symlink('repo/com.example.sym.apk.sig', 'Sym.apk.sig')
+            with mock.patch('subprocess.call', side_effect=update_awsbucket_s3cmd_call):
+                fdroidserver.deploy.update_awsbucket_s3cmd(repo_section)
+        self.assertEqual(call_iteration, 4, 'expected 4 invocations of subprocess.call')
+
+    def test_update_awsbucket_s3cmd_in_index_only_mode(self):
+        # setup parameters for this test run
+        fdroidserver.deploy.options = mock.Mock()
+        fdroidserver.deploy.options.no_checksum = True
+        fdroidserver.deploy.options.verbose = False
+        fdroidserver.deploy.options.quiet = True
+
+        config = {}
+        fdroidserver.common.fill_config_defaults(config)
+        fdroidserver.deploy.config = config
+        fdroidserver.deploy.config["awsbucket"] = "bucket"
+        fdroidserver.deploy.config["awsaccesskeyid"] = "accesskeyid"
+        fdroidserver.deploy.config["awssecretkey"] = "secretkey"
+        fdroidserver.deploy.config["s3cmd"] = "s3cmd"
+
+        repo_section = 'repo'
+
+        # setup function for asserting subprocess.call invocations
+        call_iteration = 0
+
+        def update_awsbucket_s3cmd_call(cmd):
+            nonlocal call_iteration
+            if call_iteration == 0:
+                self.assertListEqual(
+                    cmd,
+                    [
+                        's3cmd',
+                        f"--config={fdroidserver.deploy.AUTO_S3CFG}",
+                        'info',
+                        f"s3://{fdroidserver.deploy.config['awsbucket']}",
+                    ],
+                )
+            elif call_iteration == 1:
+                self.assertListEqual(
+                    cmd,
+                    [
+                        's3cmd',
+                        f"--config={fdroidserver.deploy.AUTO_S3CFG}",
+                        'sync',
+                        '--acl-public',
+                        '--quiet',
+                        '--include',
+                        'repo/entry.jar',
+                        '--include',
+                        'repo/entry.json',
+                        '--include',
+                        'repo/entry.json.asc',
+                        '--include',
+                        'repo/index-v1.jar',
+                        '--include',
+                        'repo/index-v1.json',
+                        '--include',
+                        'repo/index-v1.json.asc',
+                        '--include',
+                        'repo/index-v2.json',
+                        '--include',
+                        'repo/index-v2.json.asc',
+                        '--include',
+                        'repo/index.jar',
+                        '--include',
+                        'repo/index.xml',
+                        '--delete-removed',
+                        '--delete-after',
+                        '--no-check-md5',
+                        repo_section,
+                        f"s3://{fdroidserver.deploy.config['awsbucket']}/fdroid/",
+                    ],
+                )
+            else:
+                self.fail('unexpected subprocess.call invocation')
+            call_iteration += 1
+            return 0
+
+        with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
+            os.mkdir('repo')
+            os.symlink('repo/com.example.sym.apk', 'Sym.apk')
+            os.symlink('repo/com.example.sym.apk.asc', 'Sym.apk.asc')
+            os.symlink('repo/com.example.sym.apk.sig', 'Sym.apk.sig')
+            with mock.patch('subprocess.call', side_effect=update_awsbucket_s3cmd_call):
+                fdroidserver.deploy.update_awsbucket_s3cmd(
+                    repo_section, index_only=True
+                )
+        self.assertEqual(call_iteration, 2, 'expected 2 invocations of subprocess.call')
+
+    def test_update_awsbucket_libcloud(self):
+        from libcloud.storage.base import Container
+
+        # setup parameters for this test run
+        fdroidserver.deploy.options = mock.Mock()
+        fdroidserver.deploy.options.no_checksum = True
+        fdroidserver.deploy.options.verbose = False
+        fdroidserver.deploy.options.quiet = True
+
+        config = {}
+        fdroidserver.common.fill_config_defaults(config)
+        fdroidserver.deploy.config = config
+        fdroidserver.deploy.config["awsbucket"] = "bucket"
+        fdroidserver.deploy.config["awsaccesskeyid"] = "accesskeyid"
+        fdroidserver.deploy.config["awssecretkey"] = "secretkey"
+        fdroidserver.deploy.config["s3cmd"] = "s3cmd"
+
+        repo_section = 'repo'
+
+        os.chdir(self.testdir)
+        repo = Path('repo')
+        repo.mkdir(parents=True)
+        fake_apk = repo / 'Sym.apk'
+        with fake_apk.open('w') as fp:
+            fp.write('not an APK, but has the right filename')
+        fake_index = repo / fdroidserver.deploy.INDEX_FILES[0]
+        with fake_index.open('w') as fp:
+            fp.write('not an index, but has the right filename')
+
+        with mock.patch(
+            'libcloud.storage.drivers.s3.S3StorageDriver'
+        ) as mock_driver_class:
+            mock_driver = mock_driver_class.return_value
+            mock_container = mock.MagicMock(spec=Container)
+            mock_container.list_objects.return_value = [
+                mock.MagicMock(name='Sym.apk'),
+                mock.MagicMock(name=fdroidserver.deploy.INDEX_FILES[0]),
+            ]
+
+            mock_driver.get_container.return_value = mock_container
+            mock_driver.upload_object_via_stream.return_value = None
+
+            fdroidserver.deploy.update_awsbucket_libcloud(repo_section)
+
+            mock_driver.get_container.assert_called_once_with(
+                container_name=fdroidserver.deploy.config["awsbucket"]
+            )
+            mock_container.list_objects.assert_called_once_with()
+            files_to_upload = ['fdroid/repo/Sym.apk', 'fdroid/repo/entry.jar']
+            calls = [
+                mock.call(
+                    iterator=mock.ANY,
+                    container=mock_container,
+                    object_name=file,
+                    extra={'acl': 'public-read'},
+                )
+                for file in files_to_upload
+            ]
+            mock_driver.upload_object_via_stream.assert_has_calls(calls, any_order=True)
+            assert mock_driver.upload_object_via_stream.call_count == 2
+
+    def test_update_awsbucket_libcloud_in_index_only_mode(self):
+        from libcloud.storage.base import Container
+
+        # setup parameters for this test run
+        fdroidserver.deploy.options = mock.Mock()
+        fdroidserver.deploy.options.no_checksum = True
+        fdroidserver.deploy.options.verbose = False
+        fdroidserver.deploy.options.quiet = True
+
+        config = {}
+        fdroidserver.common.fill_config_defaults(config)
+        fdroidserver.deploy.config = config
+        fdroidserver.deploy.config["awsbucket"] = "bucket"
+        fdroidserver.deploy.config["awsaccesskeyid"] = "accesskeyid"
+        fdroidserver.deploy.config["awssecretkey"] = "secretkey"
+        fdroidserver.deploy.config["s3cmd"] = "s3cmd"
+
+        repo_section = 'repo'
+
+        os.chdir(self.testdir)
+        repo = Path('repo')
+        repo.mkdir(parents=True)
+        fake_apk = repo / 'Sym.apk'
+        with fake_apk.open('w') as fp:
+            fp.write('not an APK, but has the right filename')
+        fake_index = repo / fdroidserver.deploy.INDEX_FILES[0]
+        with fake_index.open('w') as fp:
+            fp.write('not an index, but has the right filename')
+
+        with mock.patch(
+            'libcloud.storage.drivers.s3.S3StorageDriver'
+        ) as mock_driver_class:
+            mock_driver = mock_driver_class.return_value
+            mock_container = mock.MagicMock(spec=Container)
+            mock_container.list_objects.return_value = [
+                mock.MagicMock(name='Sym.apk'),
+                mock.MagicMock(name=fdroidserver.deploy.INDEX_FILES[0]),
+            ]
+
+            mock_driver.get_container.return_value = mock_container
+            mock_driver.upload_object_via_stream.return_value = None
+
+            fdroidserver.deploy.update_awsbucket_libcloud(repo_section, index_only=True)
+
+            mock_driver.get_container.assert_called_once_with(
+                container_name=fdroidserver.deploy.config["awsbucket"]
+            )
+            mock_container.list_objects.assert_called_once_with()
+            files_to_upload = ['fdroid/repo/entry.jar']
+            calls = [
+                mock.call(
+                    iterator=mock.ANY,
+                    container=mock_container,
+                    object_name=file,
+                    extra={'acl': 'public-read'},
+                )
+                for file in files_to_upload
+            ]
+            mock_driver.upload_object_via_stream.assert_has_calls(
+                calls,
+                any_order=False,
+            )
+            assert mock_driver.upload_object_via_stream.call_count == 1
+
+    def test_update_servergitmirrors(self):
+        # setup parameters for this test run
         fdroidserver.deploy.options.identity_file = None
         fdroidserver.deploy.options.no_keep_git_mirror_archive = False
         fdroidserver.deploy.options.verbose = False
         fdroidserver.deploy.options.quiet = True
-        fdroidserver.deploy.options.index_only = False
 
         config = {}
         fdroidserver.common.fill_config_defaults(config)
@@ -317,8 +851,7 @@ class DeployTest(unittest.TestCase):
         repo_section = 'repo'
 
         # setup function for asserting subprocess.call invocations
-        update_servergitmirrors_call_iteration = 0
-        remote_push_call_iteration = 0
+        call_iteration = 0
 
         os.chdir(self.testdir)
         repo = Path('repo')
@@ -331,8 +864,8 @@ class DeployTest(unittest.TestCase):
             fp.write('not an index, but has the right filename')
 
         def update_servergitmirrors_call(cmd):
-            nonlocal update_servergitmirrors_call_iteration
-            if update_servergitmirrors_call_iteration == 0:
+            nonlocal call_iteration
+            if call_iteration == 0:
                 self.assertListEqual(
                     cmd,
                     [
@@ -351,34 +884,67 @@ class DeployTest(unittest.TestCase):
                 )
             else:
                 self.fail('unexpected subprocess.call invocation')
-            update_servergitmirrors_call_iteration += 1
+            call_iteration += 1
             return 0
 
-        def remote_push_call(ref, force=False, set_upstream=False, **_args):
-            nonlocal remote_push_call_iteration
-            if remote_push_call_iteration == 0:
-                self.assertEqual([ref, force, set_upstream], ['master', True, True])
+        with mock.patch('subprocess.call', side_effect=update_servergitmirrors_call):
+            fdroidserver.deploy.update_servergitmirrors([], repo_section)
+        self.assertEqual(call_iteration, 1, 'expected 1 invocations of subprocess.call')
+
+    def test_update_servergitmirrors_in_index_only_mode(self):
+        # setup parameters for this test run
+        fdroidserver.deploy.options.identity_file = None
+        fdroidserver.deploy.options.no_keep_git_mirror_archive = False
+        fdroidserver.deploy.options.verbose = False
+        fdroidserver.deploy.options.quiet = True
+
+        config = {}
+        fdroidserver.common.fill_config_defaults(config)
+        fdroidserver.deploy.config = config
+        fdroidserver.deploy.config["servergitmirrors"] = []
+
+        repo_section = 'repo'
+
+        # setup function for asserting subprocess.call invocations
+        call_iteration = 0
+
+        os.chdir(self.testdir)
+        repo = Path('repo')
+        repo.mkdir(parents=True)
+        fake_apk = repo / 'Sym.apk'
+        with fake_apk.open('w') as fp:
+            fp.write('not an APK, but has the right filename')
+        fake_index = repo / fdroidserver.deploy.INDEX_FILES[0]
+        with fake_index.open('w') as fp:
+            fp.write('not an index, but has the right filename')
+
+        def update_servergitmirrors_call(cmd):
+            nonlocal call_iteration
+            if call_iteration == 0:
+                self.assertListEqual(
+                    cmd,
+                    [
+                        'rsync',
+                        '--recursive',
+                        '--safe-links',
+                        '--times',
+                        '--perms',
+                        '--one-file-system',
+                        '--delete',
+                        '--chmod=Da+rx,Fa-x,a+r,u+w',
+                        '--quiet',
+                        'repo/',
+                        "git-mirror/fdroid/repo/",
+                    ],
+                )
             else:
-                self.fail('unexpected git.Remote.push invocation')
-            remote_push_call_iteration += 1
-            return []
+                self.fail('unexpected subprocess.call invocation')
+            call_iteration += 1
+            return 0
 
         with mock.patch('subprocess.call', side_effect=update_servergitmirrors_call):
-            with mock.patch(
-                'git.Remote.push', side_effect=remote_push_call
-            ) as mock_remote_push:
-                mock_remote_push.return_value = []
-                fdroidserver.deploy.update_servergitmirrors(
-                    [{'url': 'https://github.com/user/repo'}], repo_section
-                )
-        self.assertEqual(
-            update_servergitmirrors_call_iteration,
-            1,
-            'expected 1 invocations of subprocess.call',
-        )
-        self.assertEqual(
-            remote_push_call_iteration, 1, 'expected 1 invocations of git.Remote.push'
-        )
+            fdroidserver.deploy.update_servergitmirrors([], repo_section)
+        self.assertEqual(call_iteration, 1, 'expected 1 invocations of subprocess.call')
 
 
 if __name__ == "__main__":