Merge branch 'index-only-deploy-mode' into 'master'

Add index only deployment mode

Closes #1181

See merge request fdroid/fdroidserver!1420

.safety-policy.yml

@@ -17,3 +17,6 @@ security:
     62044:
       reason: "F-Droid doesn't fetch pip dependencies directly from hg/mercurial repositories: https://data.safetycli.com/v/62044/f17/"
       expires: '2025-01-31'
+    63687:
+      reason: Only affects Windows https://security-tracker.debian.org/tracker/CVE-2024-22190
+      expires: '2026-01-31'

completion/bash-completion

@@ -264,7 +264,7 @@ __complete_nightly() {
 __complete_deploy() {
 	opts="-i -v -q"
 	lopts="--identity-file --local-copy-dir --sync-from-local-copy-dir
- --verbose --quiet --no-checksum --no-keep-git-mirror-archive"
+ --verbose --quiet --no-checksum --no-keep-git-mirror-archive --index-only"
 	__complete_options
 }
 

examples/config.yml

@@ -178,6 +178,12 @@
 # serverwebroot:
 #   - foo.com:/usr/share/nginx/www/fdroid
 #   - bar.info:/var/www/fdroid
+#
+# There is a special mode to only deploy the index file:
+#
+# serverwebroot:
+#   - url: 'me@b.az:/srv/fdroid'
+#     indexOnly: true
 
 
 # When running fdroid processes on a remote server, it is possible to
@@ -192,14 +198,18 @@
 # deploy_process_logs: true
 
 # The full URL to a git remote repository. You can include
-# multiple servers to mirror to by wrapping the whole thing in {} or [], and
-# including the servergitmirrors strings in a comma-separated list.
+# multiple servers to mirror to by adding strings to a YAML list or map.
 # Servers listed here will also be automatically inserted in the mirrors list.
 #
 # servergitmirrors: https://github.com/user/repo
 # servergitmirrors:
 #   - https://github.com/user/repo
 #   - https://gitlab.com/user/repo
+#
+# servergitmirrors:
+#   - url: https://github.com/user/repo
+#   - url: https://gitlab.com/user/repo
+#     indexOnly: true
 
 # Most git hosting services have hard size limits for each git repo.
 # `fdroid deploy` will delete the git history when the git mirror repo
@@ -367,3 +377,9 @@
 #   - suss
 #   - exodus
 #   - https://example.com/signatures.json
+
+# The scanner can use signature sources from the internet. These are
+# cached locally.  To force them to be refreshed from the network on
+# every run, set this to true:
+#
+# refresh_scanner: true

fdroidserver/__init__.py

@@ -55,7 +55,9 @@ scan_apk  # NOQA: B101
 scan_repo_files  # NOQA: B101
 from fdroidserver.deploy import (update_awsbucket,
                                  update_servergitmirrors,
+                                 update_serverwebroots,
                                  update_serverwebroot)  # NOQA: E402
 update_awsbucket  # NOQA: B101
 update_servergitmirrors  # NOQA: B101
+update_serverwebroots  # NOQA: B101
 update_serverwebroot  # NOQA: B101

fdroidserver/__main__.py

@@ -23,7 +23,9 @@ import sys
 import os
 import pkgutil
 import logging
+import importlib.metadata
 
+import git
 import fdroidserver.common
 import fdroidserver.metadata
 from fdroidserver import _
@@ -142,31 +144,21 @@ def main():
             print(_("""ERROR: The "server" subcommand has been removed, use "deploy"!"""))
             sys.exit(1)
         elif command == '--version':
-            output = _('no version info found!')
-            cmddir = os.path.realpath(os.path.dirname(os.path.dirname(__file__)))
-            moduledir = os.path.realpath(os.path.dirname(fdroidserver.common.__file__) + '/..')
-            if cmddir == moduledir:
-                # running from git
-                os.chdir(cmddir)
-                if os.path.isdir('.git'):
-                    import subprocess
-                    try:
-                        output = subprocess.check_output(['git', 'describe'],
-                                                         stderr=subprocess.STDOUT,
-                                                         universal_newlines=True)
-                    except subprocess.CalledProcessError:
-                        output = 'git commit ' + subprocess.check_output(['git', 'rev-parse', 'HEAD'],
-                                                                         universal_newlines=True)
-                elif os.path.exists('setup.py'):
-                    m = re.search(r'''.*[\s,\(]+version\s*=\s*["']([0-9a-z.]+)["'].*''',
-                                  open('setup.py').read(), flags=re.MULTILINE)
-                    if m:
-                        output = m.group(1) + '\n'
-            else:
-                from pkg_resources import get_distribution
-                output = get_distribution('fdroidserver').version + '\n'
-            print(output)
-            sys.exit(0)
+            try:
+                print(importlib.metadata.version("fdroidserver"))
+                sys.exit(0)
+            except importlib.metadata.PackageNotFoundError:
+                pass
+            try:
+                print(
+                    git.repo.Repo(
+                        os.path.dirname(os.path.dirname(__file__))
+                    ).git.describe(always=True, tags=True)
+                )
+                sys.exit(0)
+            except git.exc.InvalidGitRepositoryError:
+                print(_('No version information could be found.'))
+                sys.exit(1)
         else:
             print(_("Command '%s' not recognised.\n" % command))
             print_help(available_plugins=available_plugins)

fdroidserver/build.py

@@ -868,6 +868,8 @@ def parse_commandline():
                         help=_("Don't create a source tarball, useful when testing a build"))
     parser.add_argument("--no-refresh", dest="refresh", action="store_false", default=True,
                         help=_("Don't refresh the repository, useful when testing a build with no internet connection"))
+    parser.add_argument("-r", "--refresh-scanner", dest="refresh_scanner", action="store_true", default=False,
+                        help=_("Refresh and cache scanner rules and signatures from the network"))
     parser.add_argument("-f", "--force", action="store_true", default=False,
                         help=_("Force build of disabled apps, and carries on regardless of scan problems. Only allowed in test mode."))
     parser.add_argument("-a", "--all", action="store_true", default=False,

fdroidserver/common.py

@@ -29,6 +29,7 @@
 # libraries here as they will become a requirement for all commands.
 
 import difflib
+from typing import List
 import git
 import glob
 import io
@@ -67,7 +68,7 @@ from pyasn1.error import PyAsn1Error
 import fdroidserver.metadata
 import fdroidserver.lint
 from fdroidserver import _
-from fdroidserver.exception import FDroidException, VCSException, NoSubmodulesException,\
+from fdroidserver.exception import FDroidException, VCSException, NoSubmodulesException, \
     BuildException, VerificationException, MetaDataException
 from .asynchronousfilereader import AsynchronousFileReader
 from .looseversion import LooseVersion
@@ -462,24 +463,30 @@ def read_config(opts=None):
 
     if 'serverwebroot' in config:
         if isinstance(config['serverwebroot'], str):
-            roots = [config['serverwebroot']]
+            roots = [{'url': config['serverwebroot']}]
         elif all(isinstance(item, str) for item in config['serverwebroot']):
+            roots = [{'url': i} for i in config['serverwebroot']]
+        elif all(isinstance(item, dict) for item in config['serverwebroot']):
             roots = config['serverwebroot']
         else:
             raise TypeError(_('only accepts strings, lists, and tuples'))
         rootlist = []
-        for rootstr in roots:
+        for d in roots:
             # since this is used with rsync, where trailing slashes have
             # meaning, ensure there is always a trailing slash
+            rootstr = d['url']
             if rootstr[-1] != '/':
                 rootstr += '/'
-            rootlist.append(rootstr.replace('//', '/'))
+            d['url'] = rootstr.replace('//', '/')
+            rootlist.append(d)
         config['serverwebroot'] = rootlist
 
     if 'servergitmirrors' in config:
         if isinstance(config['servergitmirrors'], str):
-            roots = [config['servergitmirrors']]
+            roots = [{"url": config['servergitmirrors']}]
         elif all(isinstance(item, str) for item in config['servergitmirrors']):
+            roots = [{'url': i} for i in config['servergitmirrors']]
+        elif all(isinstance(item, dict) for item in config['servergitmirrors']):
             roots = config['servergitmirrors']
         else:
             raise TypeError(_('only accepts strings, lists, and tuples'))
@@ -3990,7 +3997,7 @@ def get_app_display_name(app):
     return app.get('AutoName') or app['id']
 
 
-def local_rsync(options, fromdir, todir):
+def local_rsync(options, from_paths: List[str], todir: str, args: List[str] = []):
     """Rsync method for local to local copying of things.
 
     This is an rsync wrapper with all the settings for safe use within
@@ -4007,8 +4014,9 @@ def local_rsync(options, fromdir, todir):
         rsyncargs += ['--verbose']
     if options.quiet:
         rsyncargs += ['--quiet']
-    logging.debug(' '.join(rsyncargs + [fromdir, todir]))
-    if subprocess.call(rsyncargs + [fromdir, todir]) != 0:
+    rsyncargs += args
+    logging.debug(' '.join(rsyncargs + from_paths + [todir]))
+    if subprocess.call(rsyncargs + from_paths + [todir]) != 0:
         raise FDroidException()
 
 
@@ -4052,7 +4060,8 @@ def rsync_status_file_to_repo(path, repo_subdir=None):
         logging.debug(_('skip deploying full build logs: not enabled in config'))
         return
 
-    for webroot in config.get('serverwebroot', []):
+    for d in config.get('serverwebroot', []):
+        webroot = d['url']
         cmd = ['rsync',
                '--archive',
                '--delete-after',

fdroidserver/deploy.py

@@ -47,28 +47,33 @@ AUTO_S3CFG = '.fdroid-deploy-s3cfg'
 USER_S3CFG = 's3cfg'
 REMOTE_HOSTNAME_REGEX = re.compile(r'\W*\w+\W+(\w+).*')
 
+INDEX_FILES = [
+    "entry.jar",
+    "entry.json",
+    "entry.json.asc",
+    "index-v1.jar",
+    "index-v1.json",
+    "index-v1.json.asc",
+    "index-v2.json",
+    "index-v2.json.asc",
+    "index.jar",
+    "index.xml",
+]
 
-def _get_index_excludes(repo_section):
+
+def _get_index_file_paths(base_dir):
     """Return the list of files to be synced last, since they finalize the deploy.
 
     The process of pushing all the new packages to the various
     services can take a while.  So the index files should be updated
     last.  That ensures that the package files are available when the
     client learns about them from the new index files.
-
     """
-    indexes = [
-        os.path.join(repo_section, 'entry.jar'),
-        os.path.join(repo_section, 'entry.json'),
-        os.path.join(repo_section, 'entry.json.asc'),
-        os.path.join(repo_section, 'index-v1.jar'),
-        os.path.join(repo_section, 'index-v1.json'),
-        os.path.join(repo_section, 'index-v1.json.asc'),
-        os.path.join(repo_section, 'index-v2.json'),
-        os.path.join(repo_section, 'index-v2.json.asc'),
-        os.path.join(repo_section, 'index.jar'),
-        os.path.join(repo_section, 'index.xml'),
-    ]
+    return [os.path.join(base_dir, filename) for filename in INDEX_FILES]
+
+
+def _get_index_excludes(base_dir):
+    indexes = _get_index_file_paths(base_dir)
     index_excludes = []
     for f in indexes:
         index_excludes.append('--exclude')
@@ -76,6 +81,15 @@ def _get_index_excludes(repo_section):
     return index_excludes
 
 
+def _get_index_includes(base_dir):
+    indexes = _get_index_file_paths(base_dir)
+    index_includes = []
+    for f in indexes:
+        index_includes.append('--include')
+        index_includes.append(f)
+    return index_includes
+
+
 def update_awsbucket(repo_section):
     """Upload the contents of the directory `repo_section` (including subdirectories) to the AWS S3 "bucket".
 
@@ -136,33 +150,49 @@ def update_awsbucket_s3cmd(repo_section):
         s3cmd_sync += ['--quiet']
 
     s3url = s3bucketurl + '/fdroid/'
-    logging.debug('s3cmd sync new files in ' + repo_section + ' to ' + s3url)
-    logging.debug(_('Running first pass with MD5 checking disabled'))
-    excludes = _get_index_excludes(repo_section)
-    returncode = subprocess.call(
-        s3cmd_sync
-        + excludes
-        + ['--no-check-md5', '--skip-existing', repo_section, s3url]
-    )
-    if returncode != 0:
-        raise FDroidException()
-    logging.debug('s3cmd sync all files in ' + repo_section + ' to ' + s3url)
-    returncode = subprocess.call(
-        s3cmd_sync + excludes + ['--no-check-md5', repo_section, s3url]
-    )
-    if returncode != 0:
-        raise FDroidException()
 
-    logging.debug(_('s3cmd sync indexes {path} to {url} and delete')
-                  .format(path=repo_section, url=s3url))
-    s3cmd_sync.append('--delete-removed')
-    s3cmd_sync.append('--delete-after')
-    if options.no_checksum:
-        s3cmd_sync.append('--no-check-md5')
+    if options.index_only:
+        logging.debug(_('s3cmd syncs indexes from {path} to {url} and deletes removed')
+                      .format(path=repo_section, url=s3url))
+        sync_indexes_flags = []
+        sync_indexes_flags.extend(_get_index_includes(repo_section))
+        sync_indexes_flags.append('--delete-removed')
+        sync_indexes_flags.append('--delete-after')
+        if options.no_checksum:
+            sync_indexes_flags.append('--no-check-md5')
+        else:
+            sync_indexes_flags.append('--check-md5')
+        returncode = subprocess.call(s3cmd_sync + sync_indexes_flags + [repo_section, s3url])
+        if returncode != 0:
+            raise FDroidException()
     else:
-        s3cmd_sync.append('--check-md5')
-    if subprocess.call(s3cmd_sync + [repo_section, s3url]) != 0:
-        raise FDroidException()
+        logging.debug('s3cmd sync new files in ' + repo_section + ' to ' + s3url)
+        logging.debug(_('Running first pass with MD5 checking disabled'))
+        excludes = _get_index_excludes(repo_section)
+        returncode = subprocess.call(
+            s3cmd_sync
+            + excludes
+            + ['--no-check-md5', '--skip-existing', repo_section, s3url]
+        )
+        if returncode != 0:
+            raise FDroidException()
+        logging.debug('s3cmd sync all files in ' + repo_section + ' to ' + s3url)
+        returncode = subprocess.call(
+            s3cmd_sync + excludes + ['--no-check-md5', repo_section, s3url]
+        )
+        if returncode != 0:
+            raise FDroidException()
+
+        logging.debug(_('s3cmd sync indexes {path} to {url} and delete')
+                      .format(path=repo_section, url=s3url))
+        s3cmd_sync.append('--delete-removed')
+        s3cmd_sync.append('--delete-after')
+        if options.no_checksum:
+            s3cmd_sync.append('--no-check-md5')
+        else:
+            s3cmd_sync.append('--check-md5')
+        if subprocess.call(s3cmd_sync + [repo_section, s3url]) != 0:
+            raise FDroidException()
 
 
 def update_awsbucket_libcloud(repo_section):
@@ -208,47 +238,53 @@ def update_awsbucket_libcloud(repo_section):
         if obj.name.startswith(upload_dir + '/'):
             objs[obj.name] = obj
 
-    for root, dirs, files in os.walk(os.path.join(os.getcwd(), repo_section)):
-        for name in files:
-            upload = False
-            file_to_upload = os.path.join(root, name)
-            object_name = 'fdroid/' + os.path.relpath(file_to_upload, os.getcwd())
-            if object_name not in objs:
+    if options.index_only:
+        index_files = [f"{os.getcwd()}/{name}" for name in _get_index_file_paths(repo_section)]
+        files_to_upload = [os.path.join(root, name) for root, dirs, files in os.walk(os.path.join(os.getcwd(), repo_section)) for name in files]
+        files_to_upload = list(set(files_to_upload) & set(index_files))
+
+    else:
+        files_to_upload = [os.path.join(root, name) for root, dirs, files in os.walk(os.path.join(os.getcwd(), repo_section)) for name in files]
+
+    for file_to_upload in files_to_upload:
+        upload = False
+        object_name = 'fdroid/' + os.path.relpath(file_to_upload, os.getcwd())
+        if object_name not in objs:
+            upload = True
+        else:
+            obj = objs.pop(object_name)
+            if obj.size != os.path.getsize(file_to_upload):
                 upload = True
             else:
-                obj = objs.pop(object_name)
-                if obj.size != os.path.getsize(file_to_upload):
+                # if the sizes match, then compare by MD5
+                md5 = hashlib.md5()  # nosec AWS uses MD5
+                with open(file_to_upload, 'rb') as f:
+                    while True:
+                        data = f.read(8192)
+                        if not data:
+                            break
+                        md5.update(data)
+                if obj.hash != md5.hexdigest():
+                    s3url = 's3://' + awsbucket + '/' + obj.name
+                    logging.info(' deleting ' + s3url)
+                    if not driver.delete_object(obj):
+                        logging.warning('Could not delete ' + s3url)
                     upload = True
-                else:
-                    # if the sizes match, then compare by MD5
-                    md5 = hashlib.md5()  # nosec AWS uses MD5
-                    with open(file_to_upload, 'rb') as f:
-                        while True:
-                            data = f.read(8192)
-                            if not data:
-                                break
-                            md5.update(data)
-                    if obj.hash != md5.hexdigest():
-                        s3url = 's3://' + awsbucket + '/' + obj.name
-                        logging.info(' deleting ' + s3url)
-                        if not driver.delete_object(obj):
-                            logging.warning('Could not delete ' + s3url)
-                        upload = True
 
-            if upload:
-                logging.debug(' uploading "' + file_to_upload + '"...')
-                extra = {'acl': 'public-read'}
-                if file_to_upload.endswith('.sig'):
-                    extra['content_type'] = 'application/pgp-signature'
-                elif file_to_upload.endswith('.asc'):
-                    extra['content_type'] = 'application/pgp-signature'
-                logging.info(' uploading ' + os.path.relpath(file_to_upload)
-                             + ' to s3://' + awsbucket + '/' + object_name)
-                with open(file_to_upload, 'rb') as iterator:
-                    obj = driver.upload_object_via_stream(iterator=iterator,
-                                                          container=container,
-                                                          object_name=object_name,
-                                                          extra=extra)
+        if upload:
+            logging.debug(' uploading "' + file_to_upload + '"...')
+            extra = {'acl': 'public-read'}
+            if file_to_upload.endswith('.sig'):
+                extra['content_type'] = 'application/pgp-signature'
+            elif file_to_upload.endswith('.asc'):
+                extra['content_type'] = 'application/pgp-signature'
+            logging.info(' uploading ' + os.path.relpath(file_to_upload)
+                         + ' to s3://' + awsbucket + '/' + object_name)
+            with open(file_to_upload, 'rb') as iterator:
+                obj = driver.upload_object_via_stream(iterator=iterator,
+                                                      container=container,
+                                                      object_name=object_name,
+                                                      extra=extra)
     # delete the remnants in the bucket, they do not exist locally
     while objs:
         object_name, obj = objs.popitem()
@@ -284,32 +320,68 @@ def update_serverwebroot(serverwebroot, repo_section):
             _('rsync is missing or broken: {error}').format(error=e)
         ) from e
     rsyncargs = ['rsync', '--archive', '--delete-after', '--safe-links']
-    if not options.no_checksum:
+    if not options or not options.no_checksum:
         rsyncargs.append('--checksum')
-    if options.verbose:
+    if options and options.verbose:
         rsyncargs += ['--verbose']
-    if options.quiet:
+    if options and options.quiet:
         rsyncargs += ['--quiet']
-    if options.identity_file is not None:
+    if options and options.identity_file:
         rsyncargs += ['-e', 'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ' + options.identity_file]
-    elif 'identity_file' in config:
+    elif config and config.get('identity_file'):
         rsyncargs += ['-e', 'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ' + config['identity_file']]
-    logging.info('rsyncing ' + repo_section + ' to ' + serverwebroot)
-    excludes = _get_index_excludes(repo_section)
-    if subprocess.call(rsyncargs + excludes + [repo_section, serverwebroot]) != 0:
-        raise FDroidException()
-    if subprocess.call(rsyncargs + [repo_section, serverwebroot]) != 0:
-        raise FDroidException()
-    # upload "current version" symlinks if requested
-    if config['make_current_version_link'] and repo_section == 'repo':
-        links_to_upload = []
-        for f in glob.glob('*.apk') \
-                + glob.glob('*.apk.asc') + glob.glob('*.apk.sig'):
-            if os.path.islink(f):
-                links_to_upload.append(f)
-        if len(links_to_upload) > 0:
-            if subprocess.call(rsyncargs + links_to_upload + [serverwebroot]) != 0:
-                raise FDroidException()
+    url = serverwebroot['url']
+    logging.info('rsyncing ' + repo_section + ' to ' + url)
+    if options.index_only:
+        rsyncargs += _get_index_file_paths(repo_section)
+        rsyncargs += [f'{url}/{repo_section}/']
+        logging.info(rsyncargs)
+        if subprocess.call(rsyncargs) != 0:
+            raise FDroidException()
+    else:
+        excludes = _get_index_excludes(repo_section)
+        if subprocess.call(rsyncargs + excludes + [repo_section, url]) != 0:
+            raise FDroidException()
+        if subprocess.call(rsyncargs + [repo_section, url]) != 0:
+            raise FDroidException()
+        # upload "current version" symlinks if requested
+        if config['make_current_version_link'] and repo_section == 'repo':
+            links_to_upload = []
+            for f in glob.glob('*.apk') \
+                    + glob.glob('*.apk.asc') + glob.glob('*.apk.sig'):
+                if os.path.islink(f):
+                    links_to_upload.append(f)
+            if len(links_to_upload) > 0:
+                if subprocess.call(rsyncargs + links_to_upload + [url]) != 0:
+                    raise FDroidException()
+
+
+def update_serverwebroots(serverwebroots, repo_section, standardwebroot=True):
+    for d in serverwebroots:
+        # this supports both an ssh host:path and just a path
+        serverwebroot = d['url']
+        s = serverwebroot.rstrip('/').split(':')
+        if len(s) == 1:
+            fdroiddir = s[0]
+        elif len(s) == 2:
+            host, fdroiddir = s
+        else:
+            logging.error(_('Malformed serverwebroot line:') + ' ' + serverwebroot)
+            sys.exit(1)
+        repobase = os.path.basename(fdroiddir)
+        if standardwebroot and repobase != 'fdroid':
+            logging.error(
+                _(
+                    'serverwebroot: path does not end with "fdroid", perhaps you meant one of these:'
+                )
+                + '\n\t'
+                + serverwebroot.rstrip('/')
+                + '/fdroid\n\t'
+                + serverwebroot.rstrip('/').rstrip(repobase)
+                + 'fdroid'
+            )
+            sys.exit(1)
+        update_serverwebroot(d, repo_section)
 
 
 def sync_from_localcopy(repo_section, local_copy_dir):
@@ -321,11 +393,13 @@ def sync_from_localcopy(repo_section, local_copy_dir):
     push to all the servers that are configured.
 
     """
-    logging.info('Syncing from local_copy_dir to this repo.')
+    if options.index_only:
+        raise FDroidException(_('The index only mode cannot be used when syncing to the local copy filesystem'))
+
     # trailing slashes have a meaning in rsync which is not needed here, so
     # make sure both paths have exactly one trailing slash
     common.local_rsync(options,
-                       os.path.join(local_copy_dir, repo_section).rstrip('/') + '/',
+                       [os.path.join(local_copy_dir, repo_section).rstrip('/') + '/'],
                        repo_section.rstrip('/') + '/')
 
     offline_copy = os.path.join(local_copy_dir, BINARY_TRANSPARENCY_DIR)
@@ -342,8 +416,11 @@ def update_localcopy(repo_section, local_copy_dir):
     drive.
 
     """
+    if options.index_only:
+        raise FDroidException(_('The index only mode cannot be used when syncing to the local copy filesystem'))
+
     # local_copy_dir is guaranteed to have a trailing slash in main() below
-    common.local_rsync(options, repo_section, local_copy_dir)
+    common.local_rsync(options, [repo_section], local_copy_dir)
 
     offline_copy = os.path.join(os.getcwd(), BINARY_TRANSPARENCY_DIR)
     if os.path.isdir(os.path.join(offline_copy, '.git')):
@@ -403,10 +480,17 @@ def update_servergitmirrors(servergitmirrors, repo_section):
             archive_path = os.path.join(git_mirror_path, 'fdroid', 'archive')
             shutil.rmtree(archive_path, ignore_errors=True)
 
-        # rsync is very particular about trailing slashes
-        common.local_rsync(options,
-                           repo_section.rstrip('/') + '/',
-                           git_repodir.rstrip('/') + '/')
+        if options.index_only:
+            # rsync is very particular about trailing slashes
+            common.local_rsync(options,
+                               _get_index_file_paths(repo_section),
+                               git_repodir.rstrip('/') + '/')
+        else:
+            # trailing slashes have a meaning in rsync which is not needed here, so
+            # make sure both paths have exactly one trailing slash
+            common.local_rsync(options,
+                               [repo_section.rstrip('/') + '/'],
+                               git_repodir.rstrip('/') + '/')
 
         # use custom SSH command if identity_file specified
         ssh_cmd = 'ssh -oBatchMode=yes'
@@ -418,7 +502,8 @@ def update_servergitmirrors(servergitmirrors, repo_section):
         repo = git.Repo.init(git_mirror_path, initial_branch=GIT_BRANCH)
 
         enabled_remotes = []
-        for remote_url in servergitmirrors:
+        for d in servergitmirrors:
+            remote_url = d['url']
             name = REMOTE_HOSTNAME_REGEX.sub(r'\1', remote_url)
             enabled_remotes.append(name)
             r = git.remote.Remote(repo, name)
@@ -739,6 +824,12 @@ def main():
                         help=_("Don't use rsync checksums"))
     parser.add_argument("--no-keep-git-mirror-archive", action="store_true", default=False,
                         help=_("If a git mirror gets to big, allow the archive to be deleted"))
+    parser.add_argument(
+        "--index-only",
+        action="store_true",
+        default=False,
+        help="Only deploy the index files entry.* and index-v*.json",
+    )
     options = parser.parse_args()
     config = common.read_config(options)
 
@@ -747,24 +838,6 @@ def main():
     else:
         standardwebroot = True
 
-    for serverwebroot in config.get('serverwebroot', []):
-        # this supports both an ssh host:path and just a path
-        s = serverwebroot.rstrip('/').split(':')
-        if len(s) == 1:
-            fdroiddir = s[0]
-        elif len(s) == 2:
-            host, fdroiddir = s
-        else:
-            logging.error(_('Malformed serverwebroot line:') + ' ' + serverwebroot)
-            sys.exit(1)
-        repobase = os.path.basename(fdroiddir)
-        if standardwebroot and repobase != 'fdroid':
-            logging.error('serverwebroot path does not end with "fdroid", '
-                          + 'perhaps you meant one of these:\n\t'
-                          + serverwebroot.rstrip('/') + '/fdroid\n\t'
-                          + serverwebroot.rstrip('/').rstrip(repobase) + 'fdroid')
-            sys.exit(1)
-
     if options.local_copy_dir is not None:
         local_copy_dir = options.local_copy_dir
     elif config.get('local_copy_dir'):
@@ -825,12 +898,13 @@ def main():
                 sync_from_localcopy(repo_section, local_copy_dir)
             else:
                 update_localcopy(repo_section, local_copy_dir)
-        for serverwebroot in config.get('serverwebroot', []):
-            update_serverwebroot(serverwebroot, repo_section)
-        if config.get('servergitmirrors', []):
+        if config.get('serverwebroot'):
+            update_serverwebroots(
+                config['serverwebroot'], repo_section, standardwebroot
+            )
+        if config.get('servergitmirrors'):
             # update_servergitmirrors will take care of multiple mirrors so don't need a foreach
-            servergitmirrors = config.get('servergitmirrors', [])
-            update_servergitmirrors(servergitmirrors, repo_section)
+            update_servergitmirrors(config['servergitmirrors'], repo_section)
         if config.get('awsbucket'):
             update_awsbucket(repo_section)
         if config.get('androidobservatory'):

fdroidserver/index.py

@@ -1478,7 +1478,7 @@ def add_mirrors_to_repodict(repo_section, repodict):
         repodict['mirrors'].insert(0, {'isPrimary': True, 'url': repodict['address']})
 
 
-def get_mirror_service_urls(url):
+def get_mirror_service_urls(mirror):
     """Get direct URLs from git service for use by fdroidclient.
 
     Via 'servergitmirrors', fdroidserver can create and push a mirror
@@ -1496,6 +1496,7 @@ def get_mirror_service_urls(url):
     information about the repo available to end user.
 
     """
+    url = mirror['url']
     if url.startswith('git@'):
         url = re.sub(r'^git@([^:]+):(.+)', r'https://\1/\2', url)
 

fdroidserver/nightly.py

@@ -321,11 +321,11 @@ Last updated: {date}'''.format(repo_git_base=repo_git_base,
 
         os.chdir(repo_basedir)
         if os.path.isdir(git_mirror_repodir):
-            common.local_rsync(options, git_mirror_repodir + '/', 'repo/')
+            common.local_rsync(options, [git_mirror_repodir + '/'], 'repo/')
         if os.path.isdir(git_mirror_metadatadir):
-            common.local_rsync(options, git_mirror_metadatadir + '/', 'metadata/')
+            common.local_rsync(options, [git_mirror_metadatadir + '/'], 'metadata/')
         if os.path.isdir(git_mirror_statsdir):
-            common.local_rsync(options, git_mirror_statsdir + '/', 'stats/')
+            common.local_rsync(options, [git_mirror_statsdir + '/'], 'stats/')
 
         ssh_private_key_file = _ssh_key_from_debug_keystore()
         # this is needed for GitPython to find the SSH key
@@ -358,7 +358,7 @@ Last updated: {date}'''.format(repo_git_base=repo_git_base,
             'archive_url': repo_base + '/archive',
             'archive_description': 'Old nightly builds that have been archived.',
             'archive_older': options.archive_older,
-            'servergitmirrors': servergitmirror,
+            'servergitmirrors': [{"url": servergitmirror}],
             'keystore': KEYSTORE_FILE,
             'repo_keyalias': KEY_ALIAS,
             'keystorepass': PASSWORD,
@@ -427,9 +427,9 @@ Last updated: {date}'''.format(repo_git_base=repo_git_base,
             cwd=repo_basedir,
         )
         common.local_rsync(
-            options, repo_basedir + '/metadata/', git_mirror_metadatadir + '/'
+            options, [repo_basedir + '/metadata/'], git_mirror_metadatadir + '/'
         )
-        common.local_rsync(options, repo_basedir + '/stats/', git_mirror_statsdir + '/')
+        common.local_rsync(options, [repo_basedir + '/stats/'], git_mirror_statsdir + '/')
         mirror_git_repo.git.add(all=True)
         mirror_git_repo.index.commit("update app metadata")
 

fdroidserver/scanner.py

@@ -325,13 +325,17 @@ class SUSSDataController(SignatureDataController):
         self.set_data(json.loads(SUSS_DEFAULT))
 
 
-class ScannerTool():
+class ScannerTool:
     def __init__(self):
-
         # we could add support for loading additional signature source
         # definitions from config.yml here
 
         self.scanner_data_lookup()
+
+        config = common.get_config()
+        if (options and options.refresh_scanner) or config.get('refresh_scanner'):
+            self.refresh()
+
         self.load()
         self.compile_regexes()
 
@@ -784,7 +788,7 @@ def main():
                         help=_("Force scan of disabled apps and builds."))
     parser.add_argument("--json", action="store_true", default=False,
                         help=_("Output JSON to stdout."))
-    parser.add_argument("-r", "--refresh", action="store_true", default=False,
+    parser.add_argument("-r", "--refresh", dest="refresh_scanner", action="store_true", default=False,
                         help=_("fetch the latest version of signatures from the web"))
     parser.add_argument("-e", "--exit-code", action="store_true", default=False,
                         help=_("Exit with a non-zero code if problems were found"))
@@ -802,9 +806,6 @@ def main():
     # initialize/load configuration values
     common.get_config(opts=options)
 
-    if options.refresh:
-        scanner._get_tool().refresh()
-
     probcount = 0
 
     appids = []
@@ -824,6 +825,8 @@ def main():
     if not appids:
         if options.exit_code and probcount > 0:
             sys.exit(ExitCode.NONFREE_CODE)
+        if options.refresh_scanner:
+            _get_tool()
         return
 
     # Read all app and srclib metadata
@@ -923,98 +926,100 @@ if __name__ == "__main__":
     main()
 
 
-SUSS_DEFAULT = '''{
+SUSS_DEFAULT = r'''{
   "cache_duration": 86400,
   "signatures": {
-    "admob": {
-      "gradle_signatures": [
-        "admob.*sdk.*android"
-      ],
-      "license": "NonFree"
-    },
-    "androidx": {
-      "gradle_signatures": [
-        "androidx.navigation:navigation-dynamic-features",
-        "androidx.work:work-gcm"
-      ],
-      "license": "NonFree"
-    },
-    "appcenter-push": {
-      "gradle_signatures": [
-        "appcenter-push"
-      ],
-      "license": "NonFree"
-    },
-    "bugsense": {
-      "gradle_signatures": [
-        "bugsense"
-      ],
-      "license": "NonFree"
-    },
-    "cloudrail": {
-      "gradle_signatures": [
-        "cloudrail"
-      ],
-      "license": "NonFree"
-    },
-    "com.android.billing": {
-      "code_signatures": [
-        "com/android/billing"
-      ],
-      "license": "NonFree"
-    },
     "com.android.billingclient": {
+      "code_signatures": [
+        "com/android/billingclient"
+      ],
       "gradle_signatures": [
-        "com.android.billingclient"
+        "com.android.billingclient",
+        "com.google.androidbrowserhelper:billing",
+        "com.anjlab.android.iab.v3:library",
+        "com.github.penn5:donations",
+        "me.proton.core:payment-iap"
       ],
       "license": "NonFree"
     },
-    "com.anjlab.android.iab.v3": {
+    "com.bugsense": {
+      "code_signatures": [
+        "com/bugsense"
+      ],
       "gradle_signatures": [
-        "com.anjlab.android.iab.v3:library"
+        "com.bugsense"
       ],
       "license": "NonFree"
     },
-    "com.cloudinary": {
+    "com.cloudrail": {
+      "code_signature": [
+        "com/cloudrail"
+      ],
       "gradle_signatures": [
-        "com.cloudinary:cloudinary-android"
+        "com.cloudrail"
       ],
       "license": "NonFree"
     },
-    "com.evernote": {
+    "com.crashlytics.sdk.android": {
+      "code_signatures": [
+        "com/crashlytics"
+      ],
       "gradle_signatures": [
-        "com.evernote:android-job"
+        "crashlytics"
       ],
       "license": "NonFree"
     },
-    "com.facebook": {
+    "com.crittercism": {
+      "code_signatures": [
+        "com/crittercism"
+      ],
       "gradle_signatures": [
-        "[\\"']com.facebook.android['\\":]"
+        "com.crittercism"
+      ],
+      "license": "NonFree"
+    },
+    "com.facebook.android": {
+      "code_signatures": [
+        "com/facebook"
+      ],
+      "gradle_signatures": [
+        "com.facebook.android"
+      ],
+      "license": "NonFree"
+    },
+    "com.flurry.android": {
+      "code_signature": [
+        "com/flurry"
+      ],
+      "gradle_signatures": [
+        "com.flurry.android"
       ],
       "license": "NonFree"
     },
     "com.github.junrar": {
+      "code_signatures": [
+        "com/github/junrar"
+      ],
       "gradle_signatures": [
         "com.github.junrar:junrar"
       ],
       "license": "NonFree"
     },
-    "com.github.penn5": {
+    "com.github.omicronapps.7-Zip-JBinding-4Android": {
       "gradle_signatures": [
-        "com.github.penn5:donations"
+        "com.github.omicronapps:7-Zip-JBinding-4Android"
       ],
-      "license": "NonFree"
+      "license": "NonFree",
+      "name": "7-Zip-JBinding-4Android"
     },
-    "com.google.analytics": {
+    "com.google.ads": {
       "code_signatures": [
-        "com/google/analytics"
+        "com/google/ads"
       ],
-      "license": "NonFree"
-    },
-    "com.google.android.exoplayer": {
       "gradle_signatures": [
-        "com.google.android.exoplayer:extension-cast",
-        "com.google.android.exoplayer:extension-cronet"
+        "com.google.ads",
+        "com.google.android.exoplayer:extension-ima",
+        "androidx.media3:media3-exoplayer-ima"
       ],
       "license": "NonFree"
     },
@@ -1022,198 +1027,256 @@ SUSS_DEFAULT = '''{
       "code_signatures": [
         "com/google/android/gms"
       ],
+      "gradle_signatures": [
+        "com.google.android.gms",
+        "androidx.core:core-google-shortcuts",
+        "androidx.media3:media3-cast",
+        "androidx.media3:media3-datasource-cronet",
+        "androidx.work:work-gcm",
+        "com.google.android.exoplayer:extension-cast",
+        "com.google.android.exoplayer:extension-cronet",
+        "com.evernote:android-job",
+        "com.cloudinary:cloudinary-android.*:2\\.[12]\\.",
+        "com.pierfrancescosoffritti.androidyoutubeplayer:chromecast-sender",
+        "com.yayandroid:locationmanager",
+        "play-services",
+        "xyz.belvi.mobilevision:barcodescanner"
+      ],
       "license": "NonFree"
     },
-    "com.google.android.libraries.places": {
+    "com.google.android.libraries": {
+      "code_signatures": [
+        "com/google/android/libraries"
+      ],
       "gradle_signatures": [
-        "com.google.android.libraries.places:places"
+        "com.google.android.libraries"
       ],
       "license": "NonFree"
     },
     "com.google.android.play": {
-      "gradle_signatures": [
-        "com.google.android.play:app-update",
-        "com.google.android.play:core.*"
+      "anti_features": [
+        "NonFreeDep",
+        "NonFreeNet"
       ],
-      "license": "NonFree"
-    },
-    "com.google.android.play.core": {
       "code_signatures": [
         "com/google/android/play/core"
       ],
-      "license": "NonFree"
-    },
-    "com.google.firebase": {
-      "code_signatures": [
-        "com/google/firebase"
+      "documentation": [
+        "https://developer.android.com/guide/playcore"
       ],
-      "license": "NonFree"
+      "gradle_signatures": [
+        "com.google.android.play:app-update",
+        "com.google.android.play:asset-delivery",
+        "com.google.android.play:core.*",
+        "com.google.android.play:feature-delivery",
+        "com.google.android.play:review",
+        "androidx.navigation:navigation-dynamic-features",
+        "com.github.SanojPunchihewa:InAppUpdater"
+      ],
+      "license": "NonFree",
+      "name": "Google Play Core"
     },
     "com.google.mlkit": {
+      "code_signatures": [
+        "com/google/mlkit"
+      ],
       "gradle_signatures": [
         "com.google.mlkit"
       ],
       "license": "NonFree"
     },
-    "com.google.tagmanager": {
+    "com.heyzap": {
       "code_signatures": [
-        "com/google/tagmanager"
+        "com/heyzap"
       ],
       "license": "NonFree"
     },
     "com.hypertrack": {
+      "code_signatures": [
+        "com/hypertrack/(?!:hyperlog)"
+      ],
       "gradle_signatures": [
-        "com\\\\.hypertrack(?!:hyperlog)"
+        "com.hypertrack(?!:hyperlog)"
+      ],
+      "gradle_signatures_negative_examples": [
+        "com.hypertrack:hyperlog"
       ],
       "license": "NonFree"
     },
     "com.mapbox": {
-      "MaintainerNotes": "com.mapbox.mapboxsdk:mapbox-sdk-services seems to be fully under this license:\\nhttps://github.com/mapbox/mapbox-java/blob/main/LICENSE\\n",
+      "MaintainerNotes": "It seems that all libs in https://github.com/mapbox/mapbox-java is fully FOSS\nsince 3.0.0.\n",
       "gradle_signatures": [
-        "com\\\\.mapbox(?!\\\\.mapboxsdk:mapbox-sdk-services)"
+        "com\\.mapbox(?!\\.mapboxsdk:mapbox-sdk-(services|geojson|turf):([3-5]))"
+      ],
+      "gradle_signatures_negative_examples": [
+        "com.mapbox.mapboxsdk:mapbox-sdk-services:5.0.0",
+        "com.github.johan12345:mapbox-events-android:a21c324501",
+        "implementation(\"com.github.johan12345.AnyMaps:anymaps-mapbox:$anyMapsVersion\")"
+      ],
+      "gradle_signatures_positive_examples": [
+        "com.mapbox.mapboxsdk:mapbox-android-plugin-annotation-v7:0.6.0",
+        "com.mapbox.mapboxsdk:mapbox-android-plugin-annotation-v8:0.7.0",
+        "com.mapbox.mapboxsdk:mapbox-android-plugin-localization-v7:0.7.0",
+        "com.mapbox.mapboxsdk:mapbox-android-plugin-locationlayer:0.4.0",
+        "com.mapbox.mapboxsdk:mapbox-android-plugin-markerview-v8:0.3.0",
+        "com.mapbox.mapboxsdk:mapbox-android-plugin-places-v8:0.9.0",
+        "com.mapbox.mapboxsdk:mapbox-android-plugin-scalebar-v8:0.2.0",
+        "com.mapbox.mapboxsdk:mapbox-android-sdk:7.3.0"
       ],
       "license": "NonFree"
     },
     "com.onesignal": {
+      "code_signatures": [
+        "com/onesignal"
+      ],
       "gradle_signatures": [
         "com.onesignal:OneSignal"
       ],
       "license": "NonFree"
     },
+    "com.paypal.sdk": {
+      "code_signatures": [
+        "com/paypal"
+      ],
+      "gradle_signatures": [
+        "com.paypal.sdk"
+      ],
+      "license": "NonFree"
+    },
     "com.tencent.bugly": {
+      "code_signatures": [
+        "com/tencent/bugly"
+      ],
       "gradle_signatures": [
         "com.tencent.bugly"
       ],
       "license": "NonFree"
     },
     "com.umeng.umsdk": {
+      "code_signatures": [
+        "com/umeng/umsdk"
+      ],
       "gradle_signatures": [
         "com.umeng.umsdk"
       ],
       "license": "NonFree"
     },
     "com.yandex.android": {
-      "gradle_signatures": [
-        "com\\\\.yandex\\\\.android(?!:authsdk)"
+      "code_signatures": [
+        "com/yandex/android/(?!:authsdk)"
       ],
-      "license": "NonFree"
-    },
-    "com.yayandroid": {
       "gradle_signatures": [
-        "com.yayandroid:LocationManager"
+        "com\\.yandex\\.android(?!:authsdk)"
       ],
-      "license": "NonFree"
-    },
-    "crashlytics": {
-      "gradle_signatures": [
-        "crashlytics"
-      ],
-      "license": "NonFree"
-    },
-    "crittercism": {
-      "gradle_signatures": [
-        "crittercism"
+      "gradle_signatures_negative_examples": [
+        "com.yandex.android:authsdk"
       ],
       "license": "NonFree"
     },
     "firebase": {
-      "gradle_signatures": [
-        "com(\\\\.google)?\\\\.firebase[.:](?!firebase-jobdispatcher|geofire-java)"
+      "code_signatures": [
+        "com/google/firebase"
       ],
-      "license": "NonFree"
+      "documentation": [
+        "https://www.firebase.com"
+      ],
+      "gradle_signatures": [
+        "com(\\.google)?\\.firebase[.:](?!firebase-jobdispatcher|geofire-java)",
+        "com.microsoft.appcenter:appcenter-push"
+      ],
+      "gradle_signatures_negative_examples": [
+        "    compile 'com.firebase:firebase-jobdispatcher:0.8.4'",
+        "implementation 'com.firebase:geofire-java:3.0.0'",
+        "    compile 'com.firebaseui:firebase-ui-auth:3.1.3'",
+        "com.firebaseui:firebase-ui-database",
+        "com.firebaseui:firebase-ui-storage",
+        "com.github.axet:android-firebase-fake",
+        "com.github.b3er.rxfirebase:firebase-database",
+        "com.github.b3er.rxfirebase:firebase-database-kotlin",
+        "com.segment.analytics.android.integrations:firebase"
+      ],
+      "gradle_signatures_positive_examples": [
+        "\tcompile 'com.google.firebase:firebase-crash:11.0.8'",
+        "\tcompile 'com.google.firebase:firebase-core:11.0.8'",
+        "com.firebase:firebase-client-android:2.5.2",
+        "com.google.firebase.crashlytics",
+        "com.google.firebase.firebase-perf",
+        "com.google.firebase:firebase-ads",
+        "com.google.firebase:firebase-analytics",
+        "com.google.firebase:firebase-appindexing",
+        "com.google.firebase:firebase-auth",
+        "com.google.firebase:firebase-config",
+        "com.google.firebase:firebase-core",
+        "com.google.firebase:firebase-crash",
+        "com.google.firebase:firebase-crashlytics",
+        "com.google.firebase:firebase-database",
+        "com.google.firebase:firebase-dynamic-links",
+        "com.google.firebase:firebase-firestore",
+        "com.google.firebase:firebase-inappmessaging",
+        "com.google.firebase:firebase-inappmessaging-display",
+        "com.google.firebase:firebase-messaging",
+        "com.google.firebase:firebase-ml-natural-language",
+        "com.google.firebase:firebase-ml-natural-language-smart-reply-model",
+        "com.google.firebase:firebase-ml-vision",
+        "com.google.firebase:firebase-perf",
+        "com.google.firebase:firebase-plugins",
+        "com.google.firebase:firebase-storage"
+      ],
+      "license": "NonFree",
+      "name": "Firebase"
     },
-    "flurryagent": {
-      "gradle_signatures": [
-        "flurryagent"
+    "google-maps": {
+      "anti_features": [
+        "NonFreeDep",
+        "NonFreeNet"
       ],
-      "license": "NonFree"
-    },
-    "google-ad": {
-      "gradle_signatures": [
-        "google.*ad.*view"
+      "api_key_ids": [
+        "com\\.google\\.android\\.geo\\.API_KEY",
+        "com\\.google\\.android\\.maps\\.v2\\.API_KEY"
       ],
-      "license": "NonFree"
-    },
-    "google.admob": {
-      "gradle_signatures": [
-        "google.*admob"
-      ],
-      "license": "NonFree"
-    },
-    "google.play.services": {
-      "gradle_signatures": [
-        "google.*play.*services"
-      ],
-      "license": "NonFree"
-    },
-    "heyzap": {
-      "gradle_signatures": [
-        "heyzap"
-      ],
-      "license": "NonFree"
+      "license": "NonFree",
+      "name": "Google Maps"
     },
     "io.github.sinaweibosdk": {
+      "code_signatures": [
+        "com/sina"
+      ],
       "gradle_signatures": [
         "io.github.sinaweibosdk"
       ],
       "license": "NonFree"
     },
     "io.objectbox": {
+      "Name": "ObjectBox Database",
+      "code_signatures": [
+        "io/objectbox"
+      ],
       "gradle_signatures": [
         "io.objectbox:objectbox-gradle-plugin"
       ],
       "license": "NonFree"
     },
-    "jpct": {
-      "gradle_signatures": [
-        "jpct.*ae"
-      ],
-      "license": "NonFree"
-    },
-    "libspen23": {
-      "gradle_signatures": [
-        "libspen23"
-      ],
-      "license": "NonFree"
-    },
     "me.pushy": {
+      "code_signatures": [
+        "me/pushy"
+      ],
       "gradle_signatures": [
-        "me.pushy:sdk"
+        "me.pushy"
       ],
       "license": "NonFree"
     },
-    "org.jetbrains.kotlinx": {
-      "gradle_signatures": [
-        "org.jetbrains.kotlinx:kotlinx-coroutines-play-services"
+    "org.mariuszgromada.math": {
+      "documentation": [
+        "https://mathparser.org"
       ],
-      "license": "NonFree"
-    },
-    "ouya": {
       "gradle_signatures": [
-        "ouya.*sdk"
+        "org.mariuszgromada.math:MathParser.org-mXparser"
       ],
-      "license": "NonFree"
-    },
-    "paypal": {
-      "gradle_signatures": [
-        "paypal.*mpl"
-      ],
-      "license": "NonFree"
-    },
-    "xyz.belvi.mobilevision": {
-      "gradle_signatures": [
-        "xyz.belvi.mobilevision:barcodescanner"
-      ],
-      "license": "NonFree"
-    },
-    "youtube": {
-      "gradle_signatures": [
-        "youtube.*android.*player.*api"
-      ],
-      "license": "NonFree"
+      "license": "NonFree",
+      "name": "mXparser"
     }
   },
-  "timestamp": 1664480104.875586,
+  "timestamp": 1706002241.887412,
   "version": 1,
-  "last_updated": 1664480104.875586
+  "last_updated": 1706171656.496258
 }'''

gradlew-fdroid

@@ -187,6 +187,7 @@ get_sha() {
         '7.6.1')  echo '6147605a23b4eff6c334927a86ff3508cb5d6722cd624c97ded4c2e8640f1f87' ;;
         '7.6.2')  echo 'a01b6587e15fe7ed120a0ee299c25982a1eee045abd6a9dd5e216b2f628ef9ac' ;;
         '7.6.3')  echo '740c2e472ee4326c33bf75a5c9f5cd1e69ecf3f9b580f6e236c86d1f3d98cfac' ;;
+        '7.6.4')  echo 'bed1da33cca0f557ab13691c77f38bb67388119e4794d113e051039b80af9bb1' ;;
         '8.0')    echo '4159b938ec734a8388ce03f52aa8f3c7ed0d31f5438622545de4f83a89b79788' ;;
         '8.0.1')  echo '1b6b558be93f29438d3df94b7dfee02e794b94d9aca4611a92cdb79b6b88e909' ;;
         '8.0.2')  echo 'ff7bf6a86f09b9b2c40bb8f48b25fc19cf2b2664fd1d220cd7ab833ec758d0d7' ;;
@@ -197,6 +198,7 @@ get_sha() {
         '8.3')    echo '591855b517fc635b9e04de1d05d5e76ada3f89f5fc76f87978d1b245b4f69225' ;;
         '8.4')    echo '3e1af3ae886920c3ac87f7a91f816c0c7c436f276a6eefdb3da152100fef72ae' ;;
         '8.5')    echo '9d926787066a081739e8200858338b4a69e837c3a821a33aca9db09dd4a41026' ;;
+        '8.6')    echo '9631d53cf3e74bfa726893aee1f8994fee4e060c401335946dba2156f440f24c' ;;
         *) exit 1
     esac
 }
@@ -213,11 +215,11 @@ contains() {
 # (key) should accept. plugin versions are actually prefixes and catch sub-
 # versions as well. Pairs are taken from:
 # https://developer.android.com/studio/releases/gradle-plugin#updating-gradle
-d_gradle_plugin_ver_k=(8.3 8.2 8.1 8.0 7.4 7.3 7.2.0 7.1 7.0 4.2 4.1 4.0 3.6   3.5   3.4   3.3   3.2    3.1 3.0 2.3 2.2 2.1.3  2.1    2.0  1.5  1.3 1.2 1.1 1.0   0.14 0.13 0.12 0.11 0.10 0.9 0.8  0.7  0.6 0.5 0.4 0.3 0.2)
-d_plugin_min_gradle_v=(8.3 8.2 8.0 8.0 7.5 7.4 7.3.3 7.2 7.0.2 6.7.1 6.5 6.1.1 5.6.4 5.4.1 5.1.1 4.10.1 4.6 4.4 4.1 3.3 2.14.1 2.14.1 2.12 2.12 2.4 2.4 2.3 2.2.1 2.2.1 2.1 2.1 1.12 1.12 1.12 1.11 1.10 1.9 1.8 1.6 1.6 1.4 1.4)
+d_gradle_plugin_ver_k=(8.4 8.3 8.2 8.1 8.0 7.4 7.3 7.2.0 7.1 7.0 4.2 4.1 4.0 3.6   3.5   3.4   3.3   3.2    3.1 3.0 2.3 2.2 2.1.3  2.1    2.0  1.5  1.3 1.2 1.1 1.0   0.14 0.13 0.12 0.11 0.10 0.9 0.8  0.7  0.6 0.5 0.4 0.3 0.2)
+d_plugin_min_gradle_v=(8.6 8.4 8.2 8.0 8.0 7.5 7.4 7.3.3 7.2 7.0.2 6.7.1 6.5 6.1.1 5.6.4 5.4.1 5.1.1 4.10.1 4.6 4.4 4.1 3.3 2.14.1 2.14.1 2.12 2.12 2.4 2.4 2.3 2.2.1 2.2.1 2.1 2.1 1.12 1.12 1.12 1.11 1.10 1.9 1.8 1.6 1.6 1.4 1.4)
 
 # All gradle versions we know about
-plugin_v=(8.5 8.4 8.3 8.2.1 8.2 8.1.1 8.1 8.0.2 8.0.1 8.0 7.6.3 7.6.2 7.6.1 7.6 7.5.1 7.5 7.4.2 7.4.1 7.4 7.3.3 7.3.2 7.3.1 7.3 7.2 7.1.1 7.1 7.0.2 7.0.1 7.0 6.9.4 6.9.3 6.9.2 6.9.1 6.9 6.8.3 6.8.2 6.8.1 6.8 6.7.1 6.7 6.6.1 6.6 6.5.1 6.5 6.4.1 6.4 6.3 6.2.2 6.2.1 6.2 6.1.1 6.1 6.0.1 6.0 5.6.4 5.6.3 5.6.2 5.6.1 5.6 5.5.1 5.5 5.4.1 5.4 5.3.1 5.3 5.2.1 5.2 5.1.1 5.1 5.0 4.10.3 4.10.2 4.10.1 4.10 4.9 4.8.1 4.8 4.7 4.6 4.5.1 4.5 4.4.1 4.4 4.3.1 4.3 4.2.1 4.2 4.1 4.0.2 4.0.1 4.0 3.5.1 3.5 3.4.1 3.4 3.3 3.2.1 3.2 3.1 3.0 2.14.1 2.14 2.13 2.12 2.11 2.10 2.9 2.8 2.7 2.6 2.5 2.4 2.3 2.2.1 2.2 2.1 2.0 1.12 1.11 1.10 1.9 1.8 1.7 1.6 1.5 1.4 1.3 1.2 1.1 1.0 0.9.2 0.9.1 0.9 0.8 0.7)
+plugin_v=(8.6 8.5 8.4 8.3 8.2.1 8.2 8.1.1 8.1 8.0.2 8.0.1 8.0 7.6.4 7.6.3 7.6.2 7.6.1 7.6 7.5.1 7.5 7.4.2 7.4.1 7.4 7.3.3 7.3.2 7.3.1 7.3 7.2 7.1.1 7.1 7.0.2 7.0.1 7.0 6.9.4 6.9.3 6.9.2 6.9.1 6.9 6.8.3 6.8.2 6.8.1 6.8 6.7.1 6.7 6.6.1 6.6 6.5.1 6.5 6.4.1 6.4 6.3 6.2.2 6.2.1 6.2 6.1.1 6.1 6.0.1 6.0 5.6.4 5.6.3 5.6.2 5.6.1 5.6 5.5.1 5.5 5.4.1 5.4 5.3.1 5.3 5.2.1 5.2 5.1.1 5.1 5.0 4.10.3 4.10.2 4.10.1 4.10 4.9 4.8.1 4.8 4.7 4.6 4.5.1 4.5 4.4.1 4.4 4.3.1 4.3 4.2.1 4.2 4.1 4.0.2 4.0.1 4.0 3.5.1 3.5 3.4.1 3.4 3.3 3.2.1 3.2 3.1 3.0 2.14.1 2.14 2.13 2.12 2.11 2.10 2.9 2.8 2.7 2.6 2.5 2.4 2.3 2.2.1 2.2 2.1 2.0 1.12 1.11 1.10 1.9 1.8 1.7 1.6 1.5 1.4 1.3 1.2 1.1 1.0 0.9.2 0.9.1 0.9 0.8 0.7)
 
 v_all=${plugin_v[@]}
 

tests/common.TestCase

@@ -1655,8 +1655,8 @@ class CommonTest(unittest.TestCase):
         fdroidserver.common.options.quiet = False
         fdroidserver.common.config = {}
         fdroidserver.common.config['serverwebroot'] = [
-            'example.com:/var/www/fdroid/',
-            'example.com:/var/www/fbot/',
+            {'url': 'example.com:/var/www/fdroid/'},
+            {'url': 'example.com:/var/www/fbot/'},
         ]
         fdroidserver.common.config['deploy_process_logs'] = True
         fdroidserver.common.config['identity_file'] = 'ssh/id_rsa'
@@ -1718,7 +1718,7 @@ class CommonTest(unittest.TestCase):
 
         fdroidserver.common.options = mock.Mock()
         fdroidserver.common.config = {}
-        fdroidserver.common.config['serverwebroot'] = [fakeserver]
+        fdroidserver.common.config['serverwebroot'] = [{'url': fakeserver}]
         fdroidserver.common.config['identity_file'] = 'ssh/id_rsa'
 
         def assert_subprocess_call(cmd):
@@ -2868,6 +2868,30 @@ class CommonTest(unittest.TestCase):
         with self.assertRaises(TypeError):
             fdroidserver.common.config_type_check('config/mirrors.yml', str())
 
+    def test_config_serverwebroot_str(self):
+        os.chdir(self.testdir)
+        Path('config.yml').write_text("""serverwebroot: 'foo@example.com:/var/www'""")
+        self.assertEqual(
+            [{'url': 'foo@example.com:/var/www/'}],
+            fdroidserver.common.read_config()['serverwebroot'],
+        )
+
+    def test_config_serverwebroot_list(self):
+        os.chdir(self.testdir)
+        Path('config.yml').write_text("""serverwebroot:\n  - foo@example.com:/var/www""")
+        self.assertEqual(
+            [{'url': 'foo@example.com:/var/www/'}],
+            fdroidserver.common.read_config()['serverwebroot'],
+        )
+
+    def test_config_serverwebroot_dict(self):
+        os.chdir(self.testdir)
+        Path('config.yml').write_text("""serverwebroot:\n  - url: 'foo@example.com:/var/www'""")
+        self.assertEqual(
+            [{'url': 'foo@example.com:/var/www/'}],
+            fdroidserver.common.read_config()['serverwebroot'],
+        )
+
 
 if __name__ == "__main__":
     os.chdir(os.path.dirname(__file__))

tests/deploy.TestCase

@@ -32,30 +32,122 @@ class DeployTest(unittest.TestCase):
         self._td = mkdtemp()
         self.testdir = self._td.name
 
-        fdroidserver.deploy.options = mock.Mock()
-        fdroidserver.deploy.config = {}
-
     def tearDown(self):
         self._td.cleanup()
 
+    def test_update_serverwebroots_bad_None(self):
+        with self.assertRaises(TypeError):
+            fdroidserver.deploy.update_serverwebroots(None, 'repo')
+
+    def test_update_serverwebroots_bad_int(self):
+        with self.assertRaises(TypeError):
+            fdroidserver.deploy.update_serverwebroots(9, 'repo')
+
+    def test_update_serverwebroots_bad_float(self):
+        with self.assertRaises(TypeError):
+            fdroidserver.deploy.update_serverwebroots(1.0, 'repo')
+
+    def test_update_serverwebroots(self):
+        """rsync works with file paths, so this test uses paths for the URLs"""
+        os.chdir(self.testdir)
+        repo = Path('repo')
+        repo.mkdir()
+        fake_apk = repo / 'fake.apk'
+        with fake_apk.open('w') as fp:
+            fp.write('not an APK, but has the right filename')
+        url0 = Path('url0/fdroid')
+        url0.mkdir(parents=True)
+        url1 = Path('url1/fdroid')
+        url1.mkdir(parents=True)
+
+        # setup parameters for this test run
+        fdroidserver.deploy.options.identity_file = None
+        fdroidserver.deploy.options.index_only = False
+        fdroidserver.deploy.config['make_current_version_link'] = False
+
+        dest_apk0 = url0 / fake_apk
+        dest_apk1 = url1 / fake_apk
+        self.assertFalse(dest_apk0.is_file())
+        self.assertFalse(dest_apk1.is_file())
+        fdroidserver.deploy.update_serverwebroots(
+            [
+                {'url': str(url0)},
+                {'url': str(url1)},
+            ],
+            str(repo),
+        )
+        self.assertTrue(dest_apk0.is_file())
+        self.assertTrue(dest_apk1.is_file())
+
+    def test_update_serverwebroots_url_does_not_end_with_fdroid(self):
+        with self.assertRaises(SystemExit):
+            fdroidserver.deploy.update_serverwebroots([{'url': 'url'}], 'repo')
+
+    def test_update_serverwebroots_bad_ssh_url(self):
+        with self.assertRaises(SystemExit):
+            fdroidserver.deploy.update_serverwebroots(
+                [{'url': 'f@b.ar::/path/to/fdroid'}], 'repo'
+            )
+
+    def test_update_serverwebroots_unsupported_ssh_url(self):
+        with self.assertRaises(SystemExit):
+            fdroidserver.deploy.update_serverwebroots([{'url': 'ssh://nope'}], 'repo')
+
     def test_update_serverwebroot(self):
+        """rsync works with file paths, so this test uses paths for the URLs"""
         os.chdir(self.testdir)
         repo = Path('repo')
         repo.mkdir(parents=True)
         fake_apk = repo / 'fake.apk'
         with fake_apk.open('w') as fp:
             fp.write('not an APK, but has the right filename')
-        serverwebroot = Path('serverwebroot')
-        serverwebroot.mkdir()
+        fake_index = repo / fdroidserver.deploy.INDEX_FILES[0]
+        with fake_index.open('w') as fp:
+            fp.write('not an index, but has the right filename')
+        url = Path('url')
+        url.mkdir()
 
         # setup parameters for this test run
         fdroidserver.deploy.options.identity_file = None
+        fdroidserver.deploy.options.index_only = False
         fdroidserver.deploy.config['make_current_version_link'] = False
 
-        dest_apk = Path(serverwebroot) / fake_apk
+        dest_apk = Path(url) / fake_apk
+        dest_index = Path(url) / fake_index
         self.assertFalse(dest_apk.is_file())
-        fdroidserver.deploy.update_serverwebroot(str(serverwebroot), 'repo')
+        self.assertFalse(dest_index.is_file())
+
+        fdroidserver.deploy.update_serverwebroot({'url': str(url)}, 'repo')
         self.assertTrue(dest_apk.is_file())
+        self.assertTrue(dest_index.is_file())
+
+    def test_update_serverwebroot_in_index_only_mode(self):
+        os.chdir(self.testdir)
+        repo = Path('repo')
+        repo.mkdir(parents=True)
+        fake_apk = repo / 'fake.apk'
+        with fake_apk.open('w') as fp:
+            fp.write('not an APK, but has the right filename')
+        for i in fdroidserver.deploy.INDEX_FILES:
+            fake_index = repo / i
+            with fake_index.open('w') as fp:
+                fp.write('not an index, but has the right filename')
+        url = Path('url')
+        url.mkdir()
+
+        # setup parameters for this test run
+        fdroidserver.deploy.options.identity_file = None
+        fdroidserver.deploy.options.index_only = True
+        fdroidserver.deploy.config['make_current_version_link'] = False
+
+        dest_apk = Path(url) / fake_apk
+        dest_index = Path(url) / fake_index
+        self.assertFalse(dest_apk.is_file())
+        self.assertFalse(dest_index.is_file())
+
+        fdroidserver.deploy.update_serverwebroot({'url': str(url)}, 'repo')
+        self.assertFalse(dest_apk.is_file())
+        self.assertTrue(dest_index.is_file())
 
     @mock.patch.dict(os.environ, clear=True)
     def test_update_serverwebroot_no_rsync_error(self):
@@ -66,13 +158,15 @@ class DeployTest(unittest.TestCase):
 
     def test_update_serverwebroot_make_cur_version_link(self):
         # setup parameters for this test run
+        fdroidserver.deploy.options = mock.Mock()
         fdroidserver.deploy.options.no_checksum = True
         fdroidserver.deploy.options.identity_file = None
         fdroidserver.deploy.options.verbose = False
         fdroidserver.deploy.options.quiet = True
         fdroidserver.deploy.options.identity_file = None
+        fdroidserver.deploy.options.index_only = False
         fdroidserver.deploy.config['make_current_version_link'] = True
-        serverwebroot = "example.com:/var/www/fdroid"
+        url = "example.com:/var/www/fdroid"
         repo_section = 'repo'
 
         # setup function for asserting subprocess.call invocations
@@ -123,7 +217,7 @@ class DeployTest(unittest.TestCase):
                         '--safe-links',
                         '--quiet',
                         'repo',
-                        serverwebroot,
+                        url,
                     ],
                 )
             elif call_iteration == 2:
@@ -152,18 +246,131 @@ class DeployTest(unittest.TestCase):
             os.symlink('repo/com.example.sym.apk.asc', 'Sym.apk.asc')
             os.symlink('repo/com.example.sym.apk.sig', 'Sym.apk.sig')
             with mock.patch('subprocess.call', side_effect=update_server_webroot_call):
-                fdroidserver.deploy.update_serverwebroot(serverwebroot, repo_section)
+                fdroidserver.deploy.update_serverwebroot({'url': url}, repo_section)
         self.assertEqual(call_iteration, 3, 'expected 3 invocations of subprocess.call')
 
+    def test_update_serverwebroot_make_cur_version_link_in_index_only_mode(self):
+        # setup parameters for this test run
+        fdroidserver.deploy.options = mock.Mock()
+        fdroidserver.deploy.options.no_checksum = True
+        fdroidserver.deploy.options.identity_file = None
+        fdroidserver.deploy.options.verbose = False
+        fdroidserver.deploy.options.quiet = True
+        fdroidserver.deploy.options.identity_file = None
+        fdroidserver.deploy.options.index_only = True
+        fdroidserver.deploy.config['make_current_version_link'] = True
+        url = "example.com:/var/www/fdroid"
+        repo_section = 'repo'
+
+        # setup function for asserting subprocess.call invocations
+        call_iteration = 0
+
+        def update_server_webroot_call(cmd):
+            nonlocal call_iteration
+            if call_iteration == 0:
+                self.assertListEqual(
+                    cmd,
+                    [
+                        'rsync',
+                        '--archive',
+                        '--delete-after',
+                        '--safe-links',
+                        '--quiet',
+                        'repo/entry.jar',
+                        'repo/entry.json',
+                        'repo/entry.json.asc',
+                        'repo/index-v1.jar',
+                        'repo/index-v1.json',
+                        'repo/index-v1.json.asc',
+                        'repo/index-v2.json',
+                        'repo/index-v2.json.asc',
+                        'repo/index.jar',
+                        'repo/index.xml',
+                        'example.com:/var/www/fdroid/repo/',
+                    ],
+                )
+            elif call_iteration == 1:
+                self.assertListEqual(
+                    cmd,
+                    [
+                        'rsync',
+                        '--archive',
+                        '--delete-after',
+                        '--safe-links',
+                        '--quiet',
+                        'repo',
+                        url,
+                    ],
+                )
+            elif call_iteration == 2:
+                self.assertListEqual(
+                    cmd,
+                    [
+                        'rsync',
+                        '--archive',
+                        '--delete-after',
+                        '--safe-links',
+                        '--quiet',
+                        'Sym.apk',
+                        'Sym.apk.asc',
+                        'Sym.apk.sig',
+                        'example.com:/var/www/fdroid',
+                    ],
+                )
+            # elif call_iteration == 1:
+            #     self.assertListEqual(
+            #         cmd,
+            #         [
+            #             'rsync',
+            #             '--archive',
+            #             '--delete-after',
+            #             '--safe-links',
+            #             '--quiet',
+            #             'repo',
+            #             serverwebroot,
+            #         ],
+            #     )
+            # elif call_iteration == 2:
+            #     self.assertListEqual(
+            #         cmd,
+            #         [
+            #             'rsync',
+            #             '--archive',
+            #             '--delete-after',
+            #             '--safe-links',
+            #             '--quiet',
+            #             'Sym.apk',
+            #             'Sym.apk.asc',
+            #             'Sym.apk.sig',
+            #             'example.com:/var/www/fdroid',
+            #         ],
+            #     )
+            else:
+                self.fail('unexpected subprocess.call invocation')
+            call_iteration += 1
+            return 0
+
+        with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
+            os.mkdir('repo')
+            os.symlink('repo/com.example.sym.apk', 'Sym.apk')
+            os.symlink('repo/com.example.sym.apk.asc', 'Sym.apk.asc')
+            os.symlink('repo/com.example.sym.apk.sig', 'Sym.apk.sig')
+            with mock.patch('subprocess.call', side_effect=update_server_webroot_call):
+                fdroidserver.deploy.update_serverwebroot({'url': url}, repo_section)
+        self.assertEqual(call_iteration, 1, 'expected 1 invocations of subprocess.call')
+
     def test_update_serverwebroot_with_id_file(self):
         # setup parameters for this test run
-        fdroidserver.deploy.options.no_chcksum = False
+        fdroidserver.deploy.options = mock.Mock()
+        fdroidserver.deploy.options.identity_file = None
+        fdroidserver.deploy.options.no_checksum = True
         fdroidserver.deploy.options.verbose = True
         fdroidserver.deploy.options.quiet = False
         fdroidserver.deploy.options.identity_file = None
+        fdroidserver.deploy.options.index_only = False
         fdroidserver.deploy.config['identity_file'] = './id_rsa'
         fdroidserver.deploy.config['make_current_version_link'] = False
-        serverwebroot = "example.com:/var/www/fdroid"
+        url = "example.com:/var/www/fdroid"
         repo_section = 'archive'
 
         # setup function for asserting subprocess.call invocations
@@ -204,7 +411,7 @@ class DeployTest(unittest.TestCase):
                         '--exclude',
                         'archive/index.xml',
                         'archive',
-                        serverwebroot,
+                        url,
                     ],
                 )
             elif call_iteration == 1:
@@ -220,7 +427,7 @@ class DeployTest(unittest.TestCase):
                         'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i '
                         + fdroidserver.deploy.config['identity_file'],
                         'archive',
-                        serverwebroot,
+                        url,
                     ],
                 )
             else:
@@ -229,9 +436,91 @@ class DeployTest(unittest.TestCase):
             return 0
 
         with mock.patch('subprocess.call', side_effect=update_server_webroot_call):
-            fdroidserver.deploy.update_serverwebroot(serverwebroot, repo_section)
+            fdroidserver.deploy.update_serverwebroot({'url': url}, repo_section)
         self.assertEqual(call_iteration, 2, 'expected 2 invocations of subprocess.call')
 
+    def test_update_serverwebroot_with_id_file_in_index_only_mode(self):
+        # setup parameters for this test run
+        fdroidserver.deploy.options.no_chcksum = False
+        fdroidserver.deploy.options.verbose = True
+        fdroidserver.deploy.options.quiet = False
+        fdroidserver.deploy.options.identity_file = None
+        fdroidserver.deploy.options.index_only = True
+        fdroidserver.deploy.config['identity_file'] = './id_rsa'
+        fdroidserver.deploy.config['make_current_version_link'] = False
+        url = "example.com:/var/www/fdroid"
+        repo_section = 'archive'
+
+        # setup function for asserting subprocess.call invocations
+        call_iteration = 0
+
+        def update_server_webroot_call(cmd):
+            nonlocal call_iteration
+            if call_iteration == 0:
+                self.assertListEqual(
+                    cmd,
+                    [
+                        'rsync',
+                        '--archive',
+                        '--delete-after',
+                        '--safe-links',
+                        '--verbose',
+                        '-e',
+                        'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i '
+                        + fdroidserver.deploy.config['identity_file'],
+                        'archive/entry.jar',
+                        'archive/entry.json',
+                        'archive/entry.json.asc',
+                        'archive/index-v1.jar',
+                        'archive/index-v1.json',
+                        'archive/index-v1.json.asc',
+                        'archive/index-v2.json',
+                        'archive/index-v2.json.asc',
+                        'archive/index.jar',
+                        'archive/index.xml',
+                        "example.com:/var/www/fdroid/archive/",
+                    ],
+                )
+            elif call_iteration == 1:
+                self.assertListEqual(
+                    cmd,
+                    [
+                        'rsync',
+                        '--archive',
+                        '--delete-after',
+                        '--safe-links',
+                        '--verbose',
+                        '-e',
+                        'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i '
+                        + fdroidserver.deploy.config['identity_file'],
+                        "example.com:/var/www/fdroid/archive/",
+                    ],
+                )
+            # elif call_iteration == 1:
+            #     self.assertListEqual(
+            #         cmd,
+            #         [
+            #             'rsync',
+            #             '--archive',
+            #             '--delete-after',
+            #             '--safe-links',
+            #             '--verbose',
+            #             '-e',
+            #             'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i '
+            #             + fdroidserver.deploy.config['identity_file'],
+            #             'archive',
+            #             serverwebroot,
+            #         ],
+            #     )
+            else:
+                self.fail('unexpected subprocess.call invocation')
+            call_iteration += 1
+            return 0
+
+        with mock.patch('subprocess.call', side_effect=update_server_webroot_call):
+            fdroidserver.deploy.update_serverwebroot({'url': url}, repo_section)
+        self.assertEqual(call_iteration, 1, 'expected 1 invocations of subprocess.call')
+
     @unittest.skipIf(
         not os.getenv('VIRUSTOTAL_API_KEY'), 'VIRUSTOTAL_API_KEY is not set'
     )
@@ -252,6 +541,476 @@ class DeployTest(unittest.TestCase):
                 name, fdroidserver.deploy.REMOTE_HOSTNAME_REGEX.sub(r'\1', remote_url)
             )
 
+    def test_update_awsbucket_s3cmd(self):
+        # setup parameters for this test run
+        fdroidserver.deploy.options = mock.Mock()
+        fdroidserver.deploy.options.no_checksum = True
+        fdroidserver.deploy.options.verbose = False
+        fdroidserver.deploy.options.quiet = True
+        fdroidserver.deploy.options.index_only = False
+
+        config = {}
+        fdroidserver.common.fill_config_defaults(config)
+        fdroidserver.deploy.config = config
+        fdroidserver.deploy.config["awsbucket"] = "bucket"
+        fdroidserver.deploy.config["awsaccesskeyid"] = "accesskeyid"
+        fdroidserver.deploy.config["awssecretkey"] = "secretkey"
+        fdroidserver.deploy.config["s3cmd"] = "s3cmd"
+
+        repo_section = 'repo'
+
+        # setup function for asserting subprocess.call invocations
+        call_iteration = 0
+
+        def update_awsbucket_s3cmd_call(cmd):
+            nonlocal call_iteration
+            if call_iteration == 0:
+                self.assertListEqual(
+                    cmd,
+                    [
+                        's3cmd',
+                        f"--config={fdroidserver.deploy.AUTO_S3CFG}",
+                        'info',
+                        f"s3://{fdroidserver.deploy.config['awsbucket']}",
+                    ],
+                )
+            elif call_iteration == 1:
+                self.assertListEqual(
+                    cmd,
+                    [
+                        's3cmd',
+                        f"--config={fdroidserver.deploy.AUTO_S3CFG}",
+                        'sync',
+                        '--acl-public',
+                        '--quiet',
+                        '--exclude',
+                        'repo/entry.jar',
+                        '--exclude',
+                        'repo/entry.json',
+                        '--exclude',
+                        'repo/entry.json.asc',
+                        '--exclude',
+                        'repo/index-v1.jar',
+                        '--exclude',
+                        'repo/index-v1.json',
+                        '--exclude',
+                        'repo/index-v1.json.asc',
+                        '--exclude',
+                        'repo/index-v2.json',
+                        '--exclude',
+                        'repo/index-v2.json.asc',
+                        '--exclude',
+                        'repo/index.jar',
+                        '--exclude',
+                        'repo/index.xml',
+                        '--no-check-md5',
+                        '--skip-existing',
+                        repo_section,
+                        f"s3://{fdroidserver.deploy.config['awsbucket']}/fdroid/",
+                    ],
+                )
+            elif call_iteration == 2:
+                self.assertListEqual(
+                    cmd,
+                    [
+                        's3cmd',
+                        f"--config={fdroidserver.deploy.AUTO_S3CFG}",
+                        'sync',
+                        '--acl-public',
+                        '--quiet',
+                        '--exclude',
+                        'repo/entry.jar',
+                        '--exclude',
+                        'repo/entry.json',
+                        '--exclude',
+                        'repo/entry.json.asc',
+                        '--exclude',
+                        'repo/index-v1.jar',
+                        '--exclude',
+                        'repo/index-v1.json',
+                        '--exclude',
+                        'repo/index-v1.json.asc',
+                        '--exclude',
+                        'repo/index-v2.json',
+                        '--exclude',
+                        'repo/index-v2.json.asc',
+                        '--exclude',
+                        'repo/index.jar',
+                        '--exclude',
+                        'repo/index.xml',
+                        '--no-check-md5',
+                        repo_section,
+                        f"s3://{fdroidserver.deploy.config['awsbucket']}/fdroid/",
+                    ],
+                )
+            elif call_iteration == 3:
+                self.assertListEqual(
+                    cmd,
+                    [
+                        's3cmd',
+                        f"--config={fdroidserver.deploy.AUTO_S3CFG}",
+                        'sync',
+                        '--acl-public',
+                        '--quiet',
+                        '--delete-removed',
+                        '--delete-after',
+                        '--no-check-md5',
+                        repo_section,
+                        f"s3://{fdroidserver.deploy.config['awsbucket']}/fdroid/",
+                    ],
+                )
+            else:
+                self.fail('unexpected subprocess.call invocation')
+            call_iteration += 1
+            return 0
+
+        with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
+            os.mkdir('repo')
+            os.symlink('repo/com.example.sym.apk', 'Sym.apk')
+            os.symlink('repo/com.example.sym.apk.asc', 'Sym.apk.asc')
+            os.symlink('repo/com.example.sym.apk.sig', 'Sym.apk.sig')
+            with mock.patch('subprocess.call', side_effect=update_awsbucket_s3cmd_call):
+                fdroidserver.deploy.update_awsbucket_s3cmd(repo_section)
+        self.assertEqual(call_iteration, 4, 'expected 4 invocations of subprocess.call')
+
+    def test_update_awsbucket_s3cmd_in_index_only_mode(self):
+        # setup parameters for this test run
+        fdroidserver.deploy.options = mock.Mock()
+        fdroidserver.deploy.options.no_checksum = True
+        fdroidserver.deploy.options.verbose = False
+        fdroidserver.deploy.options.quiet = True
+        fdroidserver.deploy.options.index_only = True
+
+        config = {}
+        fdroidserver.common.fill_config_defaults(config)
+        fdroidserver.deploy.config = config
+        fdroidserver.deploy.config["awsbucket"] = "bucket"
+        fdroidserver.deploy.config["awsaccesskeyid"] = "accesskeyid"
+        fdroidserver.deploy.config["awssecretkey"] = "secretkey"
+        fdroidserver.deploy.config["s3cmd"] = "s3cmd"
+
+        repo_section = 'repo'
+
+        # setup function for asserting subprocess.call invocations
+        call_iteration = 0
+
+        def update_awsbucket_s3cmd_call(cmd):
+            nonlocal call_iteration
+            if call_iteration == 0:
+                self.assertListEqual(
+                    cmd,
+                    [
+                        's3cmd',
+                        f"--config={fdroidserver.deploy.AUTO_S3CFG}",
+                        'info',
+                        f"s3://{fdroidserver.deploy.config['awsbucket']}",
+                    ],
+                )
+            elif call_iteration == 1:
+                self.assertListEqual(
+                    cmd,
+                    [
+                        's3cmd',
+                        f"--config={fdroidserver.deploy.AUTO_S3CFG}",
+                        'sync',
+                        '--acl-public',
+                        '--quiet',
+                        '--include',
+                        'repo/entry.jar',
+                        '--include',
+                        'repo/entry.json',
+                        '--include',
+                        'repo/entry.json.asc',
+                        '--include',
+                        'repo/index-v1.jar',
+                        '--include',
+                        'repo/index-v1.json',
+                        '--include',
+                        'repo/index-v1.json.asc',
+                        '--include',
+                        'repo/index-v2.json',
+                        '--include',
+                        'repo/index-v2.json.asc',
+                        '--include',
+                        'repo/index.jar',
+                        '--include',
+                        'repo/index.xml',
+                        '--delete-removed',
+                        '--delete-after',
+                        '--no-check-md5',
+                        repo_section,
+                        f"s3://{fdroidserver.deploy.config['awsbucket']}/fdroid/",
+                    ],
+                )
+            else:
+                self.fail('unexpected subprocess.call invocation')
+            call_iteration += 1
+            return 0
+
+        with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
+            os.mkdir('repo')
+            os.symlink('repo/com.example.sym.apk', 'Sym.apk')
+            os.symlink('repo/com.example.sym.apk.asc', 'Sym.apk.asc')
+            os.symlink('repo/com.example.sym.apk.sig', 'Sym.apk.sig')
+            with mock.patch('subprocess.call', side_effect=update_awsbucket_s3cmd_call):
+                fdroidserver.deploy.update_awsbucket_s3cmd(repo_section)
+        self.assertEqual(call_iteration, 2, 'expected 2 invocations of subprocess.call')
+
+    def test_update_awsbucket_libcloud(self):
+        from libcloud.storage.base import Container
+
+        # setup parameters for this test run
+        fdroidserver.deploy.options = mock.Mock()
+        fdroidserver.deploy.options.no_checksum = True
+        fdroidserver.deploy.options.verbose = False
+        fdroidserver.deploy.options.quiet = True
+        fdroidserver.deploy.options.index_only = False
+
+        config = {}
+        fdroidserver.common.fill_config_defaults(config)
+        fdroidserver.deploy.config = config
+        fdroidserver.deploy.config["awsbucket"] = "bucket"
+        fdroidserver.deploy.config["awsaccesskeyid"] = "accesskeyid"
+        fdroidserver.deploy.config["awssecretkey"] = "secretkey"
+        fdroidserver.deploy.config["s3cmd"] = "s3cmd"
+
+        repo_section = 'repo'
+
+        os.chdir(self.testdir)
+        repo = Path('repo')
+        repo.mkdir(parents=True)
+        fake_apk = repo / 'Sym.apk'
+        with fake_apk.open('w') as fp:
+            fp.write('not an APK, but has the right filename')
+        fake_index = repo / fdroidserver.deploy.INDEX_FILES[0]
+        with fake_index.open('w') as fp:
+            fp.write('not an index, but has the right filename')
+
+        with mock.patch(
+            'libcloud.storage.drivers.s3.S3StorageDriver'
+        ) as mock_driver_class:
+            mock_driver = mock_driver_class.return_value
+            mock_container = mock.MagicMock(spec=Container)
+            mock_container.list_objects.return_value = [
+                mock.MagicMock(name='Sym.apk'),
+                mock.MagicMock(name=fdroidserver.deploy.INDEX_FILES[0]),
+            ]
+
+            mock_driver.get_container.return_value = mock_container
+            mock_driver.upload_object_via_stream.return_value = None
+
+            fdroidserver.deploy.update_awsbucket_libcloud(repo_section)
+
+            mock_driver.get_container.assert_called_once_with(
+                container_name=fdroidserver.deploy.config["awsbucket"]
+            )
+            mock_container.list_objects.assert_called_once_with()
+            files_to_upload = ['fdroid/repo/Sym.apk', 'fdroid/repo/entry.jar']
+            calls = [
+                mock.call(
+                    iterator=mock.ANY,
+                    container=mock_container,
+                    object_name=file,
+                    extra={'acl': 'public-read'},
+                )
+                for file in files_to_upload
+            ]
+            mock_driver.upload_object_via_stream.assert_has_calls(calls, any_order=True)
+            assert mock_driver.upload_object_via_stream.call_count == 2
+
+    def test_update_awsbucket_libcloud_in_index_only_mode(self):
+        from libcloud.storage.base import Container
+
+        # setup parameters for this test run
+        fdroidserver.deploy.options = mock.Mock()
+        fdroidserver.deploy.options.no_checksum = True
+        fdroidserver.deploy.options.verbose = False
+        fdroidserver.deploy.options.quiet = True
+        fdroidserver.deploy.options.index_only = True
+
+        config = {}
+        fdroidserver.common.fill_config_defaults(config)
+        fdroidserver.deploy.config = config
+        fdroidserver.deploy.config["awsbucket"] = "bucket"
+        fdroidserver.deploy.config["awsaccesskeyid"] = "accesskeyid"
+        fdroidserver.deploy.config["awssecretkey"] = "secretkey"
+        fdroidserver.deploy.config["s3cmd"] = "s3cmd"
+
+        repo_section = 'repo'
+
+        os.chdir(self.testdir)
+        repo = Path('repo')
+        repo.mkdir(parents=True)
+        fake_apk = repo / 'Sym.apk'
+        with fake_apk.open('w') as fp:
+            fp.write('not an APK, but has the right filename')
+        fake_index = repo / fdroidserver.deploy.INDEX_FILES[0]
+        with fake_index.open('w') as fp:
+            fp.write('not an index, but has the right filename')
+
+        with mock.patch(
+            'libcloud.storage.drivers.s3.S3StorageDriver'
+        ) as mock_driver_class:
+            mock_driver = mock_driver_class.return_value
+            mock_container = mock.MagicMock(spec=Container)
+            mock_container.list_objects.return_value = [
+                mock.MagicMock(name='Sym.apk'),
+                mock.MagicMock(name=fdroidserver.deploy.INDEX_FILES[0]),
+            ]
+
+            mock_driver.get_container.return_value = mock_container
+            mock_driver.upload_object_via_stream.return_value = None
+
+            fdroidserver.deploy.update_awsbucket_libcloud(repo_section)
+
+            mock_driver.get_container.assert_called_once_with(
+                container_name=fdroidserver.deploy.config["awsbucket"]
+            )
+            mock_container.list_objects.assert_called_once_with()
+            files_to_upload = ['fdroid/repo/entry.jar']
+            calls = [
+                mock.call(
+                    iterator=mock.ANY,
+                    container=mock_container,
+                    object_name=file,
+                    extra={'acl': 'public-read'},
+                )
+                for file in files_to_upload
+            ]
+            mock_driver.upload_object_via_stream.assert_has_calls(
+                calls,
+                any_order=False,
+            )
+            assert mock_driver.upload_object_via_stream.call_count == 1
+
+    def test_update_servergitmirrors(self):
+
+        # setup parameters for this test run
+        fdroidserver.deploy.options.identity_file = None
+        fdroidserver.deploy.options.no_keep_git_mirror_archive = False
+        fdroidserver.deploy.options.verbose = False
+        fdroidserver.deploy.options.quiet = True
+        fdroidserver.deploy.options.index_only = False
+
+        config = {}
+        fdroidserver.common.fill_config_defaults(config)
+        fdroidserver.deploy.config = config
+        fdroidserver.deploy.config["servergitmirrors"] = []
+
+        repo_section = 'repo'
+
+        # setup function for asserting subprocess.call invocations
+        call_iteration = 0
+
+        os.chdir(self.testdir)
+        repo = Path('repo')
+        repo.mkdir(parents=True)
+        fake_apk = repo / 'Sym.apk'
+        with fake_apk.open('w') as fp:
+            fp.write('not an APK, but has the right filename')
+        fake_index = repo / fdroidserver.deploy.INDEX_FILES[0]
+        with fake_index.open('w') as fp:
+            fp.write('not an index, but has the right filename')
+
+        def update_servergitmirrors_call(cmd):
+            nonlocal call_iteration
+            if call_iteration == 0:
+                self.assertListEqual(
+                    cmd,
+                    [
+                        'rsync',
+                        '--recursive',
+                        '--safe-links',
+                        '--times',
+                        '--perms',
+                        '--one-file-system',
+                        '--delete',
+                        '--chmod=Da+rx,Fa-x,a+r,u+w',
+                        '--quiet',
+                        'repo/',
+                        "git-mirror/fdroid/repo/",
+                    ],
+                )
+            else:
+                self.fail('unexpected subprocess.call invocation')
+            call_iteration += 1
+            return 0
+
+        with mock.patch(
+            'subprocess.call', side_effect=update_servergitmirrors_call
+        ):
+            fdroidserver.deploy.update_servergitmirrors([], repo_section)
+        self.assertEqual(call_iteration, 1, 'expected 1 invocations of subprocess.call')
+
+    def test_update_servergitmirrors_in_index_only_mode(self):
+
+        # setup parameters for this test run
+        fdroidserver.deploy.options.identity_file = None
+        fdroidserver.deploy.options.no_keep_git_mirror_archive = False
+        fdroidserver.deploy.options.verbose = False
+        fdroidserver.deploy.options.quiet = True
+        fdroidserver.deploy.options.index_only = True
+
+        config = {}
+        fdroidserver.common.fill_config_defaults(config)
+        fdroidserver.deploy.config = config
+        fdroidserver.deploy.config["servergitmirrors"] = []
+
+        repo_section = 'repo'
+
+        # setup function for asserting subprocess.call invocations
+        call_iteration = 0
+
+        os.chdir(self.testdir)
+        repo = Path('repo')
+        repo.mkdir(parents=True)
+        fake_apk = repo / 'Sym.apk'
+        with fake_apk.open('w') as fp:
+            fp.write('not an APK, but has the right filename')
+        fake_index = repo / fdroidserver.deploy.INDEX_FILES[0]
+        with fake_index.open('w') as fp:
+            fp.write('not an index, but has the right filename')
+
+
+        def update_servergitmirrors_call(cmd):
+            nonlocal call_iteration
+            if call_iteration == 0:
+                self.assertListEqual(
+                    cmd,
+                    [
+                        'rsync',
+                        '--recursive',
+                        '--safe-links',
+                        '--times',
+                        '--perms',
+                        '--one-file-system',
+                        '--delete',
+                        '--chmod=Da+rx,Fa-x,a+r,u+w',
+                        '--quiet',
+                        'repo/entry.jar',
+                        'repo/entry.json',
+                        'repo/entry.json.asc',
+                        'repo/index-v1.jar',
+                        'repo/index-v1.json',
+                        'repo/index-v1.json.asc',
+                        'repo/index-v2.json',
+                        'repo/index-v2.json.asc',
+                        'repo/index.jar',
+                        'repo/index.xml',
+                        "git-mirror/fdroid/repo/",
+                    ],
+                )
+            else:
+                self.fail('unexpected subprocess.call invocation')
+            call_iteration += 1
+            return 0
+
+        with mock.patch(
+            'subprocess.call', side_effect=update_servergitmirrors_call
+        ):
+            fdroidserver.deploy.update_servergitmirrors([], repo_section)
+        self.assertEqual(call_iteration, 1, 'expected 1 invocations of subprocess.call')
+
 
 if __name__ == "__main__":
     os.chdir(os.path.dirname(__file__))

tests/index.TestCase

@@ -575,7 +575,7 @@ class IndexTest(unittest.TestCase):
         ]:
             self.assertEqual(
                 ['https://raw.githubusercontent.com/foo/bar/master/fdroid'],
-                index.get_mirror_service_urls(url),
+                index.get_mirror_service_urls({"url": url}),
             )
 
     @patch.dict(os.environ, clear=True)
@@ -603,13 +603,13 @@ class IndexTest(unittest.TestCase):
                     ]
                     self.assertEqual(
                         expected,
-                        index.get_mirror_service_urls(url),
+                        index.get_mirror_service_urls({"url": url}),
                     )
                     with patch.dict(os.environ, clear=True):
                         os.environ['CI_JOB_ID'] = ci_job_id
                         self.assertEqual(
                             expected + [artifacts_url],
-                            index.get_mirror_service_urls(url),
+                            index.get_mirror_service_urls({"url": url}),
                         )
                 with patch('fdroidserver.common.GITLAB_COM_PAGES_MAX_SIZE', 10):
                     expected = [
@@ -617,13 +617,13 @@ class IndexTest(unittest.TestCase):
                     ]
                     self.assertEqual(
                         expected,
-                        index.get_mirror_service_urls(url),
+                        index.get_mirror_service_urls({"url": url}),
                     )
                     with patch.dict(os.environ, clear=True):
                         os.environ['CI_JOB_ID'] = ci_job_id
                         self.assertEqual(
                             expected + [artifacts_url],
-                            index.get_mirror_service_urls(url),
+                            index.get_mirror_service_urls({"url": url}),
                         )
 
     def test_make_website(self):

tests/nightly.TestCase

@@ -256,7 +256,7 @@ class NightlyTest(unittest.TestCase):
         self.assertEqual(called, [['ssh', '-Tvi'], ['fdroid', 'deploy']])
         self.assertFalse(os.path.exists('config.py'))
         git_url = 'git@github.com:f-droid/test-nightly'
-        mirror_url = index.get_mirror_service_urls(git_url)[0]
+        mirror_url = index.get_mirror_service_urls({"url": git_url})[0]
         expected = {
             'archive_description': 'Old nightly builds that have been archived.',
             'archive_name': 'f-droid/test-nightly archive',
@@ -271,7 +271,7 @@ class NightlyTest(unittest.TestCase):
             'repo_keyalias': 'androiddebugkey',
             'repo_name': 'f-droid/test-nightly',
             'repo_url': mirror_url + '/repo',
-            'servergitmirrors': git_url,
+            'servergitmirrors': [{"url": git_url}],
             'update_stats': True,
         }
         with open('config.yml') as fp:
@@ -344,7 +344,7 @@ class NightlyTest(unittest.TestCase):
             'repo_keyalias': 'androiddebugkey',
             'repo_name': 'fdroid/test-nightly',
             'repo_url': 'https://gitlab.com/fdroid/test-nightly/-/raw/master/fdroid/repo',
-            'servergitmirrors': 'git@gitlab.com:fdroid/test-nightly',
+            'servergitmirrors': [{"url": 'git@gitlab.com:fdroid/test-nightly'}],
             'update_stats': True,
         }
         with open('config.yml') as fp:

tests/refresh-SUSS_DEFAULT.py

@@ -0,0 +1,37 @@
+#!/usr/bin/env python3
+#
+# This will update the caches suss.json from the network, then
+# overwrite fdroidserver/scanner.py to add the contents of suss.json
+# to the SUSS_DEFAULT variable.
+
+import inspect
+import os
+import re
+import sys
+from pathlib import Path
+
+localmodule = os.path.realpath(
+    os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..')
+)
+print('localmodule: ' + localmodule)
+if localmodule not in sys.path:
+    sys.path.insert(0, localmodule)
+from fdroidserver import scanner
+
+scanner._get_tool().refresh()
+scanner_py = Path(localmodule) / 'fdroidserver/scanner.py'
+contents = scanner_py.read_text()
+scanner_py.write_text(
+    re.sub(
+        r"""SUSS_DEFAULT *= *r?'''.*""",
+        """SUSS_DEFAULT = r'''""",
+        contents,
+        flags=re.DOTALL,
+    )
+)
+os.system(  # nosec bandit B605 start_process_with_a_shell, don't judge me ;-)
+    """cat %s >> %s"""
+    % (str(scanner._scanner_cachedir() / 'suss.json'), str(scanner_py))
+)
+with scanner_py.open('a') as fp:
+    fp.write("'''\n")

tests/run-tests

@@ -1154,7 +1154,7 @@ GIT_MIRROR=$REPOROOT/git-mirror
 cd $REPOROOT
 fdroid_init_with_prebuilt_keystore
 printf '\narchive_older: 3\n' >> config.yml
-echo "servergitmirrors: $SERVER_GIT_MIRROR" >> config.yml
+printf "servergitmirrors: $SERVER_GIT_MIRROR\n" >> config.yml
 
 cp $WORKSPACE/tests/repo/com.politedroid_[345].apk repo/
 $fdroid update --create-metadata

tests/scanner.TestCase

@@ -54,11 +54,11 @@ class ScannerTest(unittest.TestCase):
             'OtakuWorld': 2,
             'Zillode': 1,
             'cn.wildfirechat.chat': 4,
-            'com.github.shadowsocks': 6,
-            'com.integreight.onesheeld': 11,
-            'com.jens.automation2': 2,
+            'com.github.shadowsocks': 7,
+            'com.integreight.onesheeld': 16,
+            'com.jens.automation2': 3,
             'firebase-suspect': 1,
-            'org.mozilla.rocket': 1,
+            'org.mozilla.rocket': 2,
             'org.tasks': 2,
             'realm': 1,
             'se.manyver': 2,
@@ -704,6 +704,20 @@ class Test_SignatureDataController(unittest.TestCase):
 
 
 class Test_ScannerTool(unittest.TestCase):
+    def setUp(self):
+        fdroidserver.common.options = None
+        fdroidserver.common.config = None
+        self.basedir = os.path.join(localmodule, 'tests')
+        os.chdir(self.basedir)
+        self._td = mkdtemp()
+        self.testdir = self._td.name
+
+    def tearDown(self):
+        fdroidserver.common.options = None
+        fdroidserver.common.config = None
+        os.chdir(self.basedir)
+        self._td.cleanup()
+
     def test_load(self):
         st = mock.Mock()
         st.sdcs = [mock.Mock(), mock.Mock()]
@@ -711,6 +725,42 @@ class Test_ScannerTool(unittest.TestCase):
         st.sdcs[0].load.assert_called_once_with()
         st.sdcs[1].load.assert_called_once_with()
 
+    def test_refresh_no_options_or_config(self):
+        """This simulates what happens when running something like scan_source()"""
+        with mock.patch('fdroidserver.scanner.ScannerTool.refresh') as refresh:
+            fdroidserver.scanner.ScannerTool()
+            refresh.assert_not_called()
+
+    def test_refresh_true(self):
+        fdroidserver.scanner.options = mock.Mock()
+        fdroidserver.scanner.options.refresh_scanner = True
+        with mock.patch('fdroidserver.scanner.ScannerTool.refresh') as refresh:
+            fdroidserver.scanner.ScannerTool()
+            refresh.assert_called_once()
+
+    def test_refresh_false(self):
+        fdroidserver.scanner.options = mock.Mock()
+        fdroidserver.scanner.options.refresh_scanner = False
+        with mock.patch('fdroidserver.scanner.ScannerTool.refresh') as refresh:
+            fdroidserver.scanner.ScannerTool()
+            refresh.assert_not_called()
+
+    def test_refresh_from_config(self):
+        os.chdir(self.testdir)
+        pathlib.Path('config.yml').write_text('refresh_scanner: true')
+        with mock.patch('fdroidserver.scanner.ScannerTool.refresh') as refresh:
+            fdroidserver.scanner.ScannerTool()
+            refresh.assert_called_once()
+
+    def test_refresh_options_overrides_config(self):
+        fdroidserver.scanner.options = mock.Mock()
+        fdroidserver.scanner.options.refresh_scanner = True
+        os.chdir(self.testdir)
+        pathlib.Path('config.yml').write_text('refresh_scanner: false')
+        with mock.patch('fdroidserver.scanner.ScannerTool.refresh') as refresh:
+            fdroidserver.scanner.ScannerTool()
+            refresh.assert_called_once()
+
 
 class Test_main(unittest.TestCase):
     def setUp(self):