From 3dee7e915d623cd159464c4a7e54fec2a60c6951 Mon Sep 17 00:00:00 2001
From: proletarius101 <proletarius101@protonmail.com>
Date: Sun, 10 Dec 2023 22:17:28 +0000
Subject: [PATCH 01/68] feat: add `--index-only` flag

---
 completion/bash-completion |  2 +-
 fdroidserver/deploy.py     | 29 ++++++++++++++---------------
 2 files changed, 15 insertions(+), 16 deletions(-)

diff --git a/completion/bash-completion b/completion/bash-completion
index 940b188b..aa68a220 100644
--- a/completion/bash-completion
+++ b/completion/bash-completion
@@ -264,7 +264,7 @@ __complete_nightly() {
 __complete_deploy() {
 	opts="-i -v -q"
 	lopts="--identity-file --local-copy-dir --sync-from-local-copy-dir
- --verbose --quiet --no-checksum --no-keep-git-mirror-archive"
+ --verbose --quiet --no-checksum --no-keep-git-mirror-archive --index-only"
 	__complete_options
 }
 
diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py
index b120999f..b1db356e 100644
--- a/fdroidserver/deploy.py
+++ b/fdroidserver/deploy.py
@@ -49,6 +49,7 @@ USER_RCLONE_CONF = None
 REMOTE_HOSTNAME_REGEX = re.compile(r'\W*\w+\W+(\w+).*')
 
 INDEX_FILES = [
+    "altstore-index.json",
     "entry.jar",
     "entry.json",
     "entry.json.asc",
@@ -62,6 +63,10 @@ INDEX_FILES = [
 ]
 
 
+def _get_index_file_paths(repo_section):
+    return [os.path.join(repo_section, filename) for filename in INDEX_FILES]
+
+
 def _get_index_excludes(repo_section):
     """Return the list of files to be synced last, since they finalize the deploy.
 
@@ -71,19 +76,7 @@ def _get_index_excludes(repo_section):
     client learns about them from the new index files.
 
     """
-    indexes = [
-        os.path.join(repo_section, 'altstore-index.json'),
-        os.path.join(repo_section, 'entry.jar'),
-        os.path.join(repo_section, 'entry.json'),
-        os.path.join(repo_section, 'entry.json.asc'),
-        os.path.join(repo_section, 'index-v1.jar'),
-        os.path.join(repo_section, 'index-v1.json'),
-        os.path.join(repo_section, 'index-v1.json.asc'),
-        os.path.join(repo_section, 'index-v2.json'),
-        os.path.join(repo_section, 'index-v2.json.asc'),
-        os.path.join(repo_section, 'index.jar'),
-        os.path.join(repo_section, 'index.xml'),
-    ]
+    indexes = _get_index_file_paths(repo_section)
     index_excludes = []
     for f in indexes:
         index_excludes.append('--exclude')
@@ -1025,8 +1018,14 @@ def main():
         default=False,
         help=_("If a git mirror gets to big, allow the archive to be deleted"),
     )
-    options = common.parse_args(parser)
-    config = common.read_config()
+    parser.add_argument(
+        "--index-only",
+        action="store_true",
+        default=False,
+        help="Only deploy the index files entry.* and index-v*.json",
+    )
+    options = parser.parse_args()
+    config = common.read_config(options)
 
     if config.get('nonstandardwebroot') is True:
         standardwebroot = False

From 1cf01855087b4afb8298204849918dbc9aef8333 Mon Sep 17 00:00:00 2001
From: proletarius101 <proletarius101@protonmail.com>
Date: Sun, 10 Dec 2023 22:58:09 +0000
Subject: [PATCH 02/68] feat(update_awsbucket_s3cmd): support the index-only
 flag

---
 fdroidserver/deploy.py | 74 +++++++++++++++++++++++++++---------------
 1 file changed, 48 insertions(+), 26 deletions(-)

diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py
index b1db356e..7ef4c102 100644
--- a/fdroidserver/deploy.py
+++ b/fdroidserver/deploy.py
@@ -64,18 +64,18 @@ INDEX_FILES = [
 
 
 def _get_index_file_paths(repo_section):
-    return [os.path.join(repo_section, filename) for filename in INDEX_FILES]
-
-
-def _get_index_excludes(repo_section):
     """Return the list of files to be synced last, since they finalize the deploy.
 
     The process of pushing all the new packages to the various
     services can take a while.  So the index files should be updated
     last.  That ensures that the package files are available when the
     client learns about them from the new index files.
-
     """
+
+    return [os.path.join(repo_section, filename) for filename in INDEX_FILES]
+
+
+def _get_index_excludes(repo_section):
     indexes = _get_index_file_paths(repo_section)
     index_excludes = []
     for f in indexes:
@@ -84,6 +84,15 @@ def _get_index_excludes(repo_section):
     return index_excludes
 
 
+def _get_index_includes(repo_section):
+    indexes = _get_index_file_paths(repo_section)
+    index_includes = []
+    for f in indexes:
+        index_includes.append('--include')
+        index_includes.append(f)
+    return index_includes
+
+
 def update_awsbucket(repo_section, verbose=False, quiet=False):
     """Upload the contents of the directory `repo_section` (including subdirectories) to the AWS S3 "bucket".
 
@@ -170,35 +179,48 @@ def update_awsbucket_s3cmd(repo_section):
         s3cmd_sync += ['--quiet']
 
     s3url = s3bucketurl + '/fdroid/'
-    logging.debug('s3cmd sync new files in ' + repo_section + ' to ' + s3url)
-    logging.debug(_('Running first pass with MD5 checking disabled'))
-    excludes = _get_index_excludes(repo_section)
-    returncode = subprocess.call(
-        s3cmd_sync
-        + excludes
-        + ['--no-check-md5', '--skip-existing', repo_section, s3url]
-    )
-    if returncode != 0:
-        raise FDroidException()
-    logging.debug('s3cmd sync all files in ' + repo_section + ' to ' + s3url)
-    returncode = subprocess.call(
-        s3cmd_sync + excludes + ['--no-check-md5', repo_section, s3url]
-    )
-    if returncode != 0:
-        raise FDroidException()
 
     logging.debug(
         _('s3cmd sync indexes {path} to {url} and delete').format(
             path=repo_section, url=s3url
         )
     )
-    s3cmd_sync.append('--delete-removed')
-    s3cmd_sync.append('--delete-after')
+    if not options.index_only:
+        logging.debug('s3cmd syncs new non-index files in ' + repo_section + ' to ' + s3url + 'and deletes removed')
+        logging.debug(_('Running first pass with MD5 checking disabled'))
+        sync_non_indexes_flags = []
+
+        # Excluded files are not considered as removed: https://github.com/s3tools/s3cmd/issues/802
+        sync_non_indexes_flags.append('--delete-removed')
+        sync_non_indexes_flags.append('--delete-after')
+
+        sync_non_indexes_flags.extend(_get_index_excludes(repo_section))
+        returncode = subprocess.call(
+            s3cmd_sync
+            + sync_non_indexes_flags
+            + ['--no-check-md5', '--skip-existing', repo_section, s3url]
+        )
+        if returncode != 0:
+            raise FDroidException()
+        logging.debug('s3cmd sync all files in ' + repo_section + ' to ' + s3url)
+        returncode = subprocess.call(
+            s3cmd_sync + sync_non_indexes_flags + ['--no-check-md5', repo_section, s3url]
+        )
+        if returncode != 0:
+            raise FDroidException()
+
+    logging.debug(_('s3cmd syncs indexes from {path} to {url} and deletes removed')
+                  .format(path=repo_section, url=s3url))
+    sync_indexes_flags = []
+    sync_indexes_flags.append('--delete-removed')
+    sync_indexes_flags.append('--delete-after')
     if options.no_checksum:
-        s3cmd_sync.append('--no-check-md5')
+        sync_indexes_flags.append('--no-check-md5')
     else:
-        s3cmd_sync.append('--check-md5')
-    if subprocess.call(s3cmd_sync + [repo_section, s3url]) != 0:
+        sync_indexes_flags.append('--check-md5')
+    sync_indexes_flags.extend(_get_index_includes(repo_section))
+    returncode = subprocess.call(s3cmd_sync + sync_indexes_flags + [repo_section, s3url])
+    if returncode != 0:
         raise FDroidException()
 
 

From 2c361f6cee1af966071049338f2f3a4805bf426a Mon Sep 17 00:00:00 2001
From: proletarius101 <proletarius101@protonmail.com>
Date: Thu, 14 Dec 2023 15:18:23 +0000
Subject: [PATCH 03/68] refactor: revert changes to the full sync mode for
 s3cmd

---
 fdroidserver/deploy.py | 54 +++++++++++++++++++++++-------------------
 1 file changed, 30 insertions(+), 24 deletions(-)

diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py
index 7ef4c102..371f4ddb 100644
--- a/fdroidserver/deploy.py
+++ b/fdroidserver/deploy.py
@@ -185,43 +185,49 @@ def update_awsbucket_s3cmd(repo_section):
             path=repo_section, url=s3url
         )
     )
-    if not options.index_only:
-        logging.debug('s3cmd syncs new non-index files in ' + repo_section + ' to ' + s3url + 'and deletes removed')
+
+    if options.index_only:
+        logging.debug(_('s3cmd syncs indexes from {path} to {url} and deletes removed')
+                      .format(path=repo_section, url=s3url))
+        sync_indexes_flags = []
+        sync_indexes_flags.append('--delete-removed')
+        sync_indexes_flags.append('--delete-after')
+        if options.no_checksum:
+            sync_indexes_flags.append('--no-check-md5')
+        else:
+            sync_indexes_flags.append('--check-md5')
+        sync_indexes_flags.extend(_get_index_includes(repo_section))
+        returncode = subprocess.call(s3cmd_sync + sync_indexes_flags + [repo_section, s3url])
+        if returncode != 0:
+            raise FDroidException()
+    else:
+        logging.debug('s3cmd sync new files in ' + repo_section + ' to ' + s3url)
         logging.debug(_('Running first pass with MD5 checking disabled'))
-        sync_non_indexes_flags = []
-
-        # Excluded files are not considered as removed: https://github.com/s3tools/s3cmd/issues/802
-        sync_non_indexes_flags.append('--delete-removed')
-        sync_non_indexes_flags.append('--delete-after')
-
-        sync_non_indexes_flags.extend(_get_index_excludes(repo_section))
+        excludes = _get_index_excludes(repo_section)
         returncode = subprocess.call(
             s3cmd_sync
-            + sync_non_indexes_flags
+            + excludes
             + ['--no-check-md5', '--skip-existing', repo_section, s3url]
         )
         if returncode != 0:
             raise FDroidException()
         logging.debug('s3cmd sync all files in ' + repo_section + ' to ' + s3url)
         returncode = subprocess.call(
-            s3cmd_sync + sync_non_indexes_flags + ['--no-check-md5', repo_section, s3url]
+            s3cmd_sync + excludes + ['--no-check-md5', repo_section, s3url]
         )
         if returncode != 0:
             raise FDroidException()
 
-    logging.debug(_('s3cmd syncs indexes from {path} to {url} and deletes removed')
-                  .format(path=repo_section, url=s3url))
-    sync_indexes_flags = []
-    sync_indexes_flags.append('--delete-removed')
-    sync_indexes_flags.append('--delete-after')
-    if options.no_checksum:
-        sync_indexes_flags.append('--no-check-md5')
-    else:
-        sync_indexes_flags.append('--check-md5')
-    sync_indexes_flags.extend(_get_index_includes(repo_section))
-    returncode = subprocess.call(s3cmd_sync + sync_indexes_flags + [repo_section, s3url])
-    if returncode != 0:
-        raise FDroidException()
+        logging.debug(_('s3cmd sync indexes {path} to {url} and delete')
+                      .format(path=repo_section, url=s3url))
+        s3cmd_sync.append('--delete-removed')
+        s3cmd_sync.append('--delete-after')
+        if options.no_checksum:
+            s3cmd_sync.append('--no-check-md5')
+        else:
+            s3cmd_sync.append('--check-md5')
+        if subprocess.call(s3cmd_sync + [repo_section, s3url]) != 0:
+            raise FDroidException()
 
 
 def update_remote_storage_with_rclone(repo_section, verbose=False, quiet=False):

From bd62007b540059afe50a5b70a4b7bf3956a49585 Mon Sep 17 00:00:00 2001
From: proletarius101 <proletarius101@protonmail.com>
Date: Thu, 14 Dec 2023 15:21:53 +0000
Subject: [PATCH 04/68] feat(deploy): support --index-only option for
 serverwebroot deployments

---
 fdroidserver/deploy.py | 32 ++++++++++++++++++--------------
 1 file changed, 18 insertions(+), 14 deletions(-)

diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py
index 371f4ddb..12b81bb6 100644
--- a/fdroidserver/deploy.py
+++ b/fdroidserver/deploy.py
@@ -498,20 +498,24 @@ def update_serverwebroot(serverwebroot, repo_section):
         ]
     url = serverwebroot['url']
     logging.info('rsyncing ' + repo_section + ' to ' + url)
-    excludes = _get_index_excludes(repo_section)
-    if subprocess.call(rsyncargs + excludes + [repo_section, url]) != 0:
-        raise FDroidException()
-    if subprocess.call(rsyncargs + [repo_section, url]) != 0:
-        raise FDroidException()
-    # upload "current version" symlinks if requested
-    if config and config.get('make_current_version_link') and repo_section == 'repo':
-        links_to_upload = []
-        for f in glob.glob('*.apk') + glob.glob('*.apk.asc') + glob.glob('*.apk.sig'):
-            if os.path.islink(f):
-                links_to_upload.append(f)
-        if len(links_to_upload) > 0:
-            if subprocess.call(rsyncargs + links_to_upload + [url]) != 0:
-                raise FDroidException()
+    if options.index_only:
+        if subprocess.call(rsyncargs + ['--excludes', "*"] + _get_index_includes(repo_section) + [repo_section, url]) != 0:
+            raise FDroidException()
+    else:
+        excludes = _get_index_excludes(repo_section)
+        if subprocess.call(rsyncargs + excludes + [repo_section, url]) != 0:
+            raise FDroidException()
+        if subprocess.call(rsyncargs + [repo_section, url]) != 0:
+            raise FDroidException()
+        # upload "current version" symlinks if requested
+        if config and config.get('make_current_version_link') and repo_section == 'repo':
+            links_to_upload = []
+            for f in glob.glob('*.apk') + glob.glob('*.apk.asc') + glob.glob('*.apk.sig'):
+                if os.path.islink(f):
+                    links_to_upload.append(f)
+            if len(links_to_upload) > 0:
+                if subprocess.call(rsyncargs + links_to_upload + [url]) != 0:
+                    raise FDroidException()
 
 
 def update_serverwebroots(serverwebroots, repo_section, standardwebroot=True):

From 3ec997d58ed6346f917aa66be04010bf2e2b88dc Mon Sep 17 00:00:00 2001
From: proletarius101 <proletarius101@protonmail.com>
Date: Thu, 14 Dec 2023 15:34:37 +0000
Subject: [PATCH 05/68] feat(deploy): support index-only mode in libcloud

---
 fdroidserver/deploy.py | 81 ++++++++++++++++++++++--------------------
 1 file changed, 42 insertions(+), 39 deletions(-)

diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py
index 12b81bb6..5a6f7c7c 100644
--- a/fdroidserver/deploy.py
+++ b/fdroidserver/deploy.py
@@ -401,49 +401,52 @@ def update_awsbucket_libcloud(repo_section):
         if obj.name.startswith(upload_dir + '/'):
             objs[obj.name] = obj
 
-    for root, dirs, files in os.walk(os.path.join(os.getcwd(), repo_section)):
-        for name in files:
-            upload = False
-            file_to_upload = os.path.join(root, name)
-            object_name = 'fdroid/' + os.path.relpath(file_to_upload, os.getcwd())
-            if object_name not in objs:
+    if options.index_only:
+        files_to_upload = _get_index_includes(repo_section)
+    else:
+        files_to_upload = [os.path.join(root, name) for root, dirs, files in os.walk(os.path.join(os.getcwd(), repo_section)) for name in files]
+
+    for file_to_upload in files_to_upload:
+        upload = False
+        object_name = 'fdroid/' + os.path.relpath(file_to_upload, os.getcwd())
+        if object_name not in objs:
+            upload = True
+        else:
+            obj = objs.pop(object_name)
+            if obj.size != os.path.getsize(file_to_upload):
                 upload = True
             else:
-                obj = objs.pop(object_name)
-                if obj.size != os.path.getsize(file_to_upload):
+                # if the sizes match, then compare by MD5
+                md5 = hashlib.md5()  # nosec AWS uses MD5
+                with open(file_to_upload, 'rb') as f:
+                    while True:
+                        data = f.read(8192)
+                        if not data:
+                            break
+                        md5.update(data)
+                if obj.hash != md5.hexdigest():
+                    s3url = 's3://' + awsbucket + '/' + obj.name
+                    logging.info(' deleting ' + s3url)
+                    if not driver.delete_object(obj):
+                        logging.warning('Could not delete ' + s3url)
                     upload = True
-                else:
-                    # if the sizes match, then compare by MD5
-                    md5 = hashlib.md5()  # nosec AWS uses MD5
-                    with open(file_to_upload, 'rb') as f:
-                        while True:
-                            data = f.read(8192)
-                            if not data:
-                                break
-                            md5.update(data)
-                    if obj.hash != md5.hexdigest():
-                        s3url = 's3://' + awsbucket + '/' + obj.name
-                        logging.info(' deleting ' + s3url)
-                        if not driver.delete_object(obj):
-                            logging.warning('Could not delete ' + s3url)
-                        upload = True
 
-            if upload:
-                logging.debug(' uploading "' + file_to_upload + '"...')
-                extra = {'acl': 'public-read'}
-                if file_to_upload.endswith('.sig'):
-                    extra['content_type'] = 'application/pgp-signature'
-                elif file_to_upload.endswith('.asc'):
-                    extra['content_type'] = 'application/pgp-signature'
-                path = os.path.relpath(file_to_upload)
-                logging.info(f' uploading {path} to s3://{awsbucket}/{object_name}')
-                with open(file_to_upload, 'rb') as iterator:
-                    obj = driver.upload_object_via_stream(
-                        iterator=iterator,
-                        container=container,
-                        object_name=object_name,
-                        extra=extra,
-                    )
+        if upload:
+            logging.debug(' uploading "' + file_to_upload + '"...')
+            extra = {'acl': 'public-read'}
+            if file_to_upload.endswith('.sig'):
+                extra['content_type'] = 'application/pgp-signature'
+            elif file_to_upload.endswith('.asc'):
+                extra['content_type'] = 'application/pgp-signature'
+            path = os.path.relpath(file_to_upload)
+            logging.info(f' uploading {path} to s3://{awsbucket}/{object_name}')
+            with open(file_to_upload, 'rb') as iterator:
+                obj = driver.upload_object_via_stream(
+                    iterator=iterator,
+                    container=container,
+                    object_name=object_name,
+                    extra=extra,
+                )
     # delete the remnants in the bucket, they do not exist locally
     while objs:
         object_name, obj = objs.popitem()

From 7071cc6f6d9d8509bdd721e42c2e46d740d0c04a Mon Sep 17 00:00:00 2001
From: proletarius101 <proletarius101@protonmail.com>
Date: Thu, 14 Dec 2023 15:50:54 +0000
Subject: [PATCH 06/68] feat(deploy): support the index-only option when
 syncing from/to local copies

---
 .vscode/settings.json  | 20 ++++----------------
 fdroidserver/deploy.py | 40 +++++++++++++++++++++++++---------------
 2 files changed, 29 insertions(+), 31 deletions(-)

diff --git a/.vscode/settings.json b/.vscode/settings.json
index da31cd7f..154dce4a 100644
--- a/.vscode/settings.json
+++ b/.vscode/settings.json
@@ -2,20 +2,8 @@
     "python.formatting.blackArgs": [
         "--config=pyproject.toml"
     ],
-    "python.formatting.provider": "black",
-    "python.linting.banditEnabled": true,
-    "python.linting.banditArgs": [
-        "-ii",
-        "--ini=.bandit",
+    "python.analysis.typeCheckingMode": "basic",
+    "conventionalCommits.scopes": [
+        "deploy"
     ],
-    "python.linting.enabled": true,
-    "python.linting.mypyArgs": [
-        "--config-file=mypy.ini"
-    ],
-    "python.linting.mypyEnabled": true,
-    "python.linting.flake8Enabled": true,
-    "python.linting.pylintArgs": [
-        "--rcfile=.pylint-rcfile"
-    ],
-    "python.linting.pylintEnabled": true,
-}
+}
\ No newline at end of file
diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py
index 5a6f7c7c..c1990b93 100644
--- a/fdroidserver/deploy.py
+++ b/fdroidserver/deploy.py
@@ -63,7 +63,7 @@ INDEX_FILES = [
 ]
 
 
-def _get_index_file_paths(repo_section):
+def _get_index_file_paths(base_dir):
     """Return the list of files to be synced last, since they finalize the deploy.
 
     The process of pushing all the new packages to the various
@@ -72,11 +72,11 @@ def _get_index_file_paths(repo_section):
     client learns about them from the new index files.
     """
 
-    return [os.path.join(repo_section, filename) for filename in INDEX_FILES]
+    return [os.path.join(base_dir, filename) for filename in INDEX_FILES]
 
 
-def _get_index_excludes(repo_section):
-    indexes = _get_index_file_paths(repo_section)
+def _get_index_excludes(base_dir):
+    indexes = _get_index_file_paths(base_dir)
     index_excludes = []
     for f in indexes:
         index_excludes.append('--exclude')
@@ -84,8 +84,8 @@ def _get_index_excludes(repo_section):
     return index_excludes
 
 
-def _get_index_includes(repo_section):
-    indexes = _get_index_file_paths(repo_section)
+def _get_index_includes(base_dir):
+    indexes = _get_index_file_paths(base_dir)
     index_includes = []
     for f in indexes:
         index_includes.append('--include')
@@ -559,13 +559,18 @@ def sync_from_localcopy(repo_section, local_copy_dir):
 
     """
     logging.info('Syncing from local_copy_dir to this repo.')
-    # trailing slashes have a meaning in rsync which is not needed here, so
-    # make sure both paths have exactly one trailing slash
-    common.local_rsync(
-        common.get_options(),
-        os.path.join(local_copy_dir, repo_section).rstrip('/') + '/',
-        repo_section.rstrip('/') + '/',
-    )
+    if options.index_only:
+        common.local_rsync(common.get_options(),
+                           _get_index_includes(local_copy_dir),
+                           repo_section.rstrip('/') + '/')
+    else:
+        # trailing slashes have a meaning in rsync which is not needed here, so
+        # make sure both paths have exactly one trailing slash
+        common.local_rsync(
+            common.get_options(),
+            os.path.join(local_copy_dir, repo_section).rstrip('/') + '/',
+            repo_section.rstrip('/') + '/',
+        )
 
     offline_copy = os.path.join(local_copy_dir, BINARY_TRANSPARENCY_DIR)
     if os.path.exists(os.path.join(offline_copy, '.git')):
@@ -581,8 +586,13 @@ def update_localcopy(repo_section, local_copy_dir):
     drive.
 
     """
-    # local_copy_dir is guaranteed to have a trailing slash in main() below
-    common.local_rsync(common.get_options(), repo_section, local_copy_dir)
+    if options.index_only:
+        common.local_rsync(common.get_options(),
+                           _get_index_includes(repo_section),
+                           repo_section.rstrip('/') + '/')
+    else:
+        # local_copy_dir is guaranteed to have a trailing slash in main() below
+        common.local_rsync(common.get_options(), repo_section, local_copy_dir)
 
     offline_copy = os.path.join(os.getcwd(), BINARY_TRANSPARENCY_DIR)
     if os.path.isdir(os.path.join(offline_copy, '.git')):

From 3b58c386a44d706b30bc7822294335fc59546e8e Mon Sep 17 00:00:00 2001
From: proletarius101 <proletarius101@protonmail.com>
Date: Thu, 14 Dec 2023 16:36:26 +0000
Subject: [PATCH 07/68] fix(deploy): fix typo

---
 fdroidserver/deploy.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py
index c1990b93..5b2327f3 100644
--- a/fdroidserver/deploy.py
+++ b/fdroidserver/deploy.py
@@ -502,7 +502,7 @@ def update_serverwebroot(serverwebroot, repo_section):
     url = serverwebroot['url']
     logging.info('rsyncing ' + repo_section + ' to ' + url)
     if options.index_only:
-        if subprocess.call(rsyncargs + ['--excludes', "*"] + _get_index_includes(repo_section) + [repo_section, url]) != 0:
+        if subprocess.call(rsyncargs + ['--exclude', "*"] + _get_index_includes(repo_section) + [repo_section, url]) != 0:
             raise FDroidException()
     else:
         excludes = _get_index_excludes(repo_section)

From 742381a923f32ac6a1b741e4de131199474e64f3 Mon Sep 17 00:00:00 2001
From: proletarius101 <proletarius101@protonmail.com>
Date: Thu, 14 Dec 2023 18:31:29 +0000
Subject: [PATCH 08/68] test(deploy): properly test non-index-only mode

---
 tests/deploy.TestCase | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/tests/deploy.TestCase b/tests/deploy.TestCase
index 7e5b123b..fe6f01cf 100755
--- a/tests/deploy.TestCase
+++ b/tests/deploy.TestCase
@@ -147,7 +147,12 @@ class DeployTest(unittest.TestCase):
         url = Path('url')
         url.mkdir()
 
-        dest_apk = url / fake_apk
+        # setup parameters for this test run
+        fdroidserver.deploy.options.identity_file = None
+        fdroidserver.deploy.options.index_only = False
+        fdroidserver.deploy.config['make_current_version_link'] = False
+
+        dest_apk = Path(url) / fake_apk
         self.assertFalse(dest_apk.is_file())
         fdroidserver.deploy.options = mock.Mock()
         fdroidserver.deploy.options.identity_file = None
@@ -170,6 +175,7 @@ class DeployTest(unittest.TestCase):
         fdroidserver.common.options.identity_file = None
         fdroidserver.common.options.verbose = False
         fdroidserver.common.options.quiet = True
+        fdroidserver.deploy.options.index_only = False
         fdroidserver.deploy.config = {'make_current_version_link': True}
         url = "example.com:/var/www/fdroid"
         repo_section = 'repo'
@@ -264,6 +270,7 @@ class DeployTest(unittest.TestCase):
         fdroidserver.common.options.verbose = True
         fdroidserver.common.options.quiet = False
         fdroidserver.common.options.identity_file = None
+        fdroidserver.deploy.options.index_only = False
         fdroidserver.deploy.config = {'identity_file': './id_rsa'}
         url = "example.com:/var/www/fdroid"
         repo_section = 'archive'

From 2b6542b231592d9dcb95b6ec2f82ee0cc4c0c380 Mon Sep 17 00:00:00 2001
From: proletarius101 <proletarius101@protonmail.com>
Date: Thu, 14 Dec 2023 18:51:47 +0000
Subject: [PATCH 09/68] fix(deploy): restore accidentally removed code

---
 fdroidserver/deploy.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py
index 5b2327f3..c0222767 100644
--- a/fdroidserver/deploy.py
+++ b/fdroidserver/deploy.py
@@ -1040,6 +1040,14 @@ def main():
 
     parser = ArgumentParser()
     common.setup_global_opts(parser)
+    parser.add_argument("-i", "--identity-file", default=None,
+                        help=_("Specify an identity file to provide to SSH for rsyncing"))
+    parser.add_argument("--local-copy-dir", default=None,
+                        help=_("Specify a local folder to sync the repo to"))
+    parser.add_argument("--no-checksum", action="store_true", default=False,
+                        help=_("Don't use rsync checksums"))
+    parser.add_argument("--no-keep-git-mirror-archive", action="store_true", default=False,
+                        help=_("If a git mirror gets to big, allow the archive to be deleted"))
     parser.add_argument(
         "-i",
         "--identity-file",

From 160c23a54744b8caec647fb140a45a07267c1496 Mon Sep 17 00:00:00 2001
From: proletarius101 <proletarius101@protonmail.com>
Date: Fri, 22 Dec 2023 17:42:10 +0000
Subject: [PATCH 10/68] fix(deploy): add test cases for server webroot mode and
 fix issues

---
 fdroidserver/deploy.py |   3 +-
 tests/deploy.TestCase  | 216 ++++++++++++++++++++++++++++++++++++++++-
 2 files changed, 216 insertions(+), 3 deletions(-)

diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py
index c0222767..b668e2b0 100644
--- a/fdroidserver/deploy.py
+++ b/fdroidserver/deploy.py
@@ -502,7 +502,8 @@ def update_serverwebroot(serverwebroot, repo_section):
     url = serverwebroot['url']
     logging.info('rsyncing ' + repo_section + ' to ' + url)
     if options.index_only:
-        if subprocess.call(rsyncargs + ['--exclude', "*"] + _get_index_includes(repo_section) + [repo_section, url]) != 0:
+        rsyncargs += ['--include', "*/"] + _get_index_includes(repo_section) + ['--exclude', '*'] + [repo_section, url]
+        if subprocess.call(rsyncargs) != 0:
             raise FDroidException()
     else:
         excludes = _get_index_excludes(repo_section)
diff --git a/tests/deploy.TestCase b/tests/deploy.TestCase
index fe6f01cf..4c0c8535 100755
--- a/tests/deploy.TestCase
+++ b/tests/deploy.TestCase
@@ -144,20 +144,54 @@ class DeployTest(unittest.TestCase):
         fake_apk = repo / 'fake.apk'
         with fake_apk.open('w') as fp:
             fp.write('not an APK, but has the right filename')
+        fake_index = repo / fdroidserver.deploy.INDEX_FILES[0]
+        with fake_index.open('w') as fp:
+            fp.write('not an index, but has the right filename')
         url = Path('url')
         url.mkdir()
 
         # setup parameters for this test run
+        fdroidserver.deploy.options = mock.Mock()
+        fdroidserver.deploy.options.identity_file = None
         fdroidserver.deploy.options.identity_file = None
         fdroidserver.deploy.options.index_only = False
         fdroidserver.deploy.config['make_current_version_link'] = False
 
         dest_apk = Path(url) / fake_apk
+        dest_index = Path(url) / fake_index
         self.assertFalse(dest_apk.is_file())
-        fdroidserver.deploy.options = mock.Mock()
-        fdroidserver.deploy.options.identity_file = None
+        self.assertFalse(dest_index.is_file())
+
         fdroidserver.deploy.update_serverwebroot({'url': str(url)}, 'repo')
         self.assertTrue(dest_apk.is_file())
+        self.assertTrue(dest_index.is_file())
+
+    def test_update_serverwebroot_in_index_only_mode(self):
+        os.chdir(self.testdir)
+        repo = Path('repo')
+        repo.mkdir(parents=True)
+        fake_apk = repo / 'fake.apk'
+        with fake_apk.open('w') as fp:
+            fp.write('not an APK, but has the right filename')
+        fake_index = repo / fdroidserver.deploy.INDEX_FILES[0]
+        with fake_index.open('w') as fp:
+            fp.write('not an index, but has the right filename')
+        serverwebroot = Path('serverwebroot')
+        serverwebroot.mkdir()
+
+        # setup parameters for this test run
+        fdroidserver.deploy.options.identity_file = None
+        fdroidserver.deploy.options.index_only = True
+        fdroidserver.deploy.config['make_current_version_link'] = False
+
+        dest_apk = Path(serverwebroot) / fake_apk
+        dest_index = Path(serverwebroot) / fake_index
+        self.assertFalse(dest_apk.is_file())
+        self.assertFalse(dest_index.is_file())
+
+        fdroidserver.deploy.update_serverwebroot(str(serverwebroot), 'repo')
+        self.assertFalse(dest_apk.is_file())
+        self.assertTrue(dest_index.is_file())
 
     @mock.patch.dict(os.environ, clear=True)
     def test_update_serverwebroot_no_rsync_error(self):
@@ -262,6 +296,102 @@ class DeployTest(unittest.TestCase):
                 fdroidserver.deploy.update_serverwebroot({'url': url}, repo_section)
         self.assertEqual(call_iteration, 3, 'expected 3 invocations of subprocess.call')
 
+    def test_update_serverwebroot_make_cur_version_link_in_index_only_mode(self):
+        # setup parameters for this test run
+        fdroidserver.deploy.options.no_checksum = True
+        fdroidserver.deploy.options.identity_file = None
+        fdroidserver.deploy.options.verbose = False
+        fdroidserver.deploy.options.quiet = True
+        fdroidserver.deploy.options.identity_file = None
+        fdroidserver.deploy.options.index_only = True
+        fdroidserver.deploy.config['make_current_version_link'] = True
+        serverwebroot = "example.com:/var/www/fdroid"
+        repo_section = 'repo'
+
+        # setup function for asserting subprocess.call invocations
+        call_iteration = 0
+
+        def update_server_webroot_call(cmd):
+            nonlocal call_iteration
+            if call_iteration == 0:
+                self.assertListEqual(
+                    cmd,
+                    [
+                        'rsync',
+                        '--archive',
+                        '--delete-after',
+                        '--safe-links',
+                        '--quiet',
+                        '--include',
+                        "*/",
+                        '--include',
+                        'repo/entry.jar',
+                        '--include',
+                        'repo/entry.json',
+                        '--include',
+                        'repo/entry.json.asc',
+                        '--include',
+                        'repo/index-v1.jar',
+                        '--include',
+                        'repo/index-v1.json',
+                        '--include',
+                        'repo/index-v1.json.asc',
+                        '--include',
+                        'repo/index-v2.json',
+                        '--include',
+                        'repo/index-v2.json.asc',
+                        '--include',
+                        'repo/index.jar',
+                        '--include',
+                        'repo/index.xml',
+                        '--exclude',
+                        '*',
+                        'repo',
+                        'example.com:/var/www/fdroid',
+                    ],
+                )
+            elif call_iteration == 1:
+                self.assertListEqual(
+                    cmd,
+                    [
+                        'rsync',
+                        '--archive',
+                        '--delete-after',
+                        '--safe-links',
+                        '--quiet',
+                        'repo',
+                        serverwebroot,
+                    ],
+                )
+            elif call_iteration == 2:
+                self.assertListEqual(
+                    cmd,
+                    [
+                        'rsync',
+                        '--archive',
+                        '--delete-after',
+                        '--safe-links',
+                        '--quiet',
+                        'Sym.apk',
+                        'Sym.apk.asc',
+                        'Sym.apk.sig',
+                        'example.com:/var/www/fdroid',
+                    ],
+                )
+            else:
+                self.fail('unexpected subprocess.call invocation')
+            call_iteration += 1
+            return 0
+
+        with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
+            os.mkdir('repo')
+            os.symlink('repo/com.example.sym.apk', 'Sym.apk')
+            os.symlink('repo/com.example.sym.apk.asc', 'Sym.apk.asc')
+            os.symlink('repo/com.example.sym.apk.sig', 'Sym.apk.sig')
+            with mock.patch('subprocess.call', side_effect=update_server_webroot_call):
+                fdroidserver.deploy.update_serverwebroot(serverwebroot, repo_section)
+        self.assertEqual(call_iteration, 1, 'expected 1 invocations of subprocess.call')
+
     def test_update_serverwebroot_with_id_file(self):
         # setup parameters for this test run
         fdroidserver.common.options = mock.Mock()
@@ -343,6 +473,88 @@ class DeployTest(unittest.TestCase):
             fdroidserver.deploy.update_serverwebroot({'url': url}, repo_section)
         self.assertEqual(call_iteration, 2, 'expected 2 invocations of subprocess.call')
 
+    def test_update_serverwebroot_with_id_file_in_index_only_mode(self):
+        # setup parameters for this test run
+        fdroidserver.deploy.options.no_chcksum = False
+        fdroidserver.deploy.options.verbose = True
+        fdroidserver.deploy.options.quiet = False
+        fdroidserver.deploy.options.identity_file = None
+        fdroidserver.deploy.options.index_only = True
+        fdroidserver.deploy.config['identity_file'] = './id_rsa'
+        fdroidserver.deploy.config['make_current_version_link'] = False
+        serverwebroot = "example.com:/var/www/fdroid"
+        repo_section = 'archive'
+
+        # setup function for asserting subprocess.call invocations
+        call_iteration = 0
+
+        def update_server_webroot_call(cmd):
+            nonlocal call_iteration
+            if call_iteration == 0:
+                self.assertListEqual(
+                    cmd,
+                    [
+                        'rsync',
+                        '--archive',
+                        '--delete-after',
+                        '--safe-links',
+                        '--verbose',
+                        '-e',
+                        'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i '
+                        + fdroidserver.deploy.config['identity_file'],
+                        '--include',
+                        "*/",
+                        '--include',
+                        'archive/entry.jar',
+                        '--include',
+                        'archive/entry.json',
+                        '--include',
+                        'archive/entry.json.asc',
+                        '--include',
+                        'archive/index-v1.jar',
+                        '--include',
+                        'archive/index-v1.json',
+                        '--include',
+                        'archive/index-v1.json.asc',
+                        '--include',
+                        'archive/index-v2.json',
+                        '--include',
+                        'archive/index-v2.json.asc',
+                        '--include',
+                        'archive/index.jar',
+                        '--include',
+                        'archive/index.xml',
+                        '--exclude',
+                        "*",
+                        'archive',
+                        serverwebroot,
+                    ],
+                )
+            elif call_iteration == 1:
+                self.assertListEqual(
+                    cmd,
+                    [
+                        'rsync',
+                        '--archive',
+                        '--delete-after',
+                        '--safe-links',
+                        '--verbose',
+                        '-e',
+                        'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i '
+                        + fdroidserver.deploy.config['identity_file'],
+                        'archive',
+                        serverwebroot,
+                    ],
+                )
+            else:
+                self.fail('unexpected subprocess.call invocation')
+            call_iteration += 1
+            return 0
+
+        with mock.patch('subprocess.call', side_effect=update_server_webroot_call):
+            fdroidserver.deploy.update_serverwebroot(serverwebroot, repo_section)
+        self.assertEqual(call_iteration, 1, 'expected 1 invocations of subprocess.call')
+
     @unittest.skipIf(
         not os.getenv('VIRUSTOTAL_API_KEY'), 'VIRUSTOTAL_API_KEY is not set'
     )

From 5e3b474a1c2818cbd2b1d14cef6c8946da80dd32 Mon Sep 17 00:00:00 2001
From: proletarius101 <proletarius101@protonmail.com>
Date: Fri, 22 Dec 2023 18:20:20 +0000
Subject: [PATCH 11/68] fix(deploy): add test cases for local copy mode and fix
 issues

---
 fdroidserver/common.py |  4 ++-
 fdroidserver/deploy.py |  8 ++++--
 tests/deploy.TestCase  | 60 ++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 69 insertions(+), 3 deletions(-)

diff --git a/fdroidserver/common.py b/fdroidserver/common.py
index 5a546386..3c42232d 100644
--- a/fdroidserver/common.py
+++ b/fdroidserver/common.py
@@ -29,6 +29,7 @@
 # libraries here as they will become a requirement for all commands.
 
 import difflib
+from typing import List
 import git
 import glob
 import io
@@ -4300,7 +4301,7 @@ def get_app_display_name(app):
     return app.get('AutoName') or app['id']
 
 
-def local_rsync(options, fromdir, todir):
+def local_rsync(options, fromdir, todir, args: List[str] = []):
     """Rsync method for local to local copying of things.
 
     This is an rsync wrapper with all the settings for safe use within
@@ -4317,6 +4318,7 @@ def local_rsync(options, fromdir, todir):
         rsyncargs += ['--verbose']
     if options.quiet:
         rsyncargs += ['--quiet']
+    rsyncargs += args
     logging.debug(' '.join(rsyncargs + [fromdir, todir]))
     if subprocess.call(rsyncargs + [fromdir, todir]) != 0:
         raise FDroidException()
diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py
index b668e2b0..4d581470 100644
--- a/fdroidserver/deploy.py
+++ b/fdroidserver/deploy.py
@@ -561,9 +561,11 @@ def sync_from_localcopy(repo_section, local_copy_dir):
     """
     logging.info('Syncing from local_copy_dir to this repo.')
     if options.index_only:
+        rsyncargs = ['--include', "*/"] + _get_index_includes(repo_section) + ['--exclude', '*']
         common.local_rsync(common.get_options(),
                            _get_index_includes(local_copy_dir),
-                           repo_section.rstrip('/') + '/')
+                           repo_section.rstrip('/') + '/',
+                           args=rsyncargs)
     else:
         # trailing slashes have a meaning in rsync which is not needed here, so
         # make sure both paths have exactly one trailing slash
@@ -588,9 +590,11 @@ def update_localcopy(repo_section, local_copy_dir):
 
     """
     if options.index_only:
+        rsyncargs = ['--include', "*/"] + _get_index_includes(repo_section) + ['--exclude', '*']
         common.local_rsync(common.get_options(),
                            _get_index_includes(repo_section),
-                           repo_section.rstrip('/') + '/')
+                           local_copy_dir,
+                           args=rsyncargs)
     else:
         # local_copy_dir is guaranteed to have a trailing slash in main() below
         common.local_rsync(common.get_options(), repo_section, local_copy_dir)
diff --git a/tests/deploy.TestCase b/tests/deploy.TestCase
index 4c0c8535..41d40088 100755
--- a/tests/deploy.TestCase
+++ b/tests/deploy.TestCase
@@ -555,6 +555,66 @@ class DeployTest(unittest.TestCase):
             fdroidserver.deploy.update_serverwebroot(serverwebroot, repo_section)
         self.assertEqual(call_iteration, 1, 'expected 1 invocations of subprocess.call')
 
+    def test_update_localcopy_in_index_only_mode(self):
+        # setup parameters for this test run
+        fdroidserver.deploy.options.no_chcksum = False
+        fdroidserver.deploy.options.verbose = True
+        fdroidserver.deploy.options.quiet = False
+        fdroidserver.deploy.options.identity_file = None
+        fdroidserver.deploy.options.index_only = True
+        repo_section = 'repo'
+
+        # setup function for asserting subprocess.call invocations
+        call_iteration = 0
+
+        with tempfile.TemporaryDirectory() as local_copy_dir:
+            def update_localcopy_call(cmd):
+                nonlocal call_iteration
+                if call_iteration == 0:
+                    self.assertListEqual(
+                        cmd,
+                        [
+                            'rsync',
+                            '--recursive', '--safe-links', '--times', '--perms',
+                            '--one-file-system', '--delete', '--chmod=Da+rx,Fa-x,a+r,u+w',
+                            '--verbose',
+                            '--include',
+                            "*/",
+                            '--include',
+                            'repo/entry.jar',
+                            '--include',
+                            'repo/entry.json',
+                            '--include',
+                            'repo/entry.json.asc',
+                            '--include',
+                            'repo/index-v1.jar',
+                            '--include',
+                            'repo/index-v1.json',
+                            '--include',
+                            'repo/index-v1.json.asc',
+                            '--include',
+                            'repo/index-v2.json',
+                            '--include',
+                            'repo/index-v2.json.asc',
+                            '--include',
+                            'repo/index.jar',
+                            '--include',
+                            'repo/index.xml',
+                            '--exclude',
+                            "*",
+                            'repo',
+                            local_copy_dir,
+                        ],
+                    )
+                else:
+                    self.fail('unexpected subprocess.call invocation')
+                call_iteration += 1
+                return 0
+
+            with mock.patch('subprocess.call', side_effect=update_localcopy_call):
+                fdroidserver.deploy.update_localcopy(repo_section, local_copy_dir)
+        self.assertEqual(call_iteration, 1, 'expected 1 invocations of subprocess.call')
+
     @unittest.skipIf(
         not os.getenv('VIRUSTOTAL_API_KEY'), 'VIRUSTOTAL_API_KEY is not set'
     )

From 91cf352b962880dcd6135b9026f79ea75e0640d5 Mon Sep 17 00:00:00 2001
From: proletarius101 <proletarius101@protonmail.com>
Date: Fri, 29 Dec 2023 22:33:15 +0800
Subject: [PATCH 12/68] test(deploy): add test cases for update_awsbucket_s3cmd

---
 fdroidserver/deploy.py |   2 +-
 tests/deploy.TestCase  | 255 ++++++++++++++++++++++++++++++++++++++---
 2 files changed, 242 insertions(+), 15 deletions(-)

diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py
index 4d581470..97602aa0 100644
--- a/fdroidserver/deploy.py
+++ b/fdroidserver/deploy.py
@@ -190,13 +190,13 @@ def update_awsbucket_s3cmd(repo_section):
         logging.debug(_('s3cmd syncs indexes from {path} to {url} and deletes removed')
                       .format(path=repo_section, url=s3url))
         sync_indexes_flags = []
+        sync_indexes_flags.extend(_get_index_includes(repo_section))
         sync_indexes_flags.append('--delete-removed')
         sync_indexes_flags.append('--delete-after')
         if options.no_checksum:
             sync_indexes_flags.append('--no-check-md5')
         else:
             sync_indexes_flags.append('--check-md5')
-        sync_indexes_flags.extend(_get_index_includes(repo_section))
         returncode = subprocess.call(s3cmd_sync + sync_indexes_flags + [repo_section, s3url])
         if returncode != 0:
             raise FDroidException()
diff --git a/tests/deploy.TestCase b/tests/deploy.TestCase
index 41d40088..908c8844 100755
--- a/tests/deploy.TestCase
+++ b/tests/deploy.TestCase
@@ -70,6 +70,11 @@ class DeployTest(unittest.TestCase):
         url1 = Path('url1/fdroid')
         url1.mkdir(parents=True)
 
+        # setup parameters for this test run
+        fdroidserver.deploy.options.identity_file = None
+        fdroidserver.deploy.options.index_only = False
+        fdroidserver.deploy.config['make_current_version_link'] = False
+
         dest_apk0 = url0 / fake_apk
         dest_apk1 = url1 / fake_apk
         self.assertFalse(dest_apk0.is_file())
@@ -176,20 +181,20 @@ class DeployTest(unittest.TestCase):
         fake_index = repo / fdroidserver.deploy.INDEX_FILES[0]
         with fake_index.open('w') as fp:
             fp.write('not an index, but has the right filename')
-        serverwebroot = Path('serverwebroot')
-        serverwebroot.mkdir()
+        url = Path('url')
+        url.mkdir()
 
         # setup parameters for this test run
         fdroidserver.deploy.options.identity_file = None
         fdroidserver.deploy.options.index_only = True
         fdroidserver.deploy.config['make_current_version_link'] = False
 
-        dest_apk = Path(serverwebroot) / fake_apk
-        dest_index = Path(serverwebroot) / fake_index
+        dest_apk = Path(url) / fake_apk
+        dest_index = Path(url) / fake_index
         self.assertFalse(dest_apk.is_file())
         self.assertFalse(dest_index.is_file())
 
-        fdroidserver.deploy.update_serverwebroot(str(serverwebroot), 'repo')
+        fdroidserver.deploy.update_serverwebroot({'url': str(url)}, 'repo')
         self.assertFalse(dest_apk.is_file())
         self.assertTrue(dest_index.is_file())
 
@@ -298,6 +303,7 @@ class DeployTest(unittest.TestCase):
 
     def test_update_serverwebroot_make_cur_version_link_in_index_only_mode(self):
         # setup parameters for this test run
+        fdroidserver.deploy.options = mock.Mock()
         fdroidserver.deploy.options.no_checksum = True
         fdroidserver.deploy.options.identity_file = None
         fdroidserver.deploy.options.verbose = False
@@ -305,7 +311,7 @@ class DeployTest(unittest.TestCase):
         fdroidserver.deploy.options.identity_file = None
         fdroidserver.deploy.options.index_only = True
         fdroidserver.deploy.config['make_current_version_link'] = True
-        serverwebroot = "example.com:/var/www/fdroid"
+        url = "example.com:/var/www/fdroid"
         repo_section = 'repo'
 
         # setup function for asserting subprocess.call invocations
@@ -360,7 +366,7 @@ class DeployTest(unittest.TestCase):
                         '--safe-links',
                         '--quiet',
                         'repo',
-                        serverwebroot,
+                        url,
                     ],
                 )
             elif call_iteration == 2:
@@ -389,7 +395,7 @@ class DeployTest(unittest.TestCase):
             os.symlink('repo/com.example.sym.apk.asc', 'Sym.apk.asc')
             os.symlink('repo/com.example.sym.apk.sig', 'Sym.apk.sig')
             with mock.patch('subprocess.call', side_effect=update_server_webroot_call):
-                fdroidserver.deploy.update_serverwebroot(serverwebroot, repo_section)
+                fdroidserver.deploy.update_serverwebroot({'url': url}, repo_section)
         self.assertEqual(call_iteration, 1, 'expected 1 invocations of subprocess.call')
 
     def test_update_serverwebroot_with_id_file(self):
@@ -482,7 +488,7 @@ class DeployTest(unittest.TestCase):
         fdroidserver.deploy.options.index_only = True
         fdroidserver.deploy.config['identity_file'] = './id_rsa'
         fdroidserver.deploy.config['make_current_version_link'] = False
-        serverwebroot = "example.com:/var/www/fdroid"
+        url = "example.com:/var/www/fdroid"
         repo_section = 'archive'
 
         # setup function for asserting subprocess.call invocations
@@ -527,7 +533,7 @@ class DeployTest(unittest.TestCase):
                         '--exclude',
                         "*",
                         'archive',
-                        serverwebroot,
+                        url,
                     ],
                 )
             elif call_iteration == 1:
@@ -543,7 +549,7 @@ class DeployTest(unittest.TestCase):
                         'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i '
                         + fdroidserver.deploy.config['identity_file'],
                         'archive',
-                        serverwebroot,
+                        url,
                     ],
                 )
             else:
@@ -552,7 +558,7 @@ class DeployTest(unittest.TestCase):
             return 0
 
         with mock.patch('subprocess.call', side_effect=update_server_webroot_call):
-            fdroidserver.deploy.update_serverwebroot(serverwebroot, repo_section)
+            fdroidserver.deploy.update_serverwebroot({'url': url}, repo_section)
         self.assertEqual(call_iteration, 1, 'expected 1 invocations of subprocess.call')
 
     def test_update_localcopy_in_index_only_mode(self):
@@ -568,6 +574,7 @@ class DeployTest(unittest.TestCase):
         call_iteration = 0
 
         with tempfile.TemporaryDirectory() as local_copy_dir:
+
             def update_localcopy_call(cmd):
                 nonlocal call_iteration
                 if call_iteration == 0:
@@ -575,8 +582,13 @@ class DeployTest(unittest.TestCase):
                         cmd,
                         [
                             'rsync',
-                            '--recursive', '--safe-links', '--times', '--perms',
-                            '--one-file-system', '--delete', '--chmod=Da+rx,Fa-x,a+r,u+w',
+                            '--recursive',
+                            '--safe-links',
+                            '--times',
+                            '--perms',
+                            '--one-file-system',
+                            '--delete',
+                            '--chmod=Da+rx,Fa-x,a+r,u+w',
                             '--verbose',
                             '--include',
                             "*/",
@@ -715,6 +727,221 @@ class DeployTest(unittest.TestCase):
             remote_push_call_iteration, 1, 'expected 1 invocations of git.Remote.push'
         )
 
+    def test_update_awsbucket_s3cmd(self):
+        # setup parameters for this test run
+        fdroidserver.deploy.options = mock.Mock()
+        fdroidserver.deploy.options.no_checksum = True
+        fdroidserver.deploy.options.verbose = False
+        fdroidserver.deploy.options.quiet = True
+        fdroidserver.deploy.options.index_only = False
+
+        config = {}
+        fdroidserver.common.fill_config_defaults(config)
+        fdroidserver.deploy.config = config
+        fdroidserver.deploy.config["awsbucket"] = "bucket"
+        fdroidserver.deploy.config["awsaccesskeyid"] = "accesskeyid"
+        fdroidserver.deploy.config["awssecretkey"] = "secretkey"
+        fdroidserver.deploy.config["s3cmd"] = "s3cmd"
+
+        repo_section = 'repo'
+
+        # setup function for asserting subprocess.call invocations
+        call_iteration = 0
+
+        def update_awsbucket_s3cmd_call(cmd):
+            nonlocal call_iteration
+            if call_iteration == 0:
+                self.assertListEqual(
+                    cmd,
+                    [
+                        's3cmd',
+                        f"--config={fdroidserver.deploy.AUTO_S3CFG}",
+                        'info',
+                        f"s3://{fdroidserver.deploy.config['awsbucket']}",
+                    ],
+                )
+            elif call_iteration == 1:
+                self.assertListEqual(
+                    cmd,
+                    [
+                        's3cmd',
+                        f"--config={fdroidserver.deploy.AUTO_S3CFG}",
+                        'sync',
+                        '--acl-public',
+                        '--quiet',
+                        '--exclude',
+                        'repo/entry.jar',
+                        '--exclude',
+                        'repo/entry.json',
+                        '--exclude',
+                        'repo/entry.json.asc',
+                        '--exclude',
+                        'repo/index-v1.jar',
+                        '--exclude',
+                        'repo/index-v1.json',
+                        '--exclude',
+                        'repo/index-v1.json.asc',
+                        '--exclude',
+                        'repo/index-v2.json',
+                        '--exclude',
+                        'repo/index-v2.json.asc',
+                        '--exclude',
+                        'repo/index.jar',
+                        '--exclude',
+                        'repo/index.xml',
+                        '--no-check-md5',
+                        '--skip-existing',
+                        repo_section,
+                        f"s3://{fdroidserver.deploy.config['awsbucket']}/fdroid/",
+                    ],
+                )
+            elif call_iteration == 2:
+                self.assertListEqual(
+                    cmd,
+                    [
+                        's3cmd',
+                        f"--config={fdroidserver.deploy.AUTO_S3CFG}",
+                        'sync',
+                        '--acl-public',
+                        '--quiet',
+                        '--exclude',
+                        'repo/entry.jar',
+                        '--exclude',
+                        'repo/entry.json',
+                        '--exclude',
+                        'repo/entry.json.asc',
+                        '--exclude',
+                        'repo/index-v1.jar',
+                        '--exclude',
+                        'repo/index-v1.json',
+                        '--exclude',
+                        'repo/index-v1.json.asc',
+                        '--exclude',
+                        'repo/index-v2.json',
+                        '--exclude',
+                        'repo/index-v2.json.asc',
+                        '--exclude',
+                        'repo/index.jar',
+                        '--exclude',
+                        'repo/index.xml',
+                        '--no-check-md5',
+                        repo_section,
+                        f"s3://{fdroidserver.deploy.config['awsbucket']}/fdroid/",
+                    ],
+                )
+            elif call_iteration == 3:
+                self.assertListEqual(
+                    cmd,
+                    [
+                        's3cmd',
+                        f"--config={fdroidserver.deploy.AUTO_S3CFG}",
+                        'sync',
+                        '--acl-public',
+                        '--quiet',
+                        '--delete-removed',
+                        '--delete-after',
+                        '--no-check-md5',
+                        repo_section,
+                        f"s3://{fdroidserver.deploy.config['awsbucket']}/fdroid/",
+                    ],
+                )
+            else:
+                self.fail('unexpected subprocess.call invocation')
+            call_iteration += 1
+            return 0
+
+        with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
+            os.mkdir('repo')
+            os.symlink('repo/com.example.sym.apk', 'Sym.apk')
+            os.symlink('repo/com.example.sym.apk.asc', 'Sym.apk.asc')
+            os.symlink('repo/com.example.sym.apk.sig', 'Sym.apk.sig')
+            with mock.patch('subprocess.call', side_effect=update_awsbucket_s3cmd_call):
+                fdroidserver.deploy.update_awsbucket_s3cmd(repo_section)
+        self.assertEqual(call_iteration, 4, 'expected 4 invocations of subprocess.call')
+
+    def test_update_awsbucket_s3cmd_in_index_only_mode(self):
+        # setup parameters for this test run
+        fdroidserver.deploy.options = mock.Mock()
+        fdroidserver.deploy.options.no_checksum = True
+        fdroidserver.deploy.options.verbose = False
+        fdroidserver.deploy.options.quiet = True
+        fdroidserver.deploy.options.index_only = True
+
+        config = {}
+        fdroidserver.common.fill_config_defaults(config)
+        fdroidserver.deploy.config = config
+        fdroidserver.deploy.config["awsbucket"] = "bucket"
+        fdroidserver.deploy.config["awsaccesskeyid"] = "accesskeyid"
+        fdroidserver.deploy.config["awssecretkey"] = "secretkey"
+        fdroidserver.deploy.config["s3cmd"] = "s3cmd"
+
+        repo_section = 'repo'
+
+        # setup function for asserting subprocess.call invocations
+        call_iteration = 0
+
+        def update_awsbucket_s3cmd_call(cmd):
+            nonlocal call_iteration
+            if call_iteration == 0:
+                self.assertListEqual(
+                    cmd,
+                    [
+                        's3cmd',
+                        f"--config={fdroidserver.deploy.AUTO_S3CFG}",
+                        'info',
+                        f"s3://{fdroidserver.deploy.config['awsbucket']}",
+                    ],
+                )
+            elif call_iteration == 1:
+                self.assertListEqual(
+                    cmd,
+                    [
+                        's3cmd',
+                        f"--config={fdroidserver.deploy.AUTO_S3CFG}",
+                        'sync',
+                        '--acl-public',
+                        '--quiet',
+                        '--include',
+                        'repo/entry.jar',
+                        '--include',
+                        'repo/entry.json',
+                        '--include',
+                        'repo/entry.json.asc',
+                        '--include',
+                        'repo/index-v1.jar',
+                        '--include',
+                        'repo/index-v1.json',
+                        '--include',
+                        'repo/index-v1.json.asc',
+                        '--include',
+                        'repo/index-v2.json',
+                        '--include',
+                        'repo/index-v2.json.asc',
+                        '--include',
+                        'repo/index.jar',
+                        '--include',
+                        'repo/index.xml',
+                        '--delete-removed',
+                        '--delete-after',
+                        '--no-check-md5',
+                        repo_section,
+                        f"s3://{fdroidserver.deploy.config['awsbucket']}/fdroid/",
+                    ],
+                )
+            else:
+                self.fail('unexpected subprocess.call invocation')
+            call_iteration += 1
+            return 0
+
+        with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
+            os.mkdir('repo')
+            os.symlink('repo/com.example.sym.apk', 'Sym.apk')
+            os.symlink('repo/com.example.sym.apk.asc', 'Sym.apk.asc')
+            os.symlink('repo/com.example.sym.apk.sig', 'Sym.apk.sig')
+            with mock.patch('subprocess.call', side_effect=update_awsbucket_s3cmd_call):
+                fdroidserver.deploy.update_awsbucket_s3cmd(repo_section)
+        self.assertEqual(call_iteration, 2, 'expected 2 invocations of subprocess.call')
+
 
 if __name__ == "__main__":
     os.chdir(os.path.dirname(__file__))

From 1f3f02765d967b360921321497f5f26e011fe6f5 Mon Sep 17 00:00:00 2001
From: proletarius101 <proletarius101@protonmail.com>
Date: Sat, 30 Dec 2023 03:18:50 +0800
Subject: [PATCH 13/68] test(deploy): add test cases for
 update_awsbucket_libcloud and fix errors

---
 fdroidserver/deploy.py |   5 +-
 tests/deploy.TestCase  | 117 +++++++++++++++++++++++++++++++++++++++++
 2 files changed, 121 insertions(+), 1 deletion(-)

diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py
index 97602aa0..d773ae58 100644
--- a/fdroidserver/deploy.py
+++ b/fdroidserver/deploy.py
@@ -402,7 +402,10 @@ def update_awsbucket_libcloud(repo_section):
             objs[obj.name] = obj
 
     if options.index_only:
-        files_to_upload = _get_index_includes(repo_section)
+        index_files = [f"{os.getcwd()}/{name}" for name in _get_index_file_paths(repo_section)]
+        files_to_upload = [os.path.join(root, name) for root, dirs, files in os.walk(os.path.join(os.getcwd(), repo_section)) for name in files]
+        files_to_upload = list(set(files_to_upload) & set(index_files))
+
     else:
         files_to_upload = [os.path.join(root, name) for root, dirs, files in os.walk(os.path.join(os.getcwd(), repo_section)) for name in files]
 
diff --git a/tests/deploy.TestCase b/tests/deploy.TestCase
index 908c8844..6a0cc258 100755
--- a/tests/deploy.TestCase
+++ b/tests/deploy.TestCase
@@ -942,6 +942,123 @@ class DeployTest(unittest.TestCase):
                 fdroidserver.deploy.update_awsbucket_s3cmd(repo_section)
         self.assertEqual(call_iteration, 2, 'expected 2 invocations of subprocess.call')
 
+    def test_update_awsbucket_libcloud(self):
+        from libcloud.storage.drivers.s3 import S3StorageDriver
+        from libcloud.storage.base import Container
+
+        # setup parameters for this test run
+        fdroidserver.deploy.options.no_checksum = True
+        fdroidserver.deploy.config["awsbucket"] = "bucket"
+        fdroidserver.deploy.config["awsaccesskeyid"] = "accesskeyid"
+        fdroidserver.deploy.config["awssecretkey"] = "secretkey"
+        fdroidserver.deploy.config["s3cmd"] = "s3cmd"
+        fdroidserver.deploy.options.verbose = False
+        fdroidserver.deploy.options.quiet = True
+        fdroidserver.deploy.options.index_only = False
+        repo_section = 'repo'
+
+        os.chdir(self.testdir)
+        repo = Path('repo')
+        repo.mkdir(parents=True)
+        fake_apk = repo / 'Sym.apk'
+        with fake_apk.open('w') as fp:
+            fp.write('not an APK, but has the right filename')
+        fake_index = repo / fdroidserver.deploy.INDEX_FILES[0]
+        with fake_index.open('w') as fp:
+            fp.write('not an index, but has the right filename')
+
+        with mock.patch(
+            'libcloud.storage.drivers.s3.S3StorageDriver'
+        ) as mock_driver_class:
+            mock_driver = mock_driver_class.return_value
+            mock_container = mock.MagicMock(spec=Container)
+            mock_container.list_objects.return_value = [
+                mock.MagicMock(name='Sym.apk'),
+                mock.MagicMock(name=fdroidserver.deploy.INDEX_FILES[0]),
+            ]
+
+            mock_driver.get_container.return_value = mock_container
+            mock_driver.upload_object_via_stream.return_value = None
+
+            fdroidserver.deploy.update_awsbucket_libcloud(repo_section)
+
+            mock_driver.get_container.assert_called_once_with(
+                container_name=fdroidserver.deploy.config["awsbucket"]
+            )
+            mock_container.list_objects.assert_called_once_with()
+            files_to_upload = ['fdroid/repo/Sym.apk', 'fdroid/repo/entry.jar']
+            calls = [
+                mock.call(
+                    iterator=mock.ANY,
+                    container=mock_container,
+                    object_name=file,
+                    extra={'acl': 'public-read'},
+                )
+                for file in files_to_upload
+            ]
+            mock_driver.upload_object_via_stream.assert_has_calls(calls, any_order=True)
+            assert mock_driver.upload_object_via_stream.call_count == 2
+
+    def test_update_awsbucket_libcloud_in_index_only_mode(self):
+        from libcloud.storage.drivers.s3 import S3StorageDriver
+        from libcloud.storage.base import Container
+
+        # setup parameters for this test run
+        fdroidserver.deploy.options.no_checksum = True
+        fdroidserver.deploy.config["awsbucket"] = "bucket"
+        fdroidserver.deploy.config["awsaccesskeyid"] = "accesskeyid"
+        fdroidserver.deploy.config["awssecretkey"] = "secretkey"
+        fdroidserver.deploy.config["s3cmd"] = "s3cmd"
+        fdroidserver.deploy.options.verbose = False
+        fdroidserver.deploy.options.quiet = True
+        fdroidserver.deploy.options.index_only = True
+        repo_section = 'repo'
+
+        os.chdir(self.testdir)
+        repo = Path('repo')
+        repo.mkdir(parents=True)
+        fake_apk = repo / 'Sym.apk'
+        with fake_apk.open('w') as fp:
+            fp.write('not an APK, but has the right filename')
+        fake_index = repo / fdroidserver.deploy.INDEX_FILES[0]
+        with fake_index.open('w') as fp:
+            fp.write('not an index, but has the right filename')
+
+        with mock.patch(
+            'libcloud.storage.drivers.s3.S3StorageDriver'
+        ) as mock_driver_class:
+            mock_driver = mock_driver_class.return_value
+            mock_container = mock.MagicMock(spec=Container)
+            mock_container.list_objects.return_value = [
+                mock.MagicMock(name='Sym.apk'),
+                mock.MagicMock(name=fdroidserver.deploy.INDEX_FILES[0]),
+            ]
+
+            mock_driver.get_container.return_value = mock_container
+            mock_driver.upload_object_via_stream.return_value = None
+
+            fdroidserver.deploy.update_awsbucket_libcloud(repo_section)
+
+            mock_driver.get_container.assert_called_once_with(
+                container_name=fdroidserver.deploy.config["awsbucket"]
+            )
+            mock_container.list_objects.assert_called_once_with()
+            files_to_upload = ['fdroid/repo/entry.jar']
+            calls = [
+                mock.call(
+                    iterator=mock.ANY,
+                    container=mock_container,
+                    object_name=file,
+                    extra={'acl': 'public-read'},
+                )
+                for file in files_to_upload
+            ]
+            mock_driver.upload_object_via_stream.assert_has_calls(
+                calls,
+                any_order=False,
+            )
+            assert mock_driver.upload_object_via_stream.call_count == 1
+
 
 if __name__ == "__main__":
     os.chdir(os.path.dirname(__file__))

From 7840d0dc3542a64506e2487cfbca6c4357921971 Mon Sep 17 00:00:00 2001
From: proletarius101 <proletarius101@protonmail.com>
Date: Sat, 30 Dec 2023 14:56:20 +0800
Subject: [PATCH 14/68] style(deploy): fix pydocstyle errors

---
 fdroidserver/deploy.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py
index d773ae58..30909a16 100644
--- a/fdroidserver/deploy.py
+++ b/fdroidserver/deploy.py
@@ -71,7 +71,6 @@ def _get_index_file_paths(base_dir):
     last.  That ensures that the package files are available when the
     client learns about them from the new index files.
     """
-
     return [os.path.join(base_dir, filename) for filename in INDEX_FILES]
 
 

From e22f7e52a852502785b90f836f09f65240192713 Mon Sep 17 00:00:00 2001
From: proletarius101 <proletarius101@protonmail.com>
Date: Sat, 30 Dec 2023 14:57:14 +0800
Subject: [PATCH 15/68] style(deploy): remove unused imports

---
 tests/deploy.TestCase | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/deploy.TestCase b/tests/deploy.TestCase
index 6a0cc258..364a566f 100755
--- a/tests/deploy.TestCase
+++ b/tests/deploy.TestCase
@@ -1000,7 +1000,6 @@ class DeployTest(unittest.TestCase):
             assert mock_driver.upload_object_via_stream.call_count == 2
 
     def test_update_awsbucket_libcloud_in_index_only_mode(self):
-        from libcloud.storage.drivers.s3 import S3StorageDriver
         from libcloud.storage.base import Container
 
         # setup parameters for this test run

From cc75f2f9a434f23738ef94fa72278be5bec44cf7 Mon Sep 17 00:00:00 2001
From: proletarius101 <proletarius101@protonmail.com>
Date: Sat, 30 Dec 2023 15:28:14 +0800
Subject: [PATCH 16/68] feat(deploy): add index only mode in
 update_servergitmirrors

---
 fdroidserver/deploy.py |  18 +++--
 tests/deploy.TestCase  | 149 +++++++++++++++++++++++++++++++++++++++++
 2 files changed, 163 insertions(+), 4 deletions(-)

diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py
index 30909a16..ae45e676 100644
--- a/fdroidserver/deploy.py
+++ b/fdroidserver/deploy.py
@@ -669,10 +669,20 @@ def update_servergitmirrors(servergitmirrors, repo_section):
             archive_path = os.path.join(git_mirror_path, 'fdroid', 'archive')
             shutil.rmtree(archive_path, ignore_errors=True)
 
-        # rsync is very particular about trailing slashes
-        common.local_rsync(
-            options, repo_section.rstrip('/') + '/', git_repodir.rstrip('/') + '/'
-        )
+        if options.index_only:
+            rsyncargs = ['--include', "*/"] + _get_index_includes(repo_section) + ['--exclude', '*']
+
+            # rsync is very particular about trailing slashes
+            common.local_rsync(common.get_options(),
+                               repo_section.rstrip('/') + '/',
+                               git_repodir.rstrip('/') + '/',
+                               args=rsyncargs)
+        else:
+            # trailing slashes have a meaning in rsync which is not needed here, so
+            # make sure both paths have exactly one trailing slash
+            common.local_rsync(common.get_options(),
+                               repo_section.rstrip('/') + '/',
+                               git_repodir.rstrip('/') + '/')
 
         # use custom SSH command if identity_file specified
         ssh_cmd = 'ssh -oBatchMode=yes'
diff --git a/tests/deploy.TestCase b/tests/deploy.TestCase
index 364a566f..ceafc4ca 100755
--- a/tests/deploy.TestCase
+++ b/tests/deploy.TestCase
@@ -1058,6 +1058,155 @@ class DeployTest(unittest.TestCase):
             )
             assert mock_driver.upload_object_via_stream.call_count == 1
 
+    def test_update_servergitmirrors(self):
+        from libcloud.storage.drivers.s3 import S3StorageDriver
+        from libcloud.storage.base import Container
+
+        # setup parameters for this test run
+        fdroidserver.deploy.options.identity_file = None
+        fdroidserver.deploy.options.no_keep_git_mirror_archive = False
+        fdroidserver.deploy.options.verbose = False
+        fdroidserver.deploy.options.quiet = True
+        fdroidserver.deploy.options.index_only = False
+
+        config = {}
+        fdroidserver.common.fill_config_defaults(config)
+        fdroidserver.deploy.config = config
+        fdroidserver.deploy.config["servergitmirrors"] = []
+
+        repo_section = 'repo'
+
+        # setup function for asserting subprocess.call invocations
+        call_iteration = 0
+
+        os.chdir(self.testdir)
+        repo = Path('repo')
+        repo.mkdir(parents=True)
+        fake_apk = repo / 'Sym.apk'
+        with fake_apk.open('w') as fp:
+            fp.write('not an APK, but has the right filename')
+        fake_index = repo / fdroidserver.deploy.INDEX_FILES[0]
+        with fake_index.open('w') as fp:
+            fp.write('not an index, but has the right filename')
+
+        with tempfile.TemporaryDirectory() as local_copy_dir:
+
+            def update_servergitmirrors_call(cmd):
+                nonlocal call_iteration
+                if call_iteration == 0:
+                    self.assertListEqual(
+                        cmd,
+                        [
+                            'rsync',
+                            '--recursive',
+                            '--safe-links',
+                            '--times',
+                            '--perms',
+                            '--one-file-system',
+                            '--delete',
+                            '--chmod=Da+rx,Fa-x,a+r,u+w',
+                            '--quiet',
+                            'repo/',
+                            "git-mirror/fdroid/repo/",
+                        ],
+                    )
+                else:
+                    self.fail('unexpected subprocess.call invocation')
+                call_iteration += 1
+                return 0
+
+            with mock.patch(
+                'subprocess.call', side_effect=update_servergitmirrors_call
+            ):
+                fdroidserver.deploy.update_servergitmirrors([], repo_section)
+        self.assertEqual(call_iteration, 1, 'expected 1 invocations of subprocess.call')
+
+    def test_update_servergitmirrors_in_index_only_mode(self):
+        from libcloud.storage.base import Container
+
+        # setup parameters for this test run
+        fdroidserver.deploy.options.identity_file = None
+        fdroidserver.deploy.options.no_keep_git_mirror_archive = False
+        fdroidserver.deploy.options.verbose = False
+        fdroidserver.deploy.options.quiet = True
+        fdroidserver.deploy.options.index_only = True
+
+        config = {}
+        fdroidserver.common.fill_config_defaults(config)
+        fdroidserver.deploy.config = config
+        fdroidserver.deploy.config["servergitmirrors"] = []
+
+        repo_section = 'repo'
+
+        # setup function for asserting subprocess.call invocations
+        call_iteration = 0
+
+        os.chdir(self.testdir)
+        repo = Path('repo')
+        repo.mkdir(parents=True)
+        fake_apk = repo / 'Sym.apk'
+        with fake_apk.open('w') as fp:
+            fp.write('not an APK, but has the right filename')
+        fake_index = repo / fdroidserver.deploy.INDEX_FILES[0]
+        with fake_index.open('w') as fp:
+            fp.write('not an index, but has the right filename')
+
+        with tempfile.TemporaryDirectory() as local_copy_dir:
+
+            def update_servergitmirrors_call(cmd):
+                nonlocal call_iteration
+                if call_iteration == 0:
+                    self.assertListEqual(
+                        cmd,
+                        [
+                            'rsync',
+                            '--recursive',
+                            '--safe-links',
+                            '--times',
+                            '--perms',
+                            '--one-file-system',
+                            '--delete',
+                            '--chmod=Da+rx,Fa-x,a+r,u+w',
+                            '--quiet',
+                            '--include',
+                            "*/",
+                            '--include',
+                            'repo/entry.jar',
+                            '--include',
+                            'repo/entry.json',
+                            '--include',
+                            'repo/entry.json.asc',
+                            '--include',
+                            'repo/index-v1.jar',
+                            '--include',
+                            'repo/index-v1.json',
+                            '--include',
+                            'repo/index-v1.json.asc',
+                            '--include',
+                            'repo/index-v2.json',
+                            '--include',
+                            'repo/index-v2.json.asc',
+                            '--include',
+                            'repo/index.jar',
+                            '--include',
+                            'repo/index.xml',
+                            '--exclude',
+                            "*",
+                            'repo/',
+                            "git-mirror/fdroid/repo/",
+                        ],
+                    )
+                else:
+                    self.fail('unexpected subprocess.call invocation')
+                call_iteration += 1
+                return 0
+
+            with mock.patch(
+                'subprocess.call', side_effect=update_servergitmirrors_call
+            ):
+                fdroidserver.deploy.update_servergitmirrors([], repo_section)
+        self.assertEqual(call_iteration, 1, 'expected 1 invocations of subprocess.call')
+
 
 if __name__ == "__main__":
     os.chdir(os.path.dirname(__file__))

From 359661a3324af3f5c4b9a607041aeccfdcb9496b Mon Sep 17 00:00:00 2001
From: proletarius101 <proletarius101@protonmail.com>
Date: Sat, 30 Dec 2023 15:30:10 +0800
Subject: [PATCH 17/68] chore: revert repository vscode settings

---
 .vscode/settings.json | 20 ++++++++++++++++----
 1 file changed, 16 insertions(+), 4 deletions(-)

diff --git a/.vscode/settings.json b/.vscode/settings.json
index 154dce4a..da31cd7f 100644
--- a/.vscode/settings.json
+++ b/.vscode/settings.json
@@ -2,8 +2,20 @@
     "python.formatting.blackArgs": [
         "--config=pyproject.toml"
     ],
-    "python.analysis.typeCheckingMode": "basic",
-    "conventionalCommits.scopes": [
-        "deploy"
+    "python.formatting.provider": "black",
+    "python.linting.banditEnabled": true,
+    "python.linting.banditArgs": [
+        "-ii",
+        "--ini=.bandit",
     ],
-}
\ No newline at end of file
+    "python.linting.enabled": true,
+    "python.linting.mypyArgs": [
+        "--config-file=mypy.ini"
+    ],
+    "python.linting.mypyEnabled": true,
+    "python.linting.flake8Enabled": true,
+    "python.linting.pylintArgs": [
+        "--rcfile=.pylint-rcfile"
+    ],
+    "python.linting.pylintEnabled": true,
+}

From c7574037a9f752e65d42ef03c61ff6b50927c32c Mon Sep 17 00:00:00 2001
From: proletarius101 <proletarius101@protonmail.com>
Date: Sat, 20 Jan 2024 17:01:36 +0800
Subject: [PATCH 18/68] refactor: remove the support of the index only mode
 when syncing to the local filesystem

---
 fdroidserver/deploy.py | 35 +++++++++-------------
 tests/deploy.TestCase  | 66 ------------------------------------------
 2 files changed, 13 insertions(+), 88 deletions(-)

diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py
index ae45e676..2046bc2d 100644
--- a/fdroidserver/deploy.py
+++ b/fdroidserver/deploy.py
@@ -561,21 +561,16 @@ def sync_from_localcopy(repo_section, local_copy_dir):
     push to all the servers that are configured.
 
     """
-    logging.info('Syncing from local_copy_dir to this repo.')
     if options.index_only:
-        rsyncargs = ['--include', "*/"] + _get_index_includes(repo_section) + ['--exclude', '*']
-        common.local_rsync(common.get_options(),
-                           _get_index_includes(local_copy_dir),
-                           repo_section.rstrip('/') + '/',
-                           args=rsyncargs)
-    else:
-        # trailing slashes have a meaning in rsync which is not needed here, so
-        # make sure both paths have exactly one trailing slash
-        common.local_rsync(
-            common.get_options(),
-            os.path.join(local_copy_dir, repo_section).rstrip('/') + '/',
-            repo_section.rstrip('/') + '/',
-        )
+        raise FDroidException(_('The index only mode cannot be used when syncing to the local copy filesystem'))
+
+    # trailing slashes have a meaning in rsync which is not needed here, so
+    # make sure both paths have exactly one trailing slash
+    common.local_rsync(
+        common.get_options(),
+        os.path.join(local_copy_dir, repo_section).rstrip('/') + '/',
+        repo_section.rstrip('/') + '/',
+    )
 
     offline_copy = os.path.join(local_copy_dir, BINARY_TRANSPARENCY_DIR)
     if os.path.exists(os.path.join(offline_copy, '.git')):
@@ -592,14 +587,10 @@ def update_localcopy(repo_section, local_copy_dir):
 
     """
     if options.index_only:
-        rsyncargs = ['--include', "*/"] + _get_index_includes(repo_section) + ['--exclude', '*']
-        common.local_rsync(common.get_options(),
-                           _get_index_includes(repo_section),
-                           local_copy_dir,
-                           args=rsyncargs)
-    else:
-        # local_copy_dir is guaranteed to have a trailing slash in main() below
-        common.local_rsync(common.get_options(), repo_section, local_copy_dir)
+        raise FDroidException(_('The index only mode cannot be used when syncing to the local copy filesystem'))
+
+    # local_copy_dir is guaranteed to have a trailing slash in main() below
+    common.local_rsync(common.get_options(), repo_section, local_copy_dir)
 
     offline_copy = os.path.join(os.getcwd(), BINARY_TRANSPARENCY_DIR)
     if os.path.isdir(os.path.join(offline_copy, '.git')):
diff --git a/tests/deploy.TestCase b/tests/deploy.TestCase
index ceafc4ca..ddd8aedd 100755
--- a/tests/deploy.TestCase
+++ b/tests/deploy.TestCase
@@ -561,72 +561,6 @@ class DeployTest(unittest.TestCase):
             fdroidserver.deploy.update_serverwebroot({'url': url}, repo_section)
         self.assertEqual(call_iteration, 1, 'expected 1 invocations of subprocess.call')
 
-    def test_update_localcopy_in_index_only_mode(self):
-        # setup parameters for this test run
-        fdroidserver.deploy.options.no_chcksum = False
-        fdroidserver.deploy.options.verbose = True
-        fdroidserver.deploy.options.quiet = False
-        fdroidserver.deploy.options.identity_file = None
-        fdroidserver.deploy.options.index_only = True
-        repo_section = 'repo'
-
-        # setup function for asserting subprocess.call invocations
-        call_iteration = 0
-
-        with tempfile.TemporaryDirectory() as local_copy_dir:
-
-            def update_localcopy_call(cmd):
-                nonlocal call_iteration
-                if call_iteration == 0:
-                    self.assertListEqual(
-                        cmd,
-                        [
-                            'rsync',
-                            '--recursive',
-                            '--safe-links',
-                            '--times',
-                            '--perms',
-                            '--one-file-system',
-                            '--delete',
-                            '--chmod=Da+rx,Fa-x,a+r,u+w',
-                            '--verbose',
-                            '--include',
-                            "*/",
-                            '--include',
-                            'repo/entry.jar',
-                            '--include',
-                            'repo/entry.json',
-                            '--include',
-                            'repo/entry.json.asc',
-                            '--include',
-                            'repo/index-v1.jar',
-                            '--include',
-                            'repo/index-v1.json',
-                            '--include',
-                            'repo/index-v1.json.asc',
-                            '--include',
-                            'repo/index-v2.json',
-                            '--include',
-                            'repo/index-v2.json.asc',
-                            '--include',
-                            'repo/index.jar',
-                            '--include',
-                            'repo/index.xml',
-                            '--exclude',
-                            "*",
-                            'repo',
-                            local_copy_dir,
-                        ],
-                    )
-                else:
-                    self.fail('unexpected subprocess.call invocation')
-                call_iteration += 1
-                return 0
-
-            with mock.patch('subprocess.call', side_effect=update_localcopy_call):
-                fdroidserver.deploy.update_localcopy(repo_section, local_copy_dir)
-        self.assertEqual(call_iteration, 1, 'expected 1 invocations of subprocess.call')
-
     @unittest.skipIf(
         not os.getenv('VIRUSTOTAL_API_KEY'), 'VIRUSTOTAL_API_KEY is not set'
     )

From 221739b653f7cca514e1f87ae3f4582ed440a6ee Mon Sep 17 00:00:00 2001
From: proletarius101 <proletarius101@protonmail.com>
Date: Sat, 20 Jan 2024 17:32:33 +0800
Subject: [PATCH 19/68] refactor: use _get_index_file_paths() instead of this
 mix of --include/--exclude

---
 fdroidserver/common.py  |   6 +-
 fdroidserver/deploy.py  |  17 +++---
 fdroidserver/nightly.py |  10 ++--
 tests/deploy.TestCase   | 127 ++++++++++++++++++++++------------------
 4 files changed, 85 insertions(+), 75 deletions(-)

diff --git a/fdroidserver/common.py b/fdroidserver/common.py
index 3c42232d..ea4f6232 100644
--- a/fdroidserver/common.py
+++ b/fdroidserver/common.py
@@ -4301,7 +4301,7 @@ def get_app_display_name(app):
     return app.get('AutoName') or app['id']
 
 
-def local_rsync(options, fromdir, todir, args: List[str] = []):
+def local_rsync(options, from_paths: List[str], todir: str, args: List[str] = []):
     """Rsync method for local to local copying of things.
 
     This is an rsync wrapper with all the settings for safe use within
@@ -4319,8 +4319,8 @@ def local_rsync(options, fromdir, todir, args: List[str] = []):
     if options.quiet:
         rsyncargs += ['--quiet']
     rsyncargs += args
-    logging.debug(' '.join(rsyncargs + [fromdir, todir]))
-    if subprocess.call(rsyncargs + [fromdir, todir]) != 0:
+    logging.debug(' '.join(rsyncargs + from_paths + [todir]))
+    if subprocess.call(rsyncargs + from_paths + [todir]) != 0:
         raise FDroidException()
 
 
diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py
index 2046bc2d..5d7b2797 100644
--- a/fdroidserver/deploy.py
+++ b/fdroidserver/deploy.py
@@ -504,7 +504,9 @@ def update_serverwebroot(serverwebroot, repo_section):
     url = serverwebroot['url']
     logging.info('rsyncing ' + repo_section + ' to ' + url)
     if options.index_only:
-        rsyncargs += ['--include', "*/"] + _get_index_includes(repo_section) + ['--exclude', '*'] + [repo_section, url]
+        rsyncargs += _get_index_file_paths(repo_section)
+        rsyncargs += [f'{url}/{repo_section}/']
+        logging.info(rsyncargs)
         if subprocess.call(rsyncargs) != 0:
             raise FDroidException()
     else:
@@ -568,7 +570,7 @@ def sync_from_localcopy(repo_section, local_copy_dir):
     # make sure both paths have exactly one trailing slash
     common.local_rsync(
         common.get_options(),
-        os.path.join(local_copy_dir, repo_section).rstrip('/') + '/',
+        [os.path.join(local_copy_dir, repo_section).rstrip('/') + '/'],
         repo_section.rstrip('/') + '/',
     )
 
@@ -590,7 +592,7 @@ def update_localcopy(repo_section, local_copy_dir):
         raise FDroidException(_('The index only mode cannot be used when syncing to the local copy filesystem'))
 
     # local_copy_dir is guaranteed to have a trailing slash in main() below
-    common.local_rsync(common.get_options(), repo_section, local_copy_dir)
+    common.local_rsync(common.get_options(), [repo_section], local_copy_dir)
 
     offline_copy = os.path.join(os.getcwd(), BINARY_TRANSPARENCY_DIR)
     if os.path.isdir(os.path.join(offline_copy, '.git')):
@@ -661,18 +663,15 @@ def update_servergitmirrors(servergitmirrors, repo_section):
             shutil.rmtree(archive_path, ignore_errors=True)
 
         if options.index_only:
-            rsyncargs = ['--include', "*/"] + _get_index_includes(repo_section) + ['--exclude', '*']
-
             # rsync is very particular about trailing slashes
             common.local_rsync(common.get_options(),
-                               repo_section.rstrip('/') + '/',
-                               git_repodir.rstrip('/') + '/',
-                               args=rsyncargs)
+                               _get_index_file_paths(repo_section),
+                               git_repodir.rstrip('/') + '/')
         else:
             # trailing slashes have a meaning in rsync which is not needed here, so
             # make sure both paths have exactly one trailing slash
             common.local_rsync(common.get_options(),
-                               repo_section.rstrip('/') + '/',
+                               [repo_section.rstrip('/') + '/'],
                                git_repodir.rstrip('/') + '/')
 
         # use custom SSH command if identity_file specified
diff --git a/fdroidserver/nightly.py b/fdroidserver/nightly.py
index 074a9eee..9c2c7175 100644
--- a/fdroidserver/nightly.py
+++ b/fdroidserver/nightly.py
@@ -378,11 +378,11 @@ Last updated: {date}'''.format(repo_git_base=repo_git_base,
 
         os.chdir(repo_basedir)
         if os.path.isdir(git_mirror_repodir):
-            common.local_rsync(options, git_mirror_repodir + '/', 'repo/')
+            common.local_rsync(options, [git_mirror_repodir + '/'], 'repo/')
         if os.path.isdir(git_mirror_metadatadir):
-            common.local_rsync(options, git_mirror_metadatadir + '/', 'metadata/')
+            common.local_rsync(options, [git_mirror_metadatadir + '/'], 'metadata/')
         if os.path.isdir(git_mirror_statsdir):
-            common.local_rsync(options, git_mirror_statsdir + '/', 'stats/')
+            common.local_rsync(options, [git_mirror_statsdir + '/'], 'stats/')
 
         ssh_private_key_file = _ssh_key_from_debug_keystore()
         # this is needed for GitPython to find the SSH key
@@ -484,9 +484,9 @@ Last updated: {date}'''.format(repo_git_base=repo_git_base,
             cwd=repo_basedir,
         )
         common.local_rsync(
-            options, repo_basedir + '/metadata/', git_mirror_metadatadir + '/'
+            options, [repo_basedir + '/metadata/'], git_mirror_metadatadir + '/'
         )
-        common.local_rsync(options, repo_basedir + '/stats/', git_mirror_statsdir + '/')
+        common.local_rsync(options, [repo_basedir + '/stats/'], git_mirror_statsdir + '/')
         mirror_git_repo.git.add(all=True)
         mirror_git_repo.index.commit("update app metadata")
 
diff --git a/tests/deploy.TestCase b/tests/deploy.TestCase
index ddd8aedd..971242e5 100755
--- a/tests/deploy.TestCase
+++ b/tests/deploy.TestCase
@@ -178,9 +178,10 @@ class DeployTest(unittest.TestCase):
         fake_apk = repo / 'fake.apk'
         with fake_apk.open('w') as fp:
             fp.write('not an APK, but has the right filename')
-        fake_index = repo / fdroidserver.deploy.INDEX_FILES[0]
-        with fake_index.open('w') as fp:
-            fp.write('not an index, but has the right filename')
+        for i in fdroidserver.deploy.INDEX_FILES:
+            fake_index = repo / i
+            with fake_index.open('w') as fp:
+                fp.write('not an index, but has the right filename')
         url = Path('url')
         url.mkdir()
 
@@ -328,32 +329,17 @@ class DeployTest(unittest.TestCase):
                         '--delete-after',
                         '--safe-links',
                         '--quiet',
-                        '--include',
-                        "*/",
-                        '--include',
                         'repo/entry.jar',
-                        '--include',
                         'repo/entry.json',
-                        '--include',
                         'repo/entry.json.asc',
-                        '--include',
                         'repo/index-v1.jar',
-                        '--include',
                         'repo/index-v1.json',
-                        '--include',
                         'repo/index-v1.json.asc',
-                        '--include',
                         'repo/index-v2.json',
-                        '--include',
                         'repo/index-v2.json.asc',
-                        '--include',
                         'repo/index.jar',
-                        '--include',
                         'repo/index.xml',
-                        '--exclude',
-                        '*',
-                        'repo',
-                        'example.com:/var/www/fdroid',
+                        'example.com:/var/www/fdroid/repo/',
                     ],
                 )
             elif call_iteration == 1:
@@ -384,6 +370,34 @@ class DeployTest(unittest.TestCase):
                         'example.com:/var/www/fdroid',
                     ],
                 )
+            # elif call_iteration == 1:
+            #     self.assertListEqual(
+            #         cmd,
+            #         [
+            #             'rsync',
+            #             '--archive',
+            #             '--delete-after',
+            #             '--safe-links',
+            #             '--quiet',
+            #             'repo',
+            #             serverwebroot,
+            #         ],
+            #     )
+            # elif call_iteration == 2:
+            #     self.assertListEqual(
+            #         cmd,
+            #         [
+            #             'rsync',
+            #             '--archive',
+            #             '--delete-after',
+            #             '--safe-links',
+            #             '--quiet',
+            #             'Sym.apk',
+            #             'Sym.apk.asc',
+            #             'Sym.apk.sig',
+            #             'example.com:/var/www/fdroid',
+            #         ],
+            #     )
             else:
                 self.fail('unexpected subprocess.call invocation')
             call_iteration += 1
@@ -508,32 +522,17 @@ class DeployTest(unittest.TestCase):
                         '-e',
                         'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i '
                         + fdroidserver.deploy.config['identity_file'],
-                        '--include',
-                        "*/",
-                        '--include',
                         'archive/entry.jar',
-                        '--include',
                         'archive/entry.json',
-                        '--include',
                         'archive/entry.json.asc',
-                        '--include',
                         'archive/index-v1.jar',
-                        '--include',
                         'archive/index-v1.json',
-                        '--include',
                         'archive/index-v1.json.asc',
-                        '--include',
                         'archive/index-v2.json',
-                        '--include',
                         'archive/index-v2.json.asc',
-                        '--include',
                         'archive/index.jar',
-                        '--include',
                         'archive/index.xml',
-                        '--exclude',
-                        "*",
-                        'archive',
-                        url,
+                        "example.com:/var/www/fdroid/archive/",
                     ],
                 )
             elif call_iteration == 1:
@@ -548,10 +547,25 @@ class DeployTest(unittest.TestCase):
                         '-e',
                         'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i '
                         + fdroidserver.deploy.config['identity_file'],
-                        'archive',
-                        url,
+                        "example.com:/var/www/fdroid/archive/",
                     ],
                 )
+            # elif call_iteration == 1:
+            #     self.assertListEqual(
+            #         cmd,
+            #         [
+            #             'rsync',
+            #             '--archive',
+            #             '--delete-after',
+            #             '--safe-links',
+            #             '--verbose',
+            #             '-e',
+            #             'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i '
+            #             + fdroidserver.deploy.config['identity_file'],
+            #             'archive',
+            #             serverwebroot,
+            #         ],
+            #     )
             else:
                 self.fail('unexpected subprocess.call invocation')
             call_iteration += 1
@@ -881,14 +895,20 @@ class DeployTest(unittest.TestCase):
         from libcloud.storage.base import Container
 
         # setup parameters for this test run
+        fdroidserver.deploy.options = mock.Mock()
         fdroidserver.deploy.options.no_checksum = True
+        fdroidserver.deploy.options.verbose = False
+        fdroidserver.deploy.options.quiet = True
+        fdroidserver.deploy.options.index_only = False
+
+        config = {}
+        fdroidserver.common.fill_config_defaults(config)
+        fdroidserver.deploy.config = config
         fdroidserver.deploy.config["awsbucket"] = "bucket"
         fdroidserver.deploy.config["awsaccesskeyid"] = "accesskeyid"
         fdroidserver.deploy.config["awssecretkey"] = "secretkey"
         fdroidserver.deploy.config["s3cmd"] = "s3cmd"
-        fdroidserver.deploy.options.verbose = False
-        fdroidserver.deploy.options.quiet = True
-        fdroidserver.deploy.options.index_only = False
+
         repo_section = 'repo'
 
         os.chdir(self.testdir)
@@ -937,14 +957,20 @@ class DeployTest(unittest.TestCase):
         from libcloud.storage.base import Container
 
         # setup parameters for this test run
+        fdroidserver.deploy.options = mock.Mock()
         fdroidserver.deploy.options.no_checksum = True
+        fdroidserver.deploy.options.verbose = False
+        fdroidserver.deploy.options.quiet = True
+        fdroidserver.deploy.options.index_only = True
+
+        config = {}
+        fdroidserver.common.fill_config_defaults(config)
+        fdroidserver.deploy.config = config
         fdroidserver.deploy.config["awsbucket"] = "bucket"
         fdroidserver.deploy.config["awsaccesskeyid"] = "accesskeyid"
         fdroidserver.deploy.config["awssecretkey"] = "secretkey"
         fdroidserver.deploy.config["s3cmd"] = "s3cmd"
-        fdroidserver.deploy.options.verbose = False
-        fdroidserver.deploy.options.quiet = True
-        fdroidserver.deploy.options.index_only = True
+
         repo_section = 'repo'
 
         os.chdir(self.testdir)
@@ -1102,31 +1128,16 @@ class DeployTest(unittest.TestCase):
                             '--delete',
                             '--chmod=Da+rx,Fa-x,a+r,u+w',
                             '--quiet',
-                            '--include',
-                            "*/",
-                            '--include',
                             'repo/entry.jar',
-                            '--include',
                             'repo/entry.json',
-                            '--include',
                             'repo/entry.json.asc',
-                            '--include',
                             'repo/index-v1.jar',
-                            '--include',
                             'repo/index-v1.json',
-                            '--include',
                             'repo/index-v1.json.asc',
-                            '--include',
                             'repo/index-v2.json',
-                            '--include',
                             'repo/index-v2.json.asc',
-                            '--include',
                             'repo/index.jar',
-                            '--include',
                             'repo/index.xml',
-                            '--exclude',
-                            "*",
-                            'repo/',
                             "git-mirror/fdroid/repo/",
                         ],
                     )

From a4247eea0d50238da4efc5950a1ede503f7c3142 Mon Sep 17 00:00:00 2001
From: proletarius101 <proletarius101@protonmail.com>
Date: Thu, 15 Feb 2024 22:27:13 +0800
Subject: [PATCH 20/68] style: fix lint

---
 tests/deploy.TestCase | 213 +++++++++++++-----------------------------
 1 file changed, 63 insertions(+), 150 deletions(-)

diff --git a/tests/deploy.TestCase b/tests/deploy.TestCase
index 971242e5..0aafb1c2 100755
--- a/tests/deploy.TestCase
+++ b/tests/deploy.TestCase
@@ -595,86 +595,6 @@ class DeployTest(unittest.TestCase):
                 name, fdroidserver.deploy.REMOTE_HOSTNAME_REGEX.sub(r'\1', remote_url)
             )
 
-    def test_update_servergitmirrors(self):
-        # setup parameters for this test run
-        fdroidserver.common.options = mock.Mock()
-        fdroidserver.common.options.identity_file = None
-        fdroidserver.common.options.no_keep_git_mirror_archive = False
-        fdroidserver.common.options.verbose = False
-        fdroidserver.common.options.quiet = True
-        fdroidserver.common.options.index_only = False
-
-        config = {}
-        fdroidserver.common.fill_config_defaults(config)
-        fdroidserver.deploy.config = config
-        fdroidserver.deploy.config["servergitmirrors"] = []
-
-        repo_section = 'repo'
-
-        # setup function for asserting subprocess.call invocations
-        update_servergitmirrors_call_iteration = 0
-        remote_push_call_iteration = 0
-
-        os.chdir(self.testdir)
-        repo = Path('repo')
-        repo.mkdir(parents=True)
-        fake_apk = repo / 'Sym.apk'
-        with fake_apk.open('w') as fp:
-            fp.write('not an APK, but has the right filename')
-        fake_index = repo / fdroidserver.deploy.INDEX_FILES[0]
-        with fake_index.open('w') as fp:
-            fp.write('not an index, but has the right filename')
-
-        def update_servergitmirrors_call(cmd):
-            nonlocal update_servergitmirrors_call_iteration
-            if update_servergitmirrors_call_iteration == 0:
-                self.assertListEqual(
-                    cmd,
-                    [
-                        'rsync',
-                        '--recursive',
-                        '--safe-links',
-                        '--times',
-                        '--perms',
-                        '--one-file-system',
-                        '--delete',
-                        '--chmod=Da+rx,Fa-x,a+r,u+w',
-                        '--quiet',
-                        'repo/',
-                        "git-mirror/fdroid/repo/",
-                    ],
-                )
-            else:
-                self.fail('unexpected subprocess.call invocation')
-            update_servergitmirrors_call_iteration += 1
-            return 0
-
-        def remote_push_call(ref, force=False, set_upstream=False, **_args):
-            nonlocal remote_push_call_iteration
-            if remote_push_call_iteration == 0:
-                self.assertEqual([ref, force, set_upstream], ['master', True, True])
-            else:
-                self.fail('unexpected git.Remote.push invocation')
-            remote_push_call_iteration += 1
-            return []
-
-        with mock.patch('subprocess.call', side_effect=update_servergitmirrors_call):
-            with mock.patch(
-                'git.Remote.push', side_effect=remote_push_call
-            ) as mock_remote_push:
-                mock_remote_push.return_value = []
-                fdroidserver.deploy.update_servergitmirrors(
-                    [{'url': 'https://github.com/user/repo'}], repo_section
-                )
-        self.assertEqual(
-            update_servergitmirrors_call_iteration,
-            1,
-            'expected 1 invocations of subprocess.call',
-        )
-        self.assertEqual(
-            remote_push_call_iteration, 1, 'expected 1 invocations of git.Remote.push'
-        )
-
     def test_update_awsbucket_s3cmd(self):
         # setup parameters for this test run
         fdroidserver.deploy.options = mock.Mock()
@@ -891,7 +811,6 @@ class DeployTest(unittest.TestCase):
         self.assertEqual(call_iteration, 2, 'expected 2 invocations of subprocess.call')
 
     def test_update_awsbucket_libcloud(self):
-        from libcloud.storage.drivers.s3 import S3StorageDriver
         from libcloud.storage.base import Container
 
         # setup parameters for this test run
@@ -1019,8 +938,6 @@ class DeployTest(unittest.TestCase):
             assert mock_driver.upload_object_via_stream.call_count == 1
 
     def test_update_servergitmirrors(self):
-        from libcloud.storage.drivers.s3 import S3StorageDriver
-        from libcloud.storage.base import Container
 
         # setup parameters for this test run
         fdroidserver.deploy.options.identity_file = None
@@ -1049,40 +966,37 @@ class DeployTest(unittest.TestCase):
         with fake_index.open('w') as fp:
             fp.write('not an index, but has the right filename')
 
-        with tempfile.TemporaryDirectory() as local_copy_dir:
+        def update_servergitmirrors_call(cmd):
+            nonlocal call_iteration
+            if call_iteration == 0:
+                self.assertListEqual(
+                    cmd,
+                    [
+                        'rsync',
+                        '--recursive',
+                        '--safe-links',
+                        '--times',
+                        '--perms',
+                        '--one-file-system',
+                        '--delete',
+                        '--chmod=Da+rx,Fa-x,a+r,u+w',
+                        '--quiet',
+                        'repo/',
+                        "git-mirror/fdroid/repo/",
+                    ],
+                )
+            else:
+                self.fail('unexpected subprocess.call invocation')
+            call_iteration += 1
+            return 0
 
-            def update_servergitmirrors_call(cmd):
-                nonlocal call_iteration
-                if call_iteration == 0:
-                    self.assertListEqual(
-                        cmd,
-                        [
-                            'rsync',
-                            '--recursive',
-                            '--safe-links',
-                            '--times',
-                            '--perms',
-                            '--one-file-system',
-                            '--delete',
-                            '--chmod=Da+rx,Fa-x,a+r,u+w',
-                            '--quiet',
-                            'repo/',
-                            "git-mirror/fdroid/repo/",
-                        ],
-                    )
-                else:
-                    self.fail('unexpected subprocess.call invocation')
-                call_iteration += 1
-                return 0
-
-            with mock.patch(
-                'subprocess.call', side_effect=update_servergitmirrors_call
-            ):
-                fdroidserver.deploy.update_servergitmirrors([], repo_section)
+        with mock.patch(
+            'subprocess.call', side_effect=update_servergitmirrors_call
+        ):
+            fdroidserver.deploy.update_servergitmirrors([], repo_section)
         self.assertEqual(call_iteration, 1, 'expected 1 invocations of subprocess.call')
 
     def test_update_servergitmirrors_in_index_only_mode(self):
-        from libcloud.storage.base import Container
 
         # setup parameters for this test run
         fdroidserver.deploy.options.identity_file = None
@@ -1111,45 +1025,44 @@ class DeployTest(unittest.TestCase):
         with fake_index.open('w') as fp:
             fp.write('not an index, but has the right filename')
 
-        with tempfile.TemporaryDirectory() as local_copy_dir:
 
-            def update_servergitmirrors_call(cmd):
-                nonlocal call_iteration
-                if call_iteration == 0:
-                    self.assertListEqual(
-                        cmd,
-                        [
-                            'rsync',
-                            '--recursive',
-                            '--safe-links',
-                            '--times',
-                            '--perms',
-                            '--one-file-system',
-                            '--delete',
-                            '--chmod=Da+rx,Fa-x,a+r,u+w',
-                            '--quiet',
-                            'repo/entry.jar',
-                            'repo/entry.json',
-                            'repo/entry.json.asc',
-                            'repo/index-v1.jar',
-                            'repo/index-v1.json',
-                            'repo/index-v1.json.asc',
-                            'repo/index-v2.json',
-                            'repo/index-v2.json.asc',
-                            'repo/index.jar',
-                            'repo/index.xml',
-                            "git-mirror/fdroid/repo/",
-                        ],
-                    )
-                else:
-                    self.fail('unexpected subprocess.call invocation')
-                call_iteration += 1
-                return 0
+        def update_servergitmirrors_call(cmd):
+            nonlocal call_iteration
+            if call_iteration == 0:
+                self.assertListEqual(
+                    cmd,
+                    [
+                        'rsync',
+                        '--recursive',
+                        '--safe-links',
+                        '--times',
+                        '--perms',
+                        '--one-file-system',
+                        '--delete',
+                        '--chmod=Da+rx,Fa-x,a+r,u+w',
+                        '--quiet',
+                        'repo/entry.jar',
+                        'repo/entry.json',
+                        'repo/entry.json.asc',
+                        'repo/index-v1.jar',
+                        'repo/index-v1.json',
+                        'repo/index-v1.json.asc',
+                        'repo/index-v2.json',
+                        'repo/index-v2.json.asc',
+                        'repo/index.jar',
+                        'repo/index.xml',
+                        "git-mirror/fdroid/repo/",
+                    ],
+                )
+            else:
+                self.fail('unexpected subprocess.call invocation')
+            call_iteration += 1
+            return 0
 
-            with mock.patch(
-                'subprocess.call', side_effect=update_servergitmirrors_call
-            ):
-                fdroidserver.deploy.update_servergitmirrors([], repo_section)
+        with mock.patch(
+            'subprocess.call', side_effect=update_servergitmirrors_call
+        ):
+            fdroidserver.deploy.update_servergitmirrors([], repo_section)
         self.assertEqual(call_iteration, 1, 'expected 1 invocations of subprocess.call')
 
 

From f2fcfeabe529627de7a1b7372d757251baa378ec Mon Sep 17 00:00:00 2001
From: proletarius101 <proletarius101@protonmail.com>
Date: Thu, 15 Feb 2024 23:31:35 +0800
Subject: [PATCH 21/68] style: fix lint

---
 tests/deploy.TestCase | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)

diff --git a/tests/deploy.TestCase b/tests/deploy.TestCase
index 0aafb1c2..dee28340 100755
--- a/tests/deploy.TestCase
+++ b/tests/deploy.TestCase
@@ -990,9 +990,7 @@ class DeployTest(unittest.TestCase):
             call_iteration += 1
             return 0
 
-        with mock.patch(
-            'subprocess.call', side_effect=update_servergitmirrors_call
-        ):
+        with mock.patch('subprocess.call', side_effect=update_servergitmirrors_call):
             fdroidserver.deploy.update_servergitmirrors([], repo_section)
         self.assertEqual(call_iteration, 1, 'expected 1 invocations of subprocess.call')
 
@@ -1025,7 +1023,6 @@ class DeployTest(unittest.TestCase):
         with fake_index.open('w') as fp:
             fp.write('not an index, but has the right filename')
 
-
         def update_servergitmirrors_call(cmd):
             nonlocal call_iteration
             if call_iteration == 0:
@@ -1059,9 +1056,7 @@ class DeployTest(unittest.TestCase):
             call_iteration += 1
             return 0
 
-        with mock.patch(
-            'subprocess.call', side_effect=update_servergitmirrors_call
-        ):
+        with mock.patch('subprocess.call', side_effect=update_servergitmirrors_call):
             fdroidserver.deploy.update_servergitmirrors([], repo_section)
         self.assertEqual(call_iteration, 1, 'expected 1 invocations of subprocess.call')
 

From 0ea0a3de789e5d97ebcf7aa9ad0d9a5543466f7e Mon Sep 17 00:00:00 2001
From: proletarius101 <proletarius101@protonmail.com>
Date: Sat, 17 Feb 2024 06:47:17 +0800
Subject: [PATCH 22/68] style: fix lint

---
 tests/build.TestCase | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/tests/build.TestCase b/tests/build.TestCase
index f6607634..ef9ecd86 100755
--- a/tests/build.TestCase
+++ b/tests/build.TestCase
@@ -622,9 +622,9 @@ class BuildTest(unittest.TestCase):
         with open(metadata_file) as fp:
             app = fdroidserver.metadata.App(yaml.safe_load(fp))
         app['RepoType'] = 'git'
-        app[
-            'Binaries'
-        ] = 'https://example.com/fdroid/repo/info.guardianproject.checkey_%v.apk'
+        app['Binaries'] = (
+            'https://example.com/fdroid/repo/info.guardianproject.checkey_%v.apk'
+        )
         build = fdroidserver.metadata.Build(
             {
                 'versionCode': 123,
@@ -712,9 +712,9 @@ class BuildTest(unittest.TestCase):
         with open(metadata_file) as fp:
             app = fdroidserver.metadata.App(yaml.safe_load(fp))
         app['RepoType'] = 'git'
-        app[
-            'Binaries'
-        ] = 'https://example.com/fdroid/repo/info.guardianproject.checkey_%v.apk'
+        app['Binaries'] = (
+            'https://example.com/fdroid/repo/info.guardianproject.checkey_%v.apk'
+        )
         build = fdroidserver.metadata.Build(
             {
                 'versionCode': 123,

From df28c1e6c3b0fdf09f334626ffc5897e06048a2d Mon Sep 17 00:00:00 2001
From: proletarius101 <proletarius101@protonmail.com>
Date: Mon, 26 Feb 2024 22:57:05 +0800
Subject: [PATCH 23/68] fix(deploy): remove the index-only mode from elsewhere
 except the config file

To enable per-remote configuration
---
 completion/bash-completion |   2 +-
 fdroidserver/common.py     |   3 +-
 fdroidserver/deploy.py     | 128 ++++++++++++++++++-------------------
 tests/deploy.TestCase      |  85 +++++-------------------
 4 files changed, 80 insertions(+), 138 deletions(-)

diff --git a/completion/bash-completion b/completion/bash-completion
index aa68a220..940b188b 100644
--- a/completion/bash-completion
+++ b/completion/bash-completion
@@ -264,7 +264,7 @@ __complete_nightly() {
 __complete_deploy() {
 	opts="-i -v -q"
 	lopts="--identity-file --local-copy-dir --sync-from-local-copy-dir
- --verbose --quiet --no-checksum --no-keep-git-mirror-archive --index-only"
+ --verbose --quiet --no-checksum --no-keep-git-mirror-archive"
 	__complete_options
 }
 
diff --git a/fdroidserver/common.py b/fdroidserver/common.py
index ea4f6232..b8374109 100644
--- a/fdroidserver/common.py
+++ b/fdroidserver/common.py
@@ -4301,7 +4301,7 @@ def get_app_display_name(app):
     return app.get('AutoName') or app['id']
 
 
-def local_rsync(options, from_paths: List[str], todir: str, args: List[str] = []):
+def local_rsync(options, from_paths: List[str], todir: str):
     """Rsync method for local to local copying of things.
 
     This is an rsync wrapper with all the settings for safe use within
@@ -4318,7 +4318,6 @@ def local_rsync(options, from_paths: List[str], todir: str, args: List[str] = []
         rsyncargs += ['--verbose']
     if options.quiet:
         rsyncargs += ['--quiet']
-    rsyncargs += args
     logging.debug(' '.join(rsyncargs + from_paths + [todir]))
     if subprocess.call(rsyncargs + from_paths + [todir]) != 0:
         raise FDroidException()
diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py
index 5d7b2797..cf6e6ab5 100644
--- a/fdroidserver/deploy.py
+++ b/fdroidserver/deploy.py
@@ -92,7 +92,7 @@ def _get_index_includes(base_dir):
     return index_includes
 
 
-def update_awsbucket(repo_section, verbose=False, quiet=False):
+def update_awsbucket(repo_section, index_only=False, verbose=False, quiet=False):
     """Upload the contents of the directory `repo_section` (including subdirectories) to the AWS S3 "bucket".
 
     The contents of that subdir of the
@@ -112,26 +112,26 @@ def update_awsbucket(repo_section, verbose=False, quiet=False):
             logging.warning(
                 'No syncing tool set in config.yml!. Defaulting to using s3cmd'
             )
-            update_awsbucket_s3cmd(repo_section)
+            update_awsbucket_s3cmd(repo_section, index_only)
         if config['s3cmd'] is True and config['rclone'] is True:
             logging.warning(
                 'Both syncing tools set in config.yml!. Defaulting to using s3cmd'
             )
-            update_awsbucket_s3cmd(repo_section)
+            update_awsbucket_s3cmd(repo_section, index_only)
         if config['s3cmd'] is True and config['rclone'] is not True:
-            update_awsbucket_s3cmd(repo_section)
+            update_awsbucket_s3cmd(repo_section, index_only)
         if config['rclone'] is True and config['s3cmd'] is not True:
             update_remote_storage_with_rclone(repo_section, verbose, quiet)
 
     elif common.set_command_in_config('s3cmd'):
-        update_awsbucket_s3cmd(repo_section)
+        update_awsbucket_s3cmd(repo_section, index_only)
     elif common.set_command_in_config('rclone'):
         update_remote_storage_with_rclone(repo_section, verbose, quiet)
     else:
-        update_awsbucket_libcloud(repo_section)
+        update_awsbucket_libcloud(repo_section, index_only)
 
 
-def update_awsbucket_s3cmd(repo_section):
+def update_awsbucket_s3cmd(repo_section, index_only=False):
     """Upload using the CLI tool s3cmd, which provides rsync-like sync.
 
     The upload is done in multiple passes to reduce the chance of
@@ -185,7 +185,7 @@ def update_awsbucket_s3cmd(repo_section):
         )
     )
 
-    if options.index_only:
+    if index_only:
         logging.debug(_('s3cmd syncs indexes from {path} to {url} and deletes removed')
                       .format(path=repo_section, url=s3url))
         sync_indexes_flags = []
@@ -352,7 +352,7 @@ def update_remote_storage_with_rclone(repo_section, verbose=False, quiet=False):
                 raise FDroidException()
 
 
-def update_awsbucket_libcloud(repo_section):
+def update_awsbucket_libcloud(repo_section, index_only=False):
     """No summary.
 
     Upload the contents of the directory `repo_section` (including
@@ -400,7 +400,7 @@ def update_awsbucket_libcloud(repo_section):
         if obj.name.startswith(upload_dir + '/'):
             objs[obj.name] = obj
 
-    if options.index_only:
+    if index_only:
         index_files = [f"{os.getcwd()}/{name}" for name in _get_index_file_paths(repo_section)]
         files_to_upload = [os.path.join(root, name) for root, dirs, files in os.walk(os.path.join(os.getcwd(), repo_section)) for name in files]
         files_to_upload = list(set(files_to_upload) & set(index_files))
@@ -502,8 +502,9 @@ def update_serverwebroot(serverwebroot, repo_section):
             'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ' + config['identity_file'],
         ]
     url = serverwebroot['url']
+    index_only = serverwebroot.get('index_only', False)
     logging.info('rsyncing ' + repo_section + ' to ' + url)
-    if options.index_only:
+    if index_only:
         rsyncargs += _get_index_file_paths(repo_section)
         rsyncargs += [f'{url}/{repo_section}/']
         logging.info(rsyncargs)
@@ -563,8 +564,7 @@ def sync_from_localcopy(repo_section, local_copy_dir):
     push to all the servers that are configured.
 
     """
-    if options.index_only:
-        raise FDroidException(_('The index only mode cannot be used when syncing to the local copy filesystem'))
+    logging.info('Syncing from local_copy_dir to this repo.')
 
     # trailing slashes have a meaning in rsync which is not needed here, so
     # make sure both paths have exactly one trailing slash
@@ -588,8 +588,6 @@ def update_localcopy(repo_section, local_copy_dir):
     drive.
 
     """
-    if options.index_only:
-        raise FDroidException(_('The index only mode cannot be used when syncing to the local copy filesystem'))
 
     # local_copy_dir is guaranteed to have a trailing slash in main() below
     common.local_rsync(common.get_options(), [repo_section], local_copy_dir)
@@ -662,17 +660,11 @@ def update_servergitmirrors(servergitmirrors, repo_section):
             archive_path = os.path.join(git_mirror_path, 'fdroid', 'archive')
             shutil.rmtree(archive_path, ignore_errors=True)
 
-        if options.index_only:
-            # rsync is very particular about trailing slashes
-            common.local_rsync(common.get_options(),
-                               _get_index_file_paths(repo_section),
-                               git_repodir.rstrip('/') + '/')
-        else:
-            # trailing slashes have a meaning in rsync which is not needed here, so
-            # make sure both paths have exactly one trailing slash
-            common.local_rsync(common.get_options(),
-                               [repo_section.rstrip('/') + '/'],
-                               git_repodir.rstrip('/') + '/')
+        # trailing slashes have a meaning in rsync which is not needed here, so
+        # make sure both paths have exactly one trailing slash
+        common.local_rsync(common.get_options(),
+                           [repo_section.rstrip('/') + '/'],
+                           git_repodir.rstrip('/') + '/')
 
         # use custom SSH command if identity_file specified
         ssh_cmd = 'ssh -oBatchMode=yes'
@@ -681,28 +673,6 @@ def update_servergitmirrors(servergitmirrors, repo_section):
         elif 'identity_file' in config:
             ssh_cmd += ' -oIdentitiesOnly=yes -i "%s"' % config['identity_file']
 
-        repo = git.Repo.init(git_mirror_path, initial_branch=GIT_BRANCH)
-
-        enabled_remotes = []
-        for d in servergitmirrors:
-            remote_url = d['url']
-            name = REMOTE_HOSTNAME_REGEX.sub(r'\1', remote_url)
-            enabled_remotes.append(name)
-            r = git.remote.Remote(repo, name)
-            if r in repo.remotes:
-                r = repo.remote(name)
-                if 'set_url' in dir(r):  # force remote URL if using GitPython 2.x
-                    r.set_url(remote_url)
-            else:
-                repo.create_remote(name, remote_url)
-            logging.info('Mirroring to: ' + remote_url)
-
-        # sadly index.add don't allow the --all parameter
-        logging.debug('Adding all files to git mirror')
-        repo.git.add(all=True)
-        logging.debug('Committing all files into git mirror')
-        repo.index.commit("fdroidserver git-mirror")
-
         if options.verbose:
             progressbar = progress.Bar()
 
@@ -715,23 +685,49 @@ def update_servergitmirrors(servergitmirrors, repo_section):
         else:
             progress = None
 
-        # only deploy to GitLab Artifacts if too big for GitLab Pages
-        if common.get_dir_size(git_fdroiddir) <= common.GITLAB_COM_PAGES_MAX_SIZE:
-            gitlab_ci_job_name = 'pages'
-        else:
-            gitlab_ci_job_name = 'GitLab Artifacts'
-            logging.warning(
-                _(
-                    'Skipping GitLab Pages mirror because the repo is too large (>%.2fGB)!'
-                )
-                % (common.GITLAB_COM_PAGES_MAX_SIZE / 1000000000)
-            )
+        repo = git.Repo.init(git_mirror_path, initial_branch=GIT_BRANCH)
+        initial_commit_ref = repo.head.ref
 
-        # push for every remote. This will overwrite the git history
-        for remote in repo.remotes:
-            if remote.name not in enabled_remotes:
-                repo.delete_remote(remote)
-                continue
+        enabled_remotes = []
+        for d in servergitmirrors:
+            index_only = d.get('index_only', False)
+            remote_url = d['url']
+            name = REMOTE_HOSTNAME_REGEX.sub(r'\1', remote_url)
+            enabled_remotes.append(name)
+            r = git.remote.Remote(repo, name)
+            if r in repo.remotes:
+                r = repo.remote(name)
+                if 'set_url' in dir(r):  # force remote URL if using GitPython 2.x
+                    r.set_url(remote_url)
+            else:
+                repo.create_remote(name, remote_url)
+            logging.info('Mirroring to: ' + remote_url)
+
+            if index_only:
+                logging.debug('Adding index files to git mirror')
+                repo.index.add(_get_index_file_paths(repo_section))
+            else:
+                # sadly index.add don't allow the --all parameter
+                logging.debug('Adding all files to git mirror')
+                repo.git.add(all=True)
+
+            logging.debug('Committing files into git mirror')
+            repo.index.commit("fdroidserver git-mirror")
+
+            # only deploy to GitLab Artifacts if too big for GitLab Pages
+            if common.get_dir_size(git_fdroiddir) <= common.GITLAB_COM_PAGES_MAX_SIZE:
+                gitlab_ci_job_name = 'pages'
+            else:
+                gitlab_ci_job_name = 'GitLab Artifacts'
+                logging.warning(
+                    _(
+                        'Skipping GitLab Pages mirror because the repo is too large (>%.2fGB)!'
+                    )
+                    % (common.GITLAB_COM_PAGES_MAX_SIZE / 1000000000)
+                )
+
+            # push. This will overwrite the git history
+            remote = repo.remote(name)
             if remote.name == 'gitlab':
                 logging.debug('Writing .gitlab-ci.yml to deploy to GitLab Pages')
                 with open(os.path.join(git_mirror_path, ".gitlab-ci.yml"), "wt") as fp:
@@ -751,7 +747,7 @@ def update_servergitmirrors(servergitmirrors, repo_section):
                         default_flow_style=False,
                     )
 
-                repo.git.add(all=True)
+                repo.index.add(['.gitlab-ci.yml'])
                 repo.index.commit("fdroidserver git-mirror: Deploy to GitLab Pages")
 
             logging.debug(_('Pushing to {url}').format(url=remote.url))
@@ -780,6 +776,8 @@ def update_servergitmirrors(servergitmirrors, repo_section):
                     else:
                         logging.debug(remote.url + ': ' + pushinfo.summary)
 
+            repo.head.reset(initial_commit_ref, index=True, working_tree=True)
+
         if progress:
             progressbar.done()
 
diff --git a/tests/deploy.TestCase b/tests/deploy.TestCase
index dee28340..8465bd10 100755
--- a/tests/deploy.TestCase
+++ b/tests/deploy.TestCase
@@ -10,6 +10,7 @@ import tempfile
 import unittest
 from pathlib import Path
 from unittest import mock
+from fdroidserver import index
 
 localmodule = os.path.realpath(
     os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..')
@@ -72,7 +73,6 @@ class DeployTest(unittest.TestCase):
 
         # setup parameters for this test run
         fdroidserver.deploy.options.identity_file = None
-        fdroidserver.deploy.options.index_only = False
         fdroidserver.deploy.config['make_current_version_link'] = False
 
         dest_apk0 = url0 / fake_apk
@@ -159,7 +159,6 @@ class DeployTest(unittest.TestCase):
         fdroidserver.deploy.options = mock.Mock()
         fdroidserver.deploy.options.identity_file = None
         fdroidserver.deploy.options.identity_file = None
-        fdroidserver.deploy.options.index_only = False
         fdroidserver.deploy.config['make_current_version_link'] = False
 
         dest_apk = Path(url) / fake_apk
@@ -187,7 +186,6 @@ class DeployTest(unittest.TestCase):
 
         # setup parameters for this test run
         fdroidserver.deploy.options.identity_file = None
-        fdroidserver.deploy.options.index_only = True
         fdroidserver.deploy.config['make_current_version_link'] = False
 
         dest_apk = Path(url) / fake_apk
@@ -195,7 +193,9 @@ class DeployTest(unittest.TestCase):
         self.assertFalse(dest_apk.is_file())
         self.assertFalse(dest_index.is_file())
 
-        fdroidserver.deploy.update_serverwebroot({'url': str(url)}, 'repo')
+        fdroidserver.deploy.update_serverwebroot(
+            {'url': str(url), 'index_only': True}, 'repo'
+        )
         self.assertFalse(dest_apk.is_file())
         self.assertTrue(dest_index.is_file())
 
@@ -310,7 +310,6 @@ class DeployTest(unittest.TestCase):
         fdroidserver.deploy.options.verbose = False
         fdroidserver.deploy.options.quiet = True
         fdroidserver.deploy.options.identity_file = None
-        fdroidserver.deploy.options.index_only = True
         fdroidserver.deploy.config['make_current_version_link'] = True
         url = "example.com:/var/www/fdroid"
         repo_section = 'repo'
@@ -370,34 +369,6 @@ class DeployTest(unittest.TestCase):
                         'example.com:/var/www/fdroid',
                     ],
                 )
-            # elif call_iteration == 1:
-            #     self.assertListEqual(
-            #         cmd,
-            #         [
-            #             'rsync',
-            #             '--archive',
-            #             '--delete-after',
-            #             '--safe-links',
-            #             '--quiet',
-            #             'repo',
-            #             serverwebroot,
-            #         ],
-            #     )
-            # elif call_iteration == 2:
-            #     self.assertListEqual(
-            #         cmd,
-            #         [
-            #             'rsync',
-            #             '--archive',
-            #             '--delete-after',
-            #             '--safe-links',
-            #             '--quiet',
-            #             'Sym.apk',
-            #             'Sym.apk.asc',
-            #             'Sym.apk.sig',
-            #             'example.com:/var/www/fdroid',
-            #         ],
-            #     )
             else:
                 self.fail('unexpected subprocess.call invocation')
             call_iteration += 1
@@ -409,7 +380,9 @@ class DeployTest(unittest.TestCase):
             os.symlink('repo/com.example.sym.apk.asc', 'Sym.apk.asc')
             os.symlink('repo/com.example.sym.apk.sig', 'Sym.apk.sig')
             with mock.patch('subprocess.call', side_effect=update_server_webroot_call):
-                fdroidserver.deploy.update_serverwebroot({'url': url}, repo_section)
+                fdroidserver.deploy.update_serverwebroot(
+                    {'url': url, 'index_only': True}, repo_section
+                )
         self.assertEqual(call_iteration, 1, 'expected 1 invocations of subprocess.call')
 
     def test_update_serverwebroot_with_id_file(self):
@@ -499,7 +472,6 @@ class DeployTest(unittest.TestCase):
         fdroidserver.deploy.options.verbose = True
         fdroidserver.deploy.options.quiet = False
         fdroidserver.deploy.options.identity_file = None
-        fdroidserver.deploy.options.index_only = True
         fdroidserver.deploy.config['identity_file'] = './id_rsa'
         fdroidserver.deploy.config['make_current_version_link'] = False
         url = "example.com:/var/www/fdroid"
@@ -550,29 +522,15 @@ class DeployTest(unittest.TestCase):
                         "example.com:/var/www/fdroid/archive/",
                     ],
                 )
-            # elif call_iteration == 1:
-            #     self.assertListEqual(
-            #         cmd,
-            #         [
-            #             'rsync',
-            #             '--archive',
-            #             '--delete-after',
-            #             '--safe-links',
-            #             '--verbose',
-            #             '-e',
-            #             'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i '
-            #             + fdroidserver.deploy.config['identity_file'],
-            #             'archive',
-            #             serverwebroot,
-            #         ],
-            #     )
             else:
                 self.fail('unexpected subprocess.call invocation')
             call_iteration += 1
             return 0
 
         with mock.patch('subprocess.call', side_effect=update_server_webroot_call):
-            fdroidserver.deploy.update_serverwebroot({'url': url}, repo_section)
+            fdroidserver.deploy.update_serverwebroot(
+                {'url': url, 'index_only': True}, repo_section
+            )
         self.assertEqual(call_iteration, 1, 'expected 1 invocations of subprocess.call')
 
     @unittest.skipIf(
@@ -601,7 +559,6 @@ class DeployTest(unittest.TestCase):
         fdroidserver.deploy.options.no_checksum = True
         fdroidserver.deploy.options.verbose = False
         fdroidserver.deploy.options.quiet = True
-        fdroidserver.deploy.options.index_only = False
 
         config = {}
         fdroidserver.common.fill_config_defaults(config)
@@ -733,7 +690,6 @@ class DeployTest(unittest.TestCase):
         fdroidserver.deploy.options.no_checksum = True
         fdroidserver.deploy.options.verbose = False
         fdroidserver.deploy.options.quiet = True
-        fdroidserver.deploy.options.index_only = True
 
         config = {}
         fdroidserver.common.fill_config_defaults(config)
@@ -807,7 +763,9 @@ class DeployTest(unittest.TestCase):
             os.symlink('repo/com.example.sym.apk.asc', 'Sym.apk.asc')
             os.symlink('repo/com.example.sym.apk.sig', 'Sym.apk.sig')
             with mock.patch('subprocess.call', side_effect=update_awsbucket_s3cmd_call):
-                fdroidserver.deploy.update_awsbucket_s3cmd(repo_section)
+                fdroidserver.deploy.update_awsbucket_s3cmd(
+                    repo_section, index_only=True
+                )
         self.assertEqual(call_iteration, 2, 'expected 2 invocations of subprocess.call')
 
     def test_update_awsbucket_libcloud(self):
@@ -818,7 +776,6 @@ class DeployTest(unittest.TestCase):
         fdroidserver.deploy.options.no_checksum = True
         fdroidserver.deploy.options.verbose = False
         fdroidserver.deploy.options.quiet = True
-        fdroidserver.deploy.options.index_only = False
 
         config = {}
         fdroidserver.common.fill_config_defaults(config)
@@ -880,7 +837,6 @@ class DeployTest(unittest.TestCase):
         fdroidserver.deploy.options.no_checksum = True
         fdroidserver.deploy.options.verbose = False
         fdroidserver.deploy.options.quiet = True
-        fdroidserver.deploy.options.index_only = True
 
         config = {}
         fdroidserver.common.fill_config_defaults(config)
@@ -915,7 +871,7 @@ class DeployTest(unittest.TestCase):
             mock_driver.get_container.return_value = mock_container
             mock_driver.upload_object_via_stream.return_value = None
 
-            fdroidserver.deploy.update_awsbucket_libcloud(repo_section)
+            fdroidserver.deploy.update_awsbucket_libcloud(repo_section, index_only=True)
 
             mock_driver.get_container.assert_called_once_with(
                 container_name=fdroidserver.deploy.config["awsbucket"]
@@ -944,7 +900,6 @@ class DeployTest(unittest.TestCase):
         fdroidserver.deploy.options.no_keep_git_mirror_archive = False
         fdroidserver.deploy.options.verbose = False
         fdroidserver.deploy.options.quiet = True
-        fdroidserver.deploy.options.index_only = False
 
         config = {}
         fdroidserver.common.fill_config_defaults(config)
@@ -1001,7 +956,6 @@ class DeployTest(unittest.TestCase):
         fdroidserver.deploy.options.no_keep_git_mirror_archive = False
         fdroidserver.deploy.options.verbose = False
         fdroidserver.deploy.options.quiet = True
-        fdroidserver.deploy.options.index_only = True
 
         config = {}
         fdroidserver.common.fill_config_defaults(config)
@@ -1038,16 +992,7 @@ class DeployTest(unittest.TestCase):
                         '--delete',
                         '--chmod=Da+rx,Fa-x,a+r,u+w',
                         '--quiet',
-                        'repo/entry.jar',
-                        'repo/entry.json',
-                        'repo/entry.json.asc',
-                        'repo/index-v1.jar',
-                        'repo/index-v1.json',
-                        'repo/index-v1.json.asc',
-                        'repo/index-v2.json',
-                        'repo/index-v2.json.asc',
-                        'repo/index.jar',
-                        'repo/index.xml',
+                        'repo/',
                         "git-mirror/fdroid/repo/",
                     ],
                 )

From 76432199be56433d49dce741096d28471ed1c20f Mon Sep 17 00:00:00 2001
From: proletarius101 <proletarius101@protonmail.com>
Date: Mon, 26 Feb 2024 23:06:04 +0800
Subject: [PATCH 24/68] lint: fix black formatting

---
 tests/build.TestCase  | 12 ++++++------
 tests/deploy.TestCase |  2 --
 2 files changed, 6 insertions(+), 8 deletions(-)

diff --git a/tests/build.TestCase b/tests/build.TestCase
index ef9ecd86..f6607634 100755
--- a/tests/build.TestCase
+++ b/tests/build.TestCase
@@ -622,9 +622,9 @@ class BuildTest(unittest.TestCase):
         with open(metadata_file) as fp:
             app = fdroidserver.metadata.App(yaml.safe_load(fp))
         app['RepoType'] = 'git'
-        app['Binaries'] = (
-            'https://example.com/fdroid/repo/info.guardianproject.checkey_%v.apk'
-        )
+        app[
+            'Binaries'
+        ] = 'https://example.com/fdroid/repo/info.guardianproject.checkey_%v.apk'
         build = fdroidserver.metadata.Build(
             {
                 'versionCode': 123,
@@ -712,9 +712,9 @@ class BuildTest(unittest.TestCase):
         with open(metadata_file) as fp:
             app = fdroidserver.metadata.App(yaml.safe_load(fp))
         app['RepoType'] = 'git'
-        app['Binaries'] = (
-            'https://example.com/fdroid/repo/info.guardianproject.checkey_%v.apk'
-        )
+        app[
+            'Binaries'
+        ] = 'https://example.com/fdroid/repo/info.guardianproject.checkey_%v.apk'
         build = fdroidserver.metadata.Build(
             {
                 'versionCode': 123,
diff --git a/tests/deploy.TestCase b/tests/deploy.TestCase
index 8465bd10..47743314 100755
--- a/tests/deploy.TestCase
+++ b/tests/deploy.TestCase
@@ -894,7 +894,6 @@ class DeployTest(unittest.TestCase):
             assert mock_driver.upload_object_via_stream.call_count == 1
 
     def test_update_servergitmirrors(self):
-
         # setup parameters for this test run
         fdroidserver.deploy.options.identity_file = None
         fdroidserver.deploy.options.no_keep_git_mirror_archive = False
@@ -950,7 +949,6 @@ class DeployTest(unittest.TestCase):
         self.assertEqual(call_iteration, 1, 'expected 1 invocations of subprocess.call')
 
     def test_update_servergitmirrors_in_index_only_mode(self):
-
         # setup parameters for this test run
         fdroidserver.deploy.options.identity_file = None
         fdroidserver.deploy.options.no_keep_git_mirror_archive = False

From 99a3823621a1d4f49465d09cc7cd90d11f2c5863 Mon Sep 17 00:00:00 2001
From: proletarius101 <proletarius101@protonmail.com>
Date: Mon, 26 Feb 2024 23:08:00 +0800
Subject: [PATCH 25/68] fix: ModuleNotFoundError

---
 tests/deploy.TestCase | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/deploy.TestCase b/tests/deploy.TestCase
index 47743314..811c61a6 100755
--- a/tests/deploy.TestCase
+++ b/tests/deploy.TestCase
@@ -10,7 +10,6 @@ import tempfile
 import unittest
 from pathlib import Path
 from unittest import mock
-from fdroidserver import index
 
 localmodule = os.path.realpath(
     os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..')

From ca5e9a089c78f721bf8d38ba28b8d398efa5f448 Mon Sep 17 00:00:00 2001
From: proletarius101 <proletarius101@protonmail.com>
Date: Mon, 26 Feb 2024 23:11:33 +0800
Subject: [PATCH 26/68] lint: fix pydocstyle

---
 fdroidserver/deploy.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py
index cf6e6ab5..fc9c8658 100644
--- a/fdroidserver/deploy.py
+++ b/fdroidserver/deploy.py
@@ -588,7 +588,6 @@ def update_localcopy(repo_section, local_copy_dir):
     drive.
 
     """
-
     # local_copy_dir is guaranteed to have a trailing slash in main() below
     common.local_rsync(common.get_options(), [repo_section], local_copy_dir)
 

From 4acab21d81de2538cedd1dcb7bcdbcf32945bff7 Mon Sep 17 00:00:00 2001
From: Hans-Christoph Steiner <hans@eds.org>
Date: Sat, 6 Apr 2024 05:49:49 +0000
Subject: [PATCH 27/68] fix: avoid making the parent directory redundantly in
 test case

---
 tests/deploy.TestCase | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/deploy.TestCase b/tests/deploy.TestCase
index 811c61a6..bbc09852 100755
--- a/tests/deploy.TestCase
+++ b/tests/deploy.TestCase
@@ -172,7 +172,7 @@ class DeployTest(unittest.TestCase):
     def test_update_serverwebroot_in_index_only_mode(self):
         os.chdir(self.testdir)
         repo = Path('repo')
-        repo.mkdir(parents=True)
+        repo.mkdir()
         fake_apk = repo / 'fake.apk'
         with fake_apk.open('w') as fp:
             fp.write('not an APK, but has the right filename')

From 3e676227362210dde29efde010759ed4486f23bb Mon Sep 17 00:00:00 2001
From: proletarius101 <proletarius101@protonmail.com>
Date: Sat, 6 Apr 2024 14:53:54 +0800
Subject: [PATCH 28/68] fix: index only mode setting doesn't work if it follows
 a full mode config

---
 fdroidserver/deploy.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py
index fc9c8658..c1423397 100644
--- a/fdroidserver/deploy.py
+++ b/fdroidserver/deploy.py
@@ -25,8 +25,9 @@ import re
 import subprocess
 import time
 import urllib
+from git import Optional
 import yaml
-from argparse import ArgumentParser
+from argparse import ArgumentParser, Namespace
 import logging
 from shlex import split
 import shutil
@@ -37,6 +38,7 @@ from . import index
 from .exception import FDroidException
 
 config = None
+options: Optional[Namespace] = None
 start_timestamp = time.gmtime()
 
 GIT_BRANCH = 'master'
@@ -685,7 +687,7 @@ def update_servergitmirrors(servergitmirrors, repo_section):
             progress = None
 
         repo = git.Repo.init(git_mirror_path, initial_branch=GIT_BRANCH)
-        initial_commit_ref = repo.head.ref
+        initial_commit_ref = repo.head.commit
 
         enabled_remotes = []
         for d in servergitmirrors:
@@ -775,6 +777,7 @@ def update_servergitmirrors(servergitmirrors, repo_section):
                     else:
                         logging.debug(remote.url + ': ' + pushinfo.summary)
 
+            # Reset the repository to the initial commit so we can choose which files to commit to the next remote
             repo.head.reset(initial_commit_ref, index=True, working_tree=True)
 
         if progress:

From 489944a503062b4c87e1090ab40cd150e51b7146 Mon Sep 17 00:00:00 2001
From: proletarius101 <proletarius101@protonmail.com>
Date: Sat, 6 Apr 2024 15:13:58 +0800
Subject: [PATCH 29/68] fix: index only mode setting doesn't work if it follows
 a full mode config

---
 fdroidserver/deploy.py | 20 ++++++++++++++++----
 1 file changed, 16 insertions(+), 4 deletions(-)

diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py
index c1423397..dd930315 100644
--- a/fdroidserver/deploy.py
+++ b/fdroidserver/deploy.py
@@ -25,7 +25,7 @@ import re
 import subprocess
 import time
 import urllib
-from git import Optional
+from typing import Optional
 import yaml
 from argparse import ArgumentParser, Namespace
 import logging
@@ -687,11 +687,22 @@ def update_servergitmirrors(servergitmirrors, repo_section):
             progress = None
 
         repo = git.Repo.init(git_mirror_path, initial_branch=GIT_BRANCH)
-        initial_commit_ref = repo.head.commit
+
+        # An initial commit of the git tree is required be for other operations
+        initial_branch_ref = repo.head.ref
+        repo.index.commit('Initial commit')
 
         enabled_remotes = []
         for d in servergitmirrors:
             index_only = d.get('index_only', False)
+            if index_only:
+                # Use a separate branch for the index only mode as it needs a different set of files to commit
+                repo.create_head('index_only', initial_branch_ref)
+                repo.heads.index_only.checkout()
+            else:
+                repo.create_head('full', initial_branch_ref)
+                repo.heads.full.checkout()
+
             remote_url = d['url']
             name = REMOTE_HOSTNAME_REGEX.sub(r'\1', remote_url)
             enabled_remotes.append(name)
@@ -777,8 +788,9 @@ def update_servergitmirrors(servergitmirrors, repo_section):
                     else:
                         logging.debug(remote.url + ': ' + pushinfo.summary)
 
-            # Reset the repository to the initial commit so we can choose which files to commit to the next remote
-            repo.head.reset(initial_commit_ref, index=True, working_tree=True)
+            # Switch to the initial branch
+            repo.head.reference = initial_branch_ref
+            repo.head.reset(index=True, working_tree=True)
 
         if progress:
             progressbar.done()

From bc2838829b8b27df38e8925ffbaab3c06fed5ffa Mon Sep 17 00:00:00 2001
From: proletarius101 <proletarius101@protonmail.com>
Date: Sat, 6 Apr 2024 16:10:00 +0800
Subject: [PATCH 30/68] fix(deploy): Reference at 'refs/heads/full' does
 already exist

---
 fdroidserver/deploy.py | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py
index dd930315..b56c5c40 100644
--- a/fdroidserver/deploy.py
+++ b/fdroidserver/deploy.py
@@ -695,13 +695,15 @@ def update_servergitmirrors(servergitmirrors, repo_section):
         enabled_remotes = []
         for d in servergitmirrors:
             index_only = d.get('index_only', False)
+
+            # Use a separate branch for the index only mode as it needs a different set of files to commit
             if index_only:
-                # Use a separate branch for the index only mode as it needs a different set of files to commit
-                repo.create_head('index_only', initial_branch_ref)
-                repo.heads.index_only.checkout()
+                branch_name = 'index_only'
             else:
-                repo.create_head('full', initial_branch_ref)
-                repo.heads.full.checkout()
+                branch_name = 'full'
+            if not branch_name in repo.heads:
+                repo.create_head(branch_name, initial_branch_ref)
+            repo.heads[branch_name].checkout()
 
             remote_url = d['url']
             name = REMOTE_HOSTNAME_REGEX.sub(r'\1', remote_url)

From fd7adaa61fed71cad3a0b79a70b0461cb46d42e8 Mon Sep 17 00:00:00 2001
From: proletarius101 <proletarius101@protonmail.com>
Date: Sat, 6 Apr 2024 16:36:38 +0800
Subject: [PATCH 31/68] fix: no files are uploaded

---
 fdroidserver/deploy.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py
index b56c5c40..68a40906 100644
--- a/fdroidserver/deploy.py
+++ b/fdroidserver/deploy.py
@@ -790,9 +790,9 @@ def update_servergitmirrors(servergitmirrors, repo_section):
                     else:
                         logging.debug(remote.url + ': ' + pushinfo.summary)
 
-            # Switch to the initial branch
+            # Switch to the initial branch and unstage all files
             repo.head.reference = initial_branch_ref
-            repo.head.reset(index=True, working_tree=True)
+            repo.head.reset(index=True, working_tree=False)
 
         if progress:
             progressbar.done()

From fcba5a42272c19ae1097b45171f4ae0b2184fbc9 Mon Sep 17 00:00:00 2001
From: proletarius101 <proletarius101@protonmail.com>
Date: Sat, 6 Apr 2024 16:45:03 +0800
Subject: [PATCH 32/68] fix(deploy): error: The following untracked working
 tree files would be overwritten by checkout

---
 fdroidserver/deploy.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py
index 68a40906..a17d7468 100644
--- a/fdroidserver/deploy.py
+++ b/fdroidserver/deploy.py
@@ -793,6 +793,7 @@ def update_servergitmirrors(servergitmirrors, repo_section):
             # Switch to the initial branch and unstage all files
             repo.head.reference = initial_branch_ref
             repo.head.reset(index=True, working_tree=False)
+            repo.delete_head(repo.branches[branch_name])
 
         if progress:
             progressbar.done()

From d30e7a0169adc4cfe6e2229e8352b0c756b1c077 Mon Sep 17 00:00:00 2001
From: proletarius101 <proletarius101@protonmail.com>
Date: Sat, 6 Apr 2024 16:49:24 +0800
Subject: [PATCH 33/68] fix(deploy): error: The branch 'full' is not fully
 merged.

---
 fdroidserver/deploy.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py
index a17d7468..2dcfb4d9 100644
--- a/fdroidserver/deploy.py
+++ b/fdroidserver/deploy.py
@@ -793,7 +793,7 @@ def update_servergitmirrors(servergitmirrors, repo_section):
             # Switch to the initial branch and unstage all files
             repo.head.reference = initial_branch_ref
             repo.head.reset(index=True, working_tree=False)
-            repo.delete_head(repo.branches[branch_name])
+            repo.delete_head(repo.branches[branch_name], force=True)
 
         if progress:
             progressbar.done()

From 4c32be112ca5b02a1b33d3ea456726609702c1a8 Mon Sep 17 00:00:00 2001
From: proletarius101 <proletarius101@protonmail.com>
Date: Sat, 6 Apr 2024 17:09:31 +0800
Subject: [PATCH 34/68] test

---
 fdroidserver/deploy.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py
index 2dcfb4d9..8ef4181d 100644
--- a/fdroidserver/deploy.py
+++ b/fdroidserver/deploy.py
@@ -703,7 +703,7 @@ def update_servergitmirrors(servergitmirrors, repo_section):
                 branch_name = 'full'
             if not branch_name in repo.heads:
                 repo.create_head(branch_name, initial_branch_ref)
-            repo.heads[branch_name].checkout()
+            repo.head.reference = repo.heads[branch_name]
 
             remote_url = d['url']
             name = REMOTE_HOSTNAME_REGEX.sub(r'\1', remote_url)
@@ -740,6 +740,9 @@ def update_servergitmirrors(servergitmirrors, repo_section):
                     % (common.GITLAB_COM_PAGES_MAX_SIZE / 1000000000)
                 )
 
+            # Test
+            print(repo.head.log())
+
             # push. This will overwrite the git history
             remote = repo.remote(name)
             if remote.name == 'gitlab':
@@ -794,7 +797,6 @@ def update_servergitmirrors(servergitmirrors, repo_section):
             repo.head.reference = initial_branch_ref
             repo.head.reset(index=True, working_tree=False)
             repo.delete_head(repo.branches[branch_name], force=True)
-
         if progress:
             progressbar.done()
 

From 4689da8d08f0f7d250b1fefb7c23c72439114800 Mon Sep 17 00:00:00 2001
From: proletarius101 <proletarius101@protonmail.com>
Date: Sat, 6 Apr 2024 17:13:24 +0800
Subject: [PATCH 35/68] test

---
 fdroidserver/deploy.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py
index 8ef4181d..474c57dd 100644
--- a/fdroidserver/deploy.py
+++ b/fdroidserver/deploy.py
@@ -741,6 +741,7 @@ def update_servergitmirrors(servergitmirrors, repo_section):
                 )
 
             # Test
+            print(repo.git.status())
             print(repo.head.log())
 
             # push. This will overwrite the git history

From 2bc584d4420125084fed9fa8262f4015702448da Mon Sep 17 00:00:00 2001
From: proletarius101 <proletarius101@protonmail.com>
Date: Sat, 6 Apr 2024 17:16:45 +0800
Subject: [PATCH 36/68] test

---
 fdroidserver/deploy.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py
index 474c57dd..1070ba12 100644
--- a/fdroidserver/deploy.py
+++ b/fdroidserver/deploy.py
@@ -728,6 +728,13 @@ def update_servergitmirrors(servergitmirrors, repo_section):
             logging.debug('Committing files into git mirror')
             repo.index.commit("fdroidserver git-mirror")
 
+            # Test
+            import os
+            os.listdir()
+            print(f"In index-only: {index_only} mode")
+            print(repo.git.status())
+            print(repo.head.log())
+
             # only deploy to GitLab Artifacts if too big for GitLab Pages
             if common.get_dir_size(git_fdroiddir) <= common.GITLAB_COM_PAGES_MAX_SIZE:
                 gitlab_ci_job_name = 'pages'
@@ -740,10 +747,6 @@ def update_servergitmirrors(servergitmirrors, repo_section):
                     % (common.GITLAB_COM_PAGES_MAX_SIZE / 1000000000)
                 )
 
-            # Test
-            print(repo.git.status())
-            print(repo.head.log())
-
             # push. This will overwrite the git history
             remote = repo.remote(name)
             if remote.name == 'gitlab':

From dbd6e4237ede69281c1f1107a243fafcabd6d219 Mon Sep 17 00:00:00 2001
From: proletarius101 <proletarius101@protonmail.com>
Date: Sat, 6 Apr 2024 17:18:23 +0800
Subject: [PATCH 37/68] test

---
 fdroidserver/deploy.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py
index 1070ba12..f5c0a033 100644
--- a/fdroidserver/deploy.py
+++ b/fdroidserver/deploy.py
@@ -729,7 +729,6 @@ def update_servergitmirrors(servergitmirrors, repo_section):
             repo.index.commit("fdroidserver git-mirror")
 
             # Test
-            import os
             os.listdir()
             print(f"In index-only: {index_only} mode")
             print(repo.git.status())

From 1d8f404ef128feaf8c784329dfe9135d28650b79 Mon Sep 17 00:00:00 2001
From: proletarius101 <proletarius101@protonmail.com>
Date: Sat, 6 Apr 2024 17:20:30 +0800
Subject: [PATCH 38/68] test

---
 fdroidserver/deploy.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py
index f5c0a033..4ec36042 100644
--- a/fdroidserver/deploy.py
+++ b/fdroidserver/deploy.py
@@ -694,6 +694,7 @@ def update_servergitmirrors(servergitmirrors, repo_section):
 
         enabled_remotes = []
         for d in servergitmirrors:
+            print(f"d: {d}")
             index_only = d.get('index_only', False)
 
             # Use a separate branch for the index only mode as it needs a different set of files to commit

From b4cfbd26e9d4f0986c3c27840c92852bfaf86a9c Mon Sep 17 00:00:00 2001
From: proletarius101 <proletarius101@protonmail.com>
Date: Sat, 6 Apr 2024 17:41:16 +0800
Subject: [PATCH 39/68] fix(deploy): align the config naming

---
 examples/config.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/examples/config.yml b/examples/config.yml
index 646726bb..0337e6f0 100644
--- a/examples/config.yml
+++ b/examples/config.yml
@@ -183,7 +183,7 @@
 #
 # serverwebroot:
 #   - url: 'me@b.az:/srv/fdroid'
-#     indexOnly: true
+#     index_only: true
 
 
 # When running fdroid processes on a remote server, it is possible to
@@ -209,7 +209,7 @@
 # servergitmirrors:
 #   - url: https://github.com/user/repo
 #   - url: https://gitlab.com/user/repo
-#     indexOnly: true
+#     index_only: true
 
 # Most git hosting services have hard size limits for each git repo.
 # `fdroid deploy` will delete the git history when the git mirror repo

From 2f37cfb9a20bc83f4ac2525678b01ba5a32af123 Mon Sep 17 00:00:00 2001
From: proletarius101 <proletarius101@protonmail.com>
Date: Sat, 6 Apr 2024 17:46:31 +0800
Subject: [PATCH 40/68] test

---
 fdroidserver/deploy.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py
index 4ec36042..bc5b018a 100644
--- a/fdroidserver/deploy.py
+++ b/fdroidserver/deploy.py
@@ -694,7 +694,10 @@ def update_servergitmirrors(servergitmirrors, repo_section):
 
         enabled_remotes = []
         for d in servergitmirrors:
+            # Test
             print(f"d: {d}")
+            print(os.listdir())
+
             index_only = d.get('index_only', False)
 
             # Use a separate branch for the index only mode as it needs a different set of files to commit
@@ -730,7 +733,6 @@ def update_servergitmirrors(servergitmirrors, repo_section):
             repo.index.commit("fdroidserver git-mirror")
 
             # Test
-            os.listdir()
             print(f"In index-only: {index_only} mode")
             print(repo.git.status())
             print(repo.head.log())

From 0eefa6fbe5010c701a25a97e0b139a250b0df2c7 Mon Sep 17 00:00:00 2001
From: proletarius101 <proletarius101@protonmail.com>
Date: Sat, 6 Apr 2024 22:17:40 +0800
Subject: [PATCH 41/68] test

---
 fdroidserver/deploy.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py
index bc5b018a..93d74906 100644
--- a/fdroidserver/deploy.py
+++ b/fdroidserver/deploy.py
@@ -696,7 +696,8 @@ def update_servergitmirrors(servergitmirrors, repo_section):
         for d in servergitmirrors:
             # Test
             print(f"d: {d}")
-            print(os.listdir())
+            print("files:")
+            print(glob.glob(my_path + '/**/*.txt', recursive=True))
 
             index_only = d.get('index_only', False)
 

From cda322445fa457622cb4d392c92f8ec78473c15f Mon Sep 17 00:00:00 2001
From: proletarius101 <proletarius101@protonmail.com>
Date: Sat, 6 Apr 2024 22:19:13 +0800
Subject: [PATCH 42/68] test

---
 fdroidserver/deploy.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py
index 93d74906..e631a595 100644
--- a/fdroidserver/deploy.py
+++ b/fdroidserver/deploy.py
@@ -697,7 +697,7 @@ def update_servergitmirrors(servergitmirrors, repo_section):
             # Test
             print(f"d: {d}")
             print("files:")
-            print(glob.glob(my_path + '/**/*.txt', recursive=True))
+            print(glob.glob('.' + '/**/*.txt', recursive=True))
 
             index_only = d.get('index_only', False)
 

From 1331d089ceda05af6ad5b1aa672cfb37ab2ebe85 Mon Sep 17 00:00:00 2001
From: proletarius101 <proletarius101@protonmail.com>
Date: Sat, 6 Apr 2024 22:23:36 +0800
Subject: [PATCH 43/68] test

---
 fdroidserver/deploy.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py
index e631a595..08c4f047 100644
--- a/fdroidserver/deploy.py
+++ b/fdroidserver/deploy.py
@@ -697,7 +697,7 @@ def update_servergitmirrors(servergitmirrors, repo_section):
             # Test
             print(f"d: {d}")
             print("files:")
-            print(glob.glob('.' + '/**/*.txt', recursive=True))
+            print(glob.glob('.' + '/**/*', recursive=True))
 
             index_only = d.get('index_only', False)
 

From 7bfcb21ba631e96fdf98b6280dfa54e22ea2b99f Mon Sep 17 00:00:00 2001
From: proletarius101 <proletarius101@protonmail.com>
Date: Sat, 6 Apr 2024 22:30:50 +0800
Subject: [PATCH 44/68] test

---
 fdroidserver/deploy.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py
index 08c4f047..ba219a5f 100644
--- a/fdroidserver/deploy.py
+++ b/fdroidserver/deploy.py
@@ -709,6 +709,9 @@ def update_servergitmirrors(servergitmirrors, repo_section):
             if not branch_name in repo.heads:
                 repo.create_head(branch_name, initial_branch_ref)
             repo.head.reference = repo.heads[branch_name]
+            
+            # test
+            print(repo.git.status())
 
             remote_url = d['url']
             name = REMOTE_HOSTNAME_REGEX.sub(r'\1', remote_url)
@@ -723,6 +726,9 @@ def update_servergitmirrors(servergitmirrors, repo_section):
             logging.info('Mirroring to: ' + remote_url)
 
             if index_only:
+                # test
+                print(glob.glob('.' + '/**/*', recursive=True))
+
                 logging.debug('Adding index files to git mirror')
                 repo.index.add(_get_index_file_paths(repo_section))
             else:

From 0e781913b8f0db4e53a51e141b45bf48726bbe5a Mon Sep 17 00:00:00 2001
From: proletarius101 <proletarius101@protonmail.com>
Date: Sat, 6 Apr 2024 22:41:01 +0800
Subject: [PATCH 45/68] fix: No such file or directory: 'repo/entry.jar'

---
 fdroidserver/deploy.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py
index ba219a5f..d2ad5802 100644
--- a/fdroidserver/deploy.py
+++ b/fdroidserver/deploy.py
@@ -709,7 +709,7 @@ def update_servergitmirrors(servergitmirrors, repo_section):
             if not branch_name in repo.heads:
                 repo.create_head(branch_name, initial_branch_ref)
             repo.head.reference = repo.heads[branch_name]
-            
+
             # test
             print(repo.git.status())
 
@@ -730,7 +730,7 @@ def update_servergitmirrors(servergitmirrors, repo_section):
                 print(glob.glob('.' + '/**/*', recursive=True))
 
                 logging.debug('Adding index files to git mirror')
-                repo.index.add(_get_index_file_paths(repo_section))
+                repo.index.add(_get_index_file_paths(os.path.join('fdroid', repo_section)))
             else:
                 # sadly index.add don't allow the --all parameter
                 logging.debug('Adding all files to git mirror')

From abf4c93a7bdafe4c758f8588164669b95e47a55b Mon Sep 17 00:00:00 2001
From: proletarius101 <proletarius101@protonmail.com>
Date: Mon, 20 May 2024 00:16:27 +0800
Subject: [PATCH 46/68] fix(deploy): don't recreate the branches

---
 fdroidserver/deploy.py | 245 +++++++++++++++++++++--------------------
 tests/deploy.TestCase  |  48 ++++++++
 2 files changed, 176 insertions(+), 117 deletions(-)

diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py
index d2ad5802..d2be4ce5 100644
--- a/fdroidserver/deploy.py
+++ b/fdroidserver/deploy.py
@@ -25,12 +25,14 @@ import re
 import subprocess
 import time
 import urllib
-from typing import Optional
+from typing import Dict, Optional, List
+from git import Repo
 import yaml
 from argparse import ArgumentParser, Namespace
 import logging
 from shlex import split
 import shutil
+import git
 
 from . import _
 from . import common
@@ -621,7 +623,6 @@ def update_servergitmirrors(servergitmirrors, repo_section):
     transparency log.
 
     """
-    import git
     from clint.textui import progress
 
     if config.get('local_copy_dir') and not config.get('sync_from_local_copy_dir'):
@@ -687,133 +688,143 @@ def update_servergitmirrors(servergitmirrors, repo_section):
             progress = None
 
         repo = git.Repo.init(git_mirror_path, initial_branch=GIT_BRANCH)
+        initial_ref = repo.head.ref
 
         # An initial commit of the git tree is required be for other operations
-        initial_branch_ref = repo.head.ref
-        repo.index.commit('Initial commit')
+        print("repo.index.entries:", len(repo.index.entries))
+        if len(repo.index.entries) == 0:
+            repo.index.commit('Initial commit')
 
         enabled_remotes = []
         for d in servergitmirrors:
-            # Test
-            print(f"d: {d}")
-            print("files:")
-            print(glob.glob('.' + '/**/*', recursive=True))
-
-            index_only = d.get('index_only', False)
-
-            # Use a separate branch for the index only mode as it needs a different set of files to commit
-            if index_only:
-                branch_name = 'index_only'
-            else:
-                branch_name = 'full'
-            if not branch_name in repo.heads:
-                repo.create_head(branch_name, initial_branch_ref)
-            repo.head.reference = repo.heads[branch_name]
-
-            # test
-            print(repo.git.status())
-
-            remote_url = d['url']
-            name = REMOTE_HOSTNAME_REGEX.sub(r'\1', remote_url)
-            enabled_remotes.append(name)
-            r = git.remote.Remote(repo, name)
-            if r in repo.remotes:
-                r = repo.remote(name)
-                if 'set_url' in dir(r):  # force remote URL if using GitPython 2.x
-                    r.set_url(remote_url)
-            else:
-                repo.create_remote(name, remote_url)
-            logging.info('Mirroring to: ' + remote_url)
-
-            if index_only:
-                # test
-                print(glob.glob('.' + '/**/*', recursive=True))
-
-                logging.debug('Adding index files to git mirror')
-                repo.index.add(_get_index_file_paths(os.path.join('fdroid', repo_section)))
-            else:
-                # sadly index.add don't allow the --all parameter
-                logging.debug('Adding all files to git mirror')
-                repo.git.add(all=True)
-
-            logging.debug('Committing files into git mirror')
-            repo.index.commit("fdroidserver git-mirror")
-
-            # Test
-            print(f"In index-only: {index_only} mode")
-            print(repo.git.status())
-            print(repo.head.log())
-
-            # only deploy to GitLab Artifacts if too big for GitLab Pages
-            if common.get_dir_size(git_fdroiddir) <= common.GITLAB_COM_PAGES_MAX_SIZE:
-                gitlab_ci_job_name = 'pages'
-            else:
-                gitlab_ci_job_name = 'GitLab Artifacts'
-                logging.warning(
-                    _(
-                        'Skipping GitLab Pages mirror because the repo is too large (>%.2fGB)!'
-                    )
-                    % (common.GITLAB_COM_PAGES_MAX_SIZE / 1000000000)
-                )
-
-            # push. This will overwrite the git history
-            remote = repo.remote(name)
-            if remote.name == 'gitlab':
-                logging.debug('Writing .gitlab-ci.yml to deploy to GitLab Pages')
-                with open(os.path.join(git_mirror_path, ".gitlab-ci.yml"), "wt") as fp:
-                    yaml.dump(
-                        {
-                            gitlab_ci_job_name: {
-                                'script': [
-                                    'mkdir .public',
-                                    'cp -r * .public/',
-                                    'mv .public public',
-                                ],
-                                'artifacts': {'paths': ['public']},
-                                'variables': {'GIT_DEPTH': 1},
-                            }
-                        },
-                        fp,
-                        default_flow_style=False,
-                    )
-
-                repo.index.add(['.gitlab-ci.yml'])
-                repo.index.commit("fdroidserver git-mirror: Deploy to GitLab Pages")
-
-            logging.debug(_('Pushing to {url}').format(url=remote.url))
-            with repo.git.custom_environment(GIT_SSH_COMMAND=ssh_cmd):
-                pushinfos = remote.push(
-                    GIT_BRANCH, force=True, set_upstream=True, progress=progress
-                )
-                for pushinfo in pushinfos:
-                    if pushinfo.flags & (
-                        git.remote.PushInfo.ERROR
-                        | git.remote.PushInfo.REJECTED
-                        | git.remote.PushInfo.REMOTE_FAILURE
-                        | git.remote.PushInfo.REMOTE_REJECTED
-                    ):
-                        # Show potentially useful messages from git remote
-                        for line in progress.other_lines:
-                            if line.startswith('remote:'):
-                                logging.debug(line)
-                        raise FDroidException(
-                            '{url} push failed: {flags} {summary}'.format(
-                                url=remote.url,
-                                flags=pushinfo.flags,
-                                summary=pushinfo.summary,
-                            )
-                        )
-                    else:
-                        logging.debug(remote.url + ': ' + pushinfo.summary)
+            upload_to_servergitmirror(mirror_config=d,
+                                      local_repo=repo,
+                                      enabled_remotes=enabled_remotes,
+                                      repo_section=repo_section,
+                                      fdroid_dir=git_fdroiddir,
+                                      git_mirror_path=git_mirror_path,
+                                      ssh_cmd=ssh_cmd)
 
             # Switch to the initial branch and unstage all files
-            repo.head.reference = initial_branch_ref
+            repo.head.reference = initial_ref
             repo.head.reset(index=True, working_tree=False)
-            repo.delete_head(repo.branches[branch_name], force=True)
         if progress:
             progressbar.done()
 
 
+def upload_to_servergitmirror(mirror_config: Dict[str, str],
+                              local_repo: Repo,
+                              enabled_remotes: List[str],
+                              repo_section: str,
+                              fdroid_dir: str,
+                              git_mirror_path: str,
+                              ssh_cmd: str):
+    # Test
+    print(f"mirror_config: {mirror_config}")
+    print("files:")
+    print(glob.glob('.' + '/**/*', recursive=True))
+
+    index_only = mirror_config.get('index_only', False)
+
+    # Use a separate branch for the index only mode as it needs a different set of files to commit
+    if index_only:
+        branch_name = 'index_only'
+    else:
+        branch_name = 'full'
+    if branch_name not in local_repo.heads:
+        local_repo.create_head(branch_name)
+    local_repo.head.reference = local_repo.heads[branch_name]
+
+    # test
+    print(local_repo.git.status())
+
+    remote_url = mirror_config['url']
+    name = REMOTE_HOSTNAME_REGEX.sub(r'\1', remote_url)
+    enabled_remotes.append(name)
+    r = git.remote.Remote(local_repo, name)
+    if r in local_repo.remotes:
+        r = local_repo.remote(name)
+        if 'set_url' in dir(r):  # force remote URL if using GitPython 2.x
+            r.set_url(remote_url)
+    else:
+        local_repo.create_remote(name, remote_url)
+    logging.info('Mirroring to: ' + remote_url)
+
+    if index_only:
+        # test
+        print(glob.glob('.' + '/**/*', recursive=True))
+
+        logging.debug('Adding index files to git mirror')
+        local_repo.index.add(_get_index_file_paths(os.path.join('fdroid', repo_section)))
+    else:
+        # sadly index.add don't allow the --all parameter
+        logging.debug('Adding all files to git mirror')
+        local_repo.git.add(all=True)
+
+    logging.debug('Committing files into git mirror')
+    local_repo.index.commit("fdroidserver git-mirror")
+
+    # Test
+    print(f"In index-only: {index_only} mode")
+    print(local_repo.git.status())
+    print(local_repo.head.log())
+
+    # only deploy to GitLab Artifacts if too big for GitLab Pages
+    if common.get_dir_size(fdroid_dir) <= common.GITLAB_COM_PAGES_MAX_SIZE:
+        gitlab_ci_job_name = 'pages'
+    else:
+        gitlab_ci_job_name = 'GitLab Artifacts'
+        logging.warning(
+            _(
+                'Skipping GitLab Pages mirror because the repo is too large (>%.2fGB)!'
+            )
+            % (common.GITLAB_COM_PAGES_MAX_SIZE / 1000000000)
+        )
+
+    # push. This will overwrite the git history
+    remote = local_repo.remote(name)
+    if remote.name == 'gitlab':
+        logging.debug('Writing .gitlab-ci.yml to deploy to GitLab Pages')
+        with open(os.path.join(git_mirror_path, ".gitlab-ci.yml"), "wt") as fp:
+            yaml.dump(
+                {
+                    gitlab_ci_job_name: {
+                        'script': [
+                            'mkdir .public',
+                            'cp -r * .public/',
+                            'mv .public public',
+                        ],
+                        'artifacts': {'paths': ['public']},
+                        'variables': {'GIT_DEPTH': 1},
+                    }
+                },
+                fp,
+                default_flow_style=False,
+            )
+
+        local_repo.index.add(['.gitlab-ci.yml'])
+        local_repo.index.commit("fdroidserver git-mirror: Deploy to GitLab Pages")
+
+    logging.debug(_('Pushing to {url}').format(url=remote.url))
+    with local_repo.git.custom_environment(GIT_SSH_COMMAND=ssh_cmd):
+        pushinfos = remote.push(
+            GIT_BRANCH, force=True, set_upstream=True, progress=progress
+        )
+        for pushinfo in pushinfos:
+            if pushinfo.flags & (git.remote.PushInfo.ERROR
+                                 | git.remote.PushInfo.REJECTED
+                                 | git.remote.PushInfo.REMOTE_FAILURE
+                                 | git.remote.PushInfo.REMOTE_REJECTED):
+                # Show potentially useful messages from git remote
+                for line in progress.other_lines:
+                    if line.startswith('remote:'):
+                        logging.debug(line)
+                raise FDroidException(remote.url + ' push failed: ' + str(pushinfo.flags)
+                                      + ' ' + pushinfo.summary)
+            else:
+                logging.debug(remote.url + ': ' + pushinfo.summary)
+
+
 def upload_to_android_observatory(repo_section):
     import requests
 
diff --git a/tests/deploy.TestCase b/tests/deploy.TestCase
index bbc09852..ffd4959d 100755
--- a/tests/deploy.TestCase
+++ b/tests/deploy.TestCase
@@ -7,10 +7,13 @@ import os
 import shutil
 import sys
 import tempfile
+from unicodedata import mirrored
 import unittest
 from pathlib import Path
 from unittest import mock
 
+import git
+
 localmodule = os.path.realpath(
     os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..')
 )
@@ -1002,6 +1005,51 @@ class DeployTest(unittest.TestCase):
             fdroidserver.deploy.update_servergitmirrors([], repo_section)
         self.assertEqual(call_iteration, 1, 'expected 1 invocations of subprocess.call')
 
+    def test_upload_to_servergitmirror_in_index_only_mode(self):
+        # setup parameters for this test run
+        fdroidserver.deploy.options.identity_file = None
+        fdroidserver.deploy.options.no_keep_git_mirror_archive = False
+        fdroidserver.deploy.options.verbose = False
+        fdroidserver.deploy.options.quiet = True
+
+        config = {}
+        fdroidserver.common.fill_config_defaults(config)
+        fdroidserver.deploy.config = config
+
+        repo_section = 'repo'
+        initial_branch = 'main'
+
+        os.chdir(self.testdir)
+
+        local_git_repo_path = Path(self.testdir)
+        local_git_repo = git.Repo.init(
+            local_git_repo_path, initial_branch=initial_branch
+        )
+        fdroid_dir = local_git_repo_path / 'fdroid'
+        repo_dir = fdroid_dir / repo_section
+        repo_dir.mkdir(parents=True)
+        for filename in fdroidserver.deploy.INDEX_FILES:
+            fake_file = repo_dir / filename
+            with fake_file.open('w') as fp:
+                fp.write('not a real one, but has the right filename')
+
+        remote_git_repo_path = Path('git-mirror')
+        remote_repo = git.Repo.init(remote_git_repo_path, initial_branch=initial_branch)
+
+        mirror_config = {"url": remote_git_repo_path, "index_only": True}
+        enabled_remotes = []
+        ssh_cmd = 'ssh -oBatchMode=yes'
+        fdroidserver.deploy.upload_to_servergitmirror(
+            mirror_config=mirror_config,
+            local_repo=local_git_repo,
+            enabled_remotes=enabled_remotes,
+            repo_section=repo_section,
+            fdroid_dir=str(fdroid_dir),
+            git_mirror_path=str(local_git_repo_path),
+            ssh_cmd=ssh_cmd,
+        )
+        assert remote_repo.is_dirty
+
 
 if __name__ == "__main__":
     os.chdir(os.path.dirname(__file__))

From 097f81a5653dc5ee9d683d7d6bf7feb988faa08d Mon Sep 17 00:00:00 2001
From: proletarius101 <proletarius101@protonmail.com>
Date: Mon, 20 May 2024 00:55:28 +0800
Subject: [PATCH 47/68] test(deploy): fix errors

---
 fdroidserver/deploy.py |  9 ++++-----
 tests/deploy.TestCase  | 25 +++++++++++++++----------
 2 files changed, 19 insertions(+), 15 deletions(-)

diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py
index d2be4ce5..548539f5 100644
--- a/fdroidserver/deploy.py
+++ b/fdroidserver/deploy.py
@@ -703,7 +703,8 @@ def update_servergitmirrors(servergitmirrors, repo_section):
                                       repo_section=repo_section,
                                       fdroid_dir=git_fdroiddir,
                                       git_mirror_path=git_mirror_path,
-                                      ssh_cmd=ssh_cmd)
+                                      ssh_cmd=ssh_cmd,
+                                      progress=progress)
 
             # Switch to the initial branch and unstage all files
             repo.head.reference = initial_ref
@@ -718,7 +719,8 @@ def upload_to_servergitmirror(mirror_config: Dict[str, str],
                               repo_section: str,
                               fdroid_dir: str,
                               git_mirror_path: str,
-                              ssh_cmd: str):
+                              ssh_cmd: str,
+                              progress: git.RemoteProgress) -> None:
     # Test
     print(f"mirror_config: {mirror_config}")
     print("files:")
@@ -735,9 +737,6 @@ def upload_to_servergitmirror(mirror_config: Dict[str, str],
         local_repo.create_head(branch_name)
     local_repo.head.reference = local_repo.heads[branch_name]
 
-    # test
-    print(local_repo.git.status())
-
     remote_url = mirror_config['url']
     name = REMOTE_HOSTNAME_REGEX.sub(r'\1', remote_url)
     enabled_remotes.append(name)
diff --git a/tests/deploy.TestCase b/tests/deploy.TestCase
index ffd4959d..285e552b 100755
--- a/tests/deploy.TestCase
+++ b/tests/deploy.TestCase
@@ -1017,15 +1017,18 @@ class DeployTest(unittest.TestCase):
         fdroidserver.deploy.config = config
 
         repo_section = 'repo'
-        initial_branch = 'main'
+        initial_branch = fdroidserver.deploy.GIT_BRANCH
 
         os.chdir(self.testdir)
 
-        local_git_repo_path = Path(self.testdir)
-        local_git_repo = git.Repo.init(
-            local_git_repo_path, initial_branch=initial_branch
+        local_repo_path = Path(self.testdir) / 'local'
+        local_repo = git.Repo.init(
+            local_repo_path, initial_branch=initial_branch
         )
-        fdroid_dir = local_git_repo_path / 'fdroid'
+        # An initial commit of the git tree is required be for other operations
+        local_repo.index.commit('Initial commit')
+
+        fdroid_dir = local_repo_path / 'fdroid'
         repo_dir = fdroid_dir / repo_section
         repo_dir.mkdir(parents=True)
         for filename in fdroidserver.deploy.INDEX_FILES:
@@ -1033,20 +1036,22 @@ class DeployTest(unittest.TestCase):
             with fake_file.open('w') as fp:
                 fp.write('not a real one, but has the right filename')
 
-        remote_git_repo_path = Path('git-mirror')
-        remote_repo = git.Repo.init(remote_git_repo_path, initial_branch=initial_branch)
+        # The remote repo must be a bare repo to allow being pushed to
+        remote_repo_path = Path(self.testdir) / 'remote'
+        remote_repo = git.Repo.init(remote_repo_path, initial_branch=initial_branch, bare=True)
 
-        mirror_config = {"url": remote_git_repo_path, "index_only": True}
+        mirror_config = {"url": str(remote_repo_path), "index_only": True}
         enabled_remotes = []
         ssh_cmd = 'ssh -oBatchMode=yes'
         fdroidserver.deploy.upload_to_servergitmirror(
             mirror_config=mirror_config,
-            local_repo=local_git_repo,
+            local_repo=local_repo,
             enabled_remotes=enabled_remotes,
             repo_section=repo_section,
             fdroid_dir=str(fdroid_dir),
-            git_mirror_path=str(local_git_repo_path),
+            git_mirror_path=str(local_repo_path),
             ssh_cmd=ssh_cmd,
+            progress=git.RemoteProgress()
         )
         assert remote_repo.is_dirty
 

From d1c881e2c3ceb6992d27f7815d64363d79873ccc Mon Sep 17 00:00:00 2001
From: proletarius101 <proletarius101@protonmail.com>
Date: Thu, 23 May 2024 22:40:51 +0800
Subject: [PATCH 48/68] test(deploy): fix fdroidserver.deploy.options is None

---
 tests/deploy.TestCase | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/tests/deploy.TestCase b/tests/deploy.TestCase
index 285e552b..be8b6a67 100755
--- a/tests/deploy.TestCase
+++ b/tests/deploy.TestCase
@@ -187,6 +187,7 @@ class DeployTest(unittest.TestCase):
         url.mkdir()
 
         # setup parameters for this test run
+        fdroidserver.deploy.options = mock.Mock()
         fdroidserver.deploy.options.identity_file = None
         fdroidserver.deploy.config['make_current_version_link'] = False
 
@@ -470,6 +471,7 @@ class DeployTest(unittest.TestCase):
 
     def test_update_serverwebroot_with_id_file_in_index_only_mode(self):
         # setup parameters for this test run
+        fdroidserver.deploy.options = mock.Mock()
         fdroidserver.deploy.options.no_chcksum = False
         fdroidserver.deploy.options.verbose = True
         fdroidserver.deploy.options.quiet = False
@@ -539,7 +541,8 @@ class DeployTest(unittest.TestCase):
         not os.getenv('VIRUSTOTAL_API_KEY'), 'VIRUSTOTAL_API_KEY is not set'
     )
     def test_upload_to_virustotal(self):
-        fdroidserver.common.options.verbose = True
+        fdroidserver.deploy.options = mock.Mock()
+        fdroidserver.deploy.options.verbose = True
         virustotal_apikey = os.getenv('VIRUSTOTAL_API_KEY')
         fdroidserver.deploy.upload_to_virustotal('repo', virustotal_apikey)
 
@@ -897,6 +900,7 @@ class DeployTest(unittest.TestCase):
 
     def test_update_servergitmirrors(self):
         # setup parameters for this test run
+        fdroidserver.deploy.options = mock.Mock()
         fdroidserver.deploy.options.identity_file = None
         fdroidserver.deploy.options.no_keep_git_mirror_archive = False
         fdroidserver.deploy.options.verbose = False
@@ -952,6 +956,7 @@ class DeployTest(unittest.TestCase):
 
     def test_update_servergitmirrors_in_index_only_mode(self):
         # setup parameters for this test run
+        fdroidserver.deploy.options = mock.Mock()
         fdroidserver.deploy.options.identity_file = None
         fdroidserver.deploy.options.no_keep_git_mirror_archive = False
         fdroidserver.deploy.options.verbose = False
@@ -1007,10 +1012,12 @@ class DeployTest(unittest.TestCase):
 
     def test_upload_to_servergitmirror_in_index_only_mode(self):
         # setup parameters for this test run
+        fdroidserver.deploy.options = mock.Mock()
         fdroidserver.deploy.options.identity_file = None
         fdroidserver.deploy.options.no_keep_git_mirror_archive = False
         fdroidserver.deploy.options.verbose = False
         fdroidserver.deploy.options.quiet = True
+        fdroidserver.deploy.options.identity_file = None
 
         config = {}
         fdroidserver.common.fill_config_defaults(config)

From 0e3bd6f63e9406877d3084d6a3be57df2a462fa0 Mon Sep 17 00:00:00 2001
From: proletarius101 <proletarius101@protonmail.com>
Date: Sun, 2 Jun 2024 19:31:13 +0800
Subject: [PATCH 49/68] fix(deploy): files are not pushed to the remote

---
 fdroidserver/deploy.py | 15 +++++++--------
 tests/deploy.TestCase  | 38 ++++++++++++++++++++++++++------------
 2 files changed, 33 insertions(+), 20 deletions(-)

diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py
index 548539f5..cdfa7222 100644
--- a/fdroidserver/deploy.py
+++ b/fdroidserver/deploy.py
@@ -730,12 +730,13 @@ def upload_to_servergitmirror(mirror_config: Dict[str, str],
 
     # Use a separate branch for the index only mode as it needs a different set of files to commit
     if index_only:
-        branch_name = 'index_only'
+        local_branch_name = 'index_only'
     else:
-        branch_name = 'full'
-    if branch_name not in local_repo.heads:
-        local_repo.create_head(branch_name)
-    local_repo.head.reference = local_repo.heads[branch_name]
+        local_branch_name = 'full'
+    if local_branch_name not in local_repo.heads:
+        local_repo.create_head(local_branch_name)
+    local_repo.head.reference = local_repo.heads[local_branch_name]
+    remote_branch_name = GIT_BRANCH
 
     remote_url = mirror_config['url']
     name = REMOTE_HOSTNAME_REGEX.sub(r'\1', remote_url)
@@ -751,8 +752,6 @@ def upload_to_servergitmirror(mirror_config: Dict[str, str],
 
     if index_only:
         # test
-        print(glob.glob('.' + '/**/*', recursive=True))
-
         logging.debug('Adding index files to git mirror')
         local_repo.index.add(_get_index_file_paths(os.path.join('fdroid', repo_section)))
     else:
@@ -807,7 +806,7 @@ def upload_to_servergitmirror(mirror_config: Dict[str, str],
     logging.debug(_('Pushing to {url}').format(url=remote.url))
     with local_repo.git.custom_environment(GIT_SSH_COMMAND=ssh_cmd):
         pushinfos = remote.push(
-            GIT_BRANCH, force=True, set_upstream=True, progress=progress
+            f"{local_branch_name}:{remote_branch_name}", force=True, set_upstream=True, progress=progress
         )
         for pushinfo in pushinfos:
             if pushinfo.flags & (git.remote.PushInfo.ERROR
diff --git a/tests/deploy.TestCase b/tests/deploy.TestCase
index be8b6a67..2df20aae 100755
--- a/tests/deploy.TestCase
+++ b/tests/deploy.TestCase
@@ -1028,14 +1028,14 @@ class DeployTest(unittest.TestCase):
 
         os.chdir(self.testdir)
 
-        local_repo_path = Path(self.testdir) / 'local'
-        local_repo = git.Repo.init(
-            local_repo_path, initial_branch=initial_branch
+        local_git_repo_path = Path(self.testdir) / 'local'
+        local_git_repo = git.Repo.init(
+            local_git_repo_path, initial_branch=initial_branch
         )
         # An initial commit of the git tree is required be for other operations
-        local_repo.index.commit('Initial commit')
+        local_git_repo.index.commit('Initial commit')
 
-        fdroid_dir = local_repo_path / 'fdroid'
+        fdroid_dir = local_git_repo_path / 'fdroid'
         repo_dir = fdroid_dir / repo_section
         repo_dir.mkdir(parents=True)
         for filename in fdroidserver.deploy.INDEX_FILES:
@@ -1044,23 +1044,37 @@ class DeployTest(unittest.TestCase):
                 fp.write('not a real one, but has the right filename')
 
         # The remote repo must be a bare repo to allow being pushed to
-        remote_repo_path = Path(self.testdir) / 'remote'
-        remote_repo = git.Repo.init(remote_repo_path, initial_branch=initial_branch, bare=True)
+        remote_git_repo_dir = Path(self.testdir) / 'remote'
+        remote_git_repo = git.Repo.init(
+            remote_git_repo_dir, initial_branch=initial_branch, bare=True
+        )
 
-        mirror_config = {"url": str(remote_repo_path), "index_only": True}
+        mirror_config = {"url": str(remote_git_repo_dir), "index_only": True}
         enabled_remotes = []
         ssh_cmd = 'ssh -oBatchMode=yes'
         fdroidserver.deploy.upload_to_servergitmirror(
             mirror_config=mirror_config,
-            local_repo=local_repo,
+            local_repo=local_git_repo,
             enabled_remotes=enabled_remotes,
             repo_section=repo_section,
             fdroid_dir=str(fdroid_dir),
-            git_mirror_path=str(local_repo_path),
+            git_mirror_path=str(local_git_repo_path),
             ssh_cmd=ssh_cmd,
-            progress=git.RemoteProgress()
+            progress=git.RemoteProgress(),
         )
-        assert remote_repo.is_dirty
+
+        verify_repo = remote_git_repo.clone(
+            Path(self.testdir) / 'verify',
+        )
+
+        for filename in fdroidserver.deploy.INDEX_FILES:
+            remote_file = f"fdroid/{repo_section}/{filename}"
+
+            self.assertIsNotNone(verify_repo.working_tree_dir)
+            if verify_repo.working_tree_dir is not None:
+                self.assertTrue(
+                    (Path(verify_repo.working_tree_dir) / remote_file).exists()
+                )
 
 
 if __name__ == "__main__":

From d619fbff26a56d403e14a4505d0d0b1788f9e7e5 Mon Sep 17 00:00:00 2001
From: proletarius101 <proletarius101@protonmail.com>
Date: Sun, 2 Jun 2024 19:39:05 +0800
Subject: [PATCH 50/68] chore(deploy): list all files in the local git repo

---
 fdroidserver/deploy.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py
index cdfa7222..9f827c3d 100644
--- a/fdroidserver/deploy.py
+++ b/fdroidserver/deploy.py
@@ -723,8 +723,9 @@ def upload_to_servergitmirror(mirror_config: Dict[str, str],
                               progress: git.RemoteProgress) -> None:
     # Test
     print(f"mirror_config: {mirror_config}")
+    # List all files in the git repo
     print("files:")
-    print(glob.glob('.' + '/**/*', recursive=True))
+    print(local_repo.git.ls_files())
 
     index_only = mirror_config.get('index_only', False)
 

From e8e8f8b9f01076682aa72ca9f44e88d02ab99b7e Mon Sep 17 00:00:00 2001
From: proletarius101 <proletarius101@protonmail.com>
Date: Sun, 2 Jun 2024 21:13:37 +0800
Subject: [PATCH 51/68] fix(deploy): remove unnecessary additional initial
 commit

and fix the test error: https://gitlab.com/proletarius101/fdroidserver/-/jobs/6997355250#L6978
---
 fdroidserver/deploy.py | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py
index 9f827c3d..848a2dc1 100644
--- a/fdroidserver/deploy.py
+++ b/fdroidserver/deploy.py
@@ -690,11 +690,6 @@ def update_servergitmirrors(servergitmirrors, repo_section):
         repo = git.Repo.init(git_mirror_path, initial_branch=GIT_BRANCH)
         initial_ref = repo.head.ref
 
-        # An initial commit of the git tree is required be for other operations
-        print("repo.index.entries:", len(repo.index.entries))
-        if len(repo.index.entries) == 0:
-            repo.index.commit('Initial commit')
-
         enabled_remotes = []
         for d in servergitmirrors:
             upload_to_servergitmirror(mirror_config=d,

From 04987a021b2fcb93020ae6460e08635f6e988e57 Mon Sep 17 00:00:00 2001
From: proletarius101 <proletarius101@protonmail.com>
Date: Sun, 2 Jun 2024 23:06:23 +0800
Subject: [PATCH 52/68] fix(deploy): local files are not synced to the local
 git repo

---
 fdroidserver/deploy.py |  25 ++++----
 tests/deploy.TestCase  | 132 +++++++++++++++++++----------------------
 2 files changed, 72 insertions(+), 85 deletions(-)

diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py
index 848a2dc1..eeec69e8 100644
--- a/fdroidserver/deploy.py
+++ b/fdroidserver/deploy.py
@@ -33,6 +33,7 @@ import logging
 from shlex import split
 import shutil
 import git
+from pathlib import Path
 
 from . import _
 from . import common
@@ -632,10 +633,11 @@ def update_servergitmirrors(servergitmirrors, repo_section):
         return
 
     options = common.get_options()
+    working_dir_root = Path(os.getcwd())
 
     # right now we support only 'repo' git-mirroring
     if repo_section == 'repo':
-        git_mirror_path = 'git-mirror'
+        git_mirror_path = working_dir_root / 'git-mirror'
         dotgit = os.path.join(git_mirror_path, '.git')
         git_fdroiddir = os.path.join(git_mirror_path, 'fdroid')
         git_repodir = os.path.join(git_fdroiddir, repo_section)
@@ -662,12 +664,6 @@ def update_servergitmirrors(servergitmirrors, repo_section):
             archive_path = os.path.join(git_mirror_path, 'fdroid', 'archive')
             shutil.rmtree(archive_path, ignore_errors=True)
 
-        # trailing slashes have a meaning in rsync which is not needed here, so
-        # make sure both paths have exactly one trailing slash
-        common.local_rsync(common.get_options(),
-                           [repo_section.rstrip('/') + '/'],
-                           git_repodir.rstrip('/') + '/')
-
         # use custom SSH command if identity_file specified
         ssh_cmd = 'ssh -oBatchMode=yes'
         if options.identity_file is not None:
@@ -692,18 +688,23 @@ def update_servergitmirrors(servergitmirrors, repo_section):
 
         enabled_remotes = []
         for d in servergitmirrors:
+            # trailing slashes have a meaning in rsync which is not needed here, so
+            # make sure both paths have exactly one trailing slash
+            common.local_rsync(common.get_options(),
+                            [str(working_dir_root / repo_section).rstrip('/') + '/'],
+                            git_repodir.rstrip('/') + '/')
+
             upload_to_servergitmirror(mirror_config=d,
                                       local_repo=repo,
                                       enabled_remotes=enabled_remotes,
                                       repo_section=repo_section,
                                       fdroid_dir=git_fdroiddir,
-                                      git_mirror_path=git_mirror_path,
+                                      git_mirror_path=str(git_mirror_path),
                                       ssh_cmd=ssh_cmd,
                                       progress=progress)
 
-            # Switch to the initial branch and unstage all files
+            # Switch to the initial branch
             repo.head.reference = initial_ref
-            repo.head.reset(index=True, working_tree=False)
         if progress:
             progressbar.done()
 
@@ -729,9 +730,7 @@ def upload_to_servergitmirror(mirror_config: Dict[str, str],
         local_branch_name = 'index_only'
     else:
         local_branch_name = 'full'
-    if local_branch_name not in local_repo.heads:
-        local_repo.create_head(local_branch_name)
-    local_repo.head.reference = local_repo.heads[local_branch_name]
+    local_repo.git.switch('-c', local_branch_name)
     remote_branch_name = GIT_BRANCH
 
     remote_url = mirror_config['url']
diff --git a/tests/deploy.TestCase b/tests/deploy.TestCase
index 2df20aae..9baaf5dc 100755
--- a/tests/deploy.TestCase
+++ b/tests/deploy.TestCase
@@ -909,50 +909,44 @@ class DeployTest(unittest.TestCase):
         config = {}
         fdroidserver.common.fill_config_defaults(config)
         fdroidserver.deploy.config = config
-        fdroidserver.deploy.config["servergitmirrors"] = []
+
+        os.chdir(self.testdir)
 
         repo_section = 'repo'
+        initial_branch = fdroidserver.deploy.GIT_BRANCH
 
-        # setup function for asserting subprocess.call invocations
-        call_iteration = 0
+        remote_repo = Path(self.testdir) / 'remote'
+        remote_repo.mkdir(parents=True)
+        remote_git_repo = git.Repo.init(
+            remote_repo, initial_branch=initial_branch, bare=True
+        )
+        fdroidserver.deploy.config["servergitmirrors"] = [{"url": str(remote_repo)}]
 
         os.chdir(self.testdir)
         repo = Path('repo')
         repo.mkdir(parents=True)
-        fake_apk = repo / 'Sym.apk'
-        with fake_apk.open('w') as fp:
-            fp.write('not an APK, but has the right filename')
-        fake_index = repo / fdroidserver.deploy.INDEX_FILES[0]
-        with fake_index.open('w') as fp:
-            fp.write('not an index, but has the right filename')
+        files_in_repo = fdroidserver.deploy.INDEX_FILES + ['Sym.apk']
+        for filename in files_in_repo:
+            fake_file = repo / filename
+            with fake_file.open('w') as fp:
+                fp.write('not a real one, but has the right filename')
 
-        def update_servergitmirrors_call(cmd):
-            nonlocal call_iteration
-            if call_iteration == 0:
-                self.assertListEqual(
-                    cmd,
-                    [
-                        'rsync',
-                        '--recursive',
-                        '--safe-links',
-                        '--times',
-                        '--perms',
-                        '--one-file-system',
-                        '--delete',
-                        '--chmod=Da+rx,Fa-x,a+r,u+w',
-                        '--quiet',
-                        'repo/',
-                        "git-mirror/fdroid/repo/",
-                    ],
+        fdroidserver.deploy.update_servergitmirrors(
+            fdroidserver.deploy.config["servergitmirrors"], repo_section
+        )
+
+        verify_repo = remote_git_repo.clone(
+            Path(self.testdir) / 'verify',
+        )
+
+        for filename in files_in_repo:
+            remote_file = f"fdroid/{repo_section}/{filename}"
+
+            self.assertIsNotNone(verify_repo.working_tree_dir)
+            if verify_repo.working_tree_dir is not None:
+                self.assertTrue(
+                    (Path(verify_repo.working_tree_dir) / remote_file).exists()
                 )
-            else:
-                self.fail('unexpected subprocess.call invocation')
-            call_iteration += 1
-            return 0
-
-        with mock.patch('subprocess.call', side_effect=update_servergitmirrors_call):
-            fdroidserver.deploy.update_servergitmirrors([], repo_section)
-        self.assertEqual(call_iteration, 1, 'expected 1 invocations of subprocess.call')
 
     def test_update_servergitmirrors_in_index_only_mode(self):
         # setup parameters for this test run
@@ -965,50 +959,46 @@ class DeployTest(unittest.TestCase):
         config = {}
         fdroidserver.common.fill_config_defaults(config)
         fdroidserver.deploy.config = config
-        fdroidserver.deploy.config["servergitmirrors"] = []
+
+        os.chdir(self.testdir)
 
         repo_section = 'repo'
+        initial_branch = fdroidserver.deploy.GIT_BRANCH
 
-        # setup function for asserting subprocess.call invocations
-        call_iteration = 0
+        remote_repo = Path(self.testdir) / 'remote'
+        remote_repo.mkdir(parents=True)
+        remote_git_repo = git.Repo.init(
+            remote_repo, initial_branch=initial_branch, bare=True
+        )
+        fdroidserver.deploy.config["servergitmirrors"] = [
+            {"url": str(remote_repo), "index_only": True}
+        ]
 
         os.chdir(self.testdir)
         repo = Path('repo')
         repo.mkdir(parents=True)
-        fake_apk = repo / 'Sym.apk'
-        with fake_apk.open('w') as fp:
-            fp.write('not an APK, but has the right filename')
-        fake_index = repo / fdroidserver.deploy.INDEX_FILES[0]
-        with fake_index.open('w') as fp:
-            fp.write('not an index, but has the right filename')
+        files_in_repo = fdroidserver.deploy.INDEX_FILES
+        for filename in files_in_repo:
+            fake_file = repo / filename
+            with fake_file.open('w') as fp:
+                fp.write('not a real one, but has the right filename')
 
-        def update_servergitmirrors_call(cmd):
-            nonlocal call_iteration
-            if call_iteration == 0:
-                self.assertListEqual(
-                    cmd,
-                    [
-                        'rsync',
-                        '--recursive',
-                        '--safe-links',
-                        '--times',
-                        '--perms',
-                        '--one-file-system',
-                        '--delete',
-                        '--chmod=Da+rx,Fa-x,a+r,u+w',
-                        '--quiet',
-                        'repo/',
-                        "git-mirror/fdroid/repo/",
-                    ],
+        fdroidserver.deploy.update_servergitmirrors(
+            fdroidserver.deploy.config["servergitmirrors"], repo_section
+        )
+
+        verify_repo = remote_git_repo.clone(
+            Path(self.testdir) / 'verify',
+        )
+
+        for filename in files_in_repo:
+            remote_file = f"fdroid/{repo_section}/{filename}"
+
+            self.assertIsNotNone(verify_repo.working_tree_dir)
+            if verify_repo.working_tree_dir is not None:
+                self.assertTrue(
+                    (Path(verify_repo.working_tree_dir) / remote_file).exists()
                 )
-            else:
-                self.fail('unexpected subprocess.call invocation')
-            call_iteration += 1
-            return 0
-
-        with mock.patch('subprocess.call', side_effect=update_servergitmirrors_call):
-            fdroidserver.deploy.update_servergitmirrors([], repo_section)
-        self.assertEqual(call_iteration, 1, 'expected 1 invocations of subprocess.call')
 
     def test_upload_to_servergitmirror_in_index_only_mode(self):
         # setup parameters for this test run
@@ -1032,8 +1022,6 @@ class DeployTest(unittest.TestCase):
         local_git_repo = git.Repo.init(
             local_git_repo_path, initial_branch=initial_branch
         )
-        # An initial commit of the git tree is required be for other operations
-        local_git_repo.index.commit('Initial commit')
 
         fdroid_dir = local_git_repo_path / 'fdroid'
         repo_dir = fdroid_dir / repo_section

From 4941e699df65d0a4e58a67d51a654aad1daf186e Mon Sep 17 00:00:00 2001
From: proletarius101 <proletarius101@protonmail.com>
Date: Sun, 2 Jun 2024 23:27:58 +0800
Subject: [PATCH 53/68] test(deploy): make sure the APK file is not uploaded in
 the index only mode

---
 tests/deploy.TestCase | 32 +++++++++++++++++++++++++-------
 1 file changed, 25 insertions(+), 7 deletions(-)

diff --git a/tests/deploy.TestCase b/tests/deploy.TestCase
index 9baaf5dc..5cbd978c 100755
--- a/tests/deploy.TestCase
+++ b/tests/deploy.TestCase
@@ -925,8 +925,9 @@ class DeployTest(unittest.TestCase):
         os.chdir(self.testdir)
         repo = Path('repo')
         repo.mkdir(parents=True)
-        files_in_repo = fdroidserver.deploy.INDEX_FILES + ['Sym.apk']
-        for filename in files_in_repo:
+        fake_apk = 'Sym.apk'
+        fake_files = fdroidserver.deploy.INDEX_FILES + [fake_apk]
+        for filename in fake_files:
             fake_file = repo / filename
             with fake_file.open('w') as fp:
                 fp.write('not a real one, but has the right filename')
@@ -939,7 +940,7 @@ class DeployTest(unittest.TestCase):
             Path(self.testdir) / 'verify',
         )
 
-        for filename in files_in_repo:
+        for filename in fake_files:
             remote_file = f"fdroid/{repo_section}/{filename}"
 
             self.assertIsNotNone(verify_repo.working_tree_dir)
@@ -977,8 +978,9 @@ class DeployTest(unittest.TestCase):
         os.chdir(self.testdir)
         repo = Path('repo')
         repo.mkdir(parents=True)
-        files_in_repo = fdroidserver.deploy.INDEX_FILES
-        for filename in files_in_repo:
+        fake_apk = 'Sym.apk'
+        fake_files = fdroidserver.deploy.INDEX_FILES + [fake_apk]
+        for filename in fake_files:
             fake_file = repo / filename
             with fake_file.open('w') as fp:
                 fp.write('not a real one, but has the right filename')
@@ -991,7 +993,7 @@ class DeployTest(unittest.TestCase):
             Path(self.testdir) / 'verify',
         )
 
-        for filename in files_in_repo:
+        for filename in fdroidserver.deploy.INDEX_FILES:
             remote_file = f"fdroid/{repo_section}/{filename}"
 
             self.assertIsNotNone(verify_repo.working_tree_dir)
@@ -1000,6 +1002,13 @@ class DeployTest(unittest.TestCase):
                     (Path(verify_repo.working_tree_dir) / remote_file).exists()
                 )
 
+        # Should not have the APK file
+        remote_file = f"fdroid/{repo_section}/{fake_apk}"
+        if verify_repo.working_tree_dir is not None:
+            self.assertFalse(
+                (Path(verify_repo.working_tree_dir) / remote_file).exists()
+            )
+
     def test_upload_to_servergitmirror_in_index_only_mode(self):
         # setup parameters for this test run
         fdroidserver.deploy.options = mock.Mock()
@@ -1026,7 +1035,9 @@ class DeployTest(unittest.TestCase):
         fdroid_dir = local_git_repo_path / 'fdroid'
         repo_dir = fdroid_dir / repo_section
         repo_dir.mkdir(parents=True)
-        for filename in fdroidserver.deploy.INDEX_FILES:
+        fake_apk = 'Sym.apk'
+        fake_files = fdroidserver.deploy.INDEX_FILES + [fake_apk]
+        for filename in fake_files:
             fake_file = repo_dir / filename
             with fake_file.open('w') as fp:
                 fp.write('not a real one, but has the right filename')
@@ -1064,6 +1075,13 @@ class DeployTest(unittest.TestCase):
                     (Path(verify_repo.working_tree_dir) / remote_file).exists()
                 )
 
+        # Should not have the APK file
+        remote_file = f"fdroid/{repo_section}/{fake_apk}"
+        if verify_repo.working_tree_dir is not None:
+            self.assertFalse(
+                (Path(verify_repo.working_tree_dir) / remote_file).exists()
+            )
+
 
 if __name__ == "__main__":
     os.chdir(os.path.dirname(__file__))

From 9c30b087c4454550ec61c6e76c5acbbda4dc9393 Mon Sep 17 00:00:00 2001
From: proletarius101 <proletarius101@protonmail.com>
Date: Sun, 2 Jun 2024 23:37:06 +0800
Subject: [PATCH 54/68] fix(deploy): fatal: A branch named 'full' already
 exists.

---
 fdroidserver/deploy.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py
index eeec69e8..545ba867 100644
--- a/fdroidserver/deploy.py
+++ b/fdroidserver/deploy.py
@@ -730,7 +730,10 @@ def upload_to_servergitmirror(mirror_config: Dict[str, str],
         local_branch_name = 'index_only'
     else:
         local_branch_name = 'full'
-    local_repo.git.switch('-c', local_branch_name)
+    if local_branch_name in local_repo.heads:
+        local_repo.git.switch(local_branch_name)
+    else:
+        local_repo.git.switch('-c', local_branch_name)
     remote_branch_name = GIT_BRANCH
 
     remote_url = mirror_config['url']

From c190b7fb560a905673ddb9e5ccbbc6d98f7908ec Mon Sep 17 00:00:00 2001
From: proletarius101 <proletarius101@protonmail.com>
Date: Mon, 3 Jun 2024 00:24:19 +0800
Subject: [PATCH 55/68] debug

---
 fdroidserver/deploy.py | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py
index 545ba867..0b48496d 100644
--- a/fdroidserver/deploy.py
+++ b/fdroidserver/deploy.py
@@ -748,6 +748,7 @@ def upload_to_servergitmirror(mirror_config: Dict[str, str],
         local_repo.create_remote(name, remote_url)
     logging.info('Mirroring to: ' + remote_url)
 
+    print("git status:", local_repo.git.status())
     if index_only:
         # test
         logging.debug('Adding index files to git mirror')
@@ -757,14 +758,11 @@ def upload_to_servergitmirror(mirror_config: Dict[str, str],
         logging.debug('Adding all files to git mirror')
         local_repo.git.add(all=True)
 
+    print("git status:", local_repo.git.status())
+
     logging.debug('Committing files into git mirror')
     local_repo.index.commit("fdroidserver git-mirror")
 
-    # Test
-    print(f"In index-only: {index_only} mode")
-    print(local_repo.git.status())
-    print(local_repo.head.log())
-
     # only deploy to GitLab Artifacts if too big for GitLab Pages
     if common.get_dir_size(fdroid_dir) <= common.GITLAB_COM_PAGES_MAX_SIZE:
         gitlab_ci_job_name = 'pages'

From bd2eaa58684d483eddc9fe99190c44e3a36ad45c Mon Sep 17 00:00:00 2001
From: proletarius101 <proletarius101@protonmail.com>
Date: Mon, 3 Jun 2024 00:30:57 +0800
Subject: [PATCH 56/68] fix(deploy): all files are uploaded if the index only
 mode deployment follows a full mode one

---
 fdroidserver/deploy.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py
index 0b48496d..517a9439 100644
--- a/fdroidserver/deploy.py
+++ b/fdroidserver/deploy.py
@@ -684,7 +684,6 @@ def update_servergitmirrors(servergitmirrors, repo_section):
             progress = None
 
         repo = git.Repo.init(git_mirror_path, initial_branch=GIT_BRANCH)
-        initial_ref = repo.head.ref
 
         enabled_remotes = []
         for d in servergitmirrors:
@@ -704,7 +703,7 @@ def update_servergitmirrors(servergitmirrors, repo_section):
                                       progress=progress)
 
             # Switch to the initial branch
-            repo.head.reference = initial_ref
+            repo.git.checkout(GIT_BRANCH)
         if progress:
             progressbar.done()
 
@@ -758,11 +757,14 @@ def upload_to_servergitmirror(mirror_config: Dict[str, str],
         logging.debug('Adding all files to git mirror')
         local_repo.git.add(all=True)
 
-    print("git status:", local_repo.git.status())
-
     logging.debug('Committing files into git mirror')
     local_repo.index.commit("fdroidserver git-mirror")
 
+    # Test
+    print(f"In index-only: {index_only} mode")
+    print(local_repo.git.status())
+    print(local_repo.head.log())
+
     # only deploy to GitLab Artifacts if too big for GitLab Pages
     if common.get_dir_size(fdroid_dir) <= common.GITLAB_COM_PAGES_MAX_SIZE:
         gitlab_ci_job_name = 'pages'

From b74c848d6edc4455b83d4f344f1b2ee2073a670f Mon Sep 17 00:00:00 2001
From: proletarius101 <proletarius101@protonmail.com>
Date: Mon, 3 Jun 2024 00:45:45 +0800
Subject: [PATCH 57/68] fix(deploy): all files are uploaded if the index only
 mode deployment follows a full mode one

let rsync only sync necessary files
---
 fdroidserver/deploy.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py
index 517a9439..b00fbdc7 100644
--- a/fdroidserver/deploy.py
+++ b/fdroidserver/deploy.py
@@ -689,8 +689,12 @@ def update_servergitmirrors(servergitmirrors, repo_section):
         for d in servergitmirrors:
             # trailing slashes have a meaning in rsync which is not needed here, so
             # make sure both paths have exactly one trailing slash
+            if d.get('index_only', False):
+                files_to_sync = [str(working_dir_root / repo_section / index_file) for index_file in INDEX_FILES]
+            else:
+                files_to_sync = [str(working_dir_root / repo_section).rstrip('/') + '/']
             common.local_rsync(common.get_options(),
-                            [str(working_dir_root / repo_section).rstrip('/') + '/'],
+                            files_to_sync,
                             git_repodir.rstrip('/') + '/')
 
             upload_to_servergitmirror(mirror_config=d,

From 5ff6cdd53b0c5020fdc9752c6d7c92dc9d0b247a Mon Sep 17 00:00:00 2001
From: proletarius101 <proletarius101@protonmail.com>
Date: Mon, 3 Jun 2024 00:51:54 +0800
Subject: [PATCH 58/68] fix(deploy): all files are uploaded if the index only
 mode deployment follows a full mode one

create new branches from groud up
---
 fdroidserver/deploy.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py
index b00fbdc7..87f4bb4c 100644
--- a/fdroidserver/deploy.py
+++ b/fdroidserver/deploy.py
@@ -707,7 +707,7 @@ def update_servergitmirrors(servergitmirrors, repo_section):
                                       progress=progress)
 
             # Switch to the initial branch
-            repo.git.checkout(GIT_BRANCH)
+            repo.git.switch(GIT_BRANCH)
         if progress:
             progressbar.done()
 
@@ -736,7 +736,7 @@ def upload_to_servergitmirror(mirror_config: Dict[str, str],
     if local_branch_name in local_repo.heads:
         local_repo.git.switch(local_branch_name)
     else:
-        local_repo.git.switch('-c', local_branch_name)
+        local_repo.git.switch('--orphan', local_branch_name)
     remote_branch_name = GIT_BRANCH
 
     remote_url = mirror_config['url']

From 77c3f83fd712bba6bf214d143c6a55193122aa9c Mon Sep 17 00:00:00 2001
From: proletarius101 <proletarius101@protonmail.com>
Date: Mon, 3 Jun 2024 00:58:17 +0800
Subject: [PATCH 59/68] chore(deploy): convert print to logging for in-order
 output

---
 fdroidserver/deploy.py | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py
index 87f4bb4c..42a7c509 100644
--- a/fdroidserver/deploy.py
+++ b/fdroidserver/deploy.py
@@ -16,6 +16,7 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+from math import log
 import sys
 import glob
 import hashlib
@@ -721,10 +722,10 @@ def upload_to_servergitmirror(mirror_config: Dict[str, str],
                               ssh_cmd: str,
                               progress: git.RemoteProgress) -> None:
     # Test
-    print(f"mirror_config: {mirror_config}")
+    logging.info(f"mirror_config: {mirror_config}")
     # List all files in the git repo
-    print("files:")
-    print(local_repo.git.ls_files())
+    logging.info("files:")
+    logging.info(local_repo.git.ls_files())
 
     index_only = mirror_config.get('index_only', False)
 
@@ -751,7 +752,7 @@ def upload_to_servergitmirror(mirror_config: Dict[str, str],
         local_repo.create_remote(name, remote_url)
     logging.info('Mirroring to: ' + remote_url)
 
-    print("git status:", local_repo.git.status())
+    logging.info("git status:", local_repo.git.status())
     if index_only:
         # test
         logging.debug('Adding index files to git mirror')
@@ -765,9 +766,9 @@ def upload_to_servergitmirror(mirror_config: Dict[str, str],
     local_repo.index.commit("fdroidserver git-mirror")
 
     # Test
-    print(f"In index-only: {index_only} mode")
-    print(local_repo.git.status())
-    print(local_repo.head.log())
+    logging.info(f"In index-only: {index_only} mode")
+    logging.info(local_repo.git.status())
+    logging.info(local_repo.head.log())
 
     # only deploy to GitLab Artifacts if too big for GitLab Pages
     if common.get_dir_size(fdroid_dir) <= common.GITLAB_COM_PAGES_MAX_SIZE:

From 01990411c7828664e342d6db21ebf18d568eff86 Mon Sep 17 00:00:00 2001
From: proletarius101 <proletarius101@protonmail.com>
Date: Mon, 3 Jun 2024 21:09:03 +0800
Subject: [PATCH 60/68] fix(deploy): all files are uploaded if the index only
 mode deployment follows a full mode one

switch the branch before rsyncing
---
 fdroidserver/deploy.py | 32 ++++++++++++++++++--------------
 tests/deploy.TestCase  |  1 +
 2 files changed, 19 insertions(+), 14 deletions(-)

diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py
index 42a7c509..79e7a8af 100644
--- a/fdroidserver/deploy.py
+++ b/fdroidserver/deploy.py
@@ -688,9 +688,21 @@ def update_servergitmirrors(servergitmirrors, repo_section):
 
         enabled_remotes = []
         for d in servergitmirrors:
+            is_index_only = d.get('index_only', False)
+
+            # Use a separate branch for the index only mode as it needs a different set of files to commit
+            if is_index_only:
+                local_branch_name = 'index_only'
+            else:
+                local_branch_name = 'full'
+            if local_branch_name in repo.heads:
+                repo.git.switch(local_branch_name)
+            else:
+                repo.git.switch('--orphan', local_branch_name)
+
             # trailing slashes have a meaning in rsync which is not needed here, so
             # make sure both paths have exactly one trailing slash
-            if d.get('index_only', False):
+            if is_index_only:
                 files_to_sync = [str(working_dir_root / repo_section / index_file) for index_file in INDEX_FILES]
             else:
                 files_to_sync = [str(working_dir_root / repo_section).rstrip('/') + '/']
@@ -702,6 +714,7 @@ def update_servergitmirrors(servergitmirrors, repo_section):
                                       local_repo=repo,
                                       enabled_remotes=enabled_remotes,
                                       repo_section=repo_section,
+                                      is_index_only=is_index_only,
                                       fdroid_dir=git_fdroiddir,
                                       git_mirror_path=str(git_mirror_path),
                                       ssh_cmd=ssh_cmd,
@@ -717,6 +730,7 @@ def upload_to_servergitmirror(mirror_config: Dict[str, str],
                               local_repo: Repo,
                               enabled_remotes: List[str],
                               repo_section: str,
+                              is_index_only: bool,
                               fdroid_dir: str,
                               git_mirror_path: str,
                               ssh_cmd: str,
@@ -727,18 +741,8 @@ def upload_to_servergitmirror(mirror_config: Dict[str, str],
     logging.info("files:")
     logging.info(local_repo.git.ls_files())
 
-    index_only = mirror_config.get('index_only', False)
-
-    # Use a separate branch for the index only mode as it needs a different set of files to commit
-    if index_only:
-        local_branch_name = 'index_only'
-    else:
-        local_branch_name = 'full'
-    if local_branch_name in local_repo.heads:
-        local_repo.git.switch(local_branch_name)
-    else:
-        local_repo.git.switch('--orphan', local_branch_name)
     remote_branch_name = GIT_BRANCH
+    local_branch_name = local_repo.active_branch.name
 
     remote_url = mirror_config['url']
     name = REMOTE_HOSTNAME_REGEX.sub(r'\1', remote_url)
@@ -753,7 +757,7 @@ def upload_to_servergitmirror(mirror_config: Dict[str, str],
     logging.info('Mirroring to: ' + remote_url)
 
     logging.info("git status:", local_repo.git.status())
-    if index_only:
+    if is_index_only:
         # test
         logging.debug('Adding index files to git mirror')
         local_repo.index.add(_get_index_file_paths(os.path.join('fdroid', repo_section)))
@@ -766,7 +770,7 @@ def upload_to_servergitmirror(mirror_config: Dict[str, str],
     local_repo.index.commit("fdroidserver git-mirror")
 
     # Test
-    logging.info(f"In index-only: {index_only} mode")
+    logging.info(f"In index-only: {is_index_only} mode")
     logging.info(local_repo.git.status())
     logging.info(local_repo.head.log())
 
diff --git a/tests/deploy.TestCase b/tests/deploy.TestCase
index 5cbd978c..b997dad2 100755
--- a/tests/deploy.TestCase
+++ b/tests/deploy.TestCase
@@ -1056,6 +1056,7 @@ class DeployTest(unittest.TestCase):
             local_repo=local_git_repo,
             enabled_remotes=enabled_remotes,
             repo_section=repo_section,
+            is_index_only=mirror_config['index_only'],
             fdroid_dir=str(fdroid_dir),
             git_mirror_path=str(local_git_repo_path),
             ssh_cmd=ssh_cmd,

From f321f629049862b0db282b86037b775fc20f3c4d Mon Sep 17 00:00:00 2001
From: proletarius101 <proletarius101@protonmail.com>
Date: Wed, 5 Jun 2024 02:38:14 +0800
Subject: [PATCH 61/68] style(deploy): fix pyflake

---
 fdroidserver/deploy.py | 1 -
 tests/deploy.TestCase  | 1 -
 2 files changed, 2 deletions(-)

diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py
index 79e7a8af..739defdf 100644
--- a/fdroidserver/deploy.py
+++ b/fdroidserver/deploy.py
@@ -16,7 +16,6 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-from math import log
 import sys
 import glob
 import hashlib
diff --git a/tests/deploy.TestCase b/tests/deploy.TestCase
index b997dad2..c5fd9b0d 100755
--- a/tests/deploy.TestCase
+++ b/tests/deploy.TestCase
@@ -7,7 +7,6 @@ import os
 import shutil
 import sys
 import tempfile
-from unicodedata import mirrored
 import unittest
 from pathlib import Path
 from unittest import mock

From f27ecd1674e846dcae983a1a076994f54fb5f0fb Mon Sep 17 00:00:00 2001
From: proletarius101 <proletarius101@protonmail.com>
Date: Wed, 5 Jun 2024 03:52:02 +0800
Subject: [PATCH 62/68] style(deploy): rename `working_dir_root` variable to
 `workspace_dir`

---
 fdroidserver/deploy.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py
index 739defdf..4c79977b 100644
--- a/fdroidserver/deploy.py
+++ b/fdroidserver/deploy.py
@@ -633,11 +633,11 @@ def update_servergitmirrors(servergitmirrors, repo_section):
         return
 
     options = common.get_options()
-    working_dir_root = Path(os.getcwd())
+    workspace_dir = Path(os.getcwd())
 
     # right now we support only 'repo' git-mirroring
     if repo_section == 'repo':
-        git_mirror_path = working_dir_root / 'git-mirror'
+        git_mirror_path = workspace_dir / 'git-mirror'
         dotgit = os.path.join(git_mirror_path, '.git')
         git_fdroiddir = os.path.join(git_mirror_path, 'fdroid')
         git_repodir = os.path.join(git_fdroiddir, repo_section)
@@ -702,9 +702,9 @@ def update_servergitmirrors(servergitmirrors, repo_section):
             # trailing slashes have a meaning in rsync which is not needed here, so
             # make sure both paths have exactly one trailing slash
             if is_index_only:
-                files_to_sync = [str(working_dir_root / repo_section / index_file) for index_file in INDEX_FILES]
+                files_to_sync = [str(workspace_dir / repo_section / index_file) for index_file in INDEX_FILES]
             else:
-                files_to_sync = [str(working_dir_root / repo_section).rstrip('/') + '/']
+                files_to_sync = [str(workspace_dir / repo_section).rstrip('/') + '/']
             common.local_rsync(common.get_options(),
                             files_to_sync,
                             git_repodir.rstrip('/') + '/')
@@ -809,7 +809,7 @@ def upload_to_servergitmirror(mirror_config: Dict[str, str],
         local_repo.index.add(['.gitlab-ci.yml'])
         local_repo.index.commit("fdroidserver git-mirror: Deploy to GitLab Pages")
 
-    logging.debug(_('Pushing to {url}').format(url=remote.url))
+    logging.info(_('Pushing to {url}').format(url=remote.url))
     with local_repo.git.custom_environment(GIT_SSH_COMMAND=ssh_cmd):
         pushinfos = remote.push(
             f"{local_branch_name}:{remote_branch_name}", force=True, set_upstream=True, progress=progress

From b1642410264905f7563ef68deda440a06a6a83ca Mon Sep 17 00:00:00 2001
From: proletarius101 <proletarius101@protonmail.com>
Date: Tue, 4 Jun 2024 20:10:13 +0800
Subject: [PATCH 63/68] fix(deploy):
 `$GIT_MIRROR/fdroid/repo/com.politedroid_6.apk` doesn't exist when the test
 is finished

---
 fdroidserver/deploy.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py
index 4c79977b..703bba35 100644
--- a/fdroidserver/deploy.py
+++ b/fdroidserver/deploy.py
@@ -718,9 +718,6 @@ def update_servergitmirrors(servergitmirrors, repo_section):
                                       git_mirror_path=str(git_mirror_path),
                                       ssh_cmd=ssh_cmd,
                                       progress=progress)
-
-            # Switch to the initial branch
-            repo.git.switch(GIT_BRANCH)
         if progress:
             progressbar.done()
 

From 9244189ce5079978c14def8db3ac54a96b70d1b1 Mon Sep 17 00:00:00 2001
From: proletarius101 <proletarius101@protonmail.com>
Date: Tue, 4 Jun 2024 20:22:56 +0800
Subject: [PATCH 64/68] fix(deploy): remove the `--index-only` flag

---
 fdroidserver/deploy.py | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py
index 703bba35..24b5dbb2 100644
--- a/fdroidserver/deploy.py
+++ b/fdroidserver/deploy.py
@@ -1120,12 +1120,6 @@ def main():
         default=False,
         help=_("If a git mirror gets to big, allow the archive to be deleted"),
     )
-    parser.add_argument(
-        "--index-only",
-        action="store_true",
-        default=False,
-        help="Only deploy the index files entry.* and index-v*.json",
-    )
     options = parser.parse_args()
     config = common.read_config(options)
 

From b48e297ec2376c1ca87770d43408b7c3b145d4b7 Mon Sep 17 00:00:00 2001
From: proletarius101 <proletarius101@protonmail.com>
Date: Tue, 4 Jun 2024 20:27:01 +0800
Subject: [PATCH 65/68] chore(deploy): clean up code

---
 fdroidserver/deploy.py | 12 ------------
 1 file changed, 12 deletions(-)

diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py
index 24b5dbb2..de0a7fab 100644
--- a/fdroidserver/deploy.py
+++ b/fdroidserver/deploy.py
@@ -731,12 +731,6 @@ def upload_to_servergitmirror(mirror_config: Dict[str, str],
                               git_mirror_path: str,
                               ssh_cmd: str,
                               progress: git.RemoteProgress) -> None:
-    # Test
-    logging.info(f"mirror_config: {mirror_config}")
-    # List all files in the git repo
-    logging.info("files:")
-    logging.info(local_repo.git.ls_files())
-
     remote_branch_name = GIT_BRANCH
     local_branch_name = local_repo.active_branch.name
 
@@ -754,8 +748,6 @@ def upload_to_servergitmirror(mirror_config: Dict[str, str],
 
     logging.info("git status:", local_repo.git.status())
     if is_index_only:
-        # test
-        logging.debug('Adding index files to git mirror')
         local_repo.index.add(_get_index_file_paths(os.path.join('fdroid', repo_section)))
     else:
         # sadly index.add don't allow the --all parameter
@@ -765,10 +757,6 @@ def upload_to_servergitmirror(mirror_config: Dict[str, str],
     logging.debug('Committing files into git mirror')
     local_repo.index.commit("fdroidserver git-mirror")
 
-    # Test
-    logging.info(f"In index-only: {is_index_only} mode")
-    logging.info(local_repo.git.status())
-    logging.info(local_repo.head.log())
 
     # only deploy to GitLab Artifacts if too big for GitLab Pages
     if common.get_dir_size(fdroid_dir) <= common.GITLAB_COM_PAGES_MAX_SIZE:

From 4c0e096a99a5099eac453054305c18ea5d9d3646 Mon Sep 17 00:00:00 2001
From: proletarius101 <proletarius101@protonmail.com>
Date: Tue, 4 Jun 2024 20:30:35 +0800
Subject: [PATCH 66/68] chore(deploy): rename `index_only` variable to
 `is_index_only` for Python idiomaticness

---
 fdroidserver/deploy.py | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py
index de0a7fab..1279683e 100644
--- a/fdroidserver/deploy.py
+++ b/fdroidserver/deploy.py
@@ -97,7 +97,7 @@ def _get_index_includes(base_dir):
     return index_includes
 
 
-def update_awsbucket(repo_section, index_only=False, verbose=False, quiet=False):
+def update_awsbucket(repo_section, is_index_only=False, verbose=False, quiet=False):
     """Upload the contents of the directory `repo_section` (including subdirectories) to the AWS S3 "bucket".
 
     The contents of that subdir of the
@@ -117,26 +117,26 @@ def update_awsbucket(repo_section, index_only=False, verbose=False, quiet=False)
             logging.warning(
                 'No syncing tool set in config.yml!. Defaulting to using s3cmd'
             )
-            update_awsbucket_s3cmd(repo_section, index_only)
+            update_awsbucket_s3cmd(repo_section, is_index_only)
         if config['s3cmd'] is True and config['rclone'] is True:
             logging.warning(
                 'Both syncing tools set in config.yml!. Defaulting to using s3cmd'
             )
-            update_awsbucket_s3cmd(repo_section, index_only)
+            update_awsbucket_s3cmd(repo_section, is_index_only)
         if config['s3cmd'] is True and config['rclone'] is not True:
-            update_awsbucket_s3cmd(repo_section, index_only)
+            update_awsbucket_s3cmd(repo_section, is_index_only)
         if config['rclone'] is True and config['s3cmd'] is not True:
             update_remote_storage_with_rclone(repo_section, verbose, quiet)
 
     elif common.set_command_in_config('s3cmd'):
-        update_awsbucket_s3cmd(repo_section, index_only)
+        update_awsbucket_s3cmd(repo_section, is_index_only)
     elif common.set_command_in_config('rclone'):
         update_remote_storage_with_rclone(repo_section, verbose, quiet)
     else:
-        update_awsbucket_libcloud(repo_section, index_only)
+        update_awsbucket_libcloud(repo_section, is_index_only)
 
 
-def update_awsbucket_s3cmd(repo_section, index_only=False):
+def update_awsbucket_s3cmd(repo_section, is_index_only=False):
     """Upload using the CLI tool s3cmd, which provides rsync-like sync.
 
     The upload is done in multiple passes to reduce the chance of
@@ -190,7 +190,7 @@ def update_awsbucket_s3cmd(repo_section, index_only=False):
         )
     )
 
-    if index_only:
+    if is_index_only:
         logging.debug(_('s3cmd syncs indexes from {path} to {url} and deletes removed')
                       .format(path=repo_section, url=s3url))
         sync_indexes_flags = []
@@ -357,7 +357,7 @@ def update_remote_storage_with_rclone(repo_section, verbose=False, quiet=False):
                 raise FDroidException()
 
 
-def update_awsbucket_libcloud(repo_section, index_only=False):
+def update_awsbucket_libcloud(repo_section, is_index_only=False):
     """No summary.
 
     Upload the contents of the directory `repo_section` (including
@@ -405,7 +405,7 @@ def update_awsbucket_libcloud(repo_section, index_only=False):
         if obj.name.startswith(upload_dir + '/'):
             objs[obj.name] = obj
 
-    if index_only:
+    if is_index_only:
         index_files = [f"{os.getcwd()}/{name}" for name in _get_index_file_paths(repo_section)]
         files_to_upload = [os.path.join(root, name) for root, dirs, files in os.walk(os.path.join(os.getcwd(), repo_section)) for name in files]
         files_to_upload = list(set(files_to_upload) & set(index_files))
@@ -507,9 +507,9 @@ def update_serverwebroot(serverwebroot, repo_section):
             'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ' + config['identity_file'],
         ]
     url = serverwebroot['url']
-    index_only = serverwebroot.get('index_only', False)
+    is_index_only = serverwebroot.get('index_only', False)
     logging.info('rsyncing ' + repo_section + ' to ' + url)
-    if index_only:
+    if is_index_only:
         rsyncargs += _get_index_file_paths(repo_section)
         rsyncargs += [f'{url}/{repo_section}/']
         logging.info(rsyncargs)

From cb18d101976cce00166d08f5a11cd9ce97556a2c Mon Sep 17 00:00:00 2001
From: proletarius101 <proletarius101@protonmail.com>
Date: Tue, 4 Jun 2024 21:15:21 +0800
Subject: [PATCH 67/68] feat(deploy): add index-only mode to the rclone
 deployment method

---
 fdroidserver/deploy.py | 94 ++++++++++++++++++++++--------------------
 tests/deploy.TestCase  | 66 +++++++++++++++++++++++++----
 2 files changed, 109 insertions(+), 51 deletions(-)

diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py
index 1279683e..afed4e30 100644
--- a/fdroidserver/deploy.py
+++ b/fdroidserver/deploy.py
@@ -126,12 +126,12 @@ def update_awsbucket(repo_section, is_index_only=False, verbose=False, quiet=Fal
         if config['s3cmd'] is True and config['rclone'] is not True:
             update_awsbucket_s3cmd(repo_section, is_index_only)
         if config['rclone'] is True and config['s3cmd'] is not True:
-            update_remote_storage_with_rclone(repo_section, verbose, quiet)
+            update_remote_storage_with_rclone(repo_section, is_index_only, verbose, quiet)
 
     elif common.set_command_in_config('s3cmd'):
         update_awsbucket_s3cmd(repo_section, is_index_only)
     elif common.set_command_in_config('rclone'):
-        update_remote_storage_with_rclone(repo_section, verbose, quiet)
+        update_remote_storage_with_rclone(repo_section, is_index_only, verbose, quiet)
     else:
         update_awsbucket_libcloud(repo_section, is_index_only)
 
@@ -234,7 +234,7 @@ def update_awsbucket_s3cmd(repo_section, is_index_only=False):
             raise FDroidException()
 
 
-def update_remote_storage_with_rclone(repo_section, verbose=False, quiet=False):
+def update_remote_storage_with_rclone(repo_section, is_index_only=False, verbose=False, quiet=False):
     """
     Upload fdroid repo folder to remote storage using rclone sync.
 
@@ -286,46 +286,18 @@ def update_remote_storage_with_rclone(repo_section, verbose=False, quiet=False):
             _('To use rclone, rclone_config and awsbucket must be set in config.yml!')
         )
 
-    if isinstance(config['rclone_config'], str):
-        rclone_sync_command = (
-            'rclone sync '
-            + repo_section
-            + ' '
-            + config['rclone_config']
-            + ':'
-            + config['awsbucket']
-            + '/'
-            + upload_dir
-        )
+    if is_index_only:
+        sources = _get_index_file_paths(repo_section)
+    else:
+        sources = repo_section
 
-        rclone_sync_command = split(rclone_sync_command)
-
-        if verbose:
-            rclone_sync_command += ['--verbose']
-        elif quiet:
-            rclone_sync_command += ['--quiet']
-
-        if configfilename:
-            rclone_sync_command += split('--config=' + configfilename)
-
-        complete_remote_path = (
-            config['rclone_config'] + ':' + config['awsbucket'] + '/' + upload_dir
-        )
-
-        logging.debug(
-            "rclone sync all files in " + repo_section + ' to ' + complete_remote_path
-        )
-
-        if subprocess.call(rclone_sync_command) != 0:
-            raise FDroidException()
-
-    if isinstance(config['rclone_config'], list):
-        for remote_config in config['rclone_config']:
+    for source in sources:
+        if isinstance(config['rclone_config'], str):
             rclone_sync_command = (
                 'rclone sync '
-                + repo_section
+                + source
                 + ' '
-                + remote_config
+                + config['rclone_config']
                 + ':'
                 + config['awsbucket']
                 + '/'
@@ -343,19 +315,53 @@ def update_remote_storage_with_rclone(repo_section, verbose=False, quiet=False):
                 rclone_sync_command += split('--config=' + configfilename)
 
             complete_remote_path = (
-                remote_config + ':' + config['awsbucket'] + '/' + upload_dir
+                config['rclone_config'] + ':' + config['awsbucket'] + '/' + upload_dir
             )
 
             logging.debug(
-                "rclone sync all files in "
-                + repo_section
-                + ' to '
-                + complete_remote_path
+                "rclone sync all files in " + source + ' to ' + complete_remote_path
             )
 
             if subprocess.call(rclone_sync_command) != 0:
                 raise FDroidException()
 
+        if isinstance(config['rclone_config'], list):
+            for remote_config in config['rclone_config']:
+                rclone_sync_command = (
+                    'rclone sync '
+                    + source
+                    + ' '
+                    + remote_config
+                    + ':'
+                    + config['awsbucket']
+                    + '/'
+                    + upload_dir
+                )
+
+                rclone_sync_command = split(rclone_sync_command)
+
+                if verbose:
+                    rclone_sync_command += ['--verbose']
+                elif quiet:
+                    rclone_sync_command += ['--quiet']
+
+                if configfilename:
+                    rclone_sync_command += split('--config=' + configfilename)
+
+                complete_remote_path = (
+                    remote_config + ':' + config['awsbucket'] + '/' + upload_dir
+                )
+
+                logging.debug(
+                    "rclone sync all files in "
+                    + source
+                    + ' to '
+                    + complete_remote_path
+                )
+
+                if subprocess.call(rclone_sync_command) != 0:
+                    raise FDroidException()
+
 
 def update_awsbucket_libcloud(repo_section, is_index_only=False):
     """No summary.
diff --git a/tests/deploy.TestCase b/tests/deploy.TestCase
index c5fd9b0d..57f15392 100755
--- a/tests/deploy.TestCase
+++ b/tests/deploy.TestCase
@@ -113,6 +113,10 @@ class DeployTest(unittest.TestCase):
         fake_apk = repo / 'another_fake.apk'
         with fake_apk.open('w') as fp:
             fp.write('not an APK, but has the right filename')
+        fake_index = repo / fdroidserver.deploy.INDEX_FILES[0]
+        with fake_index.open('w') as fp:
+            fp.write('not an index, but has the right filename')
+
 
         # write out rclone config for test use
         rclone_config = configparser.ConfigParser()
@@ -135,12 +139,61 @@ class DeployTest(unittest.TestCase):
         # write out destination path
         destination = Path('some_bucket_folder/fdroid')
         destination.mkdir(parents=True, exist_ok=True)
-        dest_path = Path(destination) / fake_apk
-        self.assertFalse(dest_path.is_file())
+        dest_apk = Path(destination) / fake_apk
+        dest_index = Path(destination) / fake_index
+        self.assertFalse(dest_apk.is_file())
+        self.assertFalse(dest_index.is_file())
         repo_section = str(repo)
         # fdroidserver.deploy.USER_RCLONE_CONF = str(rclone_file)
         fdroidserver.deploy.update_remote_storage_with_rclone(repo_section)
-        self.assertFalse(dest_path.is_file())
+        self.assertTrue(dest_apk.is_file())
+        self.assertTrue(dest_index.is_file())
+
+    @unittest.skipUnless(shutil.which('rclone'), '/usr/bin/rclone')
+    def test_update_remote_storage_with_rclone_in_index_only_mode(self):
+        os.chdir(self.testdir)
+        repo = Path('repo')
+        repo.mkdir(parents=True, exist_ok=True)
+
+        fake_apk = repo / 'another_fake.apk'
+        with fake_apk.open('w') as fp:
+            fp.write('not an APK, but has the right filename')
+        fake_index = repo / fdroidserver.deploy.INDEX_FILES[0]
+        with fake_index.open('w') as fp:
+            fp.write('not an index, but has the right filename')
+
+        # write out rclone config for test use
+        rclone_config = configparser.ConfigParser()
+        rclone_config.add_section("test-local-config")
+        rclone_config.set("test-local-config", "type", "local")
+
+        rclone_config_path = Path('rclone_config_path')
+        rclone_config_path.mkdir(parents=True, exist_ok=True)
+        rclone_file = rclone_config_path / 'rclone.conf'
+        with open(rclone_file, 'w') as configfile:
+            rclone_config.write(configfile)
+
+        # setup parameters for this test run
+        fdroidserver.deploy.config['awsbucket'] = 'test_bucket_folder'
+        fdroidserver.deploy.config['rclone'] = True
+        fdroidserver.deploy.config['rclone_config'] = 'test-local-config'
+        fdroidserver.deploy.config['path_to_custom_rclone_config'] = str(rclone_file)
+        fdroidserver.deploy.options = Options
+
+        # write out destination path
+        destination = Path('some_bucket_folder/fdroid')
+        destination.mkdir(parents=True, exist_ok=True)
+        dest_apk = Path(destination) / fake_apk
+        dest_index = Path(destination) / fake_index
+        self.assertFalse(dest_apk.is_file())
+        self.assertFalse(dest_index.is_file())
+        repo_section = str(repo)
+        # fdroidserver.deploy.USER_RCLONE_CONF = str(rclone_file)
+        fdroidserver.deploy.update_remote_storage_with_rclone(
+            repo_section, is_index_only=True
+        )
+        self.assertFalse(dest_apk.is_file())
+        self.assertTrue(dest_index.is_file())
 
     def test_update_serverwebroot(self):
         """rsync works with file paths, so this test uses paths for the URLs"""
@@ -178,10 +231,9 @@ class DeployTest(unittest.TestCase):
         fake_apk = repo / 'fake.apk'
         with fake_apk.open('w') as fp:
             fp.write('not an APK, but has the right filename')
-        for i in fdroidserver.deploy.INDEX_FILES:
-            fake_index = repo / i
-            with fake_index.open('w') as fp:
-                fp.write('not an index, but has the right filename')
+        fake_index = repo / fdroidserver.deploy.INDEX_FILES[0]
+        with fake_index.open('w') as fp:
+            fp.write('not an index, but has the right filename')
         url = Path('url')
         url.mkdir()
 

From be0f07591498149169eaa4a92455230f87a789c3 Mon Sep 17 00:00:00 2001
From: proletarius101 <proletarius101@protonmail.com>
Date: Tue, 4 Jun 2024 21:29:02 +0800
Subject: [PATCH 68/68] fix(deploy): fix errors introduced by merging

---
 fdroidserver/deploy.py | 158 +++++++++++++++++++++++++----------------
 tests/deploy.TestCase  | 130 +++++++++++++++++----------------
 2 files changed, 165 insertions(+), 123 deletions(-)

diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py
index afed4e30..64b19406 100644
--- a/fdroidserver/deploy.py
+++ b/fdroidserver/deploy.py
@@ -25,10 +25,10 @@ import re
 import subprocess
 import time
 import urllib
-from typing import Dict, Optional, List
+from typing import Dict, List
 from git import Repo
 import yaml
-from argparse import ArgumentParser, Namespace
+from argparse import ArgumentParser
 import logging
 from shlex import split
 import shutil
@@ -41,7 +41,6 @@ from . import index
 from .exception import FDroidException
 
 config = None
-options: Optional[Namespace] = None
 start_timestamp = time.gmtime()
 
 GIT_BRANCH = 'master'
@@ -126,7 +125,9 @@ def update_awsbucket(repo_section, is_index_only=False, verbose=False, quiet=Fal
         if config['s3cmd'] is True and config['rclone'] is not True:
             update_awsbucket_s3cmd(repo_section, is_index_only)
         if config['rclone'] is True and config['s3cmd'] is not True:
-            update_remote_storage_with_rclone(repo_section, is_index_only, verbose, quiet)
+            update_remote_storage_with_rclone(
+                repo_section, is_index_only, verbose, quiet
+            )
 
     elif common.set_command_in_config('s3cmd'):
         update_awsbucket_s3cmd(repo_section, is_index_only)
@@ -191,8 +192,11 @@ def update_awsbucket_s3cmd(repo_section, is_index_only=False):
     )
 
     if is_index_only:
-        logging.debug(_('s3cmd syncs indexes from {path} to {url} and deletes removed')
-                      .format(path=repo_section, url=s3url))
+        logging.debug(
+            _('s3cmd syncs indexes from {path} to {url} and deletes removed').format(
+                path=repo_section, url=s3url
+            )
+        )
         sync_indexes_flags = []
         sync_indexes_flags.extend(_get_index_includes(repo_section))
         sync_indexes_flags.append('--delete-removed')
@@ -201,7 +205,9 @@ def update_awsbucket_s3cmd(repo_section, is_index_only=False):
             sync_indexes_flags.append('--no-check-md5')
         else:
             sync_indexes_flags.append('--check-md5')
-        returncode = subprocess.call(s3cmd_sync + sync_indexes_flags + [repo_section, s3url])
+        returncode = subprocess.call(
+            s3cmd_sync + sync_indexes_flags + [repo_section, s3url]
+        )
         if returncode != 0:
             raise FDroidException()
     else:
@@ -222,8 +228,11 @@ def update_awsbucket_s3cmd(repo_section, is_index_only=False):
         if returncode != 0:
             raise FDroidException()
 
-        logging.debug(_('s3cmd sync indexes {path} to {url} and delete')
-                      .format(path=repo_section, url=s3url))
+        logging.debug(
+            _('s3cmd sync indexes {path} to {url} and delete').format(
+                path=repo_section, url=s3url
+            )
+        )
         s3cmd_sync.append('--delete-removed')
         s3cmd_sync.append('--delete-after')
         if options.no_checksum:
@@ -234,7 +243,9 @@ def update_awsbucket_s3cmd(repo_section, is_index_only=False):
             raise FDroidException()
 
 
-def update_remote_storage_with_rclone(repo_section, is_index_only=False, verbose=False, quiet=False):
+def update_remote_storage_with_rclone(
+    repo_section, is_index_only=False, verbose=False, quiet=False
+):
     """
     Upload fdroid repo folder to remote storage using rclone sync.
 
@@ -353,10 +364,7 @@ def update_remote_storage_with_rclone(repo_section, is_index_only=False, verbose
                 )
 
                 logging.debug(
-                    "rclone sync all files in "
-                    + source
-                    + ' to '
-                    + complete_remote_path
+                    "rclone sync all files in " + source + ' to ' + complete_remote_path
                 )
 
                 if subprocess.call(rclone_sync_command) != 0:
@@ -412,12 +420,22 @@ def update_awsbucket_libcloud(repo_section, is_index_only=False):
             objs[obj.name] = obj
 
     if is_index_only:
-        index_files = [f"{os.getcwd()}/{name}" for name in _get_index_file_paths(repo_section)]
-        files_to_upload = [os.path.join(root, name) for root, dirs, files in os.walk(os.path.join(os.getcwd(), repo_section)) for name in files]
+        index_files = [
+            f"{os.getcwd()}/{name}" for name in _get_index_file_paths(repo_section)
+        ]
+        files_to_upload = [
+            os.path.join(root, name)
+            for root, dirs, files in os.walk(os.path.join(os.getcwd(), repo_section))
+            for name in files
+        ]
         files_to_upload = list(set(files_to_upload) & set(index_files))
 
     else:
-        files_to_upload = [os.path.join(root, name) for root, dirs, files in os.walk(os.path.join(os.getcwd(), repo_section)) for name in files]
+        files_to_upload = [
+            os.path.join(root, name)
+            for root, dirs, files in os.walk(os.path.join(os.getcwd(), repo_section))
+            for name in files
+        ]
 
     for file_to_upload in files_to_upload:
         upload = False
@@ -528,9 +546,15 @@ def update_serverwebroot(serverwebroot, repo_section):
         if subprocess.call(rsyncargs + [repo_section, url]) != 0:
             raise FDroidException()
         # upload "current version" symlinks if requested
-        if config and config.get('make_current_version_link') and repo_section == 'repo':
+        if (
+            config
+            and config.get('make_current_version_link')
+            and repo_section == 'repo'
+        ):
             links_to_upload = []
-            for f in glob.glob('*.apk') + glob.glob('*.apk.asc') + glob.glob('*.apk.sig'):
+            for f in (
+                glob.glob('*.apk') + glob.glob('*.apk.asc') + glob.glob('*.apk.sig')
+            ):
                 if os.path.islink(f):
                     links_to_upload.append(f)
             if len(links_to_upload) > 0:
@@ -708,35 +732,42 @@ def update_servergitmirrors(servergitmirrors, repo_section):
             # trailing slashes have a meaning in rsync which is not needed here, so
             # make sure both paths have exactly one trailing slash
             if is_index_only:
-                files_to_sync = [str(workspace_dir / repo_section / index_file) for index_file in INDEX_FILES]
+                files_to_sync = [
+                    str(workspace_dir / repo_section / index_file)
+                    for index_file in INDEX_FILES
+                ]
             else:
                 files_to_sync = [str(workspace_dir / repo_section).rstrip('/') + '/']
-            common.local_rsync(common.get_options(),
-                            files_to_sync,
-                            git_repodir.rstrip('/') + '/')
+            common.local_rsync(
+                common.get_options(), files_to_sync, git_repodir.rstrip('/') + '/'
+            )
 
-            upload_to_servergitmirror(mirror_config=d,
-                                      local_repo=repo,
-                                      enabled_remotes=enabled_remotes,
-                                      repo_section=repo_section,
-                                      is_index_only=is_index_only,
-                                      fdroid_dir=git_fdroiddir,
-                                      git_mirror_path=str(git_mirror_path),
-                                      ssh_cmd=ssh_cmd,
-                                      progress=progress)
+            upload_to_servergitmirror(
+                mirror_config=d,
+                local_repo=repo,
+                enabled_remotes=enabled_remotes,
+                repo_section=repo_section,
+                is_index_only=is_index_only,
+                fdroid_dir=git_fdroiddir,
+                git_mirror_path=str(git_mirror_path),
+                ssh_cmd=ssh_cmd,
+                progress=progress,
+            )
         if progress:
             progressbar.done()
 
 
-def upload_to_servergitmirror(mirror_config: Dict[str, str],
-                              local_repo: Repo,
-                              enabled_remotes: List[str],
-                              repo_section: str,
-                              is_index_only: bool,
-                              fdroid_dir: str,
-                              git_mirror_path: str,
-                              ssh_cmd: str,
-                              progress: git.RemoteProgress) -> None:
+def upload_to_servergitmirror(
+    mirror_config: Dict[str, str],
+    local_repo: Repo,
+    enabled_remotes: List[str],
+    repo_section: str,
+    is_index_only: bool,
+    fdroid_dir: str,
+    git_mirror_path: str,
+    ssh_cmd: str,
+    progress: git.RemoteProgress,
+) -> None:
     remote_branch_name = GIT_BRANCH
     local_branch_name = local_repo.active_branch.name
 
@@ -754,7 +785,9 @@ def upload_to_servergitmirror(mirror_config: Dict[str, str],
 
     logging.info("git status:", local_repo.git.status())
     if is_index_only:
-        local_repo.index.add(_get_index_file_paths(os.path.join('fdroid', repo_section)))
+        local_repo.index.add(
+            _get_index_file_paths(os.path.join('fdroid', repo_section))
+        )
     else:
         # sadly index.add don't allow the --all parameter
         logging.debug('Adding all files to git mirror')
@@ -763,16 +796,13 @@ def upload_to_servergitmirror(mirror_config: Dict[str, str],
     logging.debug('Committing files into git mirror')
     local_repo.index.commit("fdroidserver git-mirror")
 
-
     # only deploy to GitLab Artifacts if too big for GitLab Pages
     if common.get_dir_size(fdroid_dir) <= common.GITLAB_COM_PAGES_MAX_SIZE:
         gitlab_ci_job_name = 'pages'
     else:
         gitlab_ci_job_name = 'GitLab Artifacts'
         logging.warning(
-            _(
-                'Skipping GitLab Pages mirror because the repo is too large (>%.2fGB)!'
-            )
+            _('Skipping GitLab Pages mirror because the repo is too large (>%.2fGB)!')
             % (common.GITLAB_COM_PAGES_MAX_SIZE / 1000000000)
         )
 
@@ -803,19 +833,29 @@ def upload_to_servergitmirror(mirror_config: Dict[str, str],
     logging.info(_('Pushing to {url}').format(url=remote.url))
     with local_repo.git.custom_environment(GIT_SSH_COMMAND=ssh_cmd):
         pushinfos = remote.push(
-            f"{local_branch_name}:{remote_branch_name}", force=True, set_upstream=True, progress=progress
+            f"{local_branch_name}:{remote_branch_name}",
+            force=True,
+            set_upstream=True,
+            progress=progress,
         )
         for pushinfo in pushinfos:
-            if pushinfo.flags & (git.remote.PushInfo.ERROR
-                                 | git.remote.PushInfo.REJECTED
-                                 | git.remote.PushInfo.REMOTE_FAILURE
-                                 | git.remote.PushInfo.REMOTE_REJECTED):
+            if pushinfo.flags & (
+                git.remote.PushInfo.ERROR
+                | git.remote.PushInfo.REJECTED
+                | git.remote.PushInfo.REMOTE_FAILURE
+                | git.remote.PushInfo.REMOTE_REJECTED
+            ):
                 # Show potentially useful messages from git remote
                 for line in progress.other_lines:
                     if line.startswith('remote:'):
                         logging.debug(line)
-                raise FDroidException(remote.url + ' push failed: ' + str(pushinfo.flags)
-                                      + ' ' + pushinfo.summary)
+                raise FDroidException(
+                    remote.url
+                    + ' push failed: '
+                    + str(pushinfo.flags)
+                    + ' '
+                    + pushinfo.summary
+                )
             else:
                 logging.debug(remote.url + ': ' + pushinfo.summary)
 
@@ -1083,14 +1123,6 @@ def main():
 
     parser = ArgumentParser()
     common.setup_global_opts(parser)
-    parser.add_argument("-i", "--identity-file", default=None,
-                        help=_("Specify an identity file to provide to SSH for rsyncing"))
-    parser.add_argument("--local-copy-dir", default=None,
-                        help=_("Specify a local folder to sync the repo to"))
-    parser.add_argument("--no-checksum", action="store_true", default=False,
-                        help=_("Don't use rsync checksums"))
-    parser.add_argument("--no-keep-git-mirror-archive", action="store_true", default=False,
-                        help=_("If a git mirror gets to big, allow the archive to be deleted"))
     parser.add_argument(
         "-i",
         "--identity-file",
@@ -1114,8 +1146,8 @@ def main():
         default=False,
         help=_("If a git mirror gets to big, allow the archive to be deleted"),
     )
-    options = parser.parse_args()
-    config = common.read_config(options)
+    options = common.parse_args(parser)
+    config = common.read_config()
 
     if config.get('nonstandardwebroot') is True:
         standardwebroot = False
diff --git a/tests/deploy.TestCase b/tests/deploy.TestCase
index 57f15392..860fa416 100755
--- a/tests/deploy.TestCase
+++ b/tests/deploy.TestCase
@@ -40,7 +40,7 @@ class DeployTest(unittest.TestCase):
         self._td = mkdtemp()
         self.testdir = self._td.name
 
-        fdroidserver.deploy.options = mock.Mock()
+        fdroidserver.common.options = mock.Mock()
         fdroidserver.deploy.config = {}
         fdroidserver.deploy.USER_RCLONE_CONF = False
 
@@ -73,7 +73,7 @@ class DeployTest(unittest.TestCase):
         url1.mkdir(parents=True)
 
         # setup parameters for this test run
-        fdroidserver.deploy.options.identity_file = None
+        fdroidserver.common.options.identity_file = None
         fdroidserver.deploy.config['make_current_version_link'] = False
 
         dest_apk0 = url0 / fake_apk
@@ -117,7 +117,6 @@ class DeployTest(unittest.TestCase):
         with fake_index.open('w') as fp:
             fp.write('not an index, but has the right filename')
 
-
         # write out rclone config for test use
         rclone_config = configparser.ConfigParser()
         rclone_config.add_section("test-local-config")
@@ -134,7 +133,7 @@ class DeployTest(unittest.TestCase):
         fdroidserver.deploy.config['rclone'] = True
         fdroidserver.deploy.config['rclone_config'] = 'test-local-config'
         fdroidserver.deploy.config['path_to_custom_rclone_config'] = str(rclone_file)
-        fdroidserver.deploy.options = Options
+        fdroidserver.common.options = Options
 
         # write out destination path
         destination = Path('some_bucket_folder/fdroid')
@@ -178,7 +177,7 @@ class DeployTest(unittest.TestCase):
         fdroidserver.deploy.config['rclone'] = True
         fdroidserver.deploy.config['rclone_config'] = 'test-local-config'
         fdroidserver.deploy.config['path_to_custom_rclone_config'] = str(rclone_file)
-        fdroidserver.deploy.options = Options
+        fdroidserver.common.options = Options
 
         # write out destination path
         destination = Path('some_bucket_folder/fdroid')
@@ -210,9 +209,9 @@ class DeployTest(unittest.TestCase):
         url.mkdir()
 
         # setup parameters for this test run
-        fdroidserver.deploy.options = mock.Mock()
-        fdroidserver.deploy.options.identity_file = None
-        fdroidserver.deploy.options.identity_file = None
+        fdroidserver.common.options = mock.Mock()
+        fdroidserver.common.options.identity_file = None
+        fdroidserver.common.options.identity_file = None
         fdroidserver.deploy.config['make_current_version_link'] = False
 
         dest_apk = Path(url) / fake_apk
@@ -238,8 +237,8 @@ class DeployTest(unittest.TestCase):
         url.mkdir()
 
         # setup parameters for this test run
-        fdroidserver.deploy.options = mock.Mock()
-        fdroidserver.deploy.options.identity_file = None
+        fdroidserver.common.options = mock.Mock()
+        fdroidserver.common.options.identity_file = None
         fdroidserver.deploy.config['make_current_version_link'] = False
 
         dest_apk = Path(url) / fake_apk
@@ -269,7 +268,7 @@ class DeployTest(unittest.TestCase):
         fdroidserver.common.options.identity_file = None
         fdroidserver.common.options.verbose = False
         fdroidserver.common.options.quiet = True
-        fdroidserver.deploy.options.index_only = False
+        fdroidserver.common.options.index_only = False
         fdroidserver.deploy.config = {'make_current_version_link': True}
         url = "example.com:/var/www/fdroid"
         repo_section = 'repo'
@@ -358,12 +357,12 @@ class DeployTest(unittest.TestCase):
 
     def test_update_serverwebroot_make_cur_version_link_in_index_only_mode(self):
         # setup parameters for this test run
-        fdroidserver.deploy.options = mock.Mock()
-        fdroidserver.deploy.options.no_checksum = True
-        fdroidserver.deploy.options.identity_file = None
-        fdroidserver.deploy.options.verbose = False
-        fdroidserver.deploy.options.quiet = True
-        fdroidserver.deploy.options.identity_file = None
+        fdroidserver.common.options = mock.Mock()
+        fdroidserver.common.options.no_checksum = True
+        fdroidserver.common.options.identity_file = None
+        fdroidserver.common.options.verbose = False
+        fdroidserver.common.options.quiet = True
+        fdroidserver.common.options.identity_file = None
         fdroidserver.deploy.config['make_current_version_link'] = True
         url = "example.com:/var/www/fdroid"
         repo_section = 'repo'
@@ -382,6 +381,7 @@ class DeployTest(unittest.TestCase):
                         '--delete-after',
                         '--safe-links',
                         '--quiet',
+                        'repo/altstore-index.json',
                         'repo/entry.jar',
                         'repo/entry.json',
                         'repo/entry.json.asc',
@@ -447,7 +447,7 @@ class DeployTest(unittest.TestCase):
         fdroidserver.common.options.verbose = True
         fdroidserver.common.options.quiet = False
         fdroidserver.common.options.identity_file = None
-        fdroidserver.deploy.options.index_only = False
+        fdroidserver.common.options.index_only = False
         fdroidserver.deploy.config = {'identity_file': './id_rsa'}
         url = "example.com:/var/www/fdroid"
         repo_section = 'archive'
@@ -522,11 +522,11 @@ class DeployTest(unittest.TestCase):
 
     def test_update_serverwebroot_with_id_file_in_index_only_mode(self):
         # setup parameters for this test run
-        fdroidserver.deploy.options = mock.Mock()
-        fdroidserver.deploy.options.no_chcksum = False
-        fdroidserver.deploy.options.verbose = True
-        fdroidserver.deploy.options.quiet = False
-        fdroidserver.deploy.options.identity_file = None
+        fdroidserver.common.options = mock.Mock()
+        fdroidserver.common.options.no_chcksum = False
+        fdroidserver.common.options.verbose = True
+        fdroidserver.common.options.quiet = False
+        fdroidserver.common.options.identity_file = None
         fdroidserver.deploy.config['identity_file'] = './id_rsa'
         fdroidserver.deploy.config['make_current_version_link'] = False
         url = "example.com:/var/www/fdroid"
@@ -549,6 +549,7 @@ class DeployTest(unittest.TestCase):
                         '-e',
                         'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i '
                         + fdroidserver.deploy.config['identity_file'],
+                        'archive/altstore-index.json',
                         'archive/entry.jar',
                         'archive/entry.json',
                         'archive/entry.json.asc',
@@ -592,8 +593,8 @@ class DeployTest(unittest.TestCase):
         not os.getenv('VIRUSTOTAL_API_KEY'), 'VIRUSTOTAL_API_KEY is not set'
     )
     def test_upload_to_virustotal(self):
-        fdroidserver.deploy.options = mock.Mock()
-        fdroidserver.deploy.options.verbose = True
+        fdroidserver.common.options = mock.Mock()
+        fdroidserver.common.options.verbose = True
         virustotal_apikey = os.getenv('VIRUSTOTAL_API_KEY')
         fdroidserver.deploy.upload_to_virustotal('repo', virustotal_apikey)
 
@@ -611,10 +612,10 @@ class DeployTest(unittest.TestCase):
 
     def test_update_awsbucket_s3cmd(self):
         # setup parameters for this test run
-        fdroidserver.deploy.options = mock.Mock()
-        fdroidserver.deploy.options.no_checksum = True
-        fdroidserver.deploy.options.verbose = False
-        fdroidserver.deploy.options.quiet = True
+        fdroidserver.common.options = mock.Mock()
+        fdroidserver.common.options.no_checksum = True
+        fdroidserver.common.options.verbose = False
+        fdroidserver.common.options.quiet = True
 
         config = {}
         fdroidserver.common.fill_config_defaults(config)
@@ -651,6 +652,8 @@ class DeployTest(unittest.TestCase):
                         '--acl-public',
                         '--quiet',
                         '--exclude',
+                        'repo/altstore-index.json',
+                        '--exclude',
                         'repo/entry.jar',
                         '--exclude',
                         'repo/entry.json',
@@ -686,6 +689,8 @@ class DeployTest(unittest.TestCase):
                         '--acl-public',
                         '--quiet',
                         '--exclude',
+                        'repo/altstore-index.json',
+                        '--exclude',
                         'repo/entry.jar',
                         '--exclude',
                         'repo/entry.json',
@@ -742,10 +747,10 @@ class DeployTest(unittest.TestCase):
 
     def test_update_awsbucket_s3cmd_in_index_only_mode(self):
         # setup parameters for this test run
-        fdroidserver.deploy.options = mock.Mock()
-        fdroidserver.deploy.options.no_checksum = True
-        fdroidserver.deploy.options.verbose = False
-        fdroidserver.deploy.options.quiet = True
+        fdroidserver.common.options = mock.Mock()
+        fdroidserver.common.options.no_checksum = True
+        fdroidserver.common.options.verbose = False
+        fdroidserver.common.options.quiet = True
 
         config = {}
         fdroidserver.common.fill_config_defaults(config)
@@ -782,6 +787,8 @@ class DeployTest(unittest.TestCase):
                         '--acl-public',
                         '--quiet',
                         '--include',
+                        'repo/altstore-index.json',
+                        '--include',
                         'repo/entry.jar',
                         '--include',
                         'repo/entry.json',
@@ -820,7 +827,7 @@ class DeployTest(unittest.TestCase):
             os.symlink('repo/com.example.sym.apk.sig', 'Sym.apk.sig')
             with mock.patch('subprocess.call', side_effect=update_awsbucket_s3cmd_call):
                 fdroidserver.deploy.update_awsbucket_s3cmd(
-                    repo_section, index_only=True
+                    repo_section, is_index_only=True
                 )
         self.assertEqual(call_iteration, 2, 'expected 2 invocations of subprocess.call')
 
@@ -828,10 +835,10 @@ class DeployTest(unittest.TestCase):
         from libcloud.storage.base import Container
 
         # setup parameters for this test run
-        fdroidserver.deploy.options = mock.Mock()
-        fdroidserver.deploy.options.no_checksum = True
-        fdroidserver.deploy.options.verbose = False
-        fdroidserver.deploy.options.quiet = True
+        fdroidserver.common.options = mock.Mock()
+        fdroidserver.common.options.no_checksum = True
+        fdroidserver.common.options.verbose = False
+        fdroidserver.common.options.quiet = True
 
         config = {}
         fdroidserver.common.fill_config_defaults(config)
@@ -872,7 +879,10 @@ class DeployTest(unittest.TestCase):
                 container_name=fdroidserver.deploy.config["awsbucket"]
             )
             mock_container.list_objects.assert_called_once_with()
-            files_to_upload = ['fdroid/repo/Sym.apk', 'fdroid/repo/entry.jar']
+            files_to_upload = [
+                'fdroid/repo/Sym.apk',
+                f"fdroid/repo/{fdroidserver.deploy.INDEX_FILES[0]}",
+            ]
             calls = [
                 mock.call(
                     iterator=mock.ANY,
@@ -889,10 +899,10 @@ class DeployTest(unittest.TestCase):
         from libcloud.storage.base import Container
 
         # setup parameters for this test run
-        fdroidserver.deploy.options = mock.Mock()
-        fdroidserver.deploy.options.no_checksum = True
-        fdroidserver.deploy.options.verbose = False
-        fdroidserver.deploy.options.quiet = True
+        fdroidserver.common.options = mock.Mock()
+        fdroidserver.common.options.no_checksum = True
+        fdroidserver.common.options.verbose = False
+        fdroidserver.common.options.quiet = True
 
         config = {}
         fdroidserver.common.fill_config_defaults(config)
@@ -927,7 +937,7 @@ class DeployTest(unittest.TestCase):
             mock_driver.get_container.return_value = mock_container
             mock_driver.upload_object_via_stream.return_value = None
 
-            fdroidserver.deploy.update_awsbucket_libcloud(repo_section, index_only=True)
+            fdroidserver.deploy.update_awsbucket_libcloud(repo_section, is_index_only=True)
 
             mock_driver.get_container.assert_called_once_with(
                 container_name=fdroidserver.deploy.config["awsbucket"]
@@ -951,11 +961,11 @@ class DeployTest(unittest.TestCase):
 
     def test_update_servergitmirrors(self):
         # setup parameters for this test run
-        fdroidserver.deploy.options = mock.Mock()
-        fdroidserver.deploy.options.identity_file = None
-        fdroidserver.deploy.options.no_keep_git_mirror_archive = False
-        fdroidserver.deploy.options.verbose = False
-        fdroidserver.deploy.options.quiet = True
+        fdroidserver.common.options = mock.Mock()
+        fdroidserver.common.options.identity_file = None
+        fdroidserver.common.options.no_keep_git_mirror_archive = False
+        fdroidserver.common.options.verbose = False
+        fdroidserver.common.options.quiet = True
 
         config = {}
         fdroidserver.common.fill_config_defaults(config)
@@ -1002,11 +1012,11 @@ class DeployTest(unittest.TestCase):
 
     def test_update_servergitmirrors_in_index_only_mode(self):
         # setup parameters for this test run
-        fdroidserver.deploy.options = mock.Mock()
-        fdroidserver.deploy.options.identity_file = None
-        fdroidserver.deploy.options.no_keep_git_mirror_archive = False
-        fdroidserver.deploy.options.verbose = False
-        fdroidserver.deploy.options.quiet = True
+        fdroidserver.common.options = mock.Mock()
+        fdroidserver.common.options.identity_file = None
+        fdroidserver.common.options.no_keep_git_mirror_archive = False
+        fdroidserver.common.options.verbose = False
+        fdroidserver.common.options.quiet = True
 
         config = {}
         fdroidserver.common.fill_config_defaults(config)
@@ -1062,12 +1072,12 @@ class DeployTest(unittest.TestCase):
 
     def test_upload_to_servergitmirror_in_index_only_mode(self):
         # setup parameters for this test run
-        fdroidserver.deploy.options = mock.Mock()
-        fdroidserver.deploy.options.identity_file = None
-        fdroidserver.deploy.options.no_keep_git_mirror_archive = False
-        fdroidserver.deploy.options.verbose = False
-        fdroidserver.deploy.options.quiet = True
-        fdroidserver.deploy.options.identity_file = None
+        fdroidserver.common.options = mock.Mock()
+        fdroidserver.common.options.identity_file = None
+        fdroidserver.common.options.no_keep_git_mirror_archive = False
+        fdroidserver.common.options.verbose = False
+        fdroidserver.common.options.quiet = True
+        fdroidserver.common.options.identity_file = None
 
         config = {}
         fdroidserver.common.fill_config_defaults(config)