fix(deploy): remove the index-only mode from elsewhere except the config file

To enable per-remote configuration

completion/bash-completion

@@ -264,7 +264,7 @@ __complete_nightly() {
 __complete_deploy() {
 	opts="-i -v -q"
 	lopts="--identity-file --local-copy-dir --sync-from-local-copy-dir
- --verbose --quiet --no-checksum --no-keep-git-mirror-archive --index-only"
+ --verbose --quiet --no-checksum --no-keep-git-mirror-archive"
 	__complete_options
 }
 

fdroidserver/common.py

@@ -4301,7 +4301,7 @@ def get_app_display_name(app):
     return app.get('AutoName') or app['id']
 
 
-def local_rsync(options, from_paths: List[str], todir: str, args: List[str] = []):
+def local_rsync(options, from_paths: List[str], todir: str):
     """Rsync method for local to local copying of things.
 
     This is an rsync wrapper with all the settings for safe use within
@@ -4318,7 +4318,6 @@ def local_rsync(options, from_paths: List[str], todir: str, args: List[str] = []
         rsyncargs += ['--verbose']
     if options.quiet:
         rsyncargs += ['--quiet']
-    rsyncargs += args
     logging.debug(' '.join(rsyncargs + from_paths + [todir]))
     if subprocess.call(rsyncargs + from_paths + [todir]) != 0:
         raise FDroidException()

fdroidserver/deploy.py

@@ -92,7 +92,7 @@ def _get_index_includes(base_dir):
     return index_includes
 
 
-def update_awsbucket(repo_section, verbose=False, quiet=False):
+def update_awsbucket(repo_section, index_only=False, verbose=False, quiet=False):
     """Upload the contents of the directory `repo_section` (including subdirectories) to the AWS S3 "bucket".
 
     The contents of that subdir of the
@@ -112,26 +112,26 @@ def update_awsbucket(repo_section, verbose=False, quiet=False):
             logging.warning(
                 'No syncing tool set in config.yml!. Defaulting to using s3cmd'
             )
-            update_awsbucket_s3cmd(repo_section)
+            update_awsbucket_s3cmd(repo_section, index_only)
         if config['s3cmd'] is True and config['rclone'] is True:
             logging.warning(
                 'Both syncing tools set in config.yml!. Defaulting to using s3cmd'
             )
-            update_awsbucket_s3cmd(repo_section)
+            update_awsbucket_s3cmd(repo_section, index_only)
         if config['s3cmd'] is True and config['rclone'] is not True:
-            update_awsbucket_s3cmd(repo_section)
+            update_awsbucket_s3cmd(repo_section, index_only)
         if config['rclone'] is True and config['s3cmd'] is not True:
             update_remote_storage_with_rclone(repo_section, verbose, quiet)
 
     elif common.set_command_in_config('s3cmd'):
-        update_awsbucket_s3cmd(repo_section)
+        update_awsbucket_s3cmd(repo_section, index_only)
     elif common.set_command_in_config('rclone'):
         update_remote_storage_with_rclone(repo_section, verbose, quiet)
     else:
-        update_awsbucket_libcloud(repo_section)
+        update_awsbucket_libcloud(repo_section, index_only)
 
 
-def update_awsbucket_s3cmd(repo_section):
+def update_awsbucket_s3cmd(repo_section, index_only=False):
     """Upload using the CLI tool s3cmd, which provides rsync-like sync.
 
     The upload is done in multiple passes to reduce the chance of
@@ -185,7 +185,7 @@ def update_awsbucket_s3cmd(repo_section):
         )
     )
 
-    if options.index_only:
+    if index_only:
         logging.debug(_('s3cmd syncs indexes from {path} to {url} and deletes removed')
                       .format(path=repo_section, url=s3url))
         sync_indexes_flags = []
@@ -352,7 +352,7 @@ def update_remote_storage_with_rclone(repo_section, verbose=False, quiet=False):
                 raise FDroidException()
 
 
-def update_awsbucket_libcloud(repo_section):
+def update_awsbucket_libcloud(repo_section, index_only=False):
     """No summary.
 
     Upload the contents of the directory `repo_section` (including
@@ -400,7 +400,7 @@ def update_awsbucket_libcloud(repo_section):
         if obj.name.startswith(upload_dir + '/'):
             objs[obj.name] = obj
 
-    if options.index_only:
+    if index_only:
         index_files = [f"{os.getcwd()}/{name}" for name in _get_index_file_paths(repo_section)]
         files_to_upload = [os.path.join(root, name) for root, dirs, files in os.walk(os.path.join(os.getcwd(), repo_section)) for name in files]
         files_to_upload = list(set(files_to_upload) & set(index_files))
@@ -502,8 +502,9 @@ def update_serverwebroot(serverwebroot, repo_section):
             'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ' + config['identity_file'],
         ]
     url = serverwebroot['url']
+    index_only = serverwebroot.get('index_only', False)
     logging.info('rsyncing ' + repo_section + ' to ' + url)
-    if options.index_only:
+    if index_only:
         rsyncargs += _get_index_file_paths(repo_section)
         rsyncargs += [f'{url}/{repo_section}/']
         logging.info(rsyncargs)
@@ -563,8 +564,7 @@ def sync_from_localcopy(repo_section, local_copy_dir):
     push to all the servers that are configured.
 
     """
-    if options.index_only:
+    logging.info('Syncing from local_copy_dir to this repo.')
-        raise FDroidException(_('The index only mode cannot be used when syncing to the local copy filesystem'))
 
     # trailing slashes have a meaning in rsync which is not needed here, so
     # make sure both paths have exactly one trailing slash
@@ -588,8 +588,6 @@ def update_localcopy(repo_section, local_copy_dir):
     drive.
 
     """
-    if options.index_only:
-        raise FDroidException(_('The index only mode cannot be used when syncing to the local copy filesystem'))
 
     # local_copy_dir is guaranteed to have a trailing slash in main() below
     common.local_rsync(common.get_options(), [repo_section], local_copy_dir)
@@ -662,17 +660,11 @@ def update_servergitmirrors(servergitmirrors, repo_section):
             archive_path = os.path.join(git_mirror_path, 'fdroid', 'archive')
             shutil.rmtree(archive_path, ignore_errors=True)
 
-        if options.index_only:
+        # trailing slashes have a meaning in rsync which is not needed here, so
-            # rsync is very particular about trailing slashes
+        # make sure both paths have exactly one trailing slash
-            common.local_rsync(common.get_options(),
+        common.local_rsync(common.get_options(),
-                               _get_index_file_paths(repo_section),
+                           [repo_section.rstrip('/') + '/'],
-                               git_repodir.rstrip('/') + '/')
+                           git_repodir.rstrip('/') + '/')
-        else:
-            # trailing slashes have a meaning in rsync which is not needed here, so
-            # make sure both paths have exactly one trailing slash
-            common.local_rsync(common.get_options(),
-                               [repo_section.rstrip('/') + '/'],
-                               git_repodir.rstrip('/') + '/')
 
         # use custom SSH command if identity_file specified
         ssh_cmd = 'ssh -oBatchMode=yes'
@@ -681,28 +673,6 @@ def update_servergitmirrors(servergitmirrors, repo_section):
         elif 'identity_file' in config:
             ssh_cmd += ' -oIdentitiesOnly=yes -i "%s"' % config['identity_file']
 
-        repo = git.Repo.init(git_mirror_path, initial_branch=GIT_BRANCH)
-
-        enabled_remotes = []
-        for d in servergitmirrors:
-            remote_url = d['url']
-            name = REMOTE_HOSTNAME_REGEX.sub(r'\1', remote_url)
-            enabled_remotes.append(name)
-            r = git.remote.Remote(repo, name)
-            if r in repo.remotes:
-                r = repo.remote(name)
-                if 'set_url' in dir(r):  # force remote URL if using GitPython 2.x
-                    r.set_url(remote_url)
-            else:
-                repo.create_remote(name, remote_url)
-            logging.info('Mirroring to: ' + remote_url)
-
-        # sadly index.add don't allow the --all parameter
-        logging.debug('Adding all files to git mirror')
-        repo.git.add(all=True)
-        logging.debug('Committing all files into git mirror')
-        repo.index.commit("fdroidserver git-mirror")
-
         if options.verbose:
             progressbar = progress.Bar()
 
@@ -715,23 +685,49 @@ def update_servergitmirrors(servergitmirrors, repo_section):
         else:
             progress = None
 
-        # only deploy to GitLab Artifacts if too big for GitLab Pages
+        repo = git.Repo.init(git_mirror_path, initial_branch=GIT_BRANCH)
-        if common.get_dir_size(git_fdroiddir) <= common.GITLAB_COM_PAGES_MAX_SIZE:
+        initial_commit_ref = repo.head.ref
-            gitlab_ci_job_name = 'pages'
-        else:
-            gitlab_ci_job_name = 'GitLab Artifacts'
-            logging.warning(
-                _(
-                    'Skipping GitLab Pages mirror because the repo is too large (>%.2fGB)!'
-                )
-                % (common.GITLAB_COM_PAGES_MAX_SIZE / 1000000000)
-            )
 
-        # push for every remote. This will overwrite the git history
+        enabled_remotes = []
-        for remote in repo.remotes:
+        for d in servergitmirrors:
-            if remote.name not in enabled_remotes:
+            index_only = d.get('index_only', False)
-                repo.delete_remote(remote)
+            remote_url = d['url']
-                continue
+            name = REMOTE_HOSTNAME_REGEX.sub(r'\1', remote_url)
+            enabled_remotes.append(name)
+            r = git.remote.Remote(repo, name)
+            if r in repo.remotes:
+                r = repo.remote(name)
+                if 'set_url' in dir(r):  # force remote URL if using GitPython 2.x
+                    r.set_url(remote_url)
+            else:
+                repo.create_remote(name, remote_url)
+            logging.info('Mirroring to: ' + remote_url)
+
+            if index_only:
+                logging.debug('Adding index files to git mirror')
+                repo.index.add(_get_index_file_paths(repo_section))
+            else:
+                # sadly index.add don't allow the --all parameter
+                logging.debug('Adding all files to git mirror')
+                repo.git.add(all=True)
+
+            logging.debug('Committing files into git mirror')
+            repo.index.commit("fdroidserver git-mirror")
+
+            # only deploy to GitLab Artifacts if too big for GitLab Pages
+            if common.get_dir_size(git_fdroiddir) <= common.GITLAB_COM_PAGES_MAX_SIZE:
+                gitlab_ci_job_name = 'pages'
+            else:
+                gitlab_ci_job_name = 'GitLab Artifacts'
+                logging.warning(
+                    _(
+                        'Skipping GitLab Pages mirror because the repo is too large (>%.2fGB)!'
+                    )
+                    % (common.GITLAB_COM_PAGES_MAX_SIZE / 1000000000)
+                )
+
+            # push. This will overwrite the git history
+            remote = repo.remote(name)
             if remote.name == 'gitlab':
                 logging.debug('Writing .gitlab-ci.yml to deploy to GitLab Pages')
                 with open(os.path.join(git_mirror_path, ".gitlab-ci.yml"), "wt") as fp:
@@ -751,7 +747,7 @@ def update_servergitmirrors(servergitmirrors, repo_section):
                         default_flow_style=False,
                     )
 
-                repo.git.add(all=True)
+                repo.index.add(['.gitlab-ci.yml'])
                 repo.index.commit("fdroidserver git-mirror: Deploy to GitLab Pages")
 
             logging.debug(_('Pushing to {url}').format(url=remote.url))
@@ -780,6 +776,8 @@ def update_servergitmirrors(servergitmirrors, repo_section):
                     else:
                         logging.debug(remote.url + ': ' + pushinfo.summary)
 
+            repo.head.reset(initial_commit_ref, index=True, working_tree=True)
+
         if progress:
             progressbar.done()
 

tests/deploy.TestCase

@@ -10,6 +10,7 @@ import tempfile
 import unittest
 from pathlib import Path
 from unittest import mock
+from fdroidserver import index
 
 localmodule = os.path.realpath(
     os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..')
@@ -72,7 +73,6 @@ class DeployTest(unittest.TestCase):
 
         # setup parameters for this test run
         fdroidserver.deploy.options.identity_file = None
-        fdroidserver.deploy.options.index_only = False
         fdroidserver.deploy.config['make_current_version_link'] = False
 
         dest_apk0 = url0 / fake_apk
@@ -159,7 +159,6 @@ class DeployTest(unittest.TestCase):
         fdroidserver.deploy.options = mock.Mock()
         fdroidserver.deploy.options.identity_file = None
         fdroidserver.deploy.options.identity_file = None
-        fdroidserver.deploy.options.index_only = False
         fdroidserver.deploy.config['make_current_version_link'] = False
 
         dest_apk = Path(url) / fake_apk
@@ -187,7 +186,6 @@ class DeployTest(unittest.TestCase):
 
         # setup parameters for this test run
         fdroidserver.deploy.options.identity_file = None
-        fdroidserver.deploy.options.index_only = True
         fdroidserver.deploy.config['make_current_version_link'] = False
 
         dest_apk = Path(url) / fake_apk
@@ -195,7 +193,9 @@ class DeployTest(unittest.TestCase):
         self.assertFalse(dest_apk.is_file())
         self.assertFalse(dest_index.is_file())
 
-        fdroidserver.deploy.update_serverwebroot({'url': str(url)}, 'repo')
+        fdroidserver.deploy.update_serverwebroot(
+            {'url': str(url), 'index_only': True}, 'repo'
+        )
         self.assertFalse(dest_apk.is_file())
         self.assertTrue(dest_index.is_file())
 
@@ -310,7 +310,6 @@ class DeployTest(unittest.TestCase):
         fdroidserver.deploy.options.verbose = False
         fdroidserver.deploy.options.quiet = True
         fdroidserver.deploy.options.identity_file = None
-        fdroidserver.deploy.options.index_only = True
         fdroidserver.deploy.config['make_current_version_link'] = True
         url = "example.com:/var/www/fdroid"
         repo_section = 'repo'
@@ -370,34 +369,6 @@ class DeployTest(unittest.TestCase):
                         'example.com:/var/www/fdroid',
                     ],
                 )
-            # elif call_iteration == 1:
-            #     self.assertListEqual(
-            #         cmd,
-            #         [
-            #             'rsync',
-            #             '--archive',
-            #             '--delete-after',
-            #             '--safe-links',
-            #             '--quiet',
-            #             'repo',
-            #             serverwebroot,
-            #         ],
-            #     )
-            # elif call_iteration == 2:
-            #     self.assertListEqual(
-            #         cmd,
-            #         [
-            #             'rsync',
-            #             '--archive',
-            #             '--delete-after',
-            #             '--safe-links',
-            #             '--quiet',
-            #             'Sym.apk',
-            #             'Sym.apk.asc',
-            #             'Sym.apk.sig',
-            #             'example.com:/var/www/fdroid',
-            #         ],
-            #     )
             else:
                 self.fail('unexpected subprocess.call invocation')
             call_iteration += 1
@@ -409,7 +380,9 @@ class DeployTest(unittest.TestCase):
             os.symlink('repo/com.example.sym.apk.asc', 'Sym.apk.asc')
             os.symlink('repo/com.example.sym.apk.sig', 'Sym.apk.sig')
             with mock.patch('subprocess.call', side_effect=update_server_webroot_call):
-                fdroidserver.deploy.update_serverwebroot({'url': url}, repo_section)
+                fdroidserver.deploy.update_serverwebroot(
+                    {'url': url, 'index_only': True}, repo_section
+                )
         self.assertEqual(call_iteration, 1, 'expected 1 invocations of subprocess.call')
 
     def test_update_serverwebroot_with_id_file(self):
@@ -499,7 +472,6 @@ class DeployTest(unittest.TestCase):
         fdroidserver.deploy.options.verbose = True
         fdroidserver.deploy.options.quiet = False
         fdroidserver.deploy.options.identity_file = None
-        fdroidserver.deploy.options.index_only = True
         fdroidserver.deploy.config['identity_file'] = './id_rsa'
         fdroidserver.deploy.config['make_current_version_link'] = False
         url = "example.com:/var/www/fdroid"
@@ -550,29 +522,15 @@ class DeployTest(unittest.TestCase):
                         "example.com:/var/www/fdroid/archive/",
                     ],
                 )
-            # elif call_iteration == 1:
-            #     self.assertListEqual(
-            #         cmd,
-            #         [
-            #             'rsync',
-            #             '--archive',
-            #             '--delete-after',
-            #             '--safe-links',
-            #             '--verbose',
-            #             '-e',
-            #             'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i '
-            #             + fdroidserver.deploy.config['identity_file'],
-            #             'archive',
-            #             serverwebroot,
-            #         ],
-            #     )
             else:
                 self.fail('unexpected subprocess.call invocation')
             call_iteration += 1
             return 0
 
         with mock.patch('subprocess.call', side_effect=update_server_webroot_call):
-            fdroidserver.deploy.update_serverwebroot({'url': url}, repo_section)
+            fdroidserver.deploy.update_serverwebroot(
+                {'url': url, 'index_only': True}, repo_section
+            )
         self.assertEqual(call_iteration, 1, 'expected 1 invocations of subprocess.call')
 
     @unittest.skipIf(
@@ -601,7 +559,6 @@ class DeployTest(unittest.TestCase):
         fdroidserver.deploy.options.no_checksum = True
         fdroidserver.deploy.options.verbose = False
         fdroidserver.deploy.options.quiet = True
-        fdroidserver.deploy.options.index_only = False
 
         config = {}
         fdroidserver.common.fill_config_defaults(config)
@@ -733,7 +690,6 @@ class DeployTest(unittest.TestCase):
         fdroidserver.deploy.options.no_checksum = True
         fdroidserver.deploy.options.verbose = False
         fdroidserver.deploy.options.quiet = True
-        fdroidserver.deploy.options.index_only = True
 
         config = {}
         fdroidserver.common.fill_config_defaults(config)
@@ -807,7 +763,9 @@ class DeployTest(unittest.TestCase):
             os.symlink('repo/com.example.sym.apk.asc', 'Sym.apk.asc')
             os.symlink('repo/com.example.sym.apk.sig', 'Sym.apk.sig')
             with mock.patch('subprocess.call', side_effect=update_awsbucket_s3cmd_call):
-                fdroidserver.deploy.update_awsbucket_s3cmd(repo_section)
+                fdroidserver.deploy.update_awsbucket_s3cmd(
+                    repo_section, index_only=True
+                )
         self.assertEqual(call_iteration, 2, 'expected 2 invocations of subprocess.call')
 
     def test_update_awsbucket_libcloud(self):
@@ -818,7 +776,6 @@ class DeployTest(unittest.TestCase):
         fdroidserver.deploy.options.no_checksum = True
         fdroidserver.deploy.options.verbose = False
         fdroidserver.deploy.options.quiet = True
-        fdroidserver.deploy.options.index_only = False
 
         config = {}
         fdroidserver.common.fill_config_defaults(config)
@@ -880,7 +837,6 @@ class DeployTest(unittest.TestCase):
         fdroidserver.deploy.options.no_checksum = True
         fdroidserver.deploy.options.verbose = False
         fdroidserver.deploy.options.quiet = True
-        fdroidserver.deploy.options.index_only = True
 
         config = {}
         fdroidserver.common.fill_config_defaults(config)
@@ -915,7 +871,7 @@ class DeployTest(unittest.TestCase):
             mock_driver.get_container.return_value = mock_container
             mock_driver.upload_object_via_stream.return_value = None
 
-            fdroidserver.deploy.update_awsbucket_libcloud(repo_section)
+            fdroidserver.deploy.update_awsbucket_libcloud(repo_section, index_only=True)
 
             mock_driver.get_container.assert_called_once_with(
                 container_name=fdroidserver.deploy.config["awsbucket"]
@@ -944,7 +900,6 @@ class DeployTest(unittest.TestCase):
         fdroidserver.deploy.options.no_keep_git_mirror_archive = False
         fdroidserver.deploy.options.verbose = False
         fdroidserver.deploy.options.quiet = True
-        fdroidserver.deploy.options.index_only = False
 
         config = {}
         fdroidserver.common.fill_config_defaults(config)
@@ -1001,7 +956,6 @@ class DeployTest(unittest.TestCase):
         fdroidserver.deploy.options.no_keep_git_mirror_archive = False
         fdroidserver.deploy.options.verbose = False
         fdroidserver.deploy.options.quiet = True
-        fdroidserver.deploy.options.index_only = True
 
         config = {}
         fdroidserver.common.fill_config_defaults(config)
@@ -1038,16 +992,7 @@ class DeployTest(unittest.TestCase):
                         '--delete',
                         '--chmod=Da+rx,Fa-x,a+r,u+w',
                         '--quiet',
-                        'repo/entry.jar',
+                        'repo/',
-                        'repo/entry.json',
-                        'repo/entry.json.asc',
-                        'repo/index-v1.jar',
-                        'repo/index-v1.json',
-                        'repo/index-v1.json.asc',
-                        'repo/index-v2.json',
-                        'repo/index-v2.json.asc',
-                        'repo/index.jar',
-                        'repo/index.xml',
                         "git-mirror/fdroid/repo/",
                     ],
                 )