Merge branch 'index-only-deploy-mode' into 'master'

Add index only deployment mode

Closes #1181

See merge request fdroid/fdroidserver!1420

fdroidserver/deploy.py

@@ -96,15 +96,6 @@ def _get_index_includes(base_dir):
     return index_includes
 
 
-def _remove_missing_files(files: List[str]) -> List[str]:
-    """Remove files that are missing from the file system."""
-    existing = []
-    for f in files:
-        if os.path.exists(f):
-            existing.append(f)
-    return existing
-
-
 def update_awsbucket(repo_section, is_index_only=False, verbose=False, quiet=False):
     """Upload the contents of the directory `repo_section` (including subdirectories) to the AWS S3 "bucket".
 
@@ -308,9 +299,8 @@ def update_remote_storage_with_rclone(
 
     if is_index_only:
         sources = _get_index_file_paths(repo_section)
-        sources = _remove_missing_files(sources)
     else:
-        sources = [repo_section]
+        sources = repo_section
 
     for source in sources:
         if isinstance(config['rclone_config'], str):
@@ -439,7 +429,6 @@ def update_awsbucket_libcloud(repo_section, is_index_only=False):
             for name in files
         ]
         files_to_upload = list(set(files_to_upload) & set(index_files))
-        files_to_upload = _remove_missing_files(files_to_upload)
 
     else:
         files_to_upload = [
@@ -545,10 +534,7 @@ def update_serverwebroot(serverwebroot, repo_section):
     is_index_only = serverwebroot.get('index_only', False)
     logging.info('rsyncing ' + repo_section + ' to ' + url)
     if is_index_only:
-        files_to_upload = _get_index_file_paths(repo_section)
-        files_to_upload = _remove_missing_files(files_to_upload)
-
-        rsyncargs += files_to_upload
+        rsyncargs += _get_index_file_paths(repo_section)
         rsyncargs += [f'{url}/{repo_section}/']
         logging.info(rsyncargs)
         if subprocess.call(rsyncargs) != 0:
@@ -746,8 +732,10 @@ def update_servergitmirrors(servergitmirrors, repo_section):
             # trailing slashes have a meaning in rsync which is not needed here, so
             # make sure both paths have exactly one trailing slash
             if is_index_only:
-                files_to_sync = _get_index_file_paths(str(workspace_dir / repo_section))
-                files_to_sync = _remove_missing_files(files_to_sync)
+                files_to_sync = [
+                    str(workspace_dir / repo_section / index_file)
+                    for index_file in INDEX_FILES
+                ]
             else:
                 files_to_sync = [str(workspace_dir / repo_section).rstrip('/') + '/']
             common.local_rsync(
@@ -795,12 +783,11 @@ def upload_to_servergitmirror(
         local_repo.create_remote(name, remote_url)
     logging.info('Mirroring to: ' + remote_url)
 
+    logging.info("git status:", local_repo.git.status())
     if is_index_only:
-        files_to_upload = _get_index_file_paths(
-            os.path.join(local_repo.working_tree_dir, 'fdroid', repo_section)
+        local_repo.index.add(
+            _get_index_file_paths(os.path.join('fdroid', repo_section))
         )
-        files_to_upload = _remove_missing_files(files_to_upload)
-        local_repo.index.add(files_to_upload)
     else:
         # sadly index.add don't allow the --all parameter
         logging.debug('Adding all files to git mirror')
@@ -843,7 +830,7 @@ def upload_to_servergitmirror(
         local_repo.index.add(['.gitlab-ci.yml'])
         local_repo.index.commit("fdroidserver git-mirror: Deploy to GitLab Pages")
 
-    logging.debug(_('Pushing to {url}').format(url=remote.url))
+    logging.info(_('Pushing to {url}').format(url=remote.url))
     with local_repo.git.custom_environment(GIT_SSH_COMMAND=ssh_cmd):
         pushinfos = remote.push(
             f"{local_branch_name}:{remote_branch_name}",
@@ -1089,6 +1076,8 @@ def push_binary_transparency(git_repo_path, git_remote):
     drive.
 
     """
+    import git
+
     logging.info(_('Pushing binary transparency log to {url}').format(url=git_remote))
 
     if os.path.isdir(os.path.dirname(git_remote)):

tests/deploy.TestCase

@@ -136,7 +136,7 @@ class DeployTest(unittest.TestCase):
         fdroidserver.common.options = Options
 
         # write out destination path
-        destination = Path('test_bucket_folder/fdroid')
+        destination = Path('some_bucket_folder/fdroid')
         destination.mkdir(parents=True, exist_ok=True)
         dest_apk = Path(destination) / fake_apk
         dest_index = Path(destination) / fake_index
@@ -180,7 +180,7 @@ class DeployTest(unittest.TestCase):
         fdroidserver.common.options = Options
 
         # write out destination path
-        destination = Path('test_bucket_folder/fdroid')
+        destination = Path('some_bucket_folder/fdroid')
         destination.mkdir(parents=True, exist_ok=True)
         dest_apk = Path(destination) / fake_apk
         dest_index = Path(destination) / fake_index
@@ -429,17 +429,10 @@ class DeployTest(unittest.TestCase):
             return 0
 
         with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
-            os.mkdir(repo_section)
+            os.mkdir('repo')
             os.symlink('repo/com.example.sym.apk', 'Sym.apk')
             os.symlink('repo/com.example.sym.apk.asc', 'Sym.apk.asc')
             os.symlink('repo/com.example.sym.apk.sig', 'Sym.apk.sig')
-
-            fake_files = fdroidserver.deploy.INDEX_FILES
-            for filename in fake_files:
-                fake_file = Path(repo_section) / filename
-                with fake_file.open('w') as fp:
-                    fp.write('not a real one, but has the right filename')
-
             with mock.patch('subprocess.call', side_effect=update_server_webroot_call):
                 fdroidserver.deploy.update_serverwebroot(
                     {'url': url, 'index_only': True}, repo_section
@@ -590,18 +583,10 @@ class DeployTest(unittest.TestCase):
             call_iteration += 1
             return 0
 
-        with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
-            with mock.patch('subprocess.call', side_effect=update_server_webroot_call):
-                os.mkdir(repo_section)
-                fake_files = fdroidserver.deploy.INDEX_FILES
-                for filename in fake_files:
-                    fake_file = Path(repo_section) / filename
-                    with fake_file.open('w') as fp:
-                        fp.write('not a real one, but has the right filename')
-
-                fdroidserver.deploy.update_serverwebroot(
-                    {'url': url, 'index_only': True}, repo_section
-                )
+        with mock.patch('subprocess.call', side_effect=update_server_webroot_call):
+            fdroidserver.deploy.update_serverwebroot(
+                {'url': url, 'index_only': True}, repo_section
+            )
         self.assertEqual(call_iteration, 1, 'expected 1 invocations of subprocess.call')
 
     @unittest.skipIf(
@@ -952,15 +937,13 @@ class DeployTest(unittest.TestCase):
             mock_driver.get_container.return_value = mock_container
             mock_driver.upload_object_via_stream.return_value = None
 
-            fdroidserver.deploy.update_awsbucket_libcloud(
-                repo_section, is_index_only=True
-            )
+            fdroidserver.deploy.update_awsbucket_libcloud(repo_section, is_index_only=True)
 
             mock_driver.get_container.assert_called_once_with(
                 container_name=fdroidserver.deploy.config["awsbucket"]
             )
             mock_container.list_objects.assert_called_once_with()
-            files_to_upload = [f"fdroid/repo/{fdroidserver.deploy.INDEX_FILES[0]}"]
+            files_to_upload = ['fdroid/repo/entry.jar']
             calls = [
                 mock.call(
                     iterator=mock.ANY,