style(deply): fix pylint

fdroidserver/deploy.py

@@ -96,6 +96,15 @@ def _get_index_includes(base_dir):
     return index_includes
 
 
+def _remove_missing_files(files: List[str]) -> List[str]:
+    """Remove files that are missing from the file system."""
+    existing = []
+    for f in files:
+        if os.path.exists(f):
+            existing.append(f)
+    return existing
+
+
 def update_awsbucket(repo_section, is_index_only=False, verbose=False, quiet=False):
     """Upload the contents of the directory `repo_section` (including subdirectories) to the AWS S3 "bucket".
 
@@ -299,8 +308,9 @@ def update_remote_storage_with_rclone(
 
     if is_index_only:
         sources = _get_index_file_paths(repo_section)
+        sources = _remove_missing_files(sources)
     else:
-        sources = repo_section
+        sources = [repo_section]
 
     for source in sources:
         if isinstance(config['rclone_config'], str):
@@ -429,6 +439,7 @@ def update_awsbucket_libcloud(repo_section, is_index_only=False):
             for name in files
         ]
         files_to_upload = list(set(files_to_upload) & set(index_files))
+        files_to_upload = _remove_missing_files(files_to_upload)
 
     else:
         files_to_upload = [
@@ -534,7 +545,10 @@ def update_serverwebroot(serverwebroot, repo_section):
     is_index_only = serverwebroot.get('index_only', False)
     logging.info('rsyncing ' + repo_section + ' to ' + url)
     if is_index_only:
-        rsyncargs += _get_index_file_paths(repo_section)
+        files_to_upload = _get_index_file_paths(repo_section)
+        files_to_upload = _remove_missing_files(files_to_upload)
+
+        rsyncargs += files_to_upload
         rsyncargs += [f'{url}/{repo_section}/']
         logging.info(rsyncargs)
         if subprocess.call(rsyncargs) != 0:
@@ -732,10 +746,8 @@ def update_servergitmirrors(servergitmirrors, repo_section):
             # trailing slashes have a meaning in rsync which is not needed here, so
             # make sure both paths have exactly one trailing slash
             if is_index_only:
-                files_to_sync = [
-                    str(workspace_dir / repo_section / index_file)
-                    for index_file in INDEX_FILES
-                ]
+                files_to_sync = _get_index_file_paths(str(workspace_dir / repo_section))
+                files_to_sync = _remove_missing_files(files_to_sync)
             else:
                 files_to_sync = [str(workspace_dir / repo_section).rstrip('/') + '/']
             common.local_rsync(
@@ -784,9 +796,11 @@ def upload_to_servergitmirror(
     logging.info('Mirroring to: ' + remote_url)
 
     if is_index_only:
-        local_repo.index.add(
-            _get_index_file_paths(os.path.join('fdroid', repo_section))
+        files_to_upload = _get_index_file_paths(
+            os.path.join(local_repo.working_tree_dir, 'fdroid', repo_section)
         )
+        files_to_upload = _remove_missing_files(files_to_upload)
+        local_repo.index.add(files_to_upload)
     else:
         # sadly index.add don't allow the --all parameter
         logging.debug('Adding all files to git mirror')
@@ -1075,8 +1089,6 @@ def push_binary_transparency(git_repo_path, git_remote):
     drive.
 
     """
-    import git
-
     logging.info(_('Pushing binary transparency log to {url}').format(url=git_remote))
 
     if os.path.isdir(os.path.dirname(git_remote)):

tests/deploy.TestCase

@@ -136,7 +136,7 @@ class DeployTest(unittest.TestCase):
         fdroidserver.common.options = Options
 
         # write out destination path
-        destination = Path('some_bucket_folder/fdroid')
+        destination = Path('test_bucket_folder/fdroid')
         destination.mkdir(parents=True, exist_ok=True)
         dest_apk = Path(destination) / fake_apk
         dest_index = Path(destination) / fake_index
@@ -180,7 +180,7 @@ class DeployTest(unittest.TestCase):
         fdroidserver.common.options = Options
 
         # write out destination path
-        destination = Path('some_bucket_folder/fdroid')
+        destination = Path('test_bucket_folder/fdroid')
         destination.mkdir(parents=True, exist_ok=True)
         dest_apk = Path(destination) / fake_apk
         dest_index = Path(destination) / fake_index
@@ -429,10 +429,17 @@ class DeployTest(unittest.TestCase):
             return 0
 
         with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
-            os.mkdir('repo')
+            os.mkdir(repo_section)
             os.symlink('repo/com.example.sym.apk', 'Sym.apk')
             os.symlink('repo/com.example.sym.apk.asc', 'Sym.apk.asc')
             os.symlink('repo/com.example.sym.apk.sig', 'Sym.apk.sig')
+
+            fake_files = fdroidserver.deploy.INDEX_FILES
+            for filename in fake_files:
+                fake_file = Path(repo_section) / filename
+                with fake_file.open('w') as fp:
+                    fp.write('not a real one, but has the right filename')
+
             with mock.patch('subprocess.call', side_effect=update_server_webroot_call):
                 fdroidserver.deploy.update_serverwebroot(
                     {'url': url, 'index_only': True}, repo_section
@@ -583,7 +590,15 @@ class DeployTest(unittest.TestCase):
             call_iteration += 1
             return 0
 
+        with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
             with mock.patch('subprocess.call', side_effect=update_server_webroot_call):
+                os.mkdir(repo_section)
+                fake_files = fdroidserver.deploy.INDEX_FILES
+                for filename in fake_files:
+                    fake_file = Path(repo_section) / filename
+                    with fake_file.open('w') as fp:
+                        fp.write('not a real one, but has the right filename')
+
                 fdroidserver.deploy.update_serverwebroot(
                     {'url': url, 'index_only': True}, repo_section
                 )