feat(update_awsbucket_s3cmd): support the index-only flag

fdroidserver/deploy.py

@@ -64,18 +64,18 @@ INDEX_FILES = [
 
 
 def _get_index_file_paths(repo_section):
-    return [os.path.join(repo_section, filename) for filename in INDEX_FILES]
-
-
-def _get_index_excludes(repo_section):
     """Return the list of files to be synced last, since they finalize the deploy.
 
     The process of pushing all the new packages to the various
     services can take a while.  So the index files should be updated
     last.  That ensures that the package files are available when the
     client learns about them from the new index files.
-
     """
+
+    return [os.path.join(repo_section, filename) for filename in INDEX_FILES]
+
+
+def _get_index_excludes(repo_section):
     indexes = _get_index_file_paths(repo_section)
     index_excludes = []
     for f in indexes:
@@ -84,6 +84,15 @@ def _get_index_excludes(repo_section):
     return index_excludes
 
 
+def _get_index_includes(repo_section):
+    indexes = _get_index_file_paths(repo_section)
+    index_includes = []
+    for f in indexes:
+        index_includes.append('--include')
+        index_includes.append(f)
+    return index_includes
+
+
 def update_awsbucket(repo_section, verbose=False, quiet=False):
     """Upload the contents of the directory `repo_section` (including subdirectories) to the AWS S3 "bucket".
 
@@ -170,35 +179,48 @@ def update_awsbucket_s3cmd(repo_section):
         s3cmd_sync += ['--quiet']
 
     s3url = s3bucketurl + '/fdroid/'
-    logging.debug('s3cmd sync new files in ' + repo_section + ' to ' + s3url)
-    logging.debug(_('Running first pass with MD5 checking disabled'))
-    excludes = _get_index_excludes(repo_section)
-    returncode = subprocess.call(
-        s3cmd_sync
-        + excludes
-        + ['--no-check-md5', '--skip-existing', repo_section, s3url]
-    )
-    if returncode != 0:
-        raise FDroidException()
-    logging.debug('s3cmd sync all files in ' + repo_section + ' to ' + s3url)
-    returncode = subprocess.call(
-        s3cmd_sync + excludes + ['--no-check-md5', repo_section, s3url]
-    )
-    if returncode != 0:
-        raise FDroidException()
 
     logging.debug(
         _('s3cmd sync indexes {path} to {url} and delete').format(
             path=repo_section, url=s3url
         )
     )
-    s3cmd_sync.append('--delete-removed')
+    if not options.index_only:
-    s3cmd_sync.append('--delete-after')
+        logging.debug('s3cmd syncs new non-index files in ' + repo_section + ' to ' + s3url + 'and deletes removed')
+        logging.debug(_('Running first pass with MD5 checking disabled'))
+        sync_non_indexes_flags = []
+
+        # Excluded files are not considered as removed: https://github.com/s3tools/s3cmd/issues/802
+        sync_non_indexes_flags.append('--delete-removed')
+        sync_non_indexes_flags.append('--delete-after')
+
+        sync_non_indexes_flags.extend(_get_index_excludes(repo_section))
+        returncode = subprocess.call(
+            s3cmd_sync
+            + sync_non_indexes_flags
+            + ['--no-check-md5', '--skip-existing', repo_section, s3url]
+        )
+        if returncode != 0:
+            raise FDroidException()
+        logging.debug('s3cmd sync all files in ' + repo_section + ' to ' + s3url)
+        returncode = subprocess.call(
+            s3cmd_sync + sync_non_indexes_flags + ['--no-check-md5', repo_section, s3url]
+        )
+        if returncode != 0:
+            raise FDroidException()
+
+    logging.debug(_('s3cmd syncs indexes from {path} to {url} and deletes removed')
+                  .format(path=repo_section, url=s3url))
+    sync_indexes_flags = []
+    sync_indexes_flags.append('--delete-removed')
+    sync_indexes_flags.append('--delete-after')
     if options.no_checksum:
-        s3cmd_sync.append('--no-check-md5')
+        sync_indexes_flags.append('--no-check-md5')
     else:
-        s3cmd_sync.append('--check-md5')
+        sync_indexes_flags.append('--check-md5')
-    if subprocess.call(s3cmd_sync + [repo_section, s3url]) != 0:
+    sync_indexes_flags.extend(_get_index_includes(repo_section))
+    returncode = subprocess.call(s3cmd_sync + sync_indexes_flags + [repo_section, s3url])
+    if returncode != 0:
         raise FDroidException()