mirror of
https://gitlab.com/fdroid/fdroidserver.git
synced 2024-09-21 04:10:37 +02:00
feat(update_awsbucket_s3cmd): support the index-only flag
This commit is contained in:
parent
fb0041bdc1
commit
0021c76a44
@ -62,18 +62,18 @@ INDEX_FILES = [
|
||||
|
||||
|
||||
def _get_index_file_paths(repo_section):
|
||||
return [os.path.join(repo_section, filename) for filename in INDEX_FILES]
|
||||
|
||||
|
||||
def _get_index_excludes(repo_section):
|
||||
"""Return the list of files to be synced last, since they finalize the deploy.
|
||||
|
||||
The process of pushing all the new packages to the various
|
||||
services can take a while. So the index files should be updated
|
||||
last. That ensures that the package files are available when the
|
||||
client learns about them from the new index files.
|
||||
|
||||
"""
|
||||
|
||||
return [os.path.join(repo_section, filename) for filename in INDEX_FILES]
|
||||
|
||||
|
||||
def _get_index_excludes(repo_section):
|
||||
indexes = _get_index_file_paths(repo_section)
|
||||
index_excludes = []
|
||||
for f in indexes:
|
||||
@ -82,6 +82,15 @@ def _get_index_excludes(repo_section):
|
||||
return index_excludes
|
||||
|
||||
|
||||
def _get_index_includes(repo_section):
|
||||
indexes = _get_index_file_paths(repo_section)
|
||||
index_includes = []
|
||||
for f in indexes:
|
||||
index_includes.append('--include')
|
||||
index_includes.append(f)
|
||||
return index_includes
|
||||
|
||||
|
||||
def update_awsbucket(repo_section):
|
||||
"""Upload the contents of the directory `repo_section` (including subdirectories) to the AWS S3 "bucket".
|
||||
|
||||
@ -142,32 +151,43 @@ def update_awsbucket_s3cmd(repo_section):
|
||||
s3cmd_sync += ['--quiet']
|
||||
|
||||
s3url = s3bucketurl + '/fdroid/'
|
||||
logging.debug('s3cmd sync new files in ' + repo_section + ' to ' + s3url)
|
||||
|
||||
if not options.index_only:
|
||||
logging.debug('s3cmd syncs new non-index files in ' + repo_section + ' to ' + s3url + 'and deletes removed')
|
||||
logging.debug(_('Running first pass with MD5 checking disabled'))
|
||||
excludes = _get_index_excludes(repo_section)
|
||||
sync_non_indexes_flags = []
|
||||
|
||||
# Excluded files are not considered as removed: https://github.com/s3tools/s3cmd/issues/802
|
||||
sync_non_indexes_flags.append('--delete-removed')
|
||||
sync_non_indexes_flags.append('--delete-after')
|
||||
|
||||
sync_non_indexes_flags.extend(_get_index_excludes(repo_section))
|
||||
returncode = subprocess.call(
|
||||
s3cmd_sync
|
||||
+ excludes
|
||||
+ sync_non_indexes_flags
|
||||
+ ['--no-check-md5', '--skip-existing', repo_section, s3url]
|
||||
)
|
||||
if returncode != 0:
|
||||
raise FDroidException()
|
||||
logging.debug('s3cmd sync all files in ' + repo_section + ' to ' + s3url)
|
||||
returncode = subprocess.call(
|
||||
s3cmd_sync + excludes + ['--no-check-md5', repo_section, s3url]
|
||||
s3cmd_sync + sync_non_indexes_flags + ['--no-check-md5', repo_section, s3url]
|
||||
)
|
||||
if returncode != 0:
|
||||
raise FDroidException()
|
||||
|
||||
logging.debug(_('s3cmd sync indexes {path} to {url} and delete')
|
||||
logging.debug(_('s3cmd syncs indexes from {path} to {url} and deletes removed')
|
||||
.format(path=repo_section, url=s3url))
|
||||
s3cmd_sync.append('--delete-removed')
|
||||
s3cmd_sync.append('--delete-after')
|
||||
sync_indexes_flags = []
|
||||
sync_indexes_flags.append('--delete-removed')
|
||||
sync_indexes_flags.append('--delete-after')
|
||||
if options.no_checksum:
|
||||
s3cmd_sync.append('--no-check-md5')
|
||||
sync_indexes_flags.append('--no-check-md5')
|
||||
else:
|
||||
s3cmd_sync.append('--check-md5')
|
||||
if subprocess.call(s3cmd_sync + [repo_section, s3url]) != 0:
|
||||
sync_indexes_flags.append('--check-md5')
|
||||
sync_indexes_flags.extend(_get_index_includes(repo_section))
|
||||
returncode = subprocess.call(s3cmd_sync + sync_indexes_flags + [repo_section, s3url])
|
||||
if returncode != 0:
|
||||
raise FDroidException()
|
||||
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user