mirror of
https://gitlab.com/fdroid/fdroidserver.git
synced 2024-09-21 04:10:37 +02:00
feat(update_awsbucket_s3cmd): support the index-only flag
This commit is contained in:
parent
fb0041bdc1
commit
0021c76a44
@ -62,18 +62,18 @@ INDEX_FILES = [
|
|||||||
|
|
||||||
|
|
||||||
def _get_index_file_paths(repo_section):
|
def _get_index_file_paths(repo_section):
|
||||||
return [os.path.join(repo_section, filename) for filename in INDEX_FILES]
|
|
||||||
|
|
||||||
|
|
||||||
def _get_index_excludes(repo_section):
|
|
||||||
"""Return the list of files to be synced last, since they finalize the deploy.
|
"""Return the list of files to be synced last, since they finalize the deploy.
|
||||||
|
|
||||||
The process of pushing all the new packages to the various
|
The process of pushing all the new packages to the various
|
||||||
services can take a while. So the index files should be updated
|
services can take a while. So the index files should be updated
|
||||||
last. That ensures that the package files are available when the
|
last. That ensures that the package files are available when the
|
||||||
client learns about them from the new index files.
|
client learns about them from the new index files.
|
||||||
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
return [os.path.join(repo_section, filename) for filename in INDEX_FILES]
|
||||||
|
|
||||||
|
|
||||||
|
def _get_index_excludes(repo_section):
|
||||||
indexes = _get_index_file_paths(repo_section)
|
indexes = _get_index_file_paths(repo_section)
|
||||||
index_excludes = []
|
index_excludes = []
|
||||||
for f in indexes:
|
for f in indexes:
|
||||||
@ -82,6 +82,15 @@ def _get_index_excludes(repo_section):
|
|||||||
return index_excludes
|
return index_excludes
|
||||||
|
|
||||||
|
|
||||||
|
def _get_index_includes(repo_section):
|
||||||
|
indexes = _get_index_file_paths(repo_section)
|
||||||
|
index_includes = []
|
||||||
|
for f in indexes:
|
||||||
|
index_includes.append('--include')
|
||||||
|
index_includes.append(f)
|
||||||
|
return index_includes
|
||||||
|
|
||||||
|
|
||||||
def update_awsbucket(repo_section):
|
def update_awsbucket(repo_section):
|
||||||
"""Upload the contents of the directory `repo_section` (including subdirectories) to the AWS S3 "bucket".
|
"""Upload the contents of the directory `repo_section` (including subdirectories) to the AWS S3 "bucket".
|
||||||
|
|
||||||
@ -142,32 +151,43 @@ def update_awsbucket_s3cmd(repo_section):
|
|||||||
s3cmd_sync += ['--quiet']
|
s3cmd_sync += ['--quiet']
|
||||||
|
|
||||||
s3url = s3bucketurl + '/fdroid/'
|
s3url = s3bucketurl + '/fdroid/'
|
||||||
logging.debug('s3cmd sync new files in ' + repo_section + ' to ' + s3url)
|
|
||||||
logging.debug(_('Running first pass with MD5 checking disabled'))
|
|
||||||
excludes = _get_index_excludes(repo_section)
|
|
||||||
returncode = subprocess.call(
|
|
||||||
s3cmd_sync
|
|
||||||
+ excludes
|
|
||||||
+ ['--no-check-md5', '--skip-existing', repo_section, s3url]
|
|
||||||
)
|
|
||||||
if returncode != 0:
|
|
||||||
raise FDroidException()
|
|
||||||
logging.debug('s3cmd sync all files in ' + repo_section + ' to ' + s3url)
|
|
||||||
returncode = subprocess.call(
|
|
||||||
s3cmd_sync + excludes + ['--no-check-md5', repo_section, s3url]
|
|
||||||
)
|
|
||||||
if returncode != 0:
|
|
||||||
raise FDroidException()
|
|
||||||
|
|
||||||
logging.debug(_('s3cmd sync indexes {path} to {url} and delete')
|
if not options.index_only:
|
||||||
|
logging.debug('s3cmd syncs new non-index files in ' + repo_section + ' to ' + s3url + 'and deletes removed')
|
||||||
|
logging.debug(_('Running first pass with MD5 checking disabled'))
|
||||||
|
sync_non_indexes_flags = []
|
||||||
|
|
||||||
|
# Excluded files are not considered as removed: https://github.com/s3tools/s3cmd/issues/802
|
||||||
|
sync_non_indexes_flags.append('--delete-removed')
|
||||||
|
sync_non_indexes_flags.append('--delete-after')
|
||||||
|
|
||||||
|
sync_non_indexes_flags.extend(_get_index_excludes(repo_section))
|
||||||
|
returncode = subprocess.call(
|
||||||
|
s3cmd_sync
|
||||||
|
+ sync_non_indexes_flags
|
||||||
|
+ ['--no-check-md5', '--skip-existing', repo_section, s3url]
|
||||||
|
)
|
||||||
|
if returncode != 0:
|
||||||
|
raise FDroidException()
|
||||||
|
logging.debug('s3cmd sync all files in ' + repo_section + ' to ' + s3url)
|
||||||
|
returncode = subprocess.call(
|
||||||
|
s3cmd_sync + sync_non_indexes_flags + ['--no-check-md5', repo_section, s3url]
|
||||||
|
)
|
||||||
|
if returncode != 0:
|
||||||
|
raise FDroidException()
|
||||||
|
|
||||||
|
logging.debug(_('s3cmd syncs indexes from {path} to {url} and deletes removed')
|
||||||
.format(path=repo_section, url=s3url))
|
.format(path=repo_section, url=s3url))
|
||||||
s3cmd_sync.append('--delete-removed')
|
sync_indexes_flags = []
|
||||||
s3cmd_sync.append('--delete-after')
|
sync_indexes_flags.append('--delete-removed')
|
||||||
|
sync_indexes_flags.append('--delete-after')
|
||||||
if options.no_checksum:
|
if options.no_checksum:
|
||||||
s3cmd_sync.append('--no-check-md5')
|
sync_indexes_flags.append('--no-check-md5')
|
||||||
else:
|
else:
|
||||||
s3cmd_sync.append('--check-md5')
|
sync_indexes_flags.append('--check-md5')
|
||||||
if subprocess.call(s3cmd_sync + [repo_section, s3url]) != 0:
|
sync_indexes_flags.extend(_get_index_includes(repo_section))
|
||||||
|
returncode = subprocess.call(s3cmd_sync + sync_indexes_flags + [repo_section, s3url])
|
||||||
|
if returncode != 0:
|
||||||
raise FDroidException()
|
raise FDroidException()
|
||||||
|
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user