kamp/fdroidserver
1
0
Fork 0
mirror of https://gitlab.com/fdroid/fdroidserver.git synced 2024-09-21 04:10:37 +02:00
Code Activity

feat(update_awsbucket_s3cmd): support the index-only flag

Browse Source
This commit is contained in:
proletarius101 2023-12-10 22:58:09 +00:00 committed by Hans-Christoph Steiner
parent fb0041bdc1
commit 0021c76a44
1 changed files with 47 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

74
fdroidserver/deploy.py
View File

@ -62,18 +62,18 @@ INDEX_FILES = [
def _get_index_file_paths(repo_section):
return [os.path.join(repo_section, filename) for filename in INDEX_FILES]
def _get_index_excludes(repo_section):
"""Return the list of files to be synced last, since they finalize the deploy.
The process of pushing all the new packages to the various
services can take a while. So the index files should be updated
last. That ensures that the package files are available when the
client learns about them from the new index files.
"""
return [os.path.join(repo_section, filename) for filename in INDEX_FILES]
def _get_index_excludes(repo_section):
indexes = _get_index_file_paths(repo_section)
index_excludes = []
for f in indexes:
@ -82,6 +82,15 @@ def _get_index_excludes(repo_section):
return index_excludes
def _get_index_includes(repo_section):
indexes = _get_index_file_paths(repo_section)
index_includes = []
for f in indexes:
index_includes.append('--include')
index_includes.append(f)
return index_includes
def update_awsbucket(repo_section):
"""Upload the contents of the directory `repo_section` (including subdirectories) to the AWS S3 "bucket".
@ -142,32 +151,43 @@ def update_awsbucket_s3cmd(repo_section):
s3cmd_sync += ['--quiet']
s3url = s3bucketurl + '/fdroid/'
logging.debug('s3cmd sync new files in ' + repo_section + ' to ' + s3url)
logging.debug(_('Running first pass with MD5 checking disabled'))
excludes = _get_index_excludes(repo_section)
returncode = subprocess.call(
s3cmd_sync
+ excludes
+ ['--no-check-md5', '--skip-existing', repo_section, s3url]
)
if returncode != 0:
raise FDroidException()
logging.debug('s3cmd sync all files in ' + repo_section + ' to ' + s3url)
returncode = subprocess.call(
s3cmd_sync + excludes + ['--no-check-md5', repo_section, s3url]
)
if returncode != 0:
raise FDroidException()
logging.debug(_('s3cmd sync indexes {path} to {url} and delete')
if not options.index_only:
logging.debug('s3cmd syncs new non-index files in ' + repo_section + ' to ' + s3url + 'and deletes removed')
logging.debug(_('Running first pass with MD5 checking disabled'))
sync_non_indexes_flags = []
# Excluded files are not considered as removed: https://github.com/s3tools/s3cmd/issues/802
sync_non_indexes_flags.append('--delete-removed')
sync_non_indexes_flags.append('--delete-after')
sync_non_indexes_flags.extend(_get_index_excludes(repo_section))
returncode = subprocess.call(
s3cmd_sync
+ sync_non_indexes_flags
+ ['--no-check-md5', '--skip-existing', repo_section, s3url]
)
if returncode != 0:
raise FDroidException()
logging.debug('s3cmd sync all files in ' + repo_section + ' to ' + s3url)
returncode = subprocess.call(
s3cmd_sync + sync_non_indexes_flags + ['--no-check-md5', repo_section, s3url]
)
if returncode != 0:
raise FDroidException()
logging.debug(_('s3cmd syncs indexes from {path} to {url} and deletes removed')
.format(path=repo_section, url=s3url))
s3cmd_sync.append('--delete-removed')
s3cmd_sync.append('--delete-after')
sync_indexes_flags = []
sync_indexes_flags.append('--delete-removed')
sync_indexes_flags.append('--delete-after')
if options.no_checksum:
s3cmd_sync.append('--no-check-md5')
sync_indexes_flags.append('--no-check-md5')
else:
s3cmd_sync.append('--check-md5')
if subprocess.call(s3cmd_sync + [repo_section, s3url]) != 0:
sync_indexes_flags.append('--check-md5')
sync_indexes_flags.extend(_get_index_includes(repo_section))
returncode = subprocess.call(s3cmd_sync + sync_indexes_flags + [repo_section, s3url])
if returncode != 0:
raise FDroidException()