mirror of
https://gitlab.com/fdroid/fdroidserver.git
synced 2024-11-19 21:30:10 +01:00
Merge branch 'deploy-index-v2' into 'master'
deploy: handle index-v2 files on two pass sync methods See merge request fdroid/fdroidserver!1132
This commit is contained in:
Hans-Christoph Steiner
commit
dc3175cc06
@ -46,6 +46,35 @@ USER_S3CFG = 's3cfg'
|
||||
REMOTE_HOSTNAME_REGEX = re.compile(r'\W*\w+\W+(\w+).*')
|
||||
|
||||
|
||||
def _get_index_excludes(repo_section):
|
||||
"""Return the list of files to be synced last, since they finalize the deploy.
|
||||
|
||||
The process of pushing all the new packages to the various
|
||||
services can take a while. So the index files should be updated
|
||||
last. That ensures that the package files are available when the
|
||||
client learns about them from the new index files.
|
||||
|
||||
"""
|
||||
indexes = [
|
||||
os.path.join(repo_section, 'entry.jar'),
|
||||
os.path.join(repo_section, 'entry.json'),
|
||||
os.path.join(repo_section, 'entry.json.asc'),
|
||||
os.path.join(repo_section, 'index-v1.jar'),
|
||||
os.path.join(repo_section, 'index-v1.json'),
|
||||
os.path.join(repo_section, 'index-v1.json.asc'),
|
||||
os.path.join(repo_section, 'index-v2.jar'),
|
||||
os.path.join(repo_section, 'index-v2.json'),
|
||||
os.path.join(repo_section, 'index-v2.json.asc'),
|
||||
os.path.join(repo_section, 'index.jar'),
|
||||
os.path.join(repo_section, 'index.xml'),
|
||||
]
|
||||
index_excludes = []
|
||||
for f in indexes:
|
||||
index_excludes.append('--exclude')
|
||||
index_excludes.append(f)
|
||||
return index_excludes
|
||||
|
||||
|
||||
def update_awsbucket(repo_section):
|
||||
"""Upload the contents of the directory `repo_section` (including subdirectories) to the AWS S3 "bucket".
|
||||
|
||||
@ -104,33 +133,23 @@ def update_awsbucket_s3cmd(repo_section):
|
||||
s3cmd_sync += ['--verbose']
|
||||
if options.quiet:
|
||||
s3cmd_sync += ['--quiet']
|
||||
indexxml = os.path.join(repo_section, 'index.xml')
|
||||
indexjar = os.path.join(repo_section, 'index.jar')
|
||||
indexv1jar = os.path.join(repo_section, 'index-v1.jar')
|
||||
indexv1json = os.path.join(repo_section, 'index-v1.json')
|
||||
indexv1jsonasc = os.path.join(repo_section, 'index-v1.json.asc')
|
||||
|
||||
s3url = s3bucketurl + '/fdroid/'
|
||||
logging.debug('s3cmd sync new files in ' + repo_section + ' to ' + s3url)
|
||||
logging.debug(_('Running first pass with MD5 checking disabled'))
|
||||
if subprocess.call(s3cmd_sync
|
||||
+ ['--no-check-md5', '--skip-existing',
|
||||
'--exclude', indexxml,
|
||||
'--exclude', indexjar,
|
||||
'--exclude', indexv1jar,
|
||||
'--exclude', indexv1json,
|
||||
'--exclude', indexv1jsonasc,
|
||||
repo_section, s3url]) != 0:
|
||||
excludes = _get_index_excludes(repo_section)
|
||||
returncode = subprocess.call(
|
||||
s3cmd_sync
|
||||
+ excludes
|
||||
+ ['--no-check-md5', '--skip-existing', repo_section, s3url]
|
||||
)
|
||||
if returncode != 0:
|
||||
raise FDroidException()
|
||||
logging.debug('s3cmd sync all files in ' + repo_section + ' to ' + s3url)
|
||||
if subprocess.call(s3cmd_sync
|
||||
+ ['--no-check-md5',
|
||||
'--exclude', indexxml,
|
||||
'--exclude', indexjar,
|
||||
'--exclude', indexv1jar,
|
||||
'--exclude', indexv1json,
|
||||
'--exclude', indexv1jsonasc,
|
||||
repo_section, s3url]) != 0:
|
||||
returncode = subprocess.call(
|
||||
s3cmd_sync + excludes + ['--no-check-md5', repo_section, s3url]
|
||||
)
|
||||
if returncode != 0:
|
||||
raise FDroidException()
|
||||
|
||||
logging.debug(_('s3cmd sync indexes {path} to {url} and delete')
|
||||
@ -241,8 +260,22 @@ def update_awsbucket_libcloud(repo_section):
|
||||
|
||||
|
||||
def update_serverwebroot(serverwebroot, repo_section):
|
||||
# use a checksum comparison for accurate comparisons on different
|
||||
# filesystems, for example, FAT has a low resolution timestamp
|
||||
"""Deploy the index files to the serverwebroot using rsync.
|
||||
|
||||
Upload the first time without the index files and delay the
|
||||
deletion as much as possible. That keeps the repo functional
|
||||
while this update is running. Then once it is complete, rerun the
|
||||
command again to upload the index files. Always using the same
|
||||
target with rsync allows for very strict settings on the receiving
|
||||
server, you can literally specify the one rsync command that is
|
||||
allowed to run in ~/.ssh/authorized_keys. (serverwebroot is
|
||||
guaranteed to have a trailing slash in common.py)
|
||||
|
||||
It is possible to optionally use a checksum comparison for
|
||||
accurate comparisons on different filesystems, for example, FAT
|
||||
has a low resolution timestamp
|
||||
|
||||
"""
|
||||
rsyncargs = ['rsync', '--archive', '--delete-after', '--safe-links']
|
||||
if not options.no_checksum:
|
||||
rsyncargs.append('--checksum')
|
||||
@ -254,26 +287,9 @@ def update_serverwebroot(serverwebroot, repo_section):
|
||||
rsyncargs += ['-e', 'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ' + options.identity_file]
|
||||
elif 'identity_file' in config:
|
||||
rsyncargs += ['-e', 'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ' + config['identity_file']]
|
||||
indexxml = os.path.join(repo_section, 'index.xml')
|
||||
indexjar = os.path.join(repo_section, 'index.jar')
|
||||
indexv1jar = os.path.join(repo_section, 'index-v1.jar')
|
||||
indexv1json = os.path.join(repo_section, 'index-v1.json')
|
||||
indexv1jsonasc = os.path.join(repo_section, 'index-v1.json.asc')
|
||||
# Upload the first time without the index files and delay the deletion as
|
||||
# much as possible, that keeps the repo functional while this update is
|
||||
# running. Then once it is complete, rerun the command again to upload
|
||||
# the index files. Always using the same target with rsync allows for
|
||||
# very strict settings on the receiving server, you can literally specify
|
||||
# the one rsync command that is allowed to run in ~/.ssh/authorized_keys.
|
||||
# (serverwebroot is guaranteed to have a trailing slash in common.py)
|
||||
logging.info('rsyncing ' + repo_section + ' to ' + serverwebroot)
|
||||
if subprocess.call(rsyncargs
|
||||
+ ['--exclude', indexxml,
|
||||
'--exclude', indexjar,
|
||||
'--exclude', indexv1jar,
|
||||
'--exclude', indexv1json,
|
||||
'--exclude', indexv1jsonasc,
|
||||
repo_section, serverwebroot]) != 0:
|
||||
excludes = _get_index_excludes(repo_section)
|
||||
if subprocess.call(rsyncargs + excludes + [repo_section, serverwebroot]) != 0:
|
||||
raise FDroidException()
|
||||
if subprocess.call(rsyncargs + [repo_section, serverwebroot]) != 0:
|
||||
raise FDroidException()
|
||||
|
||||
@ -57,15 +57,27 @@ class DeployTest(unittest.TestCase):
|
||||
'--safe-links',
|
||||
'--quiet',
|
||||
'--exclude',
|
||||
'repo/index.xml',
|
||||
'repo/entry.jar',
|
||||
'--exclude',
|
||||
'repo/index.jar',
|
||||
'repo/entry.json',
|
||||
'--exclude',
|
||||
'repo/entry.json.asc',
|
||||
'--exclude',
|
||||
'repo/index-v1.jar',
|
||||
'--exclude',
|
||||
'repo/index-v1.json',
|
||||
'--exclude',
|
||||
'repo/index-v1.json.asc',
|
||||
'--exclude',
|
||||
'repo/index-v2.jar',
|
||||
'--exclude',
|
||||
'repo/index-v2.json',
|
||||
'--exclude',
|
||||
'repo/index-v2.json.asc',
|
||||
'--exclude',
|
||||
'repo/index.jar',
|
||||
'--exclude',
|
||||
'repo/index.xml',
|
||||
'repo',
|
||||
'example.com:/var/www/fdroid',
|
||||
],
|
||||
@ -142,15 +154,27 @@ class DeployTest(unittest.TestCase):
|
||||
'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i '
|
||||
+ fdroidserver.deploy.config['identity_file'],
|
||||
'--exclude',
|
||||
'archive/index.xml',
|
||||
'archive/entry.jar',
|
||||
'--exclude',
|
||||
'archive/index.jar',
|
||||
'archive/entry.json',
|
||||
'--exclude',
|
||||
'archive/entry.json.asc',
|
||||
'--exclude',
|
||||
'archive/index-v1.jar',
|
||||
'--exclude',
|
||||
'archive/index-v1.json',
|
||||
'--exclude',
|
||||
'archive/index-v1.json.asc',
|
||||
'--exclude',
|
||||
'archive/index-v2.jar',
|
||||
'--exclude',
|
||||
'archive/index-v2.json',
|
||||
'--exclude',
|
||||
'archive/index-v2.json.asc',
|
||||
'--exclude',
|
||||
'archive/index.jar',
|
||||
'--exclude',
|
||||
'archive/index.xml',
|
||||
'archive',
|
||||
serverwebroot,
|
||||
],
|
||||
|
||||
Loading…
Reference in New Issue
Block a user