From 053a64718a90e37d2bc5e90c40b29e3159cd415d Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 23 May 2022 11:36:06 +0200 Subject: [PATCH 1/3] deploy: handle index-v2 files on two pass sync methods When using rsync or s3cmd, the upload is done in multiple passes. This reduces the chance of interfering with an existing client-server interaction. - rsync: In the first pass, upload without the index files and delay the deletion as much as possible. That keeps the repo functional while this update is running. Then second pass uploads the index files. - s3cmd: In the first pass, only new files are uploaded. In the second pass, changed files are uploaded, overwriting what is on the server. On the third/last pass, the indexes are uploaded, and any removed files are deleted from the server. The last pass is the only pass to use a full MD5 checksum of all files to detect changes. --- fdroidserver/deploy.py | 75 +++++++++++++++++++++++------------------- 1 file changed, 42 insertions(+), 33 deletions(-) diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py index a19bdeda..304bc2fd 100644 --- a/fdroidserver/deploy.py +++ b/fdroidserver/deploy.py @@ -46,6 +46,35 @@ USER_S3CFG = 's3cfg' REMOTE_HOSTNAME_REGEX = re.compile(r'\W*\w+\W+(\w+).*') +def _get_index_excludes(repo_section): + """The list of files to be synced last, since they finalize the deploy. + + The process of pushing all the new packages to the various + services can take a while. So the index files should be updated + last. That ensures that the package files are available when the + client learns about them from the new index files. + + """ + indexes = [ + os.path.join(repo_section, 'entry.jar'), + os.path.join(repo_section, 'entry.json'), + os.path.join(repo_section, 'entry.json.asc'), + os.path.join(repo_section, 'index-v1.jar'), + os.path.join(repo_section, 'index-v1.json'), + os.path.join(repo_section, 'index-v1.json.asc'), + os.path.join(repo_section, 'index-v2.jar'), + os.path.join(repo_section, 'index-v2.json'), + os.path.join(repo_section, 'index-v2.json.asc'), + os.path.join(repo_section, 'index.jar'), + os.path.join(repo_section, 'index.xml'), + ] + index_excludes = [] + for f in indexes: + index_excludes.append('--exclude') + index_excludes.append(f) + return index_excludes + + def update_awsbucket(repo_section): """Upload the contents of the directory `repo_section` (including subdirectories) to the AWS S3 "bucket". @@ -104,33 +133,23 @@ def update_awsbucket_s3cmd(repo_section): s3cmd_sync += ['--verbose'] if options.quiet: s3cmd_sync += ['--quiet'] - indexxml = os.path.join(repo_section, 'index.xml') - indexjar = os.path.join(repo_section, 'index.jar') - indexv1jar = os.path.join(repo_section, 'index-v1.jar') - indexv1json = os.path.join(repo_section, 'index-v1.json') - indexv1jsonasc = os.path.join(repo_section, 'index-v1.json.asc') s3url = s3bucketurl + '/fdroid/' logging.debug('s3cmd sync new files in ' + repo_section + ' to ' + s3url) logging.debug(_('Running first pass with MD5 checking disabled')) - if subprocess.call(s3cmd_sync - + ['--no-check-md5', '--skip-existing', - '--exclude', indexxml, - '--exclude', indexjar, - '--exclude', indexv1jar, - '--exclude', indexv1json, - '--exclude', indexv1jsonasc, - repo_section, s3url]) != 0: + excludes = _get_index_excludes(repo_section) + returncode = subprocess.call( + s3cmd_sync + + excludes + + ['--no-check-md5', '--skip-existing', repo_section, s3url] + ) + if returncode != 0: raise FDroidException() logging.debug('s3cmd sync all files in ' + repo_section + ' to ' + s3url) - if subprocess.call(s3cmd_sync - + ['--no-check-md5', - '--exclude', indexxml, - '--exclude', indexjar, - '--exclude', indexv1jar, - '--exclude', indexv1json, - '--exclude', indexv1jsonasc, - repo_section, s3url]) != 0: + returncode = subprocess.call( + s3cmd_sync + excludes + ['--no-check-md5', repo_section, s3url] + ) + if returncode != 0: raise FDroidException() logging.debug(_('s3cmd sync indexes {path} to {url} and delete') @@ -254,11 +273,6 @@ def update_serverwebroot(serverwebroot, repo_section): rsyncargs += ['-e', 'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ' + options.identity_file] elif 'identity_file' in config: rsyncargs += ['-e', 'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ' + config['identity_file']] - indexxml = os.path.join(repo_section, 'index.xml') - indexjar = os.path.join(repo_section, 'index.jar') - indexv1jar = os.path.join(repo_section, 'index-v1.jar') - indexv1json = os.path.join(repo_section, 'index-v1.json') - indexv1jsonasc = os.path.join(repo_section, 'index-v1.json.asc') # Upload the first time without the index files and delay the deletion as # much as possible, that keeps the repo functional while this update is # running. Then once it is complete, rerun the command again to upload @@ -267,13 +281,8 @@ def update_serverwebroot(serverwebroot, repo_section): # the one rsync command that is allowed to run in ~/.ssh/authorized_keys. # (serverwebroot is guaranteed to have a trailing slash in common.py) logging.info('rsyncing ' + repo_section + ' to ' + serverwebroot) - if subprocess.call(rsyncargs - + ['--exclude', indexxml, - '--exclude', indexjar, - '--exclude', indexv1jar, - '--exclude', indexv1json, - '--exclude', indexv1jsonasc, - repo_section, serverwebroot]) != 0: + excludes = _get_index_excludes(repo_section) + if subprocess.call(rsyncargs + excludes + [repo_section, serverwebroot]) != 0: raise FDroidException() if subprocess.call(rsyncargs + [repo_section, serverwebroot]) != 0: raise FDroidException() From 293975d08112105dbe5e955dc88ea8c777a052ce Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 23 May 2022 11:36:06 +0200 Subject: [PATCH 2/3] refactor comment into docstring for update_serverwebroot --- fdroidserver/deploy.py | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py index 304bc2fd..318250b4 100644 --- a/fdroidserver/deploy.py +++ b/fdroidserver/deploy.py @@ -260,8 +260,22 @@ def update_awsbucket_libcloud(repo_section): def update_serverwebroot(serverwebroot, repo_section): - # use a checksum comparison for accurate comparisons on different - # filesystems, for example, FAT has a low resolution timestamp + """Deploy the index files to the serverwebroot using rsync. + + Upload the first time without the index files and delay the + deletion as much as possible. That keeps the repo functional + while this update is running. Then once it is complete, rerun the + command again to upload the index files. Always using the same + target with rsync allows for very strict settings on the receiving + server, you can literally specify the one rsync command that is + allowed to run in ~/.ssh/authorized_keys. (serverwebroot is + guaranteed to have a trailing slash in common.py) + + It is possible to optionally use a checksum comparison for + accurate comparisons on different filesystems, for example, FAT + has a low resolution timestamp + + """ rsyncargs = ['rsync', '--archive', '--delete-after', '--safe-links'] if not options.no_checksum: rsyncargs.append('--checksum') @@ -273,13 +287,6 @@ def update_serverwebroot(serverwebroot, repo_section): rsyncargs += ['-e', 'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ' + options.identity_file] elif 'identity_file' in config: rsyncargs += ['-e', 'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ' + config['identity_file']] - # Upload the first time without the index files and delay the deletion as - # much as possible, that keeps the repo functional while this update is - # running. Then once it is complete, rerun the command again to upload - # the index files. Always using the same target with rsync allows for - # very strict settings on the receiving server, you can literally specify - # the one rsync command that is allowed to run in ~/.ssh/authorized_keys. - # (serverwebroot is guaranteed to have a trailing slash in common.py) logging.info('rsyncing ' + repo_section + ' to ' + serverwebroot) excludes = _get_index_excludes(repo_section) if subprocess.call(rsyncargs + excludes + [repo_section, serverwebroot]) != 0: From 2448f070e9a9e88f048370d82e4acc761f308ab2 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 23 May 2022 15:25:05 +0200 Subject: [PATCH 3/3] fix tests and docstring error --- fdroidserver/deploy.py | 2 +- tests/deploy.TestCase | 32 ++++++++++++++++++++++++++++---- 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/fdroidserver/deploy.py b/fdroidserver/deploy.py index 318250b4..953c6f71 100644 --- a/fdroidserver/deploy.py +++ b/fdroidserver/deploy.py @@ -47,7 +47,7 @@ REMOTE_HOSTNAME_REGEX = re.compile(r'\W*\w+\W+(\w+).*') def _get_index_excludes(repo_section): - """The list of files to be synced last, since they finalize the deploy. + """Return the list of files to be synced last, since they finalize the deploy. The process of pushing all the new packages to the various services can take a while. So the index files should be updated diff --git a/tests/deploy.TestCase b/tests/deploy.TestCase index f493d7ea..046dcc2b 100755 --- a/tests/deploy.TestCase +++ b/tests/deploy.TestCase @@ -57,15 +57,27 @@ class DeployTest(unittest.TestCase): '--safe-links', '--quiet', '--exclude', - 'repo/index.xml', + 'repo/entry.jar', '--exclude', - 'repo/index.jar', + 'repo/entry.json', + '--exclude', + 'repo/entry.json.asc', '--exclude', 'repo/index-v1.jar', '--exclude', 'repo/index-v1.json', '--exclude', 'repo/index-v1.json.asc', + '--exclude', + 'repo/index-v2.jar', + '--exclude', + 'repo/index-v2.json', + '--exclude', + 'repo/index-v2.json.asc', + '--exclude', + 'repo/index.jar', + '--exclude', + 'repo/index.xml', 'repo', 'example.com:/var/www/fdroid', ], @@ -142,15 +154,27 @@ class DeployTest(unittest.TestCase): 'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ' + fdroidserver.deploy.config['identity_file'], '--exclude', - 'archive/index.xml', + 'archive/entry.jar', '--exclude', - 'archive/index.jar', + 'archive/entry.json', + '--exclude', + 'archive/entry.json.asc', '--exclude', 'archive/index-v1.jar', '--exclude', 'archive/index-v1.json', '--exclude', 'archive/index-v1.json.asc', + '--exclude', + 'archive/index-v2.jar', + '--exclude', + 'archive/index-v2.json', + '--exclude', + 'archive/index-v2.json.asc', + '--exclude', + 'archive/index.jar', + '--exclude', + 'archive/index.xml', 'archive', serverwebroot, ],