From 25f6b0c246c43822a6aa38af958a804c3df611b4 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 26 Jun 2014 14:18:29 -0400 Subject: [PATCH] server: --sync-from-local-copy-dir for updating from offline signing repo To support a fully offline build/signing machine, there is the "local copy dir". The repo is generated on the offline machine and then copied to a local dir where a thumb drive or SD Card is mounted. Then on the online machine, using `fdroid server update --sync-from-local-copy-dir` allows the whole server update process to happen in a single command: 0. read config.py on online machine's repo 1. rsync from the local_copy_dir to the current dir 2. copy to serverwebroot, awsbucket, etc. --- examples/config.py | 8 ++++++++ fdroidserver/common.py | 1 + fdroidserver/server.py | 30 ++++++++++++++++++++++++------ tests/run-tests | 9 ++++++++- 4 files changed, 41 insertions(+), 7 deletions(-) diff --git a/examples/config.py b/examples/config.py index faf704bf..221b63bf 100644 --- a/examples/config.py +++ b/examples/config.py @@ -124,6 +124,14 @@ keyaliases['com.example.another.plugin'] = '@com.example.another' # local_copy_dir = '/media/MyUSBThumbDrive/fdroid' +# If you are using local_copy_dir on an offline build/signing server, once the +# thumb drive has been plugged into the online machine, it will need to be +# synced to the copy on the online machine. To make that happen +# automatically, set sync_from_local_copy_dir to True: +# +# sync_from_local_copy_dir = True + + # To upload the repo to an Amazon S3 bucket using `fdroid server update`. # Warning, this deletes and recreates the whole fdroid/ directory each # time. This is based on apache-libcloud, which supports basically all cloud diff --git a/fdroidserver/common.py b/fdroidserver/common.py index a4b3fb0f..afafffdf 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -47,6 +47,7 @@ def get_default_config(): 'mvn3': "mvn", 'gradle': 'gradle', 'archive_older': 0, + 'sync_from_local_copy_dir': False, 'update_stats': False, 'stats_to_carbon': False, 'repo_maxage': 0, diff --git a/fdroidserver/server.py b/fdroidserver/server.py index 100fff29..6e686017 100644 --- a/fdroidserver/server.py +++ b/fdroidserver/server.py @@ -140,8 +140,8 @@ def update_serverwebroot(repo_section): sys.exit(1) -def update_localcopy(repo_section, local_copy_dir): - rsyncargs = ['rsync', '--update', '--recursive', '--delete'] +def _local_sync(fromdir, todir): + rsyncargs = ['rsync', '--archive', '--one-file-system', '--delete'] # use stricter rsync checking on all files since people using offline mode # are already prioritizing security above ease and speed rsyncargs += ['--checksum'] @@ -149,11 +149,24 @@ def update_localcopy(repo_section, local_copy_dir): rsyncargs += ['--verbose'] if options.quiet: rsyncargs += ['--quiet'] - # local_copy_dir is guaranteed to have a trailing slash in main() below - if subprocess.call(rsyncargs + [repo_section, local_copy_dir]) != 0: + logging.debug(' '.join(rsyncargs + [fromdir, todir])) + if subprocess.call(rsyncargs + [fromdir, todir]) != 0: sys.exit(1) +def sync_from_localcopy(repo_section, local_copy_dir): + logging.info('Syncing from local_copy_dir to this repo.') + # trailing slashes have a meaning in rsync which is not needed here, so + # remove them all + _local_sync(os.path.join(local_copy_dir, repo_section).rstrip('/'), + repo_section.rstrip('/')) + + +def update_localcopy(repo_section, local_copy_dir): + # local_copy_dir is guaranteed to have a trailing slash in main() below + _local_sync(repo_section, local_copy_dir) + + def main(): global config, options @@ -163,6 +176,8 @@ def main(): help="Specify an identity file to provide to SSH for rsyncing") parser.add_option("--local-copy-dir", default=None, help="Specify a local folder to sync the repo to") + parser.add_option("--sync-from-local-copy-dir", action="store_true", default=False, + help="Before uploading to servers, sync from local copy dir") parser.add_option("-v", "--verbose", action="store_true", default=False, help="Spew out even more information than normal") parser.add_option("-q", "--quiet", action="store_true", default=False, @@ -251,12 +266,15 @@ def main(): sys.exit(1) elif args[0] == 'update': for repo_section in repo_sections: + if local_copy_dir is not None: + if config['sync_from_local_copy_dir'] and os.path.exists(repo_section): + sync_from_localcopy(repo_section, local_copy_dir) + else: + update_localcopy(repo_section, local_copy_dir) if config.get('serverwebroot'): update_serverwebroot(repo_section) if config.get('awsbucket'): update_awsbucket(repo_section) - if local_copy_dir is not None: - update_localcopy(repo_section, local_copy_dir) sys.exit(0) diff --git a/tests/run-tests b/tests/run-tests index 8cccc044..a5576765 100755 --- a/tests/run-tests +++ b/tests/run-tests @@ -106,7 +106,7 @@ set -e #------------------------------------------------------------------------------# -echo_header "setup a new repo from scratch using ANDROID_HOME" +echo_header "setup a new repo from scratch using ANDROID_HOME and do a local sync" REPOROOT=`create_test_dir` cd $REPOROOT @@ -115,6 +115,13 @@ copy_apks_into_repo $REPOROOT $fdroid update --create-metadata grep -F '