Merge branch 'rsync-improvements-for-fdroid-server-update' into 'master'

rsync improvements for fdroid server update

This is a couple of improvements to how `fdroid server update` uses `rsync`.

(also, please remove the branch when accepting any of my merge requests)

completion/bash-completion

@@ -247,8 +247,8 @@ __complete_stats() {
 }
 
 __complete_server() {
-	opts="-h -v -q"
-	lopts="--help --verbose --quiet update"
+	opts="-h -i -v -q"
+	lopts="--help --identity-file --verbose --quiet update"
 	__complete_options
 }
 

examples/config.py

@@ -105,6 +105,9 @@ keyaliases['com.example.another.plugin'] = '@com.example.another'
 # sub-directories (i.e. /var/www/packagerepos/fdroid).
 # serverwebroot = 'user@example:/var/www/fdroid'
 
+# optionally specific which identity file to use when using rsync over SSH
+# identity_file = '~/.ssh/fdroid_id_rsa'
+
 # To upload the repo to an Amazon S3 bucket using `fdroid server update`.
 # Warning, this deletes and recreates the whole fdroid/ directory each
 # time. This is based on apache-libcloud, which supports basically all cloud

fdroidserver/server.py

@@ -116,23 +116,28 @@ def update_awsbucket(repo_section):
 
 
 def update_serverwebroot(repo_section):
-    rsyncargs = ['rsync', '-u', '-r', '--delete']
+    rsyncargs = ['rsync', '--update', '--recursive', '--delete']
     if options.verbose:
         rsyncargs += ['--verbose']
     if options.quiet:
         rsyncargs += ['--quiet']
-    index = os.path.join(repo_section, 'index.xml')
+    if options.identity_file is not None:
+        rsyncargs += ['-e', 'ssh -i ' + options.identity_file]
+    if 'identity_file' in config:
+        rsyncargs += ['-e', 'ssh -i ' + config['identity_file']]
+    indexxml = os.path.join(repo_section, 'index.xml')
     indexjar = os.path.join(repo_section, 'index.jar')
     # serverwebroot is guaranteed to have a trailing slash in common.py
     if subprocess.call(rsyncargs +
-                       ['--exclude', index, '--exclude', indexjar,
+                       ['--exclude', indexxml, '--exclude', indexjar,
                         repo_section, config['serverwebroot']]) != 0:
         sys.exit(1)
-    if subprocess.call(rsyncargs +
-                       [index, config['serverwebroot'] + repo_section]) != 0:
+    # use stricter checking on the indexes since they provide the signature
+    rsyncargs += ['--checksum']
+    sectionpath = config['serverwebroot'] + repo_section
+    if subprocess.call(rsyncargs + [indexxml, sectionpath]) != 0:
         sys.exit(1)
-    if subprocess.call(rsyncargs +
-                       [indexjar, config['serverwebroot'] + repo_section]) != 0:
+    if subprocess.call(rsyncargs + [indexjar, sectionpath]) != 0:
         sys.exit(1)
 
 
@@ -141,6 +146,8 @@ def main():
 
     # Parse command line...
     parser = OptionParser()
+    parser.add_option("-i", "--identity-file", default=None,
+                      help="Specify an identity file to provide to SSH for rsyncing")
     parser.add_option("-v", "--verbose", action="store_true", default=False,
                       help="Spew out even more information than normal")
     parser.add_option("-q", "--quiet", action="store_true", default=False,

tests/run-tests

@@ -1,7 +1,7 @@
 #!/bin/bash
 
-set -e
-set -x
+set -e # quit script on error
+set -x # show each command as it is executed
 
 echo_header() {
     echo "=============================================================================="
@@ -9,6 +9,7 @@ echo_header() {
 }
 
 copy_apks_into_repo() {
+    set +x
     for f in `find $APKDIR -name '*.apk' | grep -F -v -e unaligned -e unsigned`; do
         name=$(basename $(dirname `dirname $f`))
         apk=`aapt dump badging "$f" | sed -n "s,^package: name='\(.*\)' versionCode='\([0-9][0-9]*\)' .*,\1_\2.apk,p"`
@@ -19,6 +20,7 @@ copy_apks_into_repo() {
                 rsync -axv $f $1/repo/$apk # rsync if hard link is not possible
         fi
     done
+    set -x
 }
 
 create_fake_android_home() {