From 597fc04c9e21ef9fe5d710bd066626ec8258353c Mon Sep 17 00:00:00 2001
From: Marcus Hoffmann <bubuiic@aol.com>
Date: Fri, 24 Nov 2017 22:41:06 +0100
Subject: [PATCH] build: rsync instead of sftp dirs to the buildserver

Sending many small files with paramiko's sftp implementation is quite
slow. There seems to be no way around this, other projects (ansible)
recommend to use rsync in this case.

Our sourcecode directories sometimes have a LOT of files, it can take
up to an hour to copy all required things inside the buildserver.

Instead we just use rsync with the ssh options we get from vagrant.
For rsync specific options we use:

* sync permissions as before
* sync symlinks (was done partially before)
* don't sync hardlinks and ownership as these don't make for things
  coming from a VCS

Closes fdroid/fdroidserver#227.
---
 fdroidserver/build.py | 29 +++++++++++++----------------
 1 file changed, 13 insertions(+), 16 deletions(-)

diff --git a/fdroidserver/build.py b/fdroidserver/build.py
index a9ee02d7..740d7f2c 100644
--- a/fdroidserver/build.py
+++ b/fdroidserver/build.py
@@ -98,22 +98,19 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force):
 
         # Helper to copy the contents of a directory to the server...
         def send_dir(path):
-            startroot = os.path.dirname(path)
-            main = os.path.basename(path)
-            ftp.mkdir(main)
-            for root, dirs, files in os.walk(path):
-                rr = os.path.relpath(root, startroot)
-                ftp.chdir(rr)
-                for d in dirs:
-                    ftp.mkdir(d)
-                for f in files:
-                    lfile = os.path.join(startroot, rr, f)
-                    if not os.path.islink(lfile):
-                        ftp.put(lfile, f)
-                        ftp.chmod(f, os.stat(lfile).st_mode)
-                for i in range(len(rr.split('/'))):
-                    ftp.chdir('..')
-            ftp.chdir('..')
+            logging.debug("rsyncing " + path + " to " + ftp.getcwd())
+            subprocess.check_call(['rsync', '-rple',
+                                   'ssh -o StrictHostKeyChecking=no' +
+                                   ' -o UserKnownHostsFile=/dev/null' +
+                                   ' -o LogLevel=FATAL' +
+                                   ' -o IdentitiesOnly=yes' +
+                                   ' -o PasswordAuthentication=no' +
+                                   ' -p ' + str(sshinfo['port']) +
+                                   ' -i ' + sshinfo['idfile'],
+                                   path,
+                                   sshinfo['user'] +
+                                   "@" + sshinfo['hostname'] +
+                                   ":" + ftp.getcwd()])
 
         logging.info("Preparing server for build...")
         serverpath = os.path.abspath(os.path.dirname(__file__))