deploying build logs to server after each individual build run

fdroidserver/build.py

@@ -75,6 +75,7 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force):
 
     sshinfo = vmtools.get_clean_builder('builder', options.reset_server)
 
+    output = None
     try:
         if not buildserverid:
             buildserverid = subprocess.check_output(['vagrant', 'ssh', '-c',
@@ -279,6 +280,13 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force):
         vm = vmtools.get_build_vm('builder')
         vm.suspend()
 
+        # deploy logfile to repository web server
+        if output:
+            common.publish_build_log_with_rsync(app.id, build.versionCode, output)
+        else:
+            logging.debug('skip publishing full build logs: '
+                          'no output present')
+
 
 def force_gradle_build_tools(build_dir, build_tools):
     for root, dirs, files in os.walk(build_dir):
@@ -1137,7 +1145,7 @@ def main():
                             raise FDroidException(
                                 'Downloading Binaries from %s failed. %s' % (url, e))
 
-                        # Now we check weather the build can be verified to
+                        # Now we check whether the build can be verified to
                         # match the supplied binary or not. Should the
                         # comparison fail, we mark this build as a failure
                         # and remove everything from the unsigend folder.

fdroidserver/common.py

@@ -3072,31 +3072,49 @@ def local_rsync(options, fromdir, todir):
         raise FDroidException()
 
 
-def publish_build_log_with_rsync(appid, vercode, log_path, timestamp=int(time.time())):
+def publish_build_log_with_rsync(appid, vercode, log_content,
-    """Upload build log of one individual app build to an fdroid repository."""
+                                 timestamp=int(time.time())):
+    """Upload build log of one individual app build to an fdroid repository.
+
+    :param appid: package name for dientifying to which app this log belongs.
+    :param vercode: version of the app to which this build belongs.
+    :param log_content: Content of the log which is about to be posted.
+                        Should be either a string or bytes. (bytes will
+                        be decoded as 'utf-8')
+    :param timestamp: timestamp for avoiding logfile name collisions.
+    """
 
     # check if publishing logs is enabled in config
-    if 'publish_build_logs' not in config:
+    if not config.get('publish_build_logs', False):
-        logging.debug('publishing full build logs not enabled')
+        logging.debug('skip publishing full build logs: not enabled in config')
         return
 
-    if not os.path.isfile(log_path):
+    if not log_content:
-        logging.warning('skip uploading "{}" (not a file)'.format(log_path))
+        logging.warning('skip publishing full build logs: log content is empty')
         return
 
+    if not (isinstance(timestamp, int) or isinstance(timestamp, float)):
+        raise ValueError("supplied timestamp '{}' is not a unix timestamp"
+                         .format(timestamp))
+
     with tempfile.TemporaryDirectory() as tmpdir:
         # gzip compress log file
         log_gz_path = os.path.join(
             tmpdir, '{pkg}_{ver}_{ts}.log.gz'.format(pkg=appid,
                                                      ver=vercode,
-                                                     ts=timestamp))
+                                                     ts=int(timestamp)))
-        with open(log_path, 'rb') as i, gzip.open(log_gz_path, 'wb') as o:
+        with gzip.open(log_gz_path, 'wb') as f:
-            shutil.copyfileobj(i, o)
+            if isinstance(log_content, str):
+                f.write(bytes(log_content, 'utf-8'))
+            else:
+                f.write(log_content)
 
         # TODO: sign compressed log file, if a signing key is configured
 
         for webroot in config.get('serverwebroot', []):
             dest_path = os.path.join(webroot, "buildlogs")
+            if not dest_path.endswith('/'):
+                dest_path += '/'  # make sure rsync knows this is a directory
             cmd = ['rsync',
                    '--archive',
                    '--delete-after',
@@ -3111,7 +3129,11 @@ def publish_build_log_with_rsync(appid, vercode, log_path, timestamp=int(time.ti
 
             # TODO: also publish signature file if present
 
-            subprocess.call(cmd)
+            retcode = subprocess.call(cmd)
+            if retcode:
+                logging.warning("failded publishing build logs to '{}'".format(webroot))
+            else:
+                logging.info("published build logs to '{}'".format(webroot))
 
 
 def get_per_app_repos():

tests/common.TestCase

@@ -13,6 +13,7 @@ import tempfile
 import unittest
 import textwrap
 import yaml
+import gzip
 from zipfile import ZipFile
 from unittest import mock
 
@@ -792,11 +793,11 @@ class CommonTest(unittest.TestCase):
 
     def test_publish_build_log_with_rsync_with_id_file(self):
 
-        mocklogcontent = textwrap.dedent("""\
+        mocklogcontent = bytes(textwrap.dedent("""\
             build started
             building...
             build completed
-            profit!""")
+            profit!"""), 'utf-8')
 
         fdroidserver.common.options = mock.Mock()
         fdroidserver.common.options.verbose = False
@@ -824,6 +825,8 @@ class CommonTest(unittest.TestCase):
                                       'example.com:/var/www/fdroid/repo/buildlogs'],
                                      cmd)
                 self.assertTrue(cmd[6].endswith('/com.example.app_4711_1.log.gz'))
+                with gzip.open(cmd[6], 'r') as f:
+                    self.assertTrue(f.read(), mocklogcontent)
             elif assert_subprocess_call_iteration == 1:
                 self.assertListEqual(['rsync',
                                       '--archive',
@@ -835,21 +838,18 @@ class CommonTest(unittest.TestCase):
                                       'example.com:/var/www/fdroid/archive/buildlogs'],
                                      cmd)
                 self.assertTrue(cmd[6].endswith('/com.example.app_4711_1.log.gz'))
+                with gzip.open(cmd[6], 'r') as f:
+                    self.assertTrue(f.read(), mocklogcontent)
             else:
                 self.fail('unexpected subprocess.call invocation ({})'
                           .format(assert_subprocess_call_iteration))
             assert_subprocess_call_iteration += 1
             return 0
 
-        with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
+        with mock.patch('subprocess.call',
-            log_path = os.path.join(tmpdir, 'mock.log')
+                        side_effect=assert_subprocess_call):
-            with open(log_path, 'w') as f:
+            fdroidserver.common.publish_build_log_with_rsync(
-                f.write(mocklogcontent)
+                'com.example.app', '4711', mocklogcontent, 1.1)
-
-            with mock.patch('subprocess.call',
-                            side_effect=assert_subprocess_call):
-                fdroidserver.common.publish_build_log_with_rsync(
-                    'com.example.app', '4711', log_path, 1)
 
 
 if __name__ == "__main__":