From 4c53c71fcf4d24763f64d7e896c6e8455a640b12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Sun, 10 Jun 2018 23:23:43 +0200 Subject: [PATCH] added helper function for uploading build logs with rsync --- examples/config.py | 5 ++++ fdroidserver/common.py | 44 ++++++++++++++++++++++++++++++ tests/common.TestCase | 62 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 111 insertions(+) diff --git a/examples/config.py b/examples/config.py index 4fcba228..d379d297 100644 --- a/examples/config.py +++ b/examples/config.py @@ -163,6 +163,11 @@ The repository of older versions of applications from the main demo repository. # 'bar.info:/var/www/fdroid', # } +# Uncomment this option if you want to publish build logs to your repository +# server(s). Logs get published to all servers configured in 'serverwebroot'. +# +# publish_build_logs = True + # The full URL to a git remote repository. You can include # multiple servers to mirror to by wrapping the whole thing in {} or [], and # including the servergitmirrors strings in a comma-separated list. diff --git a/fdroidserver/common.py b/fdroidserver/common.py index ec22d4ab..26927b78 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -25,6 +25,7 @@ import os import sys import re import ast +import gzip import shutil import glob import stat @@ -100,6 +101,7 @@ default_config = { 'per_app_repos': False, 'make_current_version_link': True, 'current_version_name_source': 'Name', + 'publish_build_logs': False, 'update_stats': False, 'stats_ignore': [], 'stats_server': None, @@ -3070,6 +3072,48 @@ def local_rsync(options, fromdir, todir): raise FDroidException() +def publish_build_log_with_rsync(appid, vercode, log_path, timestamp=int(time.time())): + """Upload build log of one individual app build to an fdroid repository.""" + + # check if publishing logs is enabled in config + if 'publish_build_logs' not in config: + logging.debug('publishing full build logs not enabled') + return + + if not os.path.isfile(log_path): + logging.warning('skip uploading "{}" (not a file)'.format(log_path)) + return + + with tempfile.TemporaryDirectory() as tmpdir: + # gzip compress log file + log_gz_path = os.path.join( + tmpdir, '{pkg}_{ver}_{ts}.log.gz'.format(pkg=appid, + ver=vercode, + ts=timestamp)) + with open(log_path, 'rb') as i, gzip.open(log_gz_path, 'wb') as o: + shutil.copyfileobj(i, o) + + # TODO: sign compressed log file, if a signing key is configured + + for webroot in config.get('serverwebroot', []): + dest_path = os.path.join(webroot, "buildlogs") + cmd = ['rsync', + '--archive', + '--delete-after', + '--safe-links'] + if options.verbose: + cmd += ['--verbose'] + if options.quiet: + cmd += ['--quiet'] + if 'identity_file' in config: + cmd += ['-e', 'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ' + config['identity_file']] + cmd += [log_gz_path, dest_path] + + # TODO: also publish signature file if present + + subprocess.call(cmd) + + def get_per_app_repos(): '''per-app repos are dirs named with the packageName of a single app''' diff --git a/tests/common.TestCase b/tests/common.TestCase index 1b9d5ee9..fca3b35b 100755 --- a/tests/common.TestCase +++ b/tests/common.TestCase @@ -14,6 +14,7 @@ import unittest import textwrap import yaml from zipfile import ZipFile +from unittest import mock localmodule = os.path.realpath( @@ -789,6 +790,67 @@ class CommonTest(unittest.TestCase): with self.assertRaises(SyntaxError): fdroidserver.common.calculate_math_string('1-1 # no comment') + def test_publish_build_log_with_rsync_with_id_file(self): + + mocklogcontent = textwrap.dedent("""\ + build started + building... + build completed + profit!""") + + fdroidserver.common.options = mock.Mock() + fdroidserver.common.options.verbose = False + fdroidserver.common.options.quiet = False + fdroidserver.common.config = {} + fdroidserver.common.config['serverwebroot'] = [ + 'example.com:/var/www/fdroid/repo/', + 'example.com:/var/www/fdroid/archive/'] + fdroidserver.common.config['publish_build_logs'] = True + fdroidserver.common.config['identity_file'] = 'ssh/id_rsa' + + assert_subprocess_call_iteration = 0 + + def assert_subprocess_call(cmd): + nonlocal assert_subprocess_call_iteration + logging.debug(cmd) + if assert_subprocess_call_iteration == 0: + self.assertListEqual(['rsync', + '--archive', + '--delete-after', + '--safe-links', + '-e', + 'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ssh/id_rsa', + cmd[6], + 'example.com:/var/www/fdroid/repo/buildlogs'], + cmd) + self.assertTrue(cmd[6].endswith('/com.example.app_4711_1.log.gz')) + elif assert_subprocess_call_iteration == 1: + self.assertListEqual(['rsync', + '--archive', + '--delete-after', + '--safe-links', + '-e', + 'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ssh/id_rsa', + cmd[6], + 'example.com:/var/www/fdroid/archive/buildlogs'], + cmd) + self.assertTrue(cmd[6].endswith('/com.example.app_4711_1.log.gz')) + else: + self.fail('unexpected subprocess.call invocation ({})' + .format(assert_subprocess_call_iteration)) + assert_subprocess_call_iteration += 1 + return 0 + + with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir): + log_path = os.path.join(tmpdir, 'mock.log') + with open(log_path, 'w') as f: + f.write(mocklogcontent) + + with mock.patch('subprocess.call', + side_effect=assert_subprocess_call): + fdroidserver.common.publish_build_log_with_rsync( + 'com.example.app', '4711', log_path, 1) + if __name__ == "__main__": os.chdir(os.path.dirname(__file__))