mirror of
https://gitlab.com/fdroid/fdroidserver.git
synced 2024-10-03 17:50:11 +02:00
deploying build logs to server after each individual build run
This commit is contained in:
parent
4c53c71fcf
commit
88e64df3ef
@ -75,6 +75,7 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force):
|
||||
|
||||
sshinfo = vmtools.get_clean_builder('builder', options.reset_server)
|
||||
|
||||
output = None
|
||||
try:
|
||||
if not buildserverid:
|
||||
buildserverid = subprocess.check_output(['vagrant', 'ssh', '-c',
|
||||
@ -279,6 +280,13 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force):
|
||||
vm = vmtools.get_build_vm('builder')
|
||||
vm.suspend()
|
||||
|
||||
# deploy logfile to repository web server
|
||||
if output:
|
||||
common.publish_build_log_with_rsync(app.id, build.versionCode, output)
|
||||
else:
|
||||
logging.debug('skip publishing full build logs: '
|
||||
'no output present')
|
||||
|
||||
|
||||
def force_gradle_build_tools(build_dir, build_tools):
|
||||
for root, dirs, files in os.walk(build_dir):
|
||||
@ -1137,7 +1145,7 @@ def main():
|
||||
raise FDroidException(
|
||||
'Downloading Binaries from %s failed. %s' % (url, e))
|
||||
|
||||
# Now we check weather the build can be verified to
|
||||
# Now we check whether the build can be verified to
|
||||
# match the supplied binary or not. Should the
|
||||
# comparison fail, we mark this build as a failure
|
||||
# and remove everything from the unsigend folder.
|
||||
|
@ -3072,31 +3072,49 @@ def local_rsync(options, fromdir, todir):
|
||||
raise FDroidException()
|
||||
|
||||
|
||||
def publish_build_log_with_rsync(appid, vercode, log_path, timestamp=int(time.time())):
|
||||
"""Upload build log of one individual app build to an fdroid repository."""
|
||||
def publish_build_log_with_rsync(appid, vercode, log_content,
|
||||
timestamp=int(time.time())):
|
||||
"""Upload build log of one individual app build to an fdroid repository.
|
||||
|
||||
:param appid: package name for dientifying to which app this log belongs.
|
||||
:param vercode: version of the app to which this build belongs.
|
||||
:param log_content: Content of the log which is about to be posted.
|
||||
Should be either a string or bytes. (bytes will
|
||||
be decoded as 'utf-8')
|
||||
:param timestamp: timestamp for avoiding logfile name collisions.
|
||||
"""
|
||||
|
||||
# check if publishing logs is enabled in config
|
||||
if 'publish_build_logs' not in config:
|
||||
logging.debug('publishing full build logs not enabled')
|
||||
if not config.get('publish_build_logs', False):
|
||||
logging.debug('skip publishing full build logs: not enabled in config')
|
||||
return
|
||||
|
||||
if not os.path.isfile(log_path):
|
||||
logging.warning('skip uploading "{}" (not a file)'.format(log_path))
|
||||
if not log_content:
|
||||
logging.warning('skip publishing full build logs: log content is empty')
|
||||
return
|
||||
|
||||
if not (isinstance(timestamp, int) or isinstance(timestamp, float)):
|
||||
raise ValueError("supplied timestamp '{}' is not a unix timestamp"
|
||||
.format(timestamp))
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
# gzip compress log file
|
||||
log_gz_path = os.path.join(
|
||||
tmpdir, '{pkg}_{ver}_{ts}.log.gz'.format(pkg=appid,
|
||||
ver=vercode,
|
||||
ts=timestamp))
|
||||
with open(log_path, 'rb') as i, gzip.open(log_gz_path, 'wb') as o:
|
||||
shutil.copyfileobj(i, o)
|
||||
ts=int(timestamp)))
|
||||
with gzip.open(log_gz_path, 'wb') as f:
|
||||
if isinstance(log_content, str):
|
||||
f.write(bytes(log_content, 'utf-8'))
|
||||
else:
|
||||
f.write(log_content)
|
||||
|
||||
# TODO: sign compressed log file, if a signing key is configured
|
||||
|
||||
for webroot in config.get('serverwebroot', []):
|
||||
dest_path = os.path.join(webroot, "buildlogs")
|
||||
if not dest_path.endswith('/'):
|
||||
dest_path += '/' # make sure rsync knows this is a directory
|
||||
cmd = ['rsync',
|
||||
'--archive',
|
||||
'--delete-after',
|
||||
@ -3111,7 +3129,11 @@ def publish_build_log_with_rsync(appid, vercode, log_path, timestamp=int(time.ti
|
||||
|
||||
# TODO: also publish signature file if present
|
||||
|
||||
subprocess.call(cmd)
|
||||
retcode = subprocess.call(cmd)
|
||||
if retcode:
|
||||
logging.warning("failded publishing build logs to '{}'".format(webroot))
|
||||
else:
|
||||
logging.info("published build logs to '{}'".format(webroot))
|
||||
|
||||
|
||||
def get_per_app_repos():
|
||||
|
@ -13,6 +13,7 @@ import tempfile
|
||||
import unittest
|
||||
import textwrap
|
||||
import yaml
|
||||
import gzip
|
||||
from zipfile import ZipFile
|
||||
from unittest import mock
|
||||
|
||||
@ -792,11 +793,11 @@ class CommonTest(unittest.TestCase):
|
||||
|
||||
def test_publish_build_log_with_rsync_with_id_file(self):
|
||||
|
||||
mocklogcontent = textwrap.dedent("""\
|
||||
mocklogcontent = bytes(textwrap.dedent("""\
|
||||
build started
|
||||
building...
|
||||
build completed
|
||||
profit!""")
|
||||
profit!"""), 'utf-8')
|
||||
|
||||
fdroidserver.common.options = mock.Mock()
|
||||
fdroidserver.common.options.verbose = False
|
||||
@ -824,6 +825,8 @@ class CommonTest(unittest.TestCase):
|
||||
'example.com:/var/www/fdroid/repo/buildlogs'],
|
||||
cmd)
|
||||
self.assertTrue(cmd[6].endswith('/com.example.app_4711_1.log.gz'))
|
||||
with gzip.open(cmd[6], 'r') as f:
|
||||
self.assertTrue(f.read(), mocklogcontent)
|
||||
elif assert_subprocess_call_iteration == 1:
|
||||
self.assertListEqual(['rsync',
|
||||
'--archive',
|
||||
@ -835,21 +838,18 @@ class CommonTest(unittest.TestCase):
|
||||
'example.com:/var/www/fdroid/archive/buildlogs'],
|
||||
cmd)
|
||||
self.assertTrue(cmd[6].endswith('/com.example.app_4711_1.log.gz'))
|
||||
with gzip.open(cmd[6], 'r') as f:
|
||||
self.assertTrue(f.read(), mocklogcontent)
|
||||
else:
|
||||
self.fail('unexpected subprocess.call invocation ({})'
|
||||
.format(assert_subprocess_call_iteration))
|
||||
assert_subprocess_call_iteration += 1
|
||||
return 0
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
|
||||
log_path = os.path.join(tmpdir, 'mock.log')
|
||||
with open(log_path, 'w') as f:
|
||||
f.write(mocklogcontent)
|
||||
|
||||
with mock.patch('subprocess.call',
|
||||
side_effect=assert_subprocess_call):
|
||||
fdroidserver.common.publish_build_log_with_rsync(
|
||||
'com.example.app', '4711', log_path, 1)
|
||||
with mock.patch('subprocess.call',
|
||||
side_effect=assert_subprocess_call):
|
||||
fdroidserver.common.publish_build_log_with_rsync(
|
||||
'com.example.app', '4711', mocklogcontent, 1.1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
Loading…
Reference in New Issue
Block a user