kamp/fdroidserver
1
0
Fork 0
mirror of https://gitlab.com/fdroid/fdroidserver.git synced 2024-10-03 17:50:11 +02:00
Code Activity

Merge branch '495-publish-build-logs-with-rsync' into 'master'

Browse Source 
publish build logs with rsync

See merge request fdroid/fdroidserver!515
This commit is contained in:
Hans-Christoph Steiner 2018-06-19 13:44:09 +00:00
commit 1d6c462605
5 changed files with 313 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
examples/config.py
View File

@ -163,6 +163,13 @@ The repository of older versions of applications from the main demo repository.
# 'bar.info:/var/www/fdroid',
# }
# Uncomment this option if you want to logs of builds and other processes to
# your repository server(s). Logs get published to all servers configured in
# 'serverwebroot'. For builds, only logs from build-jobs running inside a
# buildserver VM are supported.
#
# deploy_process_logs = True
# The full URL to a git remote repository. You can include
# multiple servers to mirror to by wrapping the whole thing in {} or [], and
# including the servergitmirrors strings in a comma-separated list.

10
fdroidserver/build.py
View File

@ -75,6 +75,7 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force):
sshinfo = vmtools.get_clean_builder('builder', options.reset_server)
output = None
try:
if not buildserverid:
buildserverid = subprocess.check_output(['vagrant', 'ssh', '-c',
@ -279,6 +280,13 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force):
vm = vmtools.get_build_vm('builder')
vm.suspend()
# deploy logfile to repository web server
if output:
common.publish_build_log_with_rsync(app.id, build.versionCode, output)
else:
logging.debug('skip publishing full build logs: '
'no output present')
def force_gradle_build_tools(build_dir, build_tools):
for root, dirs, files in os.walk(build_dir):
@ -1137,7 +1145,7 @@ def main():
raise FDroidException(
'Downloading Binaries from %s failed. %s' % (url, e))
# Now we check weather the build can be verified to
# Now we check whether the build can be verified to
# match the supplied binary or not. Should the
# comparison fail, we mark this build as a failure
# and remove everything from the unsigend folder.

66
fdroidserver/common.py
View File

@ -25,6 +25,7 @@ import os
import sys
import re
import ast
import gzip
import shutil
import glob
import stat
@ -100,6 +101,7 @@ default_config = {
'per_app_repos': False,
'make_current_version_link': True,
'current_version_name_source': 'Name',
'deploy_process_logs': False,
'update_stats': False,
'stats_ignore': [],
'stats_server': None,
@ -3070,6 +3072,70 @@ def local_rsync(options, fromdir, todir):
raise FDroidException()
def deploy_build_log_with_rsync(appid, vercode, log_content,
timestamp=int(time.time())):
"""Upload build log of one individual app build to an fdroid repository.
:param appid: package name for dientifying to which app this log belongs.
:param vercode: version of the app to which this build belongs.
:param log_content: Content of the log which is about to be posted.
Should be either a string or bytes. (bytes will
be decoded as 'utf-8')
:param timestamp: timestamp for avoiding logfile name collisions.
"""
# check if deploying logs is enabled in config
if not config.get('deploy_process_logs', False):
logging.debug(_('skip deploying full build logs: not enabled in config'))
return
if not log_content:
logging.warning(_('skip deploying full build logs: log content is empty'))
return
if not (isinstance(timestamp, int) or isinstance(timestamp, float)):
raise ValueError(_("supplied timestamp value '{timestamp}' is not a unix timestamp"
.format(timestamp=timestamp)))
with tempfile.TemporaryDirectory() as tmpdir:
# gzip compress log file
log_gz_path = os.path.join(
tmpdir, '{pkg}_{ver}_{ts}.log.gz'.format(pkg=appid,
ver=vercode,
ts=int(timestamp)))
with gzip.open(log_gz_path, 'wb') as f:
if isinstance(log_content, str):
f.write(bytes(log_content, 'utf-8'))
else:
f.write(log_content)
# TODO: sign compressed log file, if a signing key is configured
for webroot in config.get('serverwebroot', []):
dest_path = os.path.join(webroot, "buildlogs")
if not dest_path.endswith('/'):
dest_path += '/' # make sure rsync knows this is a directory
cmd = ['rsync',
'--archive',
'--delete-after',
'--safe-links']
if options.verbose:
cmd += ['--verbose']
if options.quiet:
cmd += ['--quiet']
if 'identity_file' in config:
cmd += ['-e', 'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ' + config['identity_file']]
cmd += [log_gz_path, dest_path]
# TODO: also deploy signature file if present
retcode = subprocess.call(cmd)
if retcode:
logging.warning(_("failed deploying build logs to '{path}'").format(path=webroot))
else:
logging.info(_("deployeded build logs to '{path}'").format(path=webroot))
def get_per_app_repos():
'''per-app repos are dirs named with the packageName of a single app'''

89
tests/common.TestCase
View File

@ -13,7 +13,9 @@ import tempfile
import unittest
import textwrap
import yaml
import gzip
from zipfile import ZipFile
from unittest import mock
localmodule = os.path.realpath(
@ -659,12 +661,11 @@ class CommonTest(unittest.TestCase):
self.assertEqual('b30bb971af0d134866e158ec748fcd553df97c150f58b0a963190bbafbeb0868', sig)
def test_parse_androidmanifests(self):
source_files_dir = os.path.join(os.path.dirname(__file__), 'source-files')
app = fdroidserver.metadata.App()
app.id = 'org.fdroid.fdroid'
paths = [
os.path.join(source_files_dir, 'fdroid', 'fdroidclient', 'AndroidManifest.xml'),
os.path.join(source_files_dir, 'fdroid', 'fdroidclient', 'build.gradle'),
os.path.join('source-files', 'fdroid', 'fdroidclient', 'AndroidManifest.xml'),
os.path.join('source-files', 'fdroid', 'fdroidclient', 'build.gradle'),
]
for path in paths:
self.assertTrue(os.path.isfile(path))
@ -672,16 +673,14 @@ class CommonTest(unittest.TestCase):
fdroidserver.common.parse_androidmanifests(paths, app))
def test_parse_androidmanifests_with_flavor(self):
source_files_dir = os.path.join(os.path.dirname(__file__), 'source-files')
app = fdroidserver.metadata.App()
build = fdroidserver.metadata.Build()
build.gradle = ['devVersion']
app.builds = [build]
app.id = 'org.fdroid.fdroid.dev'
paths = [
os.path.join(source_files_dir, 'fdroid', 'fdroidclient', 'AndroidManifest.xml'),
os.path.join(source_files_dir, 'fdroid', 'fdroidclient', 'build.gradle'),
os.path.join('source-files', 'fdroid', 'fdroidclient', 'AndroidManifest.xml'),
os.path.join('source-files', 'fdroid', 'fdroidclient', 'build.gradle'),
]
for path in paths:
self.assertTrue(os.path.isfile(path))
@ -694,7 +693,7 @@ class CommonTest(unittest.TestCase):
app.builds = [build]
app.id = 'eu.siacs.conversations'
paths = [
os.path.join(source_files_dir, 'eu.siacs.conversations', 'build.gradle'),
os.path.join('source-files', 'eu.siacs.conversations', 'build.gradle'),
]
for path in paths:
self.assertTrue(os.path.isfile(path))
@ -707,7 +706,7 @@ class CommonTest(unittest.TestCase):
app.builds = [build]
app.id = 'com.nextcloud.client'
paths = [
os.path.join(source_files_dir, 'com.nextcloud.client', 'build.gradle'),
os.path.join('source-files', 'com.nextcloud.client', 'build.gradle'),
]
for path in paths:
self.assertTrue(os.path.isfile(path))
@ -720,7 +719,7 @@ class CommonTest(unittest.TestCase):
app.builds = [build]
app.id = 'com.nextcloud.android.beta'
paths = [
os.path.join(source_files_dir, 'com.nextcloud.client', 'build.gradle'),
os.path.join('source-files', 'com.nextcloud.client', 'build.gradle'),
]
for path in paths:
self.assertTrue(os.path.isfile(path))
@ -733,7 +732,7 @@ class CommonTest(unittest.TestCase):
app.builds = [build]
app.id = 'at.bitfire.davdroid'
paths = [
os.path.join(source_files_dir, 'at.bitfire.davdroid', 'build.gradle'),
os.path.join('source-files', 'at.bitfire.davdroid', 'build.gradle'),
]
for path in paths:
self.assertTrue(os.path.isfile(path))
@ -746,7 +745,7 @@ class CommonTest(unittest.TestCase):
app.builds = [build]
app.id = 'com.kunzisoft.fdroidtest.applicationidsuffix.libre'
paths = [
os.path.join(source_files_dir, 'com.kunzisoft.testcase', 'build.gradle'),
os.path.join('source-files', 'com.kunzisoft.testcase', 'build.gradle'),
]
for path in paths:
self.assertTrue(os.path.isfile(path))
@ -759,7 +758,7 @@ class CommonTest(unittest.TestCase):
app.builds = [build]
app.id = 'com.kunzisoft.fdroidtest.applicationidsuffix.pro'
paths = [
os.path.join(source_files_dir, 'com.kunzisoft.testcase', 'build.gradle'),
os.path.join('source-files', 'com.kunzisoft.testcase', 'build.gradle'),
]
for path in paths:
self.assertTrue(os.path.isfile(path))
@ -772,7 +771,7 @@ class CommonTest(unittest.TestCase):
app.builds = [build]
app.id = 'com.kunzisoft.fdroidtest.applicationidsuffix'
paths = [
os.path.join(source_files_dir, 'com.kunzisoft.testcase', 'build.gradle'),
os.path.join('source-files', 'com.kunzisoft.testcase', 'build.gradle'),
]
for path in paths:
self.assertTrue(os.path.isfile(path))
@ -792,8 +791,70 @@ class CommonTest(unittest.TestCase):
with self.assertRaises(SyntaxError):
fdroidserver.common.calculate_math_string('1-1 # no comment')
def test_deploy_build_log_with_rsync_with_id_file(self):
mocklogcontent = bytes(textwrap.dedent("""\
build started
building...
build completed
profit!"""), 'utf-8')
fdroidserver.common.options = mock.Mock()
fdroidserver.common.options.verbose = False
fdroidserver.common.options.quiet = False
fdroidserver.common.config = {}
fdroidserver.common.config['serverwebroot'] = [
'example.com:/var/www/fdroid/repo/',
'example.com:/var/www/fdroid/archive/']
fdroidserver.common.config['deploy_process_logs'] = True
fdroidserver.common.config['identity_file'] = 'ssh/id_rsa'
assert_subprocess_call_iteration = 0
def assert_subprocess_call(cmd):
nonlocal assert_subprocess_call_iteration
logging.debug(cmd)
if assert_subprocess_call_iteration == 0:
self.assertListEqual(['rsync',
'--archive',
'--delete-after',
'--safe-links',
'-e',
'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ssh/id_rsa',
cmd[6],
'example.com:/var/www/fdroid/repo/buildlogs'],
cmd)
self.assertTrue(cmd[6].endswith('/com.example.app_4711_1.log.gz'))
with gzip.open(cmd[6], 'r') as f:
self.assertTrue(f.read(), mocklogcontent)
elif assert_subprocess_call_iteration == 1:
self.assertListEqual(['rsync',
'--archive',
'--delete-after',
'--safe-links',
'-e',
'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i ssh/id_rsa',
cmd[6],
'example.com:/var/www/fdroid/archive/buildlogs'],
cmd)
self.assertTrue(cmd[6].endswith('/com.example.app_4711_1.log.gz'))
with gzip.open(cmd[6], 'r') as f:
self.assertTrue(f.read(), mocklogcontent)
else:
self.fail('unexpected subprocess.call invocation ({})'
.format(assert_subprocess_call_iteration))
assert_subprocess_call_iteration += 1
return 0
with mock.patch('subprocess.call',
side_effect=assert_subprocess_call):
fdroidserver.common.deploy_build_log_with_rsync(
'com.example.app', '4711', mocklogcontent, 1.1)
if __name__ == "__main__":
os.chdir(os.path.dirname(__file__))
parser = optparse.OptionParser()
parser.add_option("-v", "--verbose", action="store_true", default=False,
help="Spew out even more information than normal")

156
tests/server.TestCase Executable file
View File

@ -0,0 +1,156 @@
#!/usr/bin/env python3
import inspect
import logging
import optparse
import os
import sys
import tempfile
import unittest
from unittest import mock
localmodule = os.path.realpath(
os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..'))
if localmodule not in sys.path:
sys.path.insert(0, localmodule)
import fdroidserver.common
import fdroidserver.server
from testcommon import TmpCwd
class ServerTest(unittest.TestCase):
'''fdroidserver/server.py'''
def setUp(self):
logging.basicConfig(level=logging.DEBUG)
self.basedir = os.path.join(localmodule, 'tests')
fdroidserver.server.options = mock.Mock()
fdroidserver.server.config = {}
def test_update_serverwebroot_make_cur_version_link(self):
# setup parameters for this test run
fdroidserver.server.options.no_chcksum = True
fdroidserver.server.options.identity_file = None
fdroidserver.server.options.verbose = False
fdroidserver.server.options.quiet = True
fdroidserver.server.options.identity_file = None
fdroidserver.server.config['make_current_version_link'] = True
serverwebroot = "example.com:/var/www/fdroid"
repo_section = 'repo'
# setup function for asserting subprocess.call invocations
call_iteration = 0
def update_server_webroot_call(cmd):
nonlocal call_iteration
if call_iteration == 0:
self.assertListEqual(cmd, ['rsync',
'--archive',
'--delete-after',
'--safe-links',
'--quiet',
'--exclude', 'repo/index.xml',
'--exclude', 'repo/index.jar',
'--exclude', 'repo/index-v1.jar',
'repo',
'example.com:/var/www/fdroid'])
elif call_iteration == 1:
self.assertListEqual(cmd, ['rsync',
'--archive',
'--delete-after',
'--safe-links',
'--quiet',
'repo',
serverwebroot])
elif call_iteration == 2:
self.assertListEqual(cmd, ['rsync',
'--archive',
'--delete-after',
'--safe-links',
'--quiet',
'Sym.apk',
'Sym.apk.asc',
'Sym.apk.sig',
'example.com:/var/www/fdroid'])
else:
self.fail('unexpected subprocess.call invocation')
call_iteration += 1
return 0
with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
os.mkdir('repo')
os.symlink('repo/com.example.sym.apk', 'Sym.apk')
os.symlink('repo/com.example.sym.apk.asc', 'Sym.apk.asc')
os.symlink('repo/com.example.sym.apk.sig', 'Sym.apk.sig')
with mock.patch('subprocess.call', side_effect=update_server_webroot_call):
fdroidserver.server.update_serverwebroot(serverwebroot,
repo_section)
self.assertEqual(call_iteration, 3, 'expected 3 invocations of subprocess.call')
def test_update_serverwebroot_with_id_file(self):
# setup parameters for this test run
fdroidserver.server.options.no_chcksum = False
fdroidserver.server.options.verbose = True
fdroidserver.server.options.quiet = False
fdroidserver.server.options.identity_file = None
fdroidserver.server.config['identity_file'] = './id_rsa'
fdroidserver.server.config['make_current_version_link'] = False
serverwebroot = "example.com:/var/www/fdroid"
repo_section = 'archive'
# setup function for asserting subprocess.call invocations
call_iteration = 0
def update_server_webroot_call(cmd):
nonlocal call_iteration
if call_iteration == 0:
self.assertListEqual(cmd, ['rsync',
'--archive',
'--delete-after',
'--safe-links',
'--verbose',
'-e',
'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i '
+ fdroidserver.server.config['identity_file'],
'--exclude', 'archive/index.xml',
'--exclude', 'archive/index.jar',
'--exclude', 'archive/index-v1.jar',
'archive',
serverwebroot])
elif call_iteration == 1:
self.assertListEqual(cmd, ['rsync',
'--archive',
'--delete-after',
'--safe-links',
'--verbose',
'-e',
'ssh -oBatchMode=yes -oIdentitiesOnly=yes -i '
+ fdroidserver.server.config['identity_file'],
'archive',
serverwebroot])
else:
self.fail('unexpected subprocess.call invocation')
call_iteration += 1
return 0
with mock.patch('subprocess.call', side_effect=update_server_webroot_call):
fdroidserver.server.update_serverwebroot(serverwebroot,
repo_section)
self.assertEqual(call_iteration, 2, 'expected 2 invocations of subprocess.call')
if __name__ == "__main__":
os.chdir(os.path.dirname(__file__))
parser = optparse.OptionParser()
parser.add_option("-v", "--verbose", action="store_true", default=False,
help="Spew out even more information than normal")
(fdroidserver.common.options, args) = parser.parse_args(['--verbose'])
newSuite = unittest.TestSuite()
newSuite.addTest(unittest.makeSuite(ServerTest))
unittest.main(failfast=False)