From c62e3fd0ccf1ac8087dbf26a5e2299df3c050fbf Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 22 May 2017 17:36:18 +0200 Subject: [PATCH 01/54] move bulk of reproducible_fdroid_build_apps.sh to jenkins-build This lets us quickly and frequently test things. --- jenkins-build | 70 ++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 69 insertions(+), 1 deletion(-) diff --git a/jenkins-build b/jenkins-build index b7e5fd05..67d6b335 100755 --- a/jenkins-build +++ b/jenkins-build @@ -9,9 +9,77 @@ if [ `dirname $0` != "." ]; then exit fi +# jenkins.debian.net slaves do not export WORKSPACE +if [ -z $WORKSPACE ]; then + export WORKSPACE=`pwd` +fi + set -e set -x # this is a local repo on the Guardian Project Jenkins server cd tests -./complete-ci-tests /var/www/fdroid +#./complete-ci-tests /var/www/fdroid + + +# report info about virtualization +(dmesg | grep -i -e hypervisor -e qemu -e kvm) || true +(lspci | grep -i -e virtio -e virtualbox -e qemu -e kvm) || true +lsmod +if systemd-detect-virt -q ; then + echo "Virtualization is used:" `systemd-detect-virt` +else + echo "No virtualization is used." +fi +sudo /bin/chmod -R a+rX /var/lib/libvirt/images +ls -ld /var/lib/libvirt/images +ls -l /var/lib/libvirt/images || echo no access +ls -lR ~/.vagrant.d/ || echo no access +virsh --connect qemu:///system list --all || echo cannot virsh list +cat /etc/issue + +/sbin/ifconfig || true +hostname || true + +# point to the Vagrant/VirtualBox configs created by reproducible_setup_fdroid_build_environment.sh +# these variables are actually set in fdroidserver/jenkins-build-makebuildserver +export SETUP_WORKSPACE=$(dirname $WORKSPACE)/fdroid/fdroidserver +export XDG_CONFIG_HOME=$SETUP_WORKSPACE +export VBOX_USER_HOME=$SETUP_WORKSPACE/VirtualBox +export VAGRANT_HOME=$SETUP_WORKSPACE/vagrant.d + +# let's see what is actually there: +find $SETUP_WORKSPACE | grep -v fdroiddata/metadata/ | cut -b43-9999 + +# the way we handle jenkins slaves doesn't copy the workspace to the slaves +# so we need to "manually" clone the git repo here… +cd $WORKSPACE + +# set up Android SDK to use the Debian packages in stretch +export ANDROID_HOME=/usr/lib/android-sdk + +# ignore username/password prompt for non-existant repos +git config --global url."https://fakeusername:fakepassword@github.com".insteadOf https://github.com +git config --global url."https://fakeusername:fakepassword@gitlab.com".insteadOf https://gitlab.com +git config --global url."https://fakeusername:fakepassword@bitbucket.org".insteadOf https://bitbucket.org + +# now build the whole archive +cd $WORKSPACE + +# this can be handled in the jenkins job, or here: +if [ -e fdroiddata ]; then + cd fdroiddata + git remote update -p + git checkout master + git reset --hard origin/master +else + git clone https://gitlab.com/fdroid/fdroiddata.git fdroiddata + cd fdroiddata +fi + +echo "build_server_always = True" > config.py +$WORKSPACE/fdroid build --verbose --latest --no-tarball --all + +vagrant global-status +cd builder +vagrant status From 7ef0d5dfd846f16ac6d6cad4723daedf032e0fb8 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Fri, 17 Jun 2016 13:03:01 +0200 Subject: [PATCH 02/54] include class like UNIX `tail -f` for displaying logs This allows fdroidserver to easily log activity while displaying it at the same time. --- fdroidserver/tail.py | 103 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100644 fdroidserver/tail.py diff --git a/fdroidserver/tail.py b/fdroidserver/tail.py new file mode 100644 index 00000000..5f705992 --- /dev/null +++ b/fdroidserver/tail.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python + +''' +Python-Tail - Unix tail follow implementation in Python. + +python-tail can be used to monitor changes to a file. + +Example: + import tail + + # Create a tail instance + t = tail.Tail('file-to-be-followed') + + # Register a callback function to be called when a new line is found in the followed file. + # If no callback function is registerd, new lines would be printed to standard out. + t.register_callback(callback_function) + + # Follow the file with 5 seconds as sleep time between iterations. + # If sleep time is not provided 1 second is used as the default time. + t.follow(s=5) ''' + +# Author - Kasun Herath +# Source - https://github.com/kasun/python-tail + +# modified by Hans-Christoph Steiner to add the +# background thread and support reading multiple lines per read cycle + +import os +import sys +import time +import threading + + +class Tail(object): + ''' Represents a tail command. ''' + def __init__(self, tailed_file): + ''' Initiate a Tail instance. + Check for file validity, assigns callback function to standard out. + + Arguments: + tailed_file - File to be followed. ''' + + self.check_file_validity(tailed_file) + self.tailed_file = tailed_file + self.callback = sys.stdout.write + self.t_stop = threading.Event() + + def start(self, s=1): + '''Start tailing a file in a background thread. + + Arguments: + s - Number of seconds to wait between each iteration; Defaults to 3. + ''' + + t = threading.Thread(target=self.follow, args=(s,)) + t.start() + + def stop(self): + '''Stop a background tail. + ''' + self.t_stop.set() + + def follow(self, s=1): + ''' Do a tail follow. If a callback function is registered it is called with every new line. + Else printed to standard out. + + Arguments: + s - Number of seconds to wait between each iteration; Defaults to 1. ''' + + with open(self.tailed_file, encoding='utf8') as file_: + # Go to the end of file + file_.seek(0, 2) + while not self.t_stop.is_set(): + curr_position = file_.tell() + lines = file_.readlines() + if len(lines) == 0: + file_.seek(curr_position) + else: + for line in lines: + self.callback(line) + time.sleep(s) + + def register_callback(self, func): + ''' Overrides default callback function to provided function. ''' + self.callback = func + + def check_file_validity(self, file_): + ''' Check whether the a given file exists, readable and is a file ''' + if not os.access(file_, os.F_OK): + raise TailError("File '%s' does not exist" % (file_)) + if not os.access(file_, os.R_OK): + raise TailError("File '%s' not readable" % (file_)) + if os.path.isdir(file_): + raise TailError("File '%s' is a directory" % (file_)) + + +class TailError(Exception): + + def __init__(self, msg): + self.message = msg + + def __str__(self): + return self.message From 4b03c3d42dd91373367bc5af31334e9742f3ecfc Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 27 Sep 2016 08:49:32 +0200 Subject: [PATCH 03/54] buildserver: replace custom code with python-vagrant I ran into some annoying issues with UTF-8 output in the vagrant logs, and it was hard to solve. So I switched to using python-vagrant, which handles it all for us. Its been around since 2012, has a number of contributors, and is still actively maintained, so it seems like a good bet. I also packaged it for Debian, including a backport in jessie-backports. On Debian/jessie, do `apt-get install python3-vagrant/jessie-backports` --- makebuildserver | 98 +++++++++++++------------------------------------ setup.py | 1 + 2 files changed, 26 insertions(+), 73 deletions(-) diff --git a/makebuildserver b/makebuildserver index 523a232a..4c196ff0 100755 --- a/makebuildserver +++ b/makebuildserver @@ -6,7 +6,7 @@ import requests import stat import sys import subprocess -import time +import vagrant import hashlib import yaml from clint.textui import progress @@ -18,34 +18,7 @@ if not os.path.exists('makebuildserver') and not os.path.exists('buildserver'): sys.exit(1) -def vagrant(params, cwd=None, printout=False): - """Run vagrant. - - :param: list of parameters to pass to vagrant - :cwd: directory to run in, or None for current directory - :printout: True to print output in realtime, False to just - return it - :returns: (ret, out) where ret is the return code, and out - is the stdout (and stderr) from vagrant - """ - p = subprocess.Popen(['vagrant'] + params, cwd=cwd, - stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - universal_newlines=True) - out = '' - if printout: - while True: - line = p.stdout.readline() - if len(line) == 0: - break - print(line.rstrip()) - out += line - p.wait() - else: - out = p.communicate()[0] - return (p.returncode, out) - - -boxfile = 'buildserver.box' +boxfile = os.path.join(os.getcwd(), 'buildserver.box') serverdir = 'buildserver' parser = OptionParser() @@ -96,12 +69,6 @@ if '__builtins__' in config: if os.path.exists(boxfile): os.remove(boxfile) -if options.clean: - vagrant(['destroy', '-f'], cwd=serverdir, printout=options.verbose) - if config['vm_provider'] == 'libvirt': - subprocess.call(['virsh', 'undefine', 'buildserver_default']) - subprocess.call(['virsh', 'vol-delete', '/var/lib/libvirt/images/buildserver_default.img']) - # Update cached files. cachedir = config['cachedir'] if not os.path.exists(cachedir): @@ -394,18 +361,28 @@ elif os.path.exists('/proc/cpuinfo'): if 'vmx' in contents or 'svm' in contents: config['hwvirtex'] = 'on' +logfilename = os.path.join(serverdir, 'up.log') +log_cm = vagrant.make_file_cm(logfilename) +v = vagrant.Vagrant(root=serverdir, out_cm=log_cm, err_cm=log_cm) + +if options.clean: + v.destroy() + if config['vm_provider'] == 'libvirt': + subprocess.call(['virsh', 'undefine', 'buildserver_default']) + subprocess.call(['virsh', 'vol-delete', '/var/lib/libvirt/images/buildserver_default.img']) + # Check against the existing Vagrantfile.yaml, and if they differ, we # need to create a new box: vf = os.path.join(serverdir, 'Vagrantfile.yaml') writevf = True if os.path.exists(vf): print('Halting', serverdir) - vagrant(['halt'], serverdir) + v.halt() with open(vf, 'r', encoding='utf-8') as f: oldconfig = yaml.load(f) if config != oldconfig: print("Server configuration has changed, rebuild from scratch is required") - vagrant(['destroy', '-f'], serverdir) + v.destroy() else: print("Re-provisioning existing server") writevf = False @@ -416,13 +393,12 @@ if writevf: yaml.dump(config, f) if config['vm_provider'] == 'libvirt': - returncode, out = vagrant(['box', 'list'], serverdir, printout=options.verbose) found_basebox = False needs_mutate = False - for line in out.splitlines(): - if line.startswith(config['basebox']): + for box in v.box_list(): + if box.name == config['basebox']: found_basebox = True - if line.split('(')[1].split(',')[0] != 'libvirt': + if box.provider != 'libvirt': needs_mutate = True continue if not found_basebox: @@ -431,24 +407,16 @@ if config['vm_provider'] == 'libvirt': else: baseboxurl = config['baseboxurl'][0] print('Adding', config['basebox'], 'from', baseboxurl) - vagrant(['box', 'add', '--name', config['basebox'], baseboxurl], - serverdir, printout=options.verbose) + v.box_add(config['basebox'], baseboxurl) needs_mutate = True if needs_mutate: print('Converting', config['basebox'], 'to libvirt format') - vagrant(['mutate', config['basebox'], 'libvirt'], - serverdir, printout=options.verbose) + v._call_vagrant_command(['mutate', config['basebox'], 'libvirt']) print('Removing virtualbox format copy of', config['basebox']) - vagrant(['box', 'remove', '--provider', 'virtualbox', config['basebox']], - serverdir, printout=options.verbose) + v.box_remove(config['basebox'], 'virtualbox') print("Configuring build server VM") -returncode, out = vagrant(['up', '--provision'], serverdir, printout=True) -with open(os.path.join(serverdir, 'up.log'), 'w') as log: - log.write(out) -if returncode != 0: - print("Failed to configure server") - sys.exit(1) +v.up(provision=True) print("Writing buildserver ID") p = subprocess.Popen(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE, @@ -461,28 +429,12 @@ subprocess.call( cwd=serverdir) print("Stopping build server VM") -vagrant(['halt'], serverdir) - -print("Waiting for build server VM to be finished") -ready = False -while not ready: - time.sleep(2) - returncode, out = vagrant(['status'], serverdir) - if returncode != 0: - print("Error while checking status") - sys.exit(1) - for line in out.splitlines(): - if line.startswith("default"): - if line.find("poweroff") != -1 or line.find("shutoff") != 1: - ready = True - else: - print("Status: " + line) +v.halt() print("Packaging") -vagrant(['package', '--output', os.path.join('..', boxfile)], serverdir, - printout=options.verbose) +v.package(output=boxfile) + print("Adding box") -vagrant(['box', 'add', 'buildserver', boxfile, '-f'], - printout=options.verbose) +v.box_add('buildserver', boxfile, force=True) os.remove(boxfile) diff --git a/setup.py b/setup.py index ab74c946..18661e95 100644 --- a/setup.py +++ b/setup.py @@ -36,6 +36,7 @@ setup(name='fdroidserver', 'apache-libcloud >= 0.14.1', 'pyasn1', 'pyasn1-modules', + 'python-vagrant', 'PyYAML', 'requests < 2.11', 'docker-py == 1.9.0', From 675500ad8804aec57608d1f143d6807ba7cebbba Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 20 Jun 2016 11:33:46 +0200 Subject: [PATCH 04/54] buildserver: display verbose logging in a background tail --- makebuildserver | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/makebuildserver b/makebuildserver index 4c196ff0..912b5a77 100755 --- a/makebuildserver +++ b/makebuildserver @@ -11,6 +11,7 @@ import hashlib import yaml from clint.textui import progress from optparse import OptionParser +import fdroidserver.tail if not os.path.exists('makebuildserver') and not os.path.exists('buildserver'): @@ -362,9 +363,15 @@ elif os.path.exists('/proc/cpuinfo'): config['hwvirtex'] = 'on' logfilename = os.path.join(serverdir, 'up.log') +if not os.path.exists(logfilename): + open(logfilename, 'a').close() # create blank file log_cm = vagrant.make_file_cm(logfilename) v = vagrant.Vagrant(root=serverdir, out_cm=log_cm, err_cm=log_cm) +if options.verbose: + tail = fdroidserver.tail.Tail(logfilename) + tail.start() + if options.clean: v.destroy() if config['vm_provider'] == 'libvirt': @@ -438,3 +445,6 @@ print("Adding box") v.box_add('buildserver', boxfile, force=True) os.remove(boxfile) + +if tail is not None: + tail.stop() From 6464ec55b776ff2eb0c72cf280122d983c675555 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Sun, 25 Sep 2016 18:55:29 +0200 Subject: [PATCH 05/54] buildserver: move code into main() method to always stop thread By running the whole program in a main() function, it can be wrapped in try/finally in order to stop the background display thread. This is also done in ./fdroid, its standard practice for Python CLI utilities. --- makebuildserver | 295 +++++++++++++++++++++++++----------------------- 1 file changed, 152 insertions(+), 143 deletions(-) diff --git a/makebuildserver b/makebuildserver index 912b5a77..828b51fc 100755 --- a/makebuildserver +++ b/makebuildserver @@ -20,7 +20,7 @@ if not os.path.exists('makebuildserver') and not os.path.exists('buildserver'): boxfile = os.path.join(os.getcwd(), 'buildserver.box') -serverdir = 'buildserver' +tail = None parser = OptionParser() parser.add_option("-v", "--verbose", action="store_true", default=False, @@ -293,158 +293,167 @@ def sha256_for_file(path): return s.hexdigest() -for srcurl, shasum in cachefiles: - filename = os.path.basename(srcurl) - local_filename = os.path.join(cachedir, filename) +def main(): + global cachedir, cachefiles, config, tail - if os.path.exists(local_filename): - local_length = os.path.getsize(local_filename) - else: - local_length = -1 + for srcurl, shasum in cachefiles: + filename = os.path.basename(srcurl) + local_filename = os.path.join(cachedir, filename) - resume_header = {} - download = True - - try: - r = requests.head(srcurl, allow_redirects=True, timeout=60) - if r.status_code == 200: - content_length = int(r.headers.get('content-length')) + if os.path.exists(local_filename): + local_length = os.path.getsize(local_filename) else: + local_length = -1 + + resume_header = {} + download = True + + try: + r = requests.head(srcurl, allow_redirects=True, timeout=60) + if r.status_code == 200: + content_length = int(r.headers.get('content-length')) + else: + content_length = local_length # skip the download + except requests.exceptions.RequestException as e: content_length = local_length # skip the download - except requests.exceptions.RequestException as e: - content_length = local_length # skip the download - print(e) + print(e) - if local_length == content_length: - download = False - elif local_length > content_length: - print('deleting corrupt file from cache: ' + local_filename) - os.remove(local_filename) - print("Downloading " + filename + " to cache") - elif local_length > -1 and local_length < content_length: - print("Resuming download of " + local_filename) - resume_header = {'Range': 'bytes=%d-%d' % (local_length, content_length)} - else: - print("Downloading " + filename + " to cache") - - if download: - r = requests.get(srcurl, headers=resume_header, - stream=True, verify=False, allow_redirects=True) - content_length = int(r.headers.get('content-length')) - with open(local_filename, 'ab') as f: - for chunk in progress.bar(r.iter_content(chunk_size=65536), - expected_size=(content_length / 65536) + 1): - if chunk: # filter out keep-alive new chunks - f.write(chunk) - - v = sha256_for_file(local_filename) - if v == shasum: - print("\t...shasum verified for " + local_filename) - else: - print("Invalid shasum of '" + v + "' detected for " + local_filename) - os.remove(local_filename) - sys.exit(1) - -local_qt_filename = os.path.join(cachedir, 'qt-opensource-linux-x64-android-5.7.0.run') -print("Setting executable bit for " + local_qt_filename) -os.chmod(local_qt_filename, 0o755) - -# use VirtualBox software virtualization if hardware is not available, -# like if this is being run in kvm or some other VM platform, like -# http://jenkins.debian.net, the values are 'on' or 'off' -if sys.platform.startswith('darwin'): - # all < 10 year old Macs work, and OSX servers as VM host are very - # rare, but this could also be auto-detected if someone codes it - config['hwvirtex'] = 'on' -elif os.path.exists('/proc/cpuinfo'): - with open('/proc/cpuinfo') as f: - contents = f.read() - if 'vmx' in contents or 'svm' in contents: - config['hwvirtex'] = 'on' - -logfilename = os.path.join(serverdir, 'up.log') -if not os.path.exists(logfilename): - open(logfilename, 'a').close() # create blank file -log_cm = vagrant.make_file_cm(logfilename) -v = vagrant.Vagrant(root=serverdir, out_cm=log_cm, err_cm=log_cm) - -if options.verbose: - tail = fdroidserver.tail.Tail(logfilename) - tail.start() - -if options.clean: - v.destroy() - if config['vm_provider'] == 'libvirt': - subprocess.call(['virsh', 'undefine', 'buildserver_default']) - subprocess.call(['virsh', 'vol-delete', '/var/lib/libvirt/images/buildserver_default.img']) - -# Check against the existing Vagrantfile.yaml, and if they differ, we -# need to create a new box: -vf = os.path.join(serverdir, 'Vagrantfile.yaml') -writevf = True -if os.path.exists(vf): - print('Halting', serverdir) - v.halt() - with open(vf, 'r', encoding='utf-8') as f: - oldconfig = yaml.load(f) - if config != oldconfig: - print("Server configuration has changed, rebuild from scratch is required") - v.destroy() - else: - print("Re-provisioning existing server") - writevf = False -else: - print("No existing server - building from scratch") -if writevf: - with open(vf, 'w', encoding='utf-8') as f: - yaml.dump(config, f) - -if config['vm_provider'] == 'libvirt': - found_basebox = False - needs_mutate = False - for box in v.box_list(): - if box.name == config['basebox']: - found_basebox = True - if box.provider != 'libvirt': - needs_mutate = True - continue - if not found_basebox: - if isinstance(config['baseboxurl'], str): - baseboxurl = config['baseboxurl'] + if local_length == content_length: + download = False + elif local_length > content_length: + print('deleting corrupt file from cache: ' + local_filename) + os.remove(local_filename) + print("Downloading " + filename + " to cache") + elif local_length > -1 and local_length < content_length: + print("Resuming download of " + local_filename) + resume_header = {'Range': 'bytes=%d-%d' % (local_length, content_length)} else: - baseboxurl = config['baseboxurl'][0] - print('Adding', config['basebox'], 'from', baseboxurl) - v.box_add(config['basebox'], baseboxurl) - needs_mutate = True - if needs_mutate: - print('Converting', config['basebox'], 'to libvirt format') - v._call_vagrant_command(['mutate', config['basebox'], 'libvirt']) - print('Removing virtualbox format copy of', config['basebox']) - v.box_remove(config['basebox'], 'virtualbox') + print("Downloading " + filename + " to cache") -print("Configuring build server VM") -v.up(provision=True) + if download: + r = requests.get(srcurl, headers=resume_header, + stream=True, verify=False, allow_redirects=True) + content_length = int(r.headers.get('content-length')) + with open(local_filename, 'ab') as f: + for chunk in progress.bar(r.iter_content(chunk_size=65536), + expected_size=(content_length / 65536) + 1): + if chunk: # filter out keep-alive new chunks + f.write(chunk) -print("Writing buildserver ID") -p = subprocess.Popen(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE, - universal_newlines=True) -buildserverid = p.communicate()[0].strip() -print("...ID is " + buildserverid) -subprocess.call( - ['vagrant', 'ssh', '-c', 'sh -c "echo {0} >/home/vagrant/buildserverid"' - .format(buildserverid)], - cwd=serverdir) + v = sha256_for_file(local_filename) + if v == shasum: + print("\t...shasum verified for " + local_filename) + else: + print("Invalid shasum of '" + v + "' detected for " + local_filename) + os.remove(local_filename) + sys.exit(1) -print("Stopping build server VM") -v.halt() + local_qt_filename = os.path.join(cachedir, 'qt-opensource-linux-x64-android-5.7.0.run') + print("Setting executable bit for " + local_qt_filename) + os.chmod(local_qt_filename, 0o755) -print("Packaging") -v.package(output=boxfile) + # use VirtualBox software virtualization if hardware is not available, + # like if this is being run in kvm or some other VM platform, like + # http://jenkins.debian.net, the values are 'on' or 'off' + if sys.platform.startswith('darwin'): + # all < 10 year old Macs work, and OSX servers as VM host are very + # rare, but this could also be auto-detected if someone codes it + config['hwvirtex'] = 'on' + elif os.path.exists('/proc/cpuinfo'): + with open('/proc/cpuinfo') as f: + contents = f.read() + if 'vmx' in contents or 'svm' in contents: + config['hwvirtex'] = 'on' -print("Adding box") -v.box_add('buildserver', boxfile, force=True) + serverdir = os.path.join(os.getcwd(), 'buildserver') + logfilename = os.path.join(serverdir, 'up.log') + if not os.path.exists(logfilename): + open(logfilename, 'a').close() # create blank file + log_cm = vagrant.make_file_cm(logfilename) + v = vagrant.Vagrant(root=serverdir, out_cm=log_cm, err_cm=log_cm) -os.remove(boxfile) + if options.verbose: + tail = fdroidserver.tail.Tail(logfilename) + tail.start() -if tail is not None: - tail.stop() + if options.clean: + v.destroy() + if config['vm_provider'] == 'libvirt': + subprocess.call(['virsh', 'undefine', 'buildserver_default']) + subprocess.call(['virsh', 'vol-delete', '/var/lib/libvirt/images/buildserver_default.img']) + + # Check against the existing Vagrantfile.yaml, and if they differ, we + # need to create a new box: + vf = os.path.join(serverdir, 'Vagrantfile.yaml') + writevf = True + if os.path.exists(vf): + print('Halting', serverdir) + v.halt() + with open(vf, 'r', encoding='utf-8') as f: + oldconfig = yaml.load(f) + if config != oldconfig: + print("Server configuration has changed, rebuild from scratch is required") + v.destroy() + else: + print("Re-provisioning existing server") + writevf = False + else: + print("No existing server - building from scratch") + if writevf: + with open(vf, 'w', encoding='utf-8') as f: + yaml.dump(config, f) + + if config['vm_provider'] == 'libvirt': + found_basebox = False + needs_mutate = False + for box in v.box_list(): + if box.name == config['basebox']: + found_basebox = True + if box.provider != 'libvirt': + needs_mutate = True + continue + if not found_basebox: + if isinstance(config['baseboxurl'], str): + baseboxurl = config['baseboxurl'] + else: + baseboxurl = config['baseboxurl'][0] + print('Adding', config['basebox'], 'from', baseboxurl) + v.box_add(config['basebox'], baseboxurl) + needs_mutate = True + if needs_mutate: + print('Converting', config['basebox'], 'to libvirt format') + v._call_vagrant_command(['mutate', config['basebox'], 'libvirt']) + print('Removing virtualbox format copy of', config['basebox']) + v.box_remove(config['basebox'], 'virtualbox') + + print("Configuring build server VM") + v.up(provision=True) + + print("Writing buildserver ID") + p = subprocess.Popen(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE, + universal_newlines=True) + buildserverid = p.communicate()[0].strip() + print("...ID is " + buildserverid) + subprocess.call( + ['vagrant', 'ssh', '-c', 'sh -c "echo {0} >/home/vagrant/buildserverid"' + .format(buildserverid)], + cwd=serverdir) + + print("Stopping build server VM") + v.halt() + + print("Packaging") + v.package(output=boxfile) + + print("Adding box") + v.box_add('buildserver', boxfile, force=True) + + os.remove(boxfile) + + +if __name__ == '__main__': + try: + main() + finally: + if tail is not None: + tail.stop() From 4cde71552fa42b47f527efd366ba55566c096be8 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 23 Jun 2016 16:38:39 +0200 Subject: [PATCH 06/54] buildserver: run_via_vagrant_ssh() to run cmds via python-vagrant This moves the last vagrant call in a subprocess. --- makebuildserver | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/makebuildserver b/makebuildserver index 828b51fc..6578e27c 100755 --- a/makebuildserver +++ b/makebuildserver @@ -293,6 +293,14 @@ def sha256_for_file(path): return s.hexdigest() +def run_via_vagrant_ssh(v, cmdlist): + if (isinstance(cmdlist, str) or isinstance(cmdlist, bytes)): + cmd = cmdlist + else: + cmd = ' '.join(cmdlist) + v._run_vagrant_command(['ssh', '-c', cmd]) + + def main(): global cachedir, cachefiles, config, tail @@ -434,10 +442,7 @@ def main(): universal_newlines=True) buildserverid = p.communicate()[0].strip() print("...ID is " + buildserverid) - subprocess.call( - ['vagrant', 'ssh', '-c', 'sh -c "echo {0} >/home/vagrant/buildserverid"' - .format(buildserverid)], - cwd=serverdir) + run_via_vagrant_ssh(v, 'sh -c "echo %s >/home/vagrant/buildserverid"' % buildserverid) print("Stopping build server VM") v.halt() From daade7656afa39c1a8f9b5e039d554c503ee8a91 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 23 Jun 2016 17:12:06 +0200 Subject: [PATCH 07/54] buildserver: add copy_caches_from_host config option For people using slow, expensive, and/or flaky internet, liberal use of caching can make a huge difference. The restricted environment of the gpjenkins box has been a good test environment for this (Tor-only, whitelist of allowed IPs to visit, home internet connection). --- examples/makebuildserver.config.py | 10 ++++++++++ jenkins-build-makebuildserver | 1 + makebuildserver | 23 +++++++++++++++++++++++ 3 files changed, 34 insertions(+) diff --git a/examples/makebuildserver.config.py b/examples/makebuildserver.config.py index b6c0183d..7015cf58 100644 --- a/examples/makebuildserver.config.py +++ b/examples/makebuildserver.config.py @@ -28,6 +28,16 @@ # # apt_package_cache = True +# The buildserver can use some local caches to speed up builds, +# especially when the internet connection is slow and/or expensive. +# If enabled, the buildserver setup will look for standard caches in +# your HOME dir and copy them to the buildserver VM. Be aware: this +# will reduce the isolation of the buildserver from your host machine, +# so the buildserver will provide an environment only as trustworthy +# as the host machine's environment. +# +# copy_caches_from_host = True + # To specify which Debian mirror the build server VM should use, by # default it uses http.debian.net, which auto-detects which is the # best mirror to use. diff --git a/jenkins-build-makebuildserver b/jenkins-build-makebuildserver index ae3391a0..a2923ad4 100755 --- a/jenkins-build-makebuildserver +++ b/jenkins-build-makebuildserver @@ -49,6 +49,7 @@ cd $WORKSPACE echo "debian_mirror = 'https://deb.debian.org/debian/'" > $WORKSPACE/makebuildserver.config.py echo "boot_timeout = 1200" >> $WORKSPACE/makebuildserver.config.py echo "apt_package_cache = True" >> $WORKSPACE/makebuildserver.config.py +echo "copy_caches_from_host = True" >> $WORKSPACE/makebuildserver.config.py ./makebuildserver --verbose --clean # this can be handled in the jenkins job, or here: diff --git a/makebuildserver b/makebuildserver index 6578e27c..4985b22f 100755 --- a/makebuildserver +++ b/makebuildserver @@ -2,6 +2,7 @@ import os import pathlib +import re import requests import stat import sys @@ -39,6 +40,7 @@ config = { ], 'debian_mirror': 'http://http.debian.net/debian/', 'apt_package_cache': False, + 'copy_caches_from_host': False, 'boot_timeout': 600, 'cachedir': cachedir, 'cpus': 1, @@ -437,6 +439,27 @@ def main(): print("Configuring build server VM") v.up(provision=True) + if config['copy_caches_from_host']: + ssh_config = v.ssh_config() + user = re.search(r'User ([^ \n]+)', ssh_config).group(1) + hostname = re.search(r'HostName ([^ \n]+)', ssh_config).group(1) + port = re.search(r'Port ([0-9]+)', ssh_config).group(1) + key = re.search(r'IdentityFile ([^ \n]+)', ssh_config).group(1) + + for d in ('.m2', '.gradle/caches', '.gradle/wrapper', '.pip_download_cache'): + fullpath = os.path.join(os.getenv('HOME'), d) + if os.path.isdir(fullpath): + # TODO newer versions of vagrant provide `vagrant rsync` + run_via_vagrant_ssh(v, ['cd ~ && test -d', d, '|| mkdir -p', d]) + subprocess.call(['rsync', '-axv', '--progress', '--delete', '-e', + 'ssh -i {0} -p {1} -oIdentitiesOnly=yes'.format(key, port), + fullpath + '/', + user + '@' + hostname + ':~/' + d + '/']) + + # this file changes every time but should not be cached + run_via_vagrant_ssh(v, ['rm', '-f', '~/.gradle/caches/modules-2/modules-2.lock']) + run_via_vagrant_ssh(v, ['rm', '-fr', '~/.gradle/caches/*/plugin-resolution/']) + print("Writing buildserver ID") p = subprocess.Popen(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE, universal_newlines=True) From 299ed82a8851eda49c9aeb2ca95fbe625bd073d5 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 26 Sep 2016 05:22:05 -0400 Subject: [PATCH 08/54] buildserver: consolidate boxfile export code into one block This is just for clarity, and moving more code into the main() function. --- makebuildserver | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/makebuildserver b/makebuildserver index 4985b22f..48828925 100755 --- a/makebuildserver +++ b/makebuildserver @@ -20,7 +20,6 @@ if not os.path.exists('makebuildserver') and not os.path.exists('buildserver'): sys.exit(1) -boxfile = os.path.join(os.getcwd(), 'buildserver.box') tail = None parser = OptionParser() @@ -69,9 +68,6 @@ elif os.path.exists('makebs.config.py'): if '__builtins__' in config: del(config['__builtins__']) # added by compile/exec -if os.path.exists(boxfile): - os.remove(boxfile) - # Update cached files. cachedir = config['cachedir'] if not os.path.exists(cachedir): @@ -471,6 +467,9 @@ def main(): v.halt() print("Packaging") + boxfile = os.path.join(os.getcwd(), 'buildserver.box') + if os.path.exists(boxfile): + os.remove(boxfile) v.package(output=boxfile) print("Adding box") From 988ac21e7fb7a0ae43ef267f913b3bb90d83ce55 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 27 Sep 2016 02:26:33 -0400 Subject: [PATCH 09/54] buildserver: make --clean destroy reliably This prevents v.destroy() from running if Vagrantfile.yaml does not exist, since that is required for vagrant to run: is the core config including the name of the box, etc. Otherwise, it would exit with an error. This also does complete cleanup when using libvirt. --- makebuildserver | 33 ++++++++++++++++++++++++++++----- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/makebuildserver b/makebuildserver index 48828925..158d718c 100755 --- a/makebuildserver +++ b/makebuildserver @@ -291,6 +291,32 @@ def sha256_for_file(path): return s.hexdigest() +def destroy_current_image(v, serverdir): + global config + + # cannot run vagrant without the config in the YAML file + if os.path.exists(os.path.join(serverdir, 'Vagrantfile.yaml')): + v.destroy() + elif options.verbose: + print('Cannot run destroy vagrant setup since Vagrantfile.yaml is not setup!') + if config['vm_provider'] == 'libvirt': + import libvirt + try: + domain = 'buildserver_default' + virConnect = libvirt.open('qemu:///system') + virDomain = virConnect.lookupByName(domain) + if virDomain: + virDomain.undefineFlags(libvirt.VIR_DOMAIN_UNDEFINE_MANAGED_SAVE + | libvirt.VIR_DOMAIN_UNDEFINE_SNAPSHOTS_METADATA + | libvirt.VIR_DOMAIN_UNDEFINE_NVRAM) + storagePool = virConnect.storagePoolLookupByName('default') + if storagePool: + for vol in storagePool.listAllVolumes(): + vol.delete() + except libvirt.libvirtError as e: + print(e) + + def run_via_vagrant_ssh(v, cmdlist): if (isinstance(cmdlist, str) or isinstance(cmdlist, bytes)): cmd = cmdlist @@ -383,10 +409,7 @@ def main(): tail.start() if options.clean: - v.destroy() - if config['vm_provider'] == 'libvirt': - subprocess.call(['virsh', 'undefine', 'buildserver_default']) - subprocess.call(['virsh', 'vol-delete', '/var/lib/libvirt/images/buildserver_default.img']) + destroy_current_image(v, serverdir) # Check against the existing Vagrantfile.yaml, and if they differ, we # need to create a new box: @@ -399,7 +422,7 @@ def main(): oldconfig = yaml.load(f) if config != oldconfig: print("Server configuration has changed, rebuild from scratch is required") - v.destroy() + destroy_current_image(v, serverdir) else: print("Re-provisioning existing server") writevf = False From 69e4b91d3fea7b33457bbf5395493b70ed2b6560 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 14 Feb 2017 22:58:35 +0100 Subject: [PATCH 10/54] makebuildserver: package up KVM VM as a vagrant box `vagrant package` does not work with KVM, so we have to hack together our own until someone implements it (suppose we should do it). This is a hacked up version based on: https://github.com/vagrant-libvirt/vagrant-libvirt/blob/d7d440ea8f24f698a93a4c5b9a5149acef469579/tools/create_box.sh #238 --- makebuildserver | 58 ++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 55 insertions(+), 3 deletions(-) diff --git a/makebuildserver b/makebuildserver index 158d718c..abc4ca73 100755 --- a/makebuildserver +++ b/makebuildserver @@ -4,9 +4,11 @@ import os import pathlib import re import requests +import shutil import stat import sys import subprocess +import tarfile import vagrant import hashlib import yaml @@ -56,6 +58,7 @@ if os.path.isfile('/usr/bin/systemd-detect-virt'): if virt == 'qemu' or virt == 'kvm' or virt == 'bochs': print('Running in a VM guest, defaulting to QEMU/KVM via libvirt') config['vm_provider'] = 'libvirt' + config['domain'] = 'buildserver_default' elif virt != 'none': print('Running in an unsupported VM guest (' + virt + ')!') @@ -302,9 +305,8 @@ def destroy_current_image(v, serverdir): if config['vm_provider'] == 'libvirt': import libvirt try: - domain = 'buildserver_default' virConnect = libvirt.open('qemu:///system') - virDomain = virConnect.lookupByName(domain) + virDomain = virConnect.lookupByName(config['domain']) if virDomain: virDomain.undefineFlags(libvirt.VIR_DOMAIN_UNDEFINE_MANAGED_SAVE | libvirt.VIR_DOMAIN_UNDEFINE_SNAPSHOTS_METADATA @@ -317,6 +319,52 @@ def destroy_current_image(v, serverdir): print(e) +def kvm_package(boxfile): + ''' + Hack to replace missing `vagrant package` for kvm, based on the script + `tools/create_box.sh from vagrant-libvirt + ''' + import libvirt + virConnect = libvirt.open('qemu:///system') + storagePool = virConnect.storagePoolLookupByName('default') + if storagePool: + vol = storagePool.storageVolLookupByName(config['domain'] + '.img') + imagepath = vol.path() + # TODO use a libvirt storage pool to ensure the img file is readable + subprocess.check_call(['sudo', '/bin/chmod', '-R', 'a+rX', '/var/lib/libvirt/images']) + shutil.copy2(imagepath, 'box.img') + subprocess.check_call(['qemu-img', 'rebase', '-p', '-b', '', 'box.img']) + metadata = """{ + "provider": "libvirt", + "format": "qcow2", + "virtual_size": 1000 +} +""" + vagrantfile = """Vagrant.configure("2") do |config| + + config.vm.provider :libvirt do |libvirt| + + libvirt.driver = "kvm" + libvirt.host = "" + libvirt.connect_via_ssh = false + libvirt.storage_pool_name = "default" + + end +end +""" + with open('metadata.json', 'w') as fp: + fp.write(metadata) + with open('Vagrantfile', 'w') as fp: + fp.write(vagrantfile) + with tarfile.open(boxfile, 'w:gz') as tar: + tar.add('metadata.json') + tar.add('Vagrantfile') + tar.add('box.img') + os.remove('metadata.json') + os.remove('Vagrantfile') + os.remove('box.img') + + def run_via_vagrant_ssh(v, cmdlist): if (isinstance(cmdlist, str) or isinstance(cmdlist, bytes)): cmd = cmdlist @@ -493,7 +541,11 @@ def main(): boxfile = os.path.join(os.getcwd(), 'buildserver.box') if os.path.exists(boxfile): os.remove(boxfile) - v.package(output=boxfile) + + if config['vm_provider'] == 'libvirt': + kvm_package(boxfile) + else: + v.package(output=boxfile) print("Adding box") v.box_add('buildserver', boxfile, force=True) From fa2d44ee94804f98bfb7b238ad5d49acdae0d0e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Thu, 23 Feb 2017 05:51:05 +0100 Subject: [PATCH 11/54] added libvirt vm-provider support to build.py --- fdroidserver/build.py | 88 +++++++++++++++++++++++++++++++------------ 1 file changed, 63 insertions(+), 25 deletions(-) diff --git a/fdroidserver/build.py b/fdroidserver/build.py index 755f5764..453a555b 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -46,12 +46,21 @@ except ImportError: pass -def get_builder_vm_id(): +def get_vm_provider(): + """Determine vm provider based on .vagrant directory content + """ + if os.path.exists(os.path.join('builder', '.vagrant', 'machines', + 'default', 'libvirt')): + return 'libvirt' + return 'virtualbox' + + +def get_builder_vm_id(provider): vd = os.path.join('builder', '.vagrant') if os.path.isdir(vd): # Vagrant 1.2 (and maybe 1.1?) it's a directory tree... with open(os.path.join(vd, 'machines', 'default', - 'virtualbox', 'id')) as vf: + provider, 'id')) as vf: id = vf.read() return id else: @@ -61,7 +70,7 @@ def get_builder_vm_id(): return v['active']['default'] -def got_valid_builder_vm(): +def got_valid_builder_vm(provider): """Returns True if we have a valid-looking builder vm """ if not os.path.exists(os.path.join('builder', 'Vagrantfile')): @@ -74,7 +83,7 @@ def got_valid_builder_vm(): return True # Vagrant 1.2 - the directory can exist, but the id can be missing... if not os.path.exists(os.path.join(vd, 'machines', 'default', - 'virtualbox', 'id')): + provider, 'id')): return False return True @@ -117,6 +126,44 @@ def get_vagrant_sshinfo(): 'idfile': idfile} +def vm_snapshot_list(provider): + if provider is 'virtualbox': + p = FDroidPopen(['VBoxManage', 'snapshot', + get_builder_vm_id(provider), 'list', + '--details'], cwd='builder') + elif provider is 'libvirt': + p = FDroidPopen(['virsh', 'snapshot-list', + get_builder_vm_id(provider)]) + return p.output + + +def vm_snapshot_clean_available(provider): + return 'fdroidclean' in vm_snapshot_list(provider) + + +def vm_snapshot_restore(provider): + """Does a rollback of the build vm. + """ + if provider is 'virtualbox': + p = FDroidPopen(['VBoxManage', 'snapshot', + get_builder_vm_id(provider), 'restore', + 'fdroidclean'], cwd='builder') + elif provider is 'libvirt': + p = FDroidPopen(['virsh', 'snapshot-revert', + get_builder_vm_id(provider), 'fdroidclean']) + return p.returncode == 0 + +def vm_snapshot_create(provider): + if provider is 'virtualbox': + p = FDroidPopen(['VBoxManage', 'snapshot', + get_builder_vm_id(provider), + 'take', 'fdroidclean'], cwd='builder') + elif provider is 'libvirt': + p = FDroidPopen(['virsh', 'snapshot-create-as', + get_builder_vm_id(provider), 'fdroidclean']) + return p.returncode != 0 + + def get_clean_vm(reset=False): """Get a clean VM ready to do a buildserver build. @@ -130,17 +177,16 @@ def get_clean_vm(reset=False): :returns: A dictionary containing 'hostname', 'port', 'user' and 'idfile' """ + provider = get_vm_provider() + # Reset existing builder machine to a clean state if possible. vm_ok = False if not reset: logging.info("Checking for valid existing build server") - if got_valid_builder_vm(): - logging.info("...VM is present") - p = FDroidPopen(['VBoxManage', 'snapshot', - get_builder_vm_id(), 'list', - '--details'], cwd='builder') - if 'fdroidclean' in p.output: + if got_valid_builder_vm(provider): + logging.info("...VM is present (%s)" % provider) + if vm_snapshot_clean_available(provider): logging.info("...snapshot exists - resetting build server to " "clean state") retcode, output = vagrant(['status'], cwd='builder') @@ -150,11 +196,8 @@ def get_clean_vm(reset=False): vagrant(['suspend'], cwd='builder') logging.info("...waiting a sec...") time.sleep(10) - p = FDroidPopen(['VBoxManage', 'snapshot', get_builder_vm_id(), - 'restore', 'fdroidclean'], - cwd='builder') - if p.returncode == 0: + if vm_snapshot_restore(provider): logging.info("...reset to snapshot - server is valid") retcode, output = vagrant(['up'], cwd='builder') if retcode != 0: @@ -167,7 +210,8 @@ def get_clean_vm(reset=False): logging.info("...failed to reset to snapshot") else: logging.info("...snapshot doesn't exist - " - "VBoxManage snapshot list:\n" + p.output) + "VBoxManage snapshot list:\n" + + vm_snapshot_list(provider)) # If we can't use the existing machine for any reason, make a # new one from scratch. @@ -187,8 +231,8 @@ def get_clean_vm(reset=False): with open(os.path.join('builder', 'Vagrantfile'), 'w') as vf: vf.write('Vagrant.configure("2") do |config|\n') - vf.write('config.vm.box = "buildserver"\n') - vf.write('config.vm.synced_folder ".", "/vagrant", disabled: true\n') + vf.write(' config.vm.box = "buildserver"\n') + vf.write(' config.vm.synced_folder ".", "/vagrant", disabled: true\n') vf.write('end\n') logging.info("Starting new build server") @@ -213,10 +257,7 @@ def get_clean_vm(reset=False): raise BuildException("Failed to suspend build server") logging.info("...waiting a sec...") time.sleep(10) - p = FDroidPopen(['VBoxManage', 'snapshot', get_builder_vm_id(), - 'take', 'fdroidclean'], - cwd='builder') - if p.returncode != 0: + if vm_snapshot_create(provider): raise BuildException("Failed to take snapshot") logging.info("...waiting a sec...") time.sleep(10) @@ -227,10 +268,7 @@ def get_clean_vm(reset=False): logging.info("...waiting a sec...") time.sleep(10) # Make sure it worked... - p = FDroidPopen(['VBoxManage', 'snapshot', get_builder_vm_id(), - 'list', '--details'], - cwd='builder') - if 'fdroidclean' not in p.output: + if not vm_snapshot_clean_available(provider): raise BuildException("Failed to take snapshot.") return sshinfo From fad98eeb7fb7fd2593f98f18272d7b6b94ddfe79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Mon, 6 Mar 2017 02:57:07 +0100 Subject: [PATCH 12/54] build: fixed kvm snapshot support; makebuildserver: setup kvm ssh credentials --- fdroidserver/build.py | 7 ++++--- makebuildserver | 2 ++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/fdroidserver/build.py b/fdroidserver/build.py index 453a555b..d9ef9a0e 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -132,7 +132,7 @@ def vm_snapshot_list(provider): get_builder_vm_id(provider), 'list', '--details'], cwd='builder') elif provider is 'libvirt': - p = FDroidPopen(['virsh', 'snapshot-list', + p = FDroidPopen(['virsh', '-c', 'qemu:///system', 'snapshot-list', get_builder_vm_id(provider)]) return p.output @@ -149,17 +149,18 @@ def vm_snapshot_restore(provider): get_builder_vm_id(provider), 'restore', 'fdroidclean'], cwd='builder') elif provider is 'libvirt': - p = FDroidPopen(['virsh', 'snapshot-revert', + p = FDroidPopen(['virsh', '-c', 'qemu:///system', 'snapshot-revert', get_builder_vm_id(provider), 'fdroidclean']) return p.returncode == 0 + def vm_snapshot_create(provider): if provider is 'virtualbox': p = FDroidPopen(['VBoxManage', 'snapshot', get_builder_vm_id(provider), 'take', 'fdroidclean'], cwd='builder') elif provider is 'libvirt': - p = FDroidPopen(['virsh', 'snapshot-create-as', + p = FDroidPopen(['virsh', '-c', 'qemu:///system', 'snapshot-create-as', get_builder_vm_id(provider), 'fdroidclean']) return p.returncode != 0 diff --git a/makebuildserver b/makebuildserver index abc4ca73..e91a9282 100755 --- a/makebuildserver +++ b/makebuildserver @@ -341,6 +341,8 @@ def kvm_package(boxfile): } """ vagrantfile = """Vagrant.configure("2") do |config| + config.ssh.username = "vagrant" + config.ssh.password = "vagrant" config.vm.provider :libvirt do |libvirt| From 34cddd3be83e608d6ae60c7dd4a8ddd83b06d7a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Tue, 7 Mar 2017 15:14:10 +0100 Subject: [PATCH 13/54] delete associated libvirt domain/image when destroying builder vm --- fdroidserver/build.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/fdroidserver/build.py b/fdroidserver/build.py index d9ef9a0e..ceaf76d0 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -220,6 +220,25 @@ def get_clean_vm(reset=False): if os.path.exists('builder'): logging.info("Removing broken/incomplete/unwanted build server") vagrant(['destroy', '-f'], cwd='builder') + if provider == 'libvirt': + import libvirt + virConnect = None + virDomain = None + try: + virConnect = libvirt.open('qemu:///system') + virDomain = virConnect.lookupByName('builder_default') + except libvirt.libvirtError: + logging.debug("no libvirt domain found, skipping delete attempt") + if virDomain: + virDomain.undefineFlags(libvirt.VIR_DOMAIN_UNDEFINE_MANAGED_SAVE + | libvirt.VIR_DOMAIN_UNDEFINE_SNAPSHOTS_METADATA + | libvirt.VIR_DOMAIN_UNDEFINE_NVRAM) + if virConnect: + storagePool = virConnect.storagePoolLookupByName('default') + if storagePool: + for vol in storagePool.listAllVolumes(): + if vol.name().startswith('builder'): + vol.delete() shutil.rmtree('builder') os.mkdir('builder') From c54e0565d7c3a63d6b9dcfdb397d6a74b361a305 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Wed, 8 Mar 2017 12:29:18 +0100 Subject: [PATCH 14/54] added some debug listings to jenkins makebuildserver script --- jenkins-build-makebuildserver | 2 ++ 1 file changed, 2 insertions(+) diff --git a/jenkins-build-makebuildserver b/jenkins-build-makebuildserver index a2923ad4..15363f1a 100755 --- a/jenkins-build-makebuildserver +++ b/jenkins-build-makebuildserver @@ -14,6 +14,8 @@ cleanup_all() { set +e echo "$(date -u) - cleanup in progress..." ps auxww | grep -e VBox -e qemu + virsh --connect qemu:///system list --all + ls -hl /var/lib/libvirt/images cd $WORKSPACE/buildserver vagrant halt sleep 5 From ce3c959ce5ce7982468ff74c4338e719b0b668d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Thu, 9 Mar 2017 13:30:30 +0100 Subject: [PATCH 15/54] scan vm provider again after re-creating builder vm --- fdroidserver/build.py | 1 + 1 file changed, 1 insertion(+) diff --git a/fdroidserver/build.py b/fdroidserver/build.py index ceaf76d0..7b16317c 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -259,6 +259,7 @@ def get_clean_vm(reset=False): retcode, _ = vagrant(['up'], cwd='builder') if retcode != 0: raise BuildException("Failed to start build server") + provider = get_vm_provider() # Open SSH connection to make sure it's working and ready... logging.info("Connecting to virtual machine...") From 413c3836d5e60f8f07101385476627a211a85de3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Wed, 15 Mar 2017 00:12:01 +0100 Subject: [PATCH 16/54] auto-reset broken builder vm --- fdroidserver/build.py | 172 ++++++++++++++++++++++++++++-------------- 1 file changed, 115 insertions(+), 57 deletions(-) diff --git a/fdroidserver/build.py b/fdroidserver/build.py index 7b16317c..50bc7ca1 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -55,7 +55,7 @@ def get_vm_provider(): return 'virtualbox' -def get_builder_vm_id(provider): +def vm_get_builder_id(provider): vd = os.path.join('builder', '.vagrant') if os.path.isdir(vd): # Vagrant 1.2 (and maybe 1.1?) it's a directory tree... @@ -70,7 +70,34 @@ def get_builder_vm_id(provider): return v['active']['default'] -def got_valid_builder_vm(provider): +def vm_get_builder_status(): + """Get the current status of builder vm. + + :returns: one of: 'running', 'paused', 'shutoff', 'not created' + If something is wrong with vagrant or the vm 'unknown' is returned. + """ + (ret, out) = vagrant(['status'], cwd='builder') + + allowed_providers = 'virtualbox|libvirt' + allowed_states = 'running|paused|shutoff|not created' + + r = re.compile('^\s*(?P\w+)\s+' + + '(?P' + allowed_states + ')' + + '\s+\((?P' + allowed_providers + ')\)\s*$') + + for line in out.split('\n'): + m = r.match(line) + if m: + s = m.group('vm_state') + if options.verbose: + logging.debug('current builder vm status: ' + s) + return s + if options.verbose: + logging.debug('current builder vm status: unknown') + return 'unknown' + + +def vm_is_builder_valid(provider): """Returns True if we have a valid-looking builder vm """ if not os.path.exists(os.path.join('builder', 'Vagrantfile')): @@ -93,10 +120,11 @@ def vagrant(params, cwd=None, printout=False): :param: list of parameters to pass to vagrant :cwd: directory to run in, or None for current directory + :printout: has not effect :returns: (ret, out) where ret is the return code, and out is the stdout (and stderr) from vagrant """ - p = FDroidPopen(['vagrant'] + params, cwd=cwd) + p = FDroidPopen(['vagrant'] + params, cwd=cwd, output=printout, stderr_to_stdout=printout) return (p.returncode, p.output) @@ -126,14 +154,25 @@ def get_vagrant_sshinfo(): 'idfile': idfile} +def vm_shutdown_builder(): + """Turn off builder vm. + """ + if options.server: + if os.path.exists(os.path.join('builder', 'Vagrantfile')): + vagrant(['halt'], cwd='builder') + + def vm_snapshot_list(provider): + output = options.verbose if provider is 'virtualbox': p = FDroidPopen(['VBoxManage', 'snapshot', - get_builder_vm_id(provider), 'list', - '--details'], cwd='builder') + vm_get_builder_id(provider), 'list', + '--details'], cwd='builder', + output=output, stderr_to_stdout=output) elif provider is 'libvirt': p = FDroidPopen(['virsh', '-c', 'qemu:///system', 'snapshot-list', - get_builder_vm_id(provider)]) + vm_get_builder_id(provider)], + output=output, stderr_to_stdout=output) return p.output @@ -144,28 +183,46 @@ def vm_snapshot_clean_available(provider): def vm_snapshot_restore(provider): """Does a rollback of the build vm. """ + output = options.verbose if provider is 'virtualbox': p = FDroidPopen(['VBoxManage', 'snapshot', - get_builder_vm_id(provider), 'restore', - 'fdroidclean'], cwd='builder') + vm_get_builder_id(provider), 'restore', + 'fdroidclean'], cwd='builder', + output=output, stderr_to_stdout=output) elif provider is 'libvirt': p = FDroidPopen(['virsh', '-c', 'qemu:///system', 'snapshot-revert', - get_builder_vm_id(provider), 'fdroidclean']) + vm_get_builder_id(provider), 'fdroidclean'], + output=output, stderr_to_stdout=output) return p.returncode == 0 def vm_snapshot_create(provider): + output = options.verbose if provider is 'virtualbox': p = FDroidPopen(['VBoxManage', 'snapshot', - get_builder_vm_id(provider), - 'take', 'fdroidclean'], cwd='builder') + vm_get_builder_id(provider), + 'take', 'fdroidclean'], cwd='builder', + output=output, stderr_to_stdout=output) elif provider is 'libvirt': p = FDroidPopen(['virsh', '-c', 'qemu:///system', 'snapshot-create-as', - get_builder_vm_id(provider), 'fdroidclean']) + vm_get_builder_id(provider), 'fdroidclean'], + output=output, stderr_to_stdout=output) return p.returncode != 0 -def get_clean_vm(reset=False): +def vm_test_ssh_into_builder(): + logging.info("Connecting to virtual machine...") + sshinfo = get_vagrant_sshinfo() + sshs = paramiko.SSHClient() + sshs.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + sshs.connect(sshinfo['hostname'], username=sshinfo['user'], + port=sshinfo['port'], timeout=300, + look_for_keys=False, + key_filename=sshinfo['idfile']) + sshs.close() + + +def vm_get_clean_builder(reset=False): """Get a clean VM ready to do a buildserver build. This might involve creating and starting a new virtual machine from @@ -184,21 +241,34 @@ def get_clean_vm(reset=False): vm_ok = False if not reset: logging.info("Checking for valid existing build server") - - if got_valid_builder_vm(provider): + if vm_is_builder_valid(provider): logging.info("...VM is present (%s)" % provider) if vm_snapshot_clean_available(provider): - logging.info("...snapshot exists - resetting build server to " + logging.info("...snapshot exists - resetting build server to " + "clean state") - retcode, output = vagrant(['status'], cwd='builder') - - if 'running' in output: - logging.info("...suspending") + status = vm_get_builder_status() + if status == 'running': + vm_test_ssh_into_builder() + logging.info("...suspending builder vm") vagrant(['suspend'], cwd='builder') logging.info("...waiting a sec...") time.sleep(10) + elif status == 'shutoff': + logging.info('...starting builder vm') + vagrant(['up'], cwd='builder') + logging.info('...waiting a sec...') + time.sleep(10) + vm_test_ssh_into_builder() + logging.info('...suspending builder vm') + vagrant(['suspend'], cwd='builder') + logging.info("...waiting a sec...") + time.sleep(10) + if options.verbose: + vm_get_builder_status() if vm_snapshot_restore(provider): + if options.verbose: + vm_get_builder_status() logging.info("...reset to snapshot - server is valid") retcode, output = vagrant(['up'], cwd='builder') if retcode != 0: @@ -213,33 +283,14 @@ def get_clean_vm(reset=False): logging.info("...snapshot doesn't exist - " "VBoxManage snapshot list:\n" + vm_snapshot_list(provider)) + else: + logging.info('...VM not present') # If we can't use the existing machine for any reason, make a # new one from scratch. if not vm_ok: - if os.path.exists('builder'): - logging.info("Removing broken/incomplete/unwanted build server") - vagrant(['destroy', '-f'], cwd='builder') - if provider == 'libvirt': - import libvirt - virConnect = None - virDomain = None - try: - virConnect = libvirt.open('qemu:///system') - virDomain = virConnect.lookupByName('builder_default') - except libvirt.libvirtError: - logging.debug("no libvirt domain found, skipping delete attempt") - if virDomain: - virDomain.undefineFlags(libvirt.VIR_DOMAIN_UNDEFINE_MANAGED_SAVE - | libvirt.VIR_DOMAIN_UNDEFINE_SNAPSHOTS_METADATA - | libvirt.VIR_DOMAIN_UNDEFINE_NVRAM) - if virConnect: - storagePool = virConnect.storagePoolLookupByName('default') - if storagePool: - for vol in storagePool.listAllVolumes(): - if vol.name().startswith('builder'): - vol.delete() - shutil.rmtree('builder') + vm_destroy_builder(provider) + os.mkdir('builder') p = subprocess.Popen(['vagrant', '--version'], @@ -260,17 +311,10 @@ def get_clean_vm(reset=False): if retcode != 0: raise BuildException("Failed to start build server") provider = get_vm_provider() + sshinfo = get_vagrant_sshinfo() # Open SSH connection to make sure it's working and ready... - logging.info("Connecting to virtual machine...") - sshinfo = get_vagrant_sshinfo() - sshs = paramiko.SSHClient() - sshs.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - sshs.connect(sshinfo['hostname'], username=sshinfo['user'], - port=sshinfo['port'], timeout=300, - look_for_keys=False, - key_filename=sshinfo['idfile']) - sshs.close() + vm_test_ssh_into_builder() logging.info("Saving clean state of new build server") retcode, _ = vagrant(['suspend'], cwd='builder') @@ -295,15 +339,29 @@ def get_clean_vm(reset=False): return sshinfo -def release_vm(): - """Release the VM previously started with get_clean_vm(). +def vm_suspend_builder(): + """Release the VM previously started with vm_get_clean_builder(). - This should always be called. + This should always be called after each individual app build attempt. """ logging.info("Suspending build server") subprocess.call(['vagrant', 'suspend'], cwd='builder') +def vm_destroy_builder(provider): + """Savely destroy the builder vm. + + """ + logging.info("Removing broken/incomplete/unwanted build server") + if os.path.exists(os.path.join('builder', 'Vagrantfile')): + vagrant(['destroy', '-f'], cwd='builder') + if os.path.isdir('builder'): + shutil.rmtree('builder') + # get rid of vm and related disk images + FDroidPopen(('virsh', '-c', 'qemu:///system', 'destroy', 'builder_default')) + FDroidPopen(('virsh', '-c', 'qemu:///system', 'undefine', 'builder_default', '--nvram', '--managed-save', '--remove-all-storage', '--snapshots-metadata')) + + # Note that 'force' here also implies test mode. def build_server(app, build, vcs, build_dir, output_dir, log_dir, force): """Do a build on the builder vm. @@ -327,7 +385,7 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force): else: logging.getLogger("paramiko").setLevel(logging.WARN) - sshinfo = get_clean_vm() + sshinfo = vm_get_clean_builder() try: if not buildserverid: @@ -516,7 +574,7 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force): finally: # Suspend the build server. - release_vm() + vm_suspend_builder() def force_gradle_build_tools(build_dir, build_tools): From 2993674aa88421453ee24667059034e392460154 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 22 May 2017 16:57:47 +0200 Subject: [PATCH 17/54] calculate correct size for buildserver-box in makebuildserver https://gitlab.com/fdroid/fdroidserver/issues/238#note_24000153 "Our hard-coded image size meta-data (1000) is for some interpreted as less than the size of the box-image by my kvm setup. This makes grub/initrd refuse to boot. So I've changed the metadata size to 9999 which resulted in an actually booting vm. I can log in on the builder-vm via virt-manager and virsh. --- makebuildserver | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/makebuildserver b/makebuildserver index e91a9282..2fc75d82 100755 --- a/makebuildserver +++ b/makebuildserver @@ -12,6 +12,8 @@ import tarfile import vagrant import hashlib import yaml +import math +import json from clint.textui import progress from optparse import OptionParser import fdroidserver.tail @@ -334,12 +336,13 @@ def kvm_package(boxfile): subprocess.check_call(['sudo', '/bin/chmod', '-R', 'a+rX', '/var/lib/libvirt/images']) shutil.copy2(imagepath, 'box.img') subprocess.check_call(['qemu-img', 'rebase', '-p', '-b', '', 'box.img']) - metadata = """{ - "provider": "libvirt", - "format": "qcow2", - "virtual_size": 1000 -} -""" + img_info_raw = subprocess.check_output('sudo qemu-img info --output=json box.img', shell=True) + img_info = json.loads(img_info_raw.decode('utf-8')) + metadata = {"provider": "libvirt", + "format": img_info['format'], + "virtual_size": math.ceil(img_info['virtual-size'] / 1024.**3), + } + vagrantfile = """Vagrant.configure("2") do |config| config.ssh.username = "vagrant" config.ssh.password = "vagrant" @@ -355,7 +358,7 @@ def kvm_package(boxfile): end """ with open('metadata.json', 'w') as fp: - fp.write(metadata) + fp.write(json.dumps(metadata)) with open('Vagrantfile', 'w') as fp: fp.write(vagrantfile) with tarfile.open(boxfile, 'w:gz') as tar: From 437ff7c3f0f36d9832731746f6113a0bbd435a1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Fri, 24 Mar 2017 00:49:02 +0100 Subject: [PATCH 18/54] jenkins makebuildserver fail if vagrant box was not created --- jenkins-build-makebuildserver | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/jenkins-build-makebuildserver b/jenkins-build-makebuildserver index 15363f1a..ff3104f7 100755 --- a/jenkins-build-makebuildserver +++ b/jenkins-build-makebuildserver @@ -54,6 +54,12 @@ echo "apt_package_cache = True" >> $WORKSPACE/makebuildserver.config.py echo "copy_caches_from_host = True" >> $WORKSPACE/makebuildserver.config.py ./makebuildserver --verbose --clean +if [ -z "`vagrant box list | egrep '^buildserver\s+\((libvirt|virtualbox), [0-9]+\)$'`" ]; then + vagrant box list + echo "ERROR: buildserver box does not exist!" + exit 1 +fi + # this can be handled in the jenkins job, or here: if [ -e fdroiddata ]; then cd fdroiddata @@ -73,7 +79,7 @@ if [ -z $ANDROID_HOME ]; then . ~/.android/bashrc else echo "ANDROID_HOME must be set!" - exit + exit 1 fi fi From 8cfd3ca770bfa086782a575837b506fcb13d7ffc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Fri, 24 Mar 2017 03:15:35 +0100 Subject: [PATCH 19/54] delete .vagrant dir when cleaning up buildserver vm --- makebuildserver | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/makebuildserver b/makebuildserver index 2fc75d82..c4e5e02e 100755 --- a/makebuildserver +++ b/makebuildserver @@ -304,6 +304,12 @@ def destroy_current_image(v, serverdir): v.destroy() elif options.verbose: print('Cannot run destroy vagrant setup since Vagrantfile.yaml is not setup!') + + try: + shutil.rmtree(os.path.join(serverdir, '.vagrant')) + except Exception as e: + print("could not delete vagrant dir: %s, %s" % (os.path.join(serverdir, '.vagrant'), e)) + if config['vm_provider'] == 'libvirt': import libvirt try: From 2aa552301124389f2efc6eb0387b13049287253a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Fri, 24 Mar 2017 04:16:30 +0100 Subject: [PATCH 20/54] makebuildserver prune gloabal vagrant status when purging broken VMs --- makebuildserver | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/makebuildserver b/makebuildserver index c4e5e02e..de0f3b47 100755 --- a/makebuildserver +++ b/makebuildserver @@ -304,6 +304,7 @@ def destroy_current_image(v, serverdir): v.destroy() elif options.verbose: print('Cannot run destroy vagrant setup since Vagrantfile.yaml is not setup!') + subprocess.check_call(['vagrant', 'global-status', '--prune']) try: shutil.rmtree(os.path.join(serverdir, '.vagrant')) @@ -346,7 +347,7 @@ def kvm_package(boxfile): img_info = json.loads(img_info_raw.decode('utf-8')) metadata = {"provider": "libvirt", "format": img_info['format'], - "virtual_size": math.ceil(img_info['virtual-size'] / 1024.**3), + "virtual_size": math.ceil(img_info['virtual-size'] / 1024. ** 3), } vagrantfile = """Vagrant.configure("2") do |config| From 440509cf8a22574bfddd93de06875925995fcbba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Fri, 24 Mar 2017 20:04:50 +0100 Subject: [PATCH 21/54] makebuildserver debugging log details --- jenkins-build-makebuildserver | 2 +- makebuildserver | 138 +++++++++++++++++++++++----------- 2 files changed, 96 insertions(+), 44 deletions(-) diff --git a/jenkins-build-makebuildserver b/jenkins-build-makebuildserver index ff3104f7..55df494b 100755 --- a/jenkins-build-makebuildserver +++ b/jenkins-build-makebuildserver @@ -52,7 +52,7 @@ echo "debian_mirror = 'https://deb.debian.org/debian/'" > $WORKSPACE/makebuildse echo "boot_timeout = 1200" >> $WORKSPACE/makebuildserver.config.py echo "apt_package_cache = True" >> $WORKSPACE/makebuildserver.config.py echo "copy_caches_from_host = True" >> $WORKSPACE/makebuildserver.config.py -./makebuildserver --verbose --clean +./makebuildserver -vv --clean if [ -z "`vagrant box list | egrep '^buildserver\s+\((libvirt|virtualbox), [0-9]+\)$'`" ]; then vagrant box list diff --git a/makebuildserver b/makebuildserver index de0f3b47..2da46f22 100755 --- a/makebuildserver +++ b/makebuildserver @@ -14,27 +14,47 @@ import hashlib import yaml import math import json +import logging from clint.textui import progress from optparse import OptionParser import fdroidserver.tail -if not os.path.exists('makebuildserver') and not os.path.exists('buildserver'): - print('This must be run as ./makebuildserver in fdroidserver.git!') - sys.exit(1) +parser = OptionParser() +#parser.add_option("-v", "--verbose", action="store_true", default=False, +parser.add_option('-v', '--verbose', action="count", dest='verbosity', default=1, + help="Spew out even more information than normal") +parser.add_option('-q', action='store_const', const=0, dest='verbosity') +parser.add_option("-c", "--clean", action="store_true", default=False, + help="Build from scratch, rather than attempting to update the existing server") +parser.add_option('--skip-cache-update', action="store_true", default=False, + help="""Skip downloading and checking cache.""" + """This assumes that the cache is already downloaded completely.""") +options, args = parser.parse_args() + +logger = logging.getLogger('fdroid-makebuildserver') +if options.verbosity >= 2: + logging.basicConfig(format='%(message)s', level=logging.DEBUG) + logger.setLevel(logging.DEBUG) +elif options.verbosity == 1: + logging.basicConfig(format='%(message)s', level=logging.INFO) + logger.setLevel(logging.INFO) +elif options.verbosity <= 0: + logging.basicConfig(format='%(message)s', level=logging.WARNING) + logger.setLevel(logging.WARNING) + + +if not os.path.exists('makebuildserver') and not os.path.exists('buildserver'): + logger.critical('This must be run as ./makebuildserver in fdroidserver.git!') + sys.exit(1) tail = None -parser = OptionParser() -parser.add_option("-v", "--verbose", action="store_true", default=False, - help="Spew out even more information than normal") -parser.add_option("-c", "--clean", action="store_true", default=False, - help="Build from scratch, rather than attempting to update the existing server") -options, args = parser.parse_args() - # set up default config cachedir = os.path.join(os.getenv('HOME'), '.cache', 'fdroidserver') +logger.debug('cachedir set to: %s', cachedir) + config = { 'basebox': 'jessie64', 'baseboxurl': [ @@ -58,11 +78,12 @@ if os.path.isfile('/usr/bin/systemd-detect-virt'): except subprocess.CalledProcessError as e: virt = 'none' if virt == 'qemu' or virt == 'kvm' or virt == 'bochs': - print('Running in a VM guest, defaulting to QEMU/KVM via libvirt') + logger.info('Running in a VM guest, defaulting to QEMU/KVM via libvirt') config['vm_provider'] = 'libvirt' config['domain'] = 'buildserver_default' elif virt != 'none': - print('Running in an unsupported VM guest (' + virt + ')!') + logger.info('Running in an unsupported VM guest (%s)!', virt) +logger.debug('deceted virt: %s', virt) # load config file, if present if os.path.exists('makebuildserver.config.py'): @@ -72,24 +93,28 @@ elif os.path.exists('makebs.config.py'): exec(compile(open('makebs.config.py').read(), 'makebs.config.py', 'exec'), config) if '__builtins__' in config: del(config['__builtins__']) # added by compile/exec +logger.debug("makebuildserver.config.py parsed -> %s", json.dumps(config, indent=4, sort_keys=True)) # Update cached files. cachedir = config['cachedir'] if not os.path.exists(cachedir): os.makedirs(cachedir, 0o755) + logger.debug('created cachedir %s because it did not exists.', cachedir) if config['vm_provider'] == 'libvirt': tmp = cachedir while tmp != '/': mode = os.stat(tmp).st_mode if not (stat.S_IXUSR & mode and stat.S_IXGRP & mode and stat.S_IXOTH & mode): - print('ERROR:', tmp, 'will not be accessible to the VM! To fix, run:') - print(' chmod a+X', tmp) + logger.critical('ERROR: %s will not be accessible to the VM! To fix, run:', tmp) + logger.critical(' chmod a+X %s', tmp) sys.exit(1) tmp = os.path.dirname(tmp) + logger.debug('cache dir %s is accessible for libvirt vm.', cachedir) if config['apt_package_cache']: config['aptcachedir'] = cachedir + '/apt/archives' + logger.debug('aptcachedir is set to %s', config['aptcachedir']) cachefiles = [ ('https://dl.google.com/android/repository/tools_r25.2.3-linux.zip', @@ -299,17 +324,20 @@ def sha256_for_file(path): def destroy_current_image(v, serverdir): global config + logger.info('destroying buildserver vm, removing images and vagrand-configs...') + # cannot run vagrant without the config in the YAML file if os.path.exists(os.path.join(serverdir, 'Vagrantfile.yaml')): v.destroy() - elif options.verbose: - print('Cannot run destroy vagrant setup since Vagrantfile.yaml is not setup!') + logger.debug('vagrant destroy completed') + if logger.level <= logging.DEBUG: + logger.debug('Cannot run destroy vagrant setup since Vagrantfile.yaml is not setup!') subprocess.check_call(['vagrant', 'global-status', '--prune']) try: shutil.rmtree(os.path.join(serverdir, '.vagrant')) except Exception as e: - print("could not delete vagrant dir: %s, %s" % (os.path.join(serverdir, '.vagrant'), e)) + logger.debug("could not delete vagrant dir: %s, %s", os.path.join(serverdir, '.vagrant'), e) if config['vm_provider'] == 'libvirt': import libvirt @@ -325,7 +353,7 @@ def destroy_current_image(v, serverdir): for vol in storagePool.listAllVolumes(): vol.delete() except libvirt.libvirtError as e: - print(e) + logger.warn('%s', e) def kvm_package(boxfile): @@ -385,9 +413,7 @@ def run_via_vagrant_ssh(v, cmdlist): v._run_vagrant_command(['ssh', '-c', cmd]) -def main(): - global cachedir, cachefiles, config, tail - +def update_cache(cachedir, cachefiles): for srcurl, shasum in cachefiles: filename = os.path.basename(srcurl) local_filename = os.path.join(cachedir, filename) @@ -408,19 +434,19 @@ def main(): content_length = local_length # skip the download except requests.exceptions.RequestException as e: content_length = local_length # skip the download - print(e) + logger.warn('%s', e) if local_length == content_length: download = False elif local_length > content_length: - print('deleting corrupt file from cache: ' + local_filename) + logger.info('deleting corrupt file from cache: %s', local_filename) os.remove(local_filename) - print("Downloading " + filename + " to cache") + logger.info("Downloading %s to cache", filename) elif local_length > -1 and local_length < content_length: - print("Resuming download of " + local_filename) + logger.info("Resuming download of %s", local_filename) resume_header = {'Range': 'bytes=%d-%d' % (local_length, content_length)} else: - print("Downloading " + filename + " to cache") + logger.info("Downloading %s to cache", filename) if download: r = requests.get(srcurl, headers=resume_header, @@ -434,14 +460,37 @@ def main(): v = sha256_for_file(local_filename) if v == shasum: - print("\t...shasum verified for " + local_filename) + logger.info("\t...shasum verified for %s", local_filename) else: - print("Invalid shasum of '" + v + "' detected for " + local_filename) + logger.critical("Invalid shasum of '%s' detected for %s", v, local_filename) os.remove(local_filename) sys.exit(1) + +def debug_log_vagrant_vm(vm_dir, vm_name): + if options.verbosity >= 3: + _vagrant_dir = os.path.join(vm_dir, '.vagrant') + logger.debug('check %s dir exists? -> %r', _vagrant_dir, os.path.isdir(_vagrant_dir)) + logger.debug('> vagrant status') + subprocess.call(['vagrant', 'status'], cwd=vm_dir) + logger.debug('> vagrant box list') + subprocess.call(['vagrant', 'box', 'list']) + logger.debug('> virsh -c qmeu:///system list --all') + subprocess.call(['virsh', '-c', 'qemu:///system', 'list', '--all']) + logger.debug('> virsh -c qemu:///system snapshot-list %s', vm_name) + subprocess.call(['virsh', '-c', 'qemu:///system', 'snapshot-list', vm_name]) + + +def main(): + global cachedir, cachefiles, config, tail + + if options.skip_cache_update: + logger.info('skipping cache update and verification...') + else: + update_cache(cachedir, cachefiles) + local_qt_filename = os.path.join(cachedir, 'qt-opensource-linux-x64-android-5.7.0.run') - print("Setting executable bit for " + local_qt_filename) + logger.info("Setting executable bit for %s", local_qt_filename) os.chmod(local_qt_filename, 0o755) # use VirtualBox software virtualization if hardware is not available, @@ -451,11 +500,13 @@ def main(): # all < 10 year old Macs work, and OSX servers as VM host are very # rare, but this could also be auto-detected if someone codes it config['hwvirtex'] = 'on' + logger.info('platform is darwnin -> hwvirtex = \'on\'') elif os.path.exists('/proc/cpuinfo'): with open('/proc/cpuinfo') as f: contents = f.read() if 'vmx' in contents or 'svm' in contents: config['hwvirtex'] = 'on' + logger.info('found \'vmx\' or \'svm\' in /proc/cpuinfo -> hwvirtex = \'on\'') serverdir = os.path.join(os.getcwd(), 'buildserver') logfilename = os.path.join(serverdir, 'up.log') @@ -464,7 +515,7 @@ def main(): log_cm = vagrant.make_file_cm(logfilename) v = vagrant.Vagrant(root=serverdir, out_cm=log_cm, err_cm=log_cm) - if options.verbose: + if options.verbosity >= 2: tail = fdroidserver.tail.Tail(logfilename) tail.start() @@ -476,18 +527,18 @@ def main(): vf = os.path.join(serverdir, 'Vagrantfile.yaml') writevf = True if os.path.exists(vf): - print('Halting', serverdir) + logger.info('Halting %s', serverdir) v.halt() with open(vf, 'r', encoding='utf-8') as f: oldconfig = yaml.load(f) if config != oldconfig: - print("Server configuration has changed, rebuild from scratch is required") + logger.info("Server configuration has changed, rebuild from scratch is required") destroy_current_image(v, serverdir) else: - print("Re-provisioning existing server") + logger.info("Re-provisioning existing server") writevf = False else: - print("No existing server - building from scratch") + logger.info("No existing server - building from scratch") if writevf: with open(vf, 'w', encoding='utf-8') as f: yaml.dump(config, f) @@ -506,17 +557,19 @@ def main(): baseboxurl = config['baseboxurl'] else: baseboxurl = config['baseboxurl'][0] - print('Adding', config['basebox'], 'from', baseboxurl) + logger.info('Adding %s from %s', config['basebox'], baseboxurl) v.box_add(config['basebox'], baseboxurl) needs_mutate = True if needs_mutate: - print('Converting', config['basebox'], 'to libvirt format') + logger.info('Converting %s to libvirt format', config['basebox']) v._call_vagrant_command(['mutate', config['basebox'], 'libvirt']) - print('Removing virtualbox format copy of', config['basebox']) + logger.info('Removing virtualbox format copy of %s', config['basebox']) v.box_remove(config['basebox'], 'virtualbox') - print("Configuring build server VM") + logger.info("Configuring build server VM") + debug_log_vagrant_vm(serverdir, 'buildserver_default') v.up(provision=True) + debug_log_vagrant_vm(serverdir, 'buildserver_default') if config['copy_caches_from_host']: ssh_config = v.ssh_config() @@ -539,17 +592,16 @@ def main(): run_via_vagrant_ssh(v, ['rm', '-f', '~/.gradle/caches/modules-2/modules-2.lock']) run_via_vagrant_ssh(v, ['rm', '-fr', '~/.gradle/caches/*/plugin-resolution/']) - print("Writing buildserver ID") p = subprocess.Popen(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE, universal_newlines=True) buildserverid = p.communicate()[0].strip() - print("...ID is " + buildserverid) + logger.info("Writing buildserver ID ...ID is %s", buildserverid) run_via_vagrant_ssh(v, 'sh -c "echo %s >/home/vagrant/buildserverid"' % buildserverid) - print("Stopping build server VM") + logger.info("Stopping build server VM") v.halt() - print("Packaging") + logger.info("Packaging") boxfile = os.path.join(os.getcwd(), 'buildserver.box') if os.path.exists(boxfile): os.remove(boxfile) @@ -559,7 +611,7 @@ def main(): else: v.package(output=boxfile) - print("Adding box") + logger.info("Adding box") v.box_add('buildserver', boxfile, force=True) os.remove(boxfile) From 16b609215ecc9ba6b48a2b688f0f464667dc5d00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Sat, 25 Mar 2017 01:40:41 +0100 Subject: [PATCH 22/54] overhauled makebuildserver libvirt vm cleanup --- makebuildserver | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/makebuildserver b/makebuildserver index 2da46f22..bb4a6b1b 100755 --- a/makebuildserver +++ b/makebuildserver @@ -20,9 +20,7 @@ from optparse import OptionParser import fdroidserver.tail - parser = OptionParser() -#parser.add_option("-v", "--verbose", action="store_true", default=False, parser.add_option('-v', '--verbose', action="count", dest='verbosity', default=1, help="Spew out even more information than normal") parser.add_option('-q', action='store_const', const=0, dest='verbosity') @@ -342,18 +340,25 @@ def destroy_current_image(v, serverdir): if config['vm_provider'] == 'libvirt': import libvirt try: - virConnect = libvirt.open('qemu:///system') - virDomain = virConnect.lookupByName(config['domain']) - if virDomain: - virDomain.undefineFlags(libvirt.VIR_DOMAIN_UNDEFINE_MANAGED_SAVE - | libvirt.VIR_DOMAIN_UNDEFINE_SNAPSHOTS_METADATA - | libvirt.VIR_DOMAIN_UNDEFINE_NVRAM) - storagePool = virConnect.storagePoolLookupByName('default') - if storagePool: - for vol in storagePool.listAllVolumes(): - vol.delete() + conn = libvirt.open('qemu:///system') + try: + dom = conn.lookupByName(config['domain']) + try: + dom.destroy() + except libvirt.libvirtError as e: + logging.info("could not force libvirt domain '%s' off: %s", dom.name(), e) + # libvirt python bindings do not support all flags required + # for undefining domains correctly. + try: + logger.debug('virsh -c qemu:///system undefine builder_defaul --nvram --managed-save --remove-all-storage --snapshots-metadata') + subprocess.check_call(('virsh', '-c', 'qemu:///system', 'undefine', 'builder_default', '--nvram', '--managed-save', '--remove-all-storage', '--snapshots-metadata')) + except subprocess.CalledProcessError as e: + logger.info("could not undefine libvirt domain'%s': %s", dom.name(), e) + except libvirt.libvirtError as e: + logging.info("finding libvirt domain '%s' failed. (%s)", config['domain'], e) except libvirt.libvirtError as e: - logger.warn('%s', e) + logging.critical('could not connect to libvirtd: %s', e) + sys.exit(1) def kvm_package(boxfile): From 8e5446068bcda747fdd4bdf8b079372a7b8d99cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Sat, 25 Mar 2017 01:56:20 +0100 Subject: [PATCH 23/54] makebuildserver debug logging when initial provisioning fails --- makebuildserver | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/makebuildserver b/makebuildserver index bb4a6b1b..04e649aa 100755 --- a/makebuildserver +++ b/makebuildserver @@ -353,7 +353,7 @@ def destroy_current_image(v, serverdir): logger.debug('virsh -c qemu:///system undefine builder_defaul --nvram --managed-save --remove-all-storage --snapshots-metadata') subprocess.check_call(('virsh', '-c', 'qemu:///system', 'undefine', 'builder_default', '--nvram', '--managed-save', '--remove-all-storage', '--snapshots-metadata')) except subprocess.CalledProcessError as e: - logger.info("could not undefine libvirt domain'%s': %s", dom.name(), e) + logger.info("could not undefine libvirt domain '%s': %s", dom.name(), e) except libvirt.libvirtError as e: logging.info("finding libvirt domain '%s' failed. (%s)", config['domain'], e) except libvirt.libvirtError as e: @@ -573,8 +573,13 @@ def main(): logger.info("Configuring build server VM") debug_log_vagrant_vm(serverdir, 'buildserver_default') - v.up(provision=True) - debug_log_vagrant_vm(serverdir, 'buildserver_default') + try: + #subprocess.check_call(['vagrant', 'up', '--provision'], pwd=serverdir) + v.up(provision=True) + except subprocess.CalledProcessError as e: + debug_log_vagrant_vm(serverdir, 'buildserver_default') + logging.critical('could not bring buildserver vm up. %s', e) + sys.exit(1) if config['copy_caches_from_host']: ssh_config = v.ssh_config() From 70a827d59a9165ae49598987a292f97896ba5ed2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Sat, 25 Mar 2017 02:19:15 +0100 Subject: [PATCH 24/54] makebuildserver use virsh instead of libvirt for forcing domain off --- makebuildserver | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/makebuildserver b/makebuildserver index 04e649aa..cd4213ae 100755 --- a/makebuildserver +++ b/makebuildserver @@ -344,14 +344,15 @@ def destroy_current_image(v, serverdir): try: dom = conn.lookupByName(config['domain']) try: - dom.destroy() - except libvirt.libvirtError as e: - logging.info("could not force libvirt domain '%s' off: %s", dom.name(), e) - # libvirt python bindings do not support all flags required - # for undefining domains correctly. + logger.debug('virsh -c qemu:///system destroy %s', config['domain']) + subprocess.check_call(['virsh', '-c', 'qemu:///system', 'destroy', config['domain']]) + except subprocess.CalledProcessError as e: + logging.info("could not force libvirt domain '%s' off: %s", config['domain'], e) try: - logger.debug('virsh -c qemu:///system undefine builder_defaul --nvram --managed-save --remove-all-storage --snapshots-metadata') - subprocess.check_call(('virsh', '-c', 'qemu:///system', 'undefine', 'builder_default', '--nvram', '--managed-save', '--remove-all-storage', '--snapshots-metadata')) + # libvirt python bindings do not support all flags required + # for undefining domains correctly. + logger.debug('virsh -c qemu:///system undefine %s --nvram --managed-save --remove-all-storage --snapshots-metadata', config['domain']) + subprocess.check_call(('virsh', '-c', 'qemu:///system', 'undefine', config['domain'], '--nvram', '--managed-save', '--remove-all-storage', '--snapshots-metadata')) except subprocess.CalledProcessError as e: logger.info("could not undefine libvirt domain '%s': %s", dom.name(), e) except libvirt.libvirtError as e: @@ -574,7 +575,6 @@ def main(): logger.info("Configuring build server VM") debug_log_vagrant_vm(serverdir, 'buildserver_default') try: - #subprocess.check_call(['vagrant', 'up', '--provision'], pwd=serverdir) v.up(provision=True) except subprocess.CalledProcessError as e: debug_log_vagrant_vm(serverdir, 'buildserver_default') From 7e8f7c65bc33ec7a6185907a2131a6c2e5e90132 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Sat, 25 Mar 2017 02:37:08 +0100 Subject: [PATCH 25/54] makebuildserver added failsafe when destroy vagrant vm --- makebuildserver | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/makebuildserver b/makebuildserver index cd4213ae..b5e65167 100755 --- a/makebuildserver +++ b/makebuildserver @@ -322,12 +322,15 @@ def sha256_for_file(path): def destroy_current_image(v, serverdir): global config - logger.info('destroying buildserver vm, removing images and vagrand-configs...') + logger.info('destroying buildserver vm, removing images and vagrant-configs...') # cannot run vagrant without the config in the YAML file if os.path.exists(os.path.join(serverdir, 'Vagrantfile.yaml')): - v.destroy() - logger.debug('vagrant destroy completed') + try: + v.destroy() + logger.debug('vagrant destroy completed') + except subprocess.CalledProcessError as e: + logger.debug('vagrant destroy failed: %s', e) if logger.level <= logging.DEBUG: logger.debug('Cannot run destroy vagrant setup since Vagrantfile.yaml is not setup!') subprocess.check_call(['vagrant', 'global-status', '--prune']) @@ -573,11 +576,14 @@ def main(): v.box_remove(config['basebox'], 'virtualbox') logger.info("Configuring build server VM") - debug_log_vagrant_vm(serverdir, 'buildserver_default') + debug_log_vagrant_vm(serverdir, config['domain']) try: - v.up(provision=True) + try: + v.up(provision=True) + except subprocess.CalledProcessError as e: + v.up(provision=True) except subprocess.CalledProcessError as e: - debug_log_vagrant_vm(serverdir, 'buildserver_default') + debug_log_vagrant_vm(serverdir, config['domain']) logging.critical('could not bring buildserver vm up. %s', e) sys.exit(1) From 3c4b1dec849e20172a7bdaefe4cd29f018c6300c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Sat, 25 Mar 2017 02:48:00 +0100 Subject: [PATCH 26/54] makebuildserver more robust codepath for vagrant destroy --- makebuildserver | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/makebuildserver b/makebuildserver index b5e65167..19ea04bd 100755 --- a/makebuildserver +++ b/makebuildserver @@ -324,16 +324,15 @@ def destroy_current_image(v, serverdir): logger.info('destroying buildserver vm, removing images and vagrant-configs...') - # cannot run vagrant without the config in the YAML file - if os.path.exists(os.path.join(serverdir, 'Vagrantfile.yaml')): - try: - v.destroy() - logger.debug('vagrant destroy completed') - except subprocess.CalledProcessError as e: - logger.debug('vagrant destroy failed: %s', e) - if logger.level <= logging.DEBUG: - logger.debug('Cannot run destroy vagrant setup since Vagrantfile.yaml is not setup!') + try: + v.destroy() + logger.debug('vagrant destroy completed') + except subprocess.CalledProcessError as e: + logger.debug('vagrant destroy failed: %s', e) + try: subprocess.check_call(['vagrant', 'global-status', '--prune']) + except subprocess.CalledProcessError as e: + logger.debug('pruning global vagrant status failed: %s', e) try: shutil.rmtree(os.path.join(serverdir, '.vagrant')) From 718d01dea2399de8f211fdc024c990cdd593cc75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Sat, 25 Mar 2017 03:50:22 +0100 Subject: [PATCH 27/54] makebuildserver added sleep after destroy/undefine --- makebuildserver | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/makebuildserver b/makebuildserver index 19ea04bd..c3a6ab81 100755 --- a/makebuildserver +++ b/makebuildserver @@ -14,6 +14,7 @@ import hashlib import yaml import math import json +import time import logging from clint.textui import progress from optparse import OptionParser @@ -348,6 +349,8 @@ def destroy_current_image(v, serverdir): try: logger.debug('virsh -c qemu:///system destroy %s', config['domain']) subprocess.check_call(['virsh', '-c', 'qemu:///system', 'destroy', config['domain']]) + logging.info("...waiting a sec...") + time.sleep(10) except subprocess.CalledProcessError as e: logging.info("could not force libvirt domain '%s' off: %s", config['domain'], e) try: @@ -355,6 +358,8 @@ def destroy_current_image(v, serverdir): # for undefining domains correctly. logger.debug('virsh -c qemu:///system undefine %s --nvram --managed-save --remove-all-storage --snapshots-metadata', config['domain']) subprocess.check_call(('virsh', '-c', 'qemu:///system', 'undefine', config['domain'], '--nvram', '--managed-save', '--remove-all-storage', '--snapshots-metadata')) + logging.info("...waiting a sec...") + time.sleep(10) except subprocess.CalledProcessError as e: logger.info("could not undefine libvirt domain '%s': %s", dom.name(), e) except libvirt.libvirtError as e: From fb03e1784940a5eb1924bc916cc978a0ae0620ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Sat, 25 Mar 2017 04:19:36 +0100 Subject: [PATCH 28/54] fdroid build: added sleep after destroy/undefine --- fdroidserver/build.py | 4 ++++ makebuildserver | 12 ++++++------ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/fdroidserver/build.py b/fdroidserver/build.py index 50bc7ca1..135e6054 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -359,7 +359,11 @@ def vm_destroy_builder(provider): shutil.rmtree('builder') # get rid of vm and related disk images FDroidPopen(('virsh', '-c', 'qemu:///system', 'destroy', 'builder_default')) + logging.info("...waiting a sec...") + time.sleep(10) FDroidPopen(('virsh', '-c', 'qemu:///system', 'undefine', 'builder_default', '--nvram', '--managed-save', '--remove-all-storage', '--snapshots-metadata')) + logging.info("...waiting a sec...") + time.sleep(10) # Note that 'force' here also implies test mode. diff --git a/makebuildserver b/makebuildserver index c3a6ab81..3b14bd7d 100755 --- a/makebuildserver +++ b/makebuildserver @@ -349,23 +349,23 @@ def destroy_current_image(v, serverdir): try: logger.debug('virsh -c qemu:///system destroy %s', config['domain']) subprocess.check_call(['virsh', '-c', 'qemu:///system', 'destroy', config['domain']]) - logging.info("...waiting a sec...") + logger.info("...waiting a sec...") time.sleep(10) except subprocess.CalledProcessError as e: - logging.info("could not force libvirt domain '%s' off: %s", config['domain'], e) + logger.info("could not force libvirt domain '%s' off: %s", config['domain'], e) try: # libvirt python bindings do not support all flags required # for undefining domains correctly. logger.debug('virsh -c qemu:///system undefine %s --nvram --managed-save --remove-all-storage --snapshots-metadata', config['domain']) subprocess.check_call(('virsh', '-c', 'qemu:///system', 'undefine', config['domain'], '--nvram', '--managed-save', '--remove-all-storage', '--snapshots-metadata')) - logging.info("...waiting a sec...") + logger.info("...waiting a sec...") time.sleep(10) except subprocess.CalledProcessError as e: logger.info("could not undefine libvirt domain '%s': %s", dom.name(), e) except libvirt.libvirtError as e: - logging.info("finding libvirt domain '%s' failed. (%s)", config['domain'], e) + logger.info("finding libvirt domain '%s' failed. (%s)", config['domain'], e) except libvirt.libvirtError as e: - logging.critical('could not connect to libvirtd: %s', e) + logger.critical('could not connect to libvirtd: %s', e) sys.exit(1) @@ -588,7 +588,7 @@ def main(): v.up(provision=True) except subprocess.CalledProcessError as e: debug_log_vagrant_vm(serverdir, config['domain']) - logging.critical('could not bring buildserver vm up. %s', e) + logger.critical('could not bring buildserver vm up. %s', e) sys.exit(1) if config['copy_caches_from_host']: From 92fada803e5aa2ecf7516398bee011666f395521 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Sun, 26 Mar 2017 01:41:39 +0100 Subject: [PATCH 29/54] overhauled and moved destroying builder vm to vmtools.py --- fdroidserver/build.py | 2 +- fdroidserver/vmtools.py | 165 ++++++++++++++++++++++++++++++++++++++++ makebuildserver | 57 ++------------ 3 files changed, 171 insertions(+), 53 deletions(-) create mode 100644 fdroidserver/vmtools.py diff --git a/fdroidserver/build.py b/fdroidserver/build.py index 135e6054..de8c3377 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -1110,7 +1110,7 @@ def trybuild(app, build, build_dir, output_dir, log_dir, also_check_dir, this is the 'unsigned' directory. :param repo_dir: The repo directory - used for checking if the build is necessary. - :paaram also_check_dir: An additional location for checking if the build + :param also_check_dir: An additional location for checking if the build is necessary (usually the archive repo) :param test: True if building in test mode, in which case the build will always happen, even if the output already exists. In test mode, the diff --git a/fdroidserver/vmtools.py b/fdroidserver/vmtools.py new file mode 100644 index 00000000..a5c0f5ba --- /dev/null +++ b/fdroidserver/vmtools.py @@ -0,0 +1,165 @@ +#!/usr/bin/env python3 +# +# vmtools.py - part of the FDroid server tools +# Copyright (C) 2017 Michael Poehn +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from os.path import isdir, isfile, join as joinpath, basename, abspath +import time +import shutil +import vagrant +import subprocess +from .common import FDroidException +from logging import getLogger + +logger = getLogger('fdroidserver-vmtools') + + +def get_build_vm(srvdir, provider=None): + """Factory function for getting FDroidBuildVm instances. + + This function tries to figure out what hypervisor should be used + and creates an object for controlling a build VM. + + :param srvdir: path to a directory which contains a Vagrantfile + :param provider: optionally this parameter allows specifiying an + spesific vagrant provider. + :returns: FDroidBuildVm instance. + """ + abssrvdir = abspath(srvdir) + if provider: + if provider == 'libvirt': + logger.debug('build vm provider \'libvirt\' selected') + return LibvirtBuildVm(abssrvdir) + elif provider == 'virtualbox': + logger.debug('build vm provider \'virtualbox\' selected') + return VirtualboxBuildVm(abssrvdir) + else: + logger.warn('unsupported provider \'%s\' requested', provider) + has_libvirt_machine = isdir(joinpath(abssrvdir, '.vagrant', + 'machines', 'default', 'libvirt')) + has_vbox_machine = isdir(joinpath(abssrvdir, '.vagrant', + 'machines', 'default', 'libvirt')) + if has_libvirt_machine and has_vbox_machine: + logger.info('build vm provider lookup found virtualbox and libvirt, defaulting to \'virtualbox\'') + return VirtualboxBuildVm(abssrvdir) + elif has_libvirt_machine: + logger.debug('build vm provider lookup found \'libvirt\'') + return LibvirtBuildVm(abssrvdir) + elif has_vbox_machine: + logger.debug('build vm provider lookup found \'virtualbox\'') + return VirtualboxBuildVm(abssrvdir) + + logger.info('build vm provider lookup could not determine provider, defaulting to \'virtualbox\'') + return VirtualboxBuildVm(abssrvdir) + + +class FDroidBuildVmException(FDroidException): + pass + + +class FDroidBuildVm(): + """Abstract base class for working with FDroids build-servers. + + Use the factory method `fdroidserver.vmtools.get_build_vm()` for + getting correct instances of this class. + + This is intended to be a hypervisor independant, fault tolerant + wrapper around the vagrant functions we use. + """ + + def __init__(self, srvdir): + """Create new server class. + """ + self.srvdir = srvdir + self.srvname = basename(srvdir) + '_default' + self.vgrntfile = joinpath(srvdir, 'Vagrantfile') + if not isdir(srvdir): + raise FDroidBuildVmException("Can not init vagrant, directory %s not present" % (srvdir)) + if not isfile(self.vgrntfile): + raise FDroidBuildVmException("Can not init vagrant, '%s' not present" % (self.vgrntfile)) + self.vgrnt = vagrant.Vagrant(root=srvdir, out_cm=vagrant.stdout_cm, err_cm=vagrant.stdout_cm) + + def isUpAndRunning(self): + raise NotImplementedError('TODO implement this') + + def up(self, provision=True): + try: + self.vgrnt.up(provision=provision) + except subprocess.CalledProcessError as e: + logger.info('could not bring vm up: %s', e) + + def destroy(self): + """Remove every trace of this VM from the system. + + This includes deleting: + * hypervisor specific definitions + * vagrant state informations (eg. `.vagrant` folder) + * images related to this vm + """ + try: + self.vgrnt.destroy() + logger.debug('vagrant destroy completed') + except subprocess.CalledProcessError as e: + logger.debug('vagrant destroy failed: %s', e) + vgrntdir = joinpath(self.srvdir, '.vagrant') + try: + shutil.rmtree(vgrntdir) + logger.debug('deleted vagrant dir: %s', vgrntdir) + except Exception as e: + logger.debug("could not delete vagrant dir: %s, %s", vgrntdir, e) + try: + subprocess.check_call(['vagrant', 'global-status', '--prune']) + except subprocess.CalledProcessError as e: + logger.debug('pruning global vagrant status failed: %s', e) + + +class LibvirtBuildVm(FDroidBuildVm): + def __init__(self, srvdir): + super().__init__(srvdir) + import libvirt + + try: + self.conn = libvirt.open('qemu:///system') + except libvirt.libvirtError as e: + logger.critical('could not connect to libvirtd: %s', e) + + def destroy(self): + + super().destroy() + + # resorting to virsh instead of libvirt python bindings, because + # this is way more easy and therefore fault tolerant. + # (eg. lookupByName only works on running VMs) + try: + logger.debug('virsh -c qemu:///system destroy', self.srvname) + subprocess.check_call(('virsh', '-c', 'qemu:///system', 'destroy')) + logger.info("...waiting a sec...") + time.sleep(10) + except subprocess.CalledProcessError as e: + logger.info("could not force libvirt domain '%s' off: %s", self.srvname, e) + try: + # libvirt python bindings do not support all flags required + # for undefining domains correctly. + logger.debug('virsh -c qemu:///system undefine %s --nvram --managed-save --remove-all-storage --snapshots-metadata', self.srvname) + subprocess.check_call(('virsh', '-c', 'qemu:///system', 'undefine', self.srvname, '--nvram', '--managed-save', '--remove-all-storage', '--snapshots-metadata')) + logger.info("...waiting a sec...") + time.sleep(10) + except subprocess.CalledProcessError as e: + logger.info("could not undefine libvirt domain '%s': %s", self.srvname, e) + + +class VirtualboxBuildVm(FDroidBuildVm): + pass diff --git a/makebuildserver b/makebuildserver index 3b14bd7d..f8b892e6 100755 --- a/makebuildserver +++ b/makebuildserver @@ -19,6 +19,7 @@ import logging from clint.textui import progress from optparse import OptionParser import fdroidserver.tail +import fdroidserver.vmtools parser = OptionParser() @@ -32,7 +33,7 @@ parser.add_option('--skip-cache-update', action="store_true", default=False, """This assumes that the cache is already downloaded completely.""") options, args = parser.parse_args() -logger = logging.getLogger('fdroid-makebuildserver') +logger = logging.getLogger('fdroidserver-makebuildserver') if options.verbosity >= 2: logging.basicConfig(format='%(message)s', level=logging.DEBUG) logger.setLevel(logging.DEBUG) @@ -320,55 +321,6 @@ def sha256_for_file(path): return s.hexdigest() -def destroy_current_image(v, serverdir): - global config - - logger.info('destroying buildserver vm, removing images and vagrant-configs...') - - try: - v.destroy() - logger.debug('vagrant destroy completed') - except subprocess.CalledProcessError as e: - logger.debug('vagrant destroy failed: %s', e) - try: - subprocess.check_call(['vagrant', 'global-status', '--prune']) - except subprocess.CalledProcessError as e: - logger.debug('pruning global vagrant status failed: %s', e) - - try: - shutil.rmtree(os.path.join(serverdir, '.vagrant')) - except Exception as e: - logger.debug("could not delete vagrant dir: %s, %s", os.path.join(serverdir, '.vagrant'), e) - - if config['vm_provider'] == 'libvirt': - import libvirt - try: - conn = libvirt.open('qemu:///system') - try: - dom = conn.lookupByName(config['domain']) - try: - logger.debug('virsh -c qemu:///system destroy %s', config['domain']) - subprocess.check_call(['virsh', '-c', 'qemu:///system', 'destroy', config['domain']]) - logger.info("...waiting a sec...") - time.sleep(10) - except subprocess.CalledProcessError as e: - logger.info("could not force libvirt domain '%s' off: %s", config['domain'], e) - try: - # libvirt python bindings do not support all flags required - # for undefining domains correctly. - logger.debug('virsh -c qemu:///system undefine %s --nvram --managed-save --remove-all-storage --snapshots-metadata', config['domain']) - subprocess.check_call(('virsh', '-c', 'qemu:///system', 'undefine', config['domain'], '--nvram', '--managed-save', '--remove-all-storage', '--snapshots-metadata')) - logger.info("...waiting a sec...") - time.sleep(10) - except subprocess.CalledProcessError as e: - logger.info("could not undefine libvirt domain '%s': %s", dom.name(), e) - except libvirt.libvirtError as e: - logger.info("finding libvirt domain '%s' failed. (%s)", config['domain'], e) - except libvirt.libvirtError as e: - logger.critical('could not connect to libvirtd: %s', e) - sys.exit(1) - - def kvm_package(boxfile): ''' Hack to replace missing `vagrant package` for kvm, based on the script @@ -532,8 +484,9 @@ def main(): tail = fdroidserver.tail.Tail(logfilename) tail.start() + vm = fdroidserver.vmtools.get_build_vm(serverdir) if options.clean: - destroy_current_image(v, serverdir) + vm.destroy() # Check against the existing Vagrantfile.yaml, and if they differ, we # need to create a new box: @@ -546,7 +499,7 @@ def main(): oldconfig = yaml.load(f) if config != oldconfig: logger.info("Server configuration has changed, rebuild from scratch is required") - destroy_current_image(v, serverdir) + vm.destroy() else: logger.info("Re-provisioning existing server") writevf = False From a414aa00ff4fe97680036ac89e4f735cae217d96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Sun, 26 Mar 2017 01:51:28 +0100 Subject: [PATCH 30/54] use overhauled mv destroy code in build.py --- fdroidserver/build.py | 27 ++++++--------------------- makebuildserver | 1 - 2 files changed, 6 insertions(+), 22 deletions(-) diff --git a/fdroidserver/build.py b/fdroidserver/build.py index de8c3377..63923584 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -37,6 +37,7 @@ from . import common from . import net from . import metadata from . import scanner +from . import vmtools from .common import FDroidPopen, SdkToolsPopen from .exception import FDroidException, BuildException, VCSException @@ -289,9 +290,11 @@ def vm_get_clean_builder(reset=False): # If we can't use the existing machine for any reason, make a # new one from scratch. if not vm_ok: - vm_destroy_builder(provider) - - os.mkdir('builder') + if os.path.isdir('builder'): + vm = vmtools.get_build_vm('builder') + vm.destroy() + else: + os.mkdir('builder') p = subprocess.Popen(['vagrant', '--version'], universal_newlines=True, @@ -348,24 +351,6 @@ def vm_suspend_builder(): subprocess.call(['vagrant', 'suspend'], cwd='builder') -def vm_destroy_builder(provider): - """Savely destroy the builder vm. - - """ - logging.info("Removing broken/incomplete/unwanted build server") - if os.path.exists(os.path.join('builder', 'Vagrantfile')): - vagrant(['destroy', '-f'], cwd='builder') - if os.path.isdir('builder'): - shutil.rmtree('builder') - # get rid of vm and related disk images - FDroidPopen(('virsh', '-c', 'qemu:///system', 'destroy', 'builder_default')) - logging.info("...waiting a sec...") - time.sleep(10) - FDroidPopen(('virsh', '-c', 'qemu:///system', 'undefine', 'builder_default', '--nvram', '--managed-save', '--remove-all-storage', '--snapshots-metadata')) - logging.info("...waiting a sec...") - time.sleep(10) - - # Note that 'force' here also implies test mode. def build_server(app, build, vcs, build_dir, output_dir, log_dir, force): """Do a build on the builder vm. diff --git a/makebuildserver b/makebuildserver index f8b892e6..703cbdd6 100755 --- a/makebuildserver +++ b/makebuildserver @@ -14,7 +14,6 @@ import hashlib import yaml import math import json -import time import logging from clint.textui import progress from optparse import OptionParser From 4347c10d9d0e6d304bbcb862354aca3d586bf07e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Sun, 26 Mar 2017 03:15:33 +0200 Subject: [PATCH 31/54] use configured vm provider in when calling destroy in makebuildserver --- fdroidserver/vmtools.py | 2 +- makebuildserver | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/fdroidserver/vmtools.py b/fdroidserver/vmtools.py index a5c0f5ba..01e000dd 100644 --- a/fdroidserver/vmtools.py +++ b/fdroidserver/vmtools.py @@ -47,7 +47,7 @@ def get_build_vm(srvdir, provider=None): logger.debug('build vm provider \'virtualbox\' selected') return VirtualboxBuildVm(abssrvdir) else: - logger.warn('unsupported provider \'%s\' requested', provider) + logger.warn('build vm provider not supported: \'%s\'', provider) has_libvirt_machine = isdir(joinpath(abssrvdir, '.vagrant', 'machines', 'default', 'libvirt')) has_vbox_machine = isdir(joinpath(abssrvdir, '.vagrant', diff --git a/makebuildserver b/makebuildserver index 703cbdd6..c5522028 100755 --- a/makebuildserver +++ b/makebuildserver @@ -483,7 +483,7 @@ def main(): tail = fdroidserver.tail.Tail(logfilename) tail.start() - vm = fdroidserver.vmtools.get_build_vm(serverdir) + vm = fdroidserver.vmtools.get_build_vm(serverdir, provider=config['vm_provider']) if options.clean: vm.destroy() From d180aa26580b400540374d70382bc9370e02557e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Sun, 26 Mar 2017 03:34:29 +0200 Subject: [PATCH 32/54] fix virsh destroy parameters --- fdroidserver/vmtools.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fdroidserver/vmtools.py b/fdroidserver/vmtools.py index 01e000dd..05b176b9 100644 --- a/fdroidserver/vmtools.py +++ b/fdroidserver/vmtools.py @@ -144,8 +144,8 @@ class LibvirtBuildVm(FDroidBuildVm): # this is way more easy and therefore fault tolerant. # (eg. lookupByName only works on running VMs) try: - logger.debug('virsh -c qemu:///system destroy', self.srvname) - subprocess.check_call(('virsh', '-c', 'qemu:///system', 'destroy')) + logger.debug('virsh -c qemu:///system destroy %s', self.srvname) + subprocess.check_call(('virsh', '-c', 'qemu:///system', 'destroy', self.srvname)) logger.info("...waiting a sec...") time.sleep(10) except subprocess.CalledProcessError as e: From 5580a685dbef817920ea7e755531a01024a00d9f Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 22 May 2017 17:29:12 +0200 Subject: [PATCH 33/54] added makebuildserver option for keeping vagrant box This is very useful for debugging this process, and also for people who might want to keep a working copy of the box. --- makebuildserver | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/makebuildserver b/makebuildserver index c5522028..9b9ad5ad 100755 --- a/makebuildserver +++ b/makebuildserver @@ -30,6 +30,8 @@ parser.add_option("-c", "--clean", action="store_true", default=False, parser.add_option('--skip-cache-update', action="store_true", default=False, help="""Skip downloading and checking cache.""" """This assumes that the cache is already downloaded completely.""") +parser.add_option('--keep-box-file', action="store_true", default=False, + help="""Box file will not be deleted after adding it to box storage.""") options, args = parser.parse_args() logger = logging.getLogger('fdroidserver-makebuildserver') @@ -329,6 +331,14 @@ def kvm_package(boxfile): virConnect = libvirt.open('qemu:///system') storagePool = virConnect.storagePoolLookupByName('default') if storagePool: + + if os.path.isfile('metadata.json'): + os.remove('metadata.json') + if os.path.isfile('Vagrantfile'): + os.remove('Vagrantfile') + if os.path.isfile('box.img'): + os.remove('box.img') + vol = storagePool.storageVolLookupByName(config['domain'] + '.img') imagepath = vol.path() # TODO use a libvirt storage pool to ensure the img file is readable @@ -356,6 +366,7 @@ def kvm_package(boxfile): end end """ + with open('metadata.json', 'w') as fp: fp.write(json.dumps(metadata)) with open('Vagrantfile', 'w') as fp: @@ -364,9 +375,15 @@ end tar.add('metadata.json') tar.add('Vagrantfile') tar.add('box.img') - os.remove('metadata.json') - os.remove('Vagrantfile') - os.remove('box.img') + if not options.keep_box_file: + logger.debug('box packaging complete, removing temporary files.') + os.remove('metadata.json') + os.remove('Vagrantfile') + os.remove('box.img') + + else: + logger.warn('could not connect to storage-pool \'default\',' + + 'skipping packaging buildserver box') def run_via_vagrant_ssh(v, cmdlist): @@ -586,7 +603,11 @@ def main(): logger.info("Adding box") v.box_add('buildserver', boxfile, force=True) - os.remove(boxfile) + if not options.keep_box_file: + logger.debug('box added to vagrant, ' + + 'removing generated box file \'%s\'', + boxfile) + os.remove(boxfile) if __name__ == '__main__': From 01b64738239b16d2a4d99c4030cca3a9e775c54d Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 22 May 2017 17:12:34 +0200 Subject: [PATCH 34/54] refactored kvm_package to vmtools --- fdroidserver/vmtools.py | 70 +++++++++++++++++++++++++++++++++++ makebuildserver | 81 +++-------------------------------------- 2 files changed, 76 insertions(+), 75 deletions(-) diff --git a/fdroidserver/vmtools.py b/fdroidserver/vmtools.py index 05b176b9..c64e9994 100644 --- a/fdroidserver/vmtools.py +++ b/fdroidserver/vmtools.py @@ -16,7 +16,11 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +from os import remove as rmfile from os.path import isdir, isfile, join as joinpath, basename, abspath +import math +import json +import tarfile import time import shutil import vagrant @@ -125,6 +129,9 @@ class FDroidBuildVm(): except subprocess.CalledProcessError as e: logger.debug('pruning global vagrant status failed: %s', e) + def package(self, output=None, keep_box_file=None): + self.vgrnt.package(output=output) + class LibvirtBuildVm(FDroidBuildVm): def __init__(self, srvdir): @@ -160,6 +167,69 @@ class LibvirtBuildVm(FDroidBuildVm): except subprocess.CalledProcessError as e: logger.info("could not undefine libvirt domain '%s': %s", self.srvname, e) + def package(self, output=None, keep_box_file=False): + if not output: + output = "buildserver.box" + logger.debug('no output name set for packaging \'%s\',' + + 'defaulting to %s', self.srvname, output) + import libvirt + virConnect = libvirt.open('qemu:///system') + storagePool = virConnect.storagePoolLookupByName('default') + if storagePool: + + if isfile('metadata.json'): + rmfile('metadata.json') + if isfile('Vagrantfile'): + rmfile('Vagrantfile') + if isfile('box.img'): + rmfile('box.img') + + vol = storagePool.storageVolLookupByName(self.srvname + '.img') + imagepath = vol.path() + # TODO use a libvirt storage pool to ensure the img file is readable + subprocess.check_call(['sudo', '/bin/chmod', '-R', 'a+rX', '/var/lib/libvirt/images']) + shutil.copy2(imagepath, 'box.img') + subprocess.check_call(['qemu-img', 'rebase', '-p', '-b', '', 'box.img']) + img_info_raw = subprocess.check_output('sudo qemu-img info --output=json box.img', shell=True) + img_info = json.loads(img_info_raw.decode('utf-8')) + metadata = {"provider": "libvirt", + "format": img_info['format'], + "virtual_size": math.ceil(img_info['virtual-size'] / (1024. ** 3)) + 1, + } + + vagrantfile = """Vagrant.configure("2") do |config| + config.ssh.username = "vagrant" + config.ssh.password = "vagrant" + + config.vm.provider :libvirt do |libvirt| + + libvirt.driver = "kvm" + libvirt.host = "" + libvirt.connect_via_ssh = false + libvirt.storage_pool_name = "default" + + end +end +""" + with open('metadata.json', 'w') as fp: + fp.write(json.dumps(metadata)) + with open('Vagrantfile', 'w') as fp: + fp.write(vagrantfile) + with tarfile.open(output, 'w:gz') as tar: + tar.add('metadata.json') + tar.add('Vagrantfile') + tar.add('box.img') + + if not keep_box_file: + logger.debug('box packaging complete, removing temporary files.') + rmfile('metadata.json') + rmfile('Vagrantfile') + rmfile('box.img') + + else: + logger.warn('could not connect to storage-pool \'default\',' + + 'skipping packaging buildserver box') + class VirtualboxBuildVm(FDroidBuildVm): pass diff --git a/makebuildserver b/makebuildserver index 9b9ad5ad..469e8f1c 100755 --- a/makebuildserver +++ b/makebuildserver @@ -4,15 +4,12 @@ import os import pathlib import re import requests -import shutil import stat import sys import subprocess -import tarfile import vagrant import hashlib import yaml -import math import json import logging from clint.textui import progress @@ -322,70 +319,6 @@ def sha256_for_file(path): return s.hexdigest() -def kvm_package(boxfile): - ''' - Hack to replace missing `vagrant package` for kvm, based on the script - `tools/create_box.sh from vagrant-libvirt - ''' - import libvirt - virConnect = libvirt.open('qemu:///system') - storagePool = virConnect.storagePoolLookupByName('default') - if storagePool: - - if os.path.isfile('metadata.json'): - os.remove('metadata.json') - if os.path.isfile('Vagrantfile'): - os.remove('Vagrantfile') - if os.path.isfile('box.img'): - os.remove('box.img') - - vol = storagePool.storageVolLookupByName(config['domain'] + '.img') - imagepath = vol.path() - # TODO use a libvirt storage pool to ensure the img file is readable - subprocess.check_call(['sudo', '/bin/chmod', '-R', 'a+rX', '/var/lib/libvirt/images']) - shutil.copy2(imagepath, 'box.img') - subprocess.check_call(['qemu-img', 'rebase', '-p', '-b', '', 'box.img']) - img_info_raw = subprocess.check_output('sudo qemu-img info --output=json box.img', shell=True) - img_info = json.loads(img_info_raw.decode('utf-8')) - metadata = {"provider": "libvirt", - "format": img_info['format'], - "virtual_size": math.ceil(img_info['virtual-size'] / 1024. ** 3), - } - - vagrantfile = """Vagrant.configure("2") do |config| - config.ssh.username = "vagrant" - config.ssh.password = "vagrant" - - config.vm.provider :libvirt do |libvirt| - - libvirt.driver = "kvm" - libvirt.host = "" - libvirt.connect_via_ssh = false - libvirt.storage_pool_name = "default" - - end -end -""" - - with open('metadata.json', 'w') as fp: - fp.write(json.dumps(metadata)) - with open('Vagrantfile', 'w') as fp: - fp.write(vagrantfile) - with tarfile.open(boxfile, 'w:gz') as tar: - tar.add('metadata.json') - tar.add('Vagrantfile') - tar.add('box.img') - if not options.keep_box_file: - logger.debug('box packaging complete, removing temporary files.') - os.remove('metadata.json') - os.remove('Vagrantfile') - os.remove('box.img') - - else: - logger.warn('could not connect to storage-pool \'default\',' + - 'skipping packaging buildserver box') - - def run_via_vagrant_ssh(v, cmdlist): if (isinstance(cmdlist, str) or isinstance(cmdlist, bytes)): cmd = cmdlist @@ -551,10 +484,7 @@ def main(): logger.info("Configuring build server VM") debug_log_vagrant_vm(serverdir, config['domain']) try: - try: - v.up(provision=True) - except subprocess.CalledProcessError as e: - v.up(provision=True) + v.up(provision=True) except subprocess.CalledProcessError as e: debug_log_vagrant_vm(serverdir, config['domain']) logger.critical('could not bring buildserver vm up. %s', e) @@ -595,14 +525,15 @@ def main(): if os.path.exists(boxfile): os.remove(boxfile) - if config['vm_provider'] == 'libvirt': - kvm_package(boxfile) - else: - v.package(output=boxfile) + vm.package(output=boxfile, debug_keep_box_file=options.debug_keep_box_file) logger.info("Adding box") v.box_add('buildserver', boxfile, force=True) + if not 'buildserver' in subprocess.check_output(['vagrant', 'box', 'list']).decode('utf-8'): + logger.critical('could not add box \'%s\' as \'buildserver\', terminating', boxfile) + sys.exit(1) + if not options.keep_box_file: logger.debug('box added to vagrant, ' + 'removing generated box file \'%s\'', From 5dbcd0e9bd3768106fbab695ffe1ede717441106 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 22 May 2017 17:14:48 +0200 Subject: [PATCH 35/54] added box handling to vmtools --- fdroidserver/vmtools.py | 61 +++++++++++++++++++++++++++-------------- makebuildserver | 22 +++++++++++++-- 2 files changed, 60 insertions(+), 23 deletions(-) diff --git a/fdroidserver/vmtools.py b/fdroidserver/vmtools.py index c64e9994..5eedbaa7 100644 --- a/fdroidserver/vmtools.py +++ b/fdroidserver/vmtools.py @@ -105,6 +105,9 @@ class FDroidBuildVm(): except subprocess.CalledProcessError as e: logger.info('could not bring vm up: %s', e) + def halt(self): + self.vgrnt.halt(force=True) + def destroy(self): """Remove every trace of this VM from the system. @@ -125,13 +128,37 @@ class FDroidBuildVm(): except Exception as e: logger.debug("could not delete vagrant dir: %s, %s", vgrntdir, e) try: - subprocess.check_call(['vagrant', 'global-status', '--prune']) + self._check_call(['vagrant', 'global-status', '--prune']) except subprocess.CalledProcessError as e: logger.debug('pruning global vagrant status failed: %s', e) def package(self, output=None, keep_box_file=None): self.vgrnt.package(output=output) + def box_add(self, boxname, boxfile, force=True): + """Add vagrant box to vagrant. + + :param boxname: name assigned to local deployment of box + :param boxfile: path to box file + :param force: overwrite existing box image (default: True) + """ + boxfile = abspath(boxfile) + if not isfile(boxfile): + raise FDroidBuildVmException('supplied boxfile \'%s\' does not exist', boxfile) + self.vgrnt.box_add(boxname, abspath(boxfile), force=force) + + def box_remove(self, boxname): + try: + self._check_call(['vagrant', 'box', 'remove', '--all', '--force', boxname]) + except subprocess.CalledProcessError as e: + logger.debug('tried removing box %s, but is did not exist: %s', boxname, e) + # TODO: remove box files manually + # nesessary when Vagrantfile in ~/.vagrant.d/... is broken. + + def _check_call(self, cmd): + logger.debug(' '.join(cmd)) + return subprocess.check_call(cmd) + class LibvirtBuildVm(FDroidBuildVm): def __init__(self, srvdir): @@ -141,7 +168,7 @@ class LibvirtBuildVm(FDroidBuildVm): try: self.conn = libvirt.open('qemu:///system') except libvirt.libvirtError as e: - logger.critical('could not connect to libvirtd: %s', e) + raise FDroidBuildVmException('could not connect to libvirtd: %s' % (e)) def destroy(self): @@ -151,8 +178,7 @@ class LibvirtBuildVm(FDroidBuildVm): # this is way more easy and therefore fault tolerant. # (eg. lookupByName only works on running VMs) try: - logger.debug('virsh -c qemu:///system destroy %s', self.srvname) - subprocess.check_call(('virsh', '-c', 'qemu:///system', 'destroy', self.srvname)) + self._check_call(('virsh', '-c', 'qemu:///system', 'destroy', self.srvname)) logger.info("...waiting a sec...") time.sleep(10) except subprocess.CalledProcessError as e: @@ -160,14 +186,13 @@ class LibvirtBuildVm(FDroidBuildVm): try: # libvirt python bindings do not support all flags required # for undefining domains correctly. - logger.debug('virsh -c qemu:///system undefine %s --nvram --managed-save --remove-all-storage --snapshots-metadata', self.srvname) - subprocess.check_call(('virsh', '-c', 'qemu:///system', 'undefine', self.srvname, '--nvram', '--managed-save', '--remove-all-storage', '--snapshots-metadata')) + self._check_call(('virsh', '-c', 'qemu:///system', 'undefine', self.srvname, '--nvram', '--managed-save', '--remove-all-storage', '--snapshots-metadata')) logger.info("...waiting a sec...") time.sleep(10) except subprocess.CalledProcessError as e: logger.info("could not undefine libvirt domain '%s': %s", self.srvname, e) - def package(self, output=None, keep_box_file=False): + def package(self, output=None, vagrantfile=None, keep_box_file=False): if not output: output = "buildserver.box" logger.debug('no output name set for packaging \'%s\',' + @@ -197,20 +222,9 @@ class LibvirtBuildVm(FDroidBuildVm): "virtual_size": math.ceil(img_info['virtual-size'] / (1024. ** 3)) + 1, } - vagrantfile = """Vagrant.configure("2") do |config| - config.ssh.username = "vagrant" - config.ssh.password = "vagrant" + if not vagrantfile: + vagrantfile = 'Vagrant.configure("2") do |config|\nend' - config.vm.provider :libvirt do |libvirt| - - libvirt.driver = "kvm" - libvirt.host = "" - libvirt.connect_via_ssh = false - libvirt.storage_pool_name = "default" - - end -end -""" with open('metadata.json', 'w') as fp: fp.write(json.dumps(metadata)) with open('Vagrantfile', 'w') as fp: @@ -230,6 +244,13 @@ end logger.warn('could not connect to storage-pool \'default\',' + 'skipping packaging buildserver box') + def box_remove(self, boxname): + super().box_remove(boxname) + try: + self._check_call(['virsh', '-c', 'qemu:///system', 'vol-delete', '--pool', 'default', '%s_vagrant_box_image_0.img' % (boxname)]) + except subprocess.CalledProcessError as e: + logger.info('tired removing \'%s\', file was not present in first place: %s', boxname, e) + class VirtualboxBuildVm(FDroidBuildVm): pass diff --git a/makebuildserver b/makebuildserver index 469e8f1c..c88d665a 100755 --- a/makebuildserver +++ b/makebuildserver @@ -12,6 +12,7 @@ import hashlib import yaml import json import logging +import textwrap from clint.textui import progress from optparse import OptionParser import fdroidserver.tail @@ -525,12 +526,27 @@ def main(): if os.path.exists(boxfile): os.remove(boxfile) - vm.package(output=boxfile, debug_keep_box_file=options.debug_keep_box_file) + vagrantfile = textwrap.dedent("""\ + Vagrant.configure("2") do |config| + config.ssh.username = "vagrant" + config.ssh.password = "vagrant" + + config.vm.provider :libvirt do |libvirt| + + libvirt.driver = "kvm" + libvirt.host = "" + libvirt.connect_via_ssh = false + libvirt.storage_pool_name = "default" + + end + end""") + + vm.package(output=boxfile, vagrantfile=vagrantfile, keep_box_file=options.keep_box_file) logger.info("Adding box") - v.box_add('buildserver', boxfile, force=True) + vm.box_add('buildserver', boxfile, force=True) - if not 'buildserver' in subprocess.check_output(['vagrant', 'box', 'list']).decode('utf-8'): + if 'buildserver' not in subprocess.check_output(['vagrant', 'box', 'list']).decode('utf-8'): logger.critical('could not add box \'%s\' as \'buildserver\', terminating', boxfile) sys.exit(1) From 1bd51966b81d83a9c8adbd8ad3062a4caffc8ea7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Wed, 29 Mar 2017 17:36:04 +0200 Subject: [PATCH 36/54] vmtools debug logging for check_output calls --- fdroidserver/vmtools.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/fdroidserver/vmtools.py b/fdroidserver/vmtools.py index 5eedbaa7..cb7e2833 100644 --- a/fdroidserver/vmtools.py +++ b/fdroidserver/vmtools.py @@ -155,9 +155,13 @@ class FDroidBuildVm(): # TODO: remove box files manually # nesessary when Vagrantfile in ~/.vagrant.d/... is broken. - def _check_call(self, cmd): + def _check_call(self, cmd, shell=False): logger.debug(' '.join(cmd)) - return subprocess.check_call(cmd) + return subprocess.check_call(cmd, shell=shell) + + def _check_output(self, cmd, shell=False): + logger.debug(' '.join(cmd)) + return subprocess.check_output(cmd, shell=shell) class LibvirtBuildVm(FDroidBuildVm): @@ -212,10 +216,10 @@ class LibvirtBuildVm(FDroidBuildVm): vol = storagePool.storageVolLookupByName(self.srvname + '.img') imagepath = vol.path() # TODO use a libvirt storage pool to ensure the img file is readable - subprocess.check_call(['sudo', '/bin/chmod', '-R', 'a+rX', '/var/lib/libvirt/images']) + self._check_call(['sudo', '/bin/chmod', '-R', 'a+rX', '/var/lib/libvirt/images']) shutil.copy2(imagepath, 'box.img') - subprocess.check_call(['qemu-img', 'rebase', '-p', '-b', '', 'box.img']) - img_info_raw = subprocess.check_output('sudo qemu-img info --output=json box.img', shell=True) + self._check_call(['qemu-img', 'rebase', '-p', '-b', '', 'box.img']) + img_info_raw = self._check_output(['sudo qemu-img info --output=json box.img'], shell=True) img_info = json.loads(img_info_raw.decode('utf-8')) metadata = {"provider": "libvirt", "format": img_info['format'], From bba6b8ab0af8b7251280ffdb3d5877ecf9b4d604 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 22 May 2017 17:17:01 +0200 Subject: [PATCH 37/54] fixed reading libvirt box image size --- fdroidserver/vmtools.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/fdroidserver/vmtools.py b/fdroidserver/vmtools.py index cb7e2833..1a72b495 100644 --- a/fdroidserver/vmtools.py +++ b/fdroidserver/vmtools.py @@ -213,13 +213,14 @@ class LibvirtBuildVm(FDroidBuildVm): if isfile('box.img'): rmfile('box.img') + logger.debug('preparing box.img for box %s', output) vol = storagePool.storageVolLookupByName(self.srvname + '.img') imagepath = vol.path() # TODO use a libvirt storage pool to ensure the img file is readable self._check_call(['sudo', '/bin/chmod', '-R', 'a+rX', '/var/lib/libvirt/images']) shutil.copy2(imagepath, 'box.img') self._check_call(['qemu-img', 'rebase', '-p', '-b', '', 'box.img']) - img_info_raw = self._check_output(['sudo qemu-img info --output=json box.img'], shell=True) + img_info_raw = self._check_output(['qemu-img', 'info', '--output=json', 'box.img']) img_info = json.loads(img_info_raw.decode('utf-8')) metadata = {"provider": "libvirt", "format": img_info['format'], @@ -227,15 +228,21 @@ class LibvirtBuildVm(FDroidBuildVm): } if not vagrantfile: + logger.debug('no Vagrantfile supplied for box, generating a minimal one...') vagrantfile = 'Vagrant.configure("2") do |config|\nend' + logger.debug('preparing metadata.json for box %s', output) with open('metadata.json', 'w') as fp: fp.write(json.dumps(metadata)) + logger.debug('preparing Vagrantfile for box %s', output) with open('Vagrantfile', 'w') as fp: fp.write(vagrantfile) with tarfile.open(output, 'w:gz') as tar: + logger.debug('adding metadata.json to box %s ...', output) tar.add('metadata.json') + logger.debug('adding Vagrantfile to box %s ...', output) tar.add('Vagrantfile') + logger.debug('adding box.img to box %s ...', output) tar.add('box.img') if not keep_box_file: From 8abd3f1cbcba361e2bf59f860b05280fa3aedf99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Sat, 1 Apr 2017 12:27:26 +0200 Subject: [PATCH 38/54] auto-lookup vm provider based on available executables; more fault tolerant vagrant package --- fdroidserver/vmtools.py | 70 ++++++++++++++++++++++++++++++----------- 1 file changed, 52 insertions(+), 18 deletions(-) diff --git a/fdroidserver/vmtools.py b/fdroidserver/vmtools.py index 1a72b495..ba8008c4 100644 --- a/fdroidserver/vmtools.py +++ b/fdroidserver/vmtools.py @@ -31,6 +31,16 @@ from logging import getLogger logger = getLogger('fdroidserver-vmtools') +def _check_call(cmd, shell=False): + logger.debug(' '.join(cmd)) + return subprocess.check_call(cmd, shell=shell) + + +def _check_output(cmd, shell=False): + logger.debug(' '.join(cmd)) + return subprocess.check_output(cmd, shell=shell) + + def get_build_vm(srvdir, provider=None): """Factory function for getting FDroidBuildVm instances. @@ -43,6 +53,8 @@ def get_build_vm(srvdir, provider=None): :returns: FDroidBuildVm instance. """ abssrvdir = abspath(srvdir) + + # use supplied provider if provider: if provider == 'libvirt': logger.debug('build vm provider \'libvirt\' selected') @@ -52,6 +64,32 @@ def get_build_vm(srvdir, provider=None): return VirtualboxBuildVm(abssrvdir) else: logger.warn('build vm provider not supported: \'%s\'', provider) + + # try guessing provider from installed software + try: + kvm_installed = 0 == _check_call(['which', 'kvm']) + except subprocess.CalledProcessError: + kvm_installed = False + try: + kvm_installed |= 0 == _check_call(['which', 'qemu']) + except subprocess.CalledProcessError: + pass + try: + vbox_installed = 0 == _check_call(['which', 'VBoxHeadless'], shell=True) + except subprocess.CalledProcessError: + vbox_installed = False + if kvm_installed and vbox_installed: + logger.debug('both kvm and vbox are installed.') + elif kvm_installed: + logger.debug('libvirt is the sole installed and supported vagrant provider, selecting \'libvirt\'') + return LibvirtBuildVm(abssrvdir) + elif vbox_installed: + logger.debug('virtualbox is the sole installed and supported vagrant provider, selecting \'virtualbox\'') + return VirtualboxBuildVm(abssrvdir) + else: + logger.debug('could not confirm that either virtualbox or kvm/libvirt are installed') + + # try guessing provider from .../srvdir/.vagrant internals has_libvirt_machine = isdir(joinpath(abssrvdir, '.vagrant', 'machines', 'default', 'libvirt')) has_vbox_machine = isdir(joinpath(abssrvdir, '.vagrant', @@ -128,12 +166,16 @@ class FDroidBuildVm(): except Exception as e: logger.debug("could not delete vagrant dir: %s, %s", vgrntdir, e) try: - self._check_call(['vagrant', 'global-status', '--prune']) + _check_call(['vagrant', 'global-status', '--prune']) except subprocess.CalledProcessError as e: logger.debug('pruning global vagrant status failed: %s', e) - def package(self, output=None, keep_box_file=None): - self.vgrnt.package(output=output) + def package(self, output=None, vagrantfile=None, keep_box_file=None): + previous_tmp_dir = joinpath(self.srvdir, '_tmp_package') + if isdir(previous_tmp_dir): + logger.info('found previous vagrant package temp dir \'%s\', deleting it', previous_tmp_dir) + shutil.rmtree(previous_tmp_dir) + self.vgrnt.package(output=output, vagrantfile=vagrantfile) def box_add(self, boxname, boxfile, force=True): """Add vagrant box to vagrant. @@ -149,20 +191,12 @@ class FDroidBuildVm(): def box_remove(self, boxname): try: - self._check_call(['vagrant', 'box', 'remove', '--all', '--force', boxname]) + _check_call(['vagrant', 'box', 'remove', '--all', '--force', boxname]) except subprocess.CalledProcessError as e: logger.debug('tried removing box %s, but is did not exist: %s', boxname, e) # TODO: remove box files manually # nesessary when Vagrantfile in ~/.vagrant.d/... is broken. - def _check_call(self, cmd, shell=False): - logger.debug(' '.join(cmd)) - return subprocess.check_call(cmd, shell=shell) - - def _check_output(self, cmd, shell=False): - logger.debug(' '.join(cmd)) - return subprocess.check_output(cmd, shell=shell) - class LibvirtBuildVm(FDroidBuildVm): def __init__(self, srvdir): @@ -182,7 +216,7 @@ class LibvirtBuildVm(FDroidBuildVm): # this is way more easy and therefore fault tolerant. # (eg. lookupByName only works on running VMs) try: - self._check_call(('virsh', '-c', 'qemu:///system', 'destroy', self.srvname)) + _check_call(('virsh', '-c', 'qemu:///system', 'destroy', self.srvname)) logger.info("...waiting a sec...") time.sleep(10) except subprocess.CalledProcessError as e: @@ -190,7 +224,7 @@ class LibvirtBuildVm(FDroidBuildVm): try: # libvirt python bindings do not support all flags required # for undefining domains correctly. - self._check_call(('virsh', '-c', 'qemu:///system', 'undefine', self.srvname, '--nvram', '--managed-save', '--remove-all-storage', '--snapshots-metadata')) + _check_call(('virsh', '-c', 'qemu:///system', 'undefine', self.srvname, '--nvram', '--managed-save', '--remove-all-storage', '--snapshots-metadata')) logger.info("...waiting a sec...") time.sleep(10) except subprocess.CalledProcessError as e: @@ -217,10 +251,10 @@ class LibvirtBuildVm(FDroidBuildVm): vol = storagePool.storageVolLookupByName(self.srvname + '.img') imagepath = vol.path() # TODO use a libvirt storage pool to ensure the img file is readable - self._check_call(['sudo', '/bin/chmod', '-R', 'a+rX', '/var/lib/libvirt/images']) + _check_call(['sudo', '/bin/chmod', '-R', 'a+rX', '/var/lib/libvirt/images']) shutil.copy2(imagepath, 'box.img') - self._check_call(['qemu-img', 'rebase', '-p', '-b', '', 'box.img']) - img_info_raw = self._check_output(['qemu-img', 'info', '--output=json', 'box.img']) + _check_call(['qemu-img', 'rebase', '-p', '-b', '', 'box.img']) + img_info_raw = _check_output(['qemu-img', 'info', '--output=json', 'box.img']) img_info = json.loads(img_info_raw.decode('utf-8')) metadata = {"provider": "libvirt", "format": img_info['format'], @@ -258,7 +292,7 @@ class LibvirtBuildVm(FDroidBuildVm): def box_remove(self, boxname): super().box_remove(boxname) try: - self._check_call(['virsh', '-c', 'qemu:///system', 'vol-delete', '--pool', 'default', '%s_vagrant_box_image_0.img' % (boxname)]) + _check_call(['virsh', '-c', 'qemu:///system', 'vol-delete', '--pool', 'default', '%s_vagrant_box_image_0.img' % (boxname)]) except subprocess.CalledProcessError as e: logger.info('tired removing \'%s\', file was not present in first place: %s', boxname, e) From 3187d2cbcff8289cd8a2a4a1fcaa4b34debdd48d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Thu, 6 Apr 2017 11:42:35 +0200 Subject: [PATCH 39/54] revised build server creation --- fdroidserver/build.py | 43 ++++++++++++++++- fdroidserver/vmtools.py | 102 +++++++++++++++++++++++++++++++++++----- 2 files changed, 133 insertions(+), 12 deletions(-) diff --git a/fdroidserver/build.py b/fdroidserver/build.py index 63923584..4e5be385 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -29,6 +29,7 @@ import time import json import requests import tempfile +import textwrap from configparser import ConfigParser from argparse import ArgumentParser import logging @@ -223,6 +224,45 @@ def vm_test_ssh_into_builder(): sshs.close() +def vm_new_get_clean_builder(serverdir, reset=False): + if not os.path.isdir(serverdir): + logging.info("buildserver path does not exists, creating %s", serverdir) + os.makedirs(serverdir) + vagrantfile = os.path.join(serverdir, 'Vagrantfile') + if not os.path.isfile(vagrantfile): + with open(os.path.join('builder', 'Vagrantfile'), 'w') as f: + f.write(textwrap.dedent("""\ + # generated file, do not change. + + Vagrant.configure("2") do |config| + config.vm.box = "buildserver" + config.vm.synced_folder ".", "/vagrant", disabled: true + end + """)) + vm = vmtools.get_build_vm(serverdir) + if reset: + logging.info('resetting buildserver by request') + elif not vm.check_okay(): + logging.info('resetting buildserver because it appears to be absent or broken') + reset = True + elif not vm.snapshot_exists('fdroidclean'): + logging.info("resetting buildserver, because snapshot 'fdroidclean' is not present") + reset = True + + if reset: + vm.destroy() + vm.up() + vm.suspend() + + if reset: + vm.snapshot_create('fdroidclean') + else: + vm.snapshot_revert('droidclean') + vm.resume() + + return get_vagrant_sshinfo() + + def vm_get_clean_builder(reset=False): """Get a clean VM ready to do a buildserver build. @@ -374,7 +414,8 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force): else: logging.getLogger("paramiko").setLevel(logging.WARN) - sshinfo = vm_get_clean_builder() + # sshinfo = vm_get_clean_builder() + sshinfo = vm_new_get_clean_builder('builder') try: if not buildserverid: diff --git a/fdroidserver/vmtools.py b/fdroidserver/vmtools.py index ba8008c4..c209914c 100644 --- a/fdroidserver/vmtools.py +++ b/fdroidserver/vmtools.py @@ -17,13 +17,12 @@ # along with this program. If not, see . from os import remove as rmfile -from os.path import isdir, isfile, join as joinpath, basename, abspath +from os.path import isdir, isfile, join as joinpath, basename, abspath, expanduser import math import json import tarfile import time import shutil -import vagrant import subprocess from .common import FDroidException from logging import getLogger @@ -132,10 +131,11 @@ class FDroidBuildVm(): raise FDroidBuildVmException("Can not init vagrant, directory %s not present" % (srvdir)) if not isfile(self.vgrntfile): raise FDroidBuildVmException("Can not init vagrant, '%s' not present" % (self.vgrntfile)) + import vagrant self.vgrnt = vagrant.Vagrant(root=srvdir, out_cm=vagrant.stdout_cm, err_cm=vagrant.stdout_cm) - def isUpAndRunning(self): - raise NotImplementedError('TODO implement this') + def check_okay(self): + return True def up(self, provision=True): try: @@ -143,6 +143,15 @@ class FDroidBuildVm(): except subprocess.CalledProcessError as e: logger.info('could not bring vm up: %s', e) + def snapshot_create(self, name): + raise NotImplementedError('not implemented, please use a sub-type instance') + + def suspend(self): + self.vgrnt.suspend() + + def resume(self): + self.vgrnt.resume() + def halt(self): self.vgrnt.halt(force=True) @@ -177,6 +186,9 @@ class FDroidBuildVm(): shutil.rmtree(previous_tmp_dir) self.vgrnt.package(output=output, vagrantfile=vagrantfile) + def _vagrant_file_name(self, name): + return name.replace('/', '-VAGRANTSLASH-') + def box_add(self, boxname, boxfile, force=True): """Add vagrant box to vagrant. @@ -194,8 +206,12 @@ class FDroidBuildVm(): _check_call(['vagrant', 'box', 'remove', '--all', '--force', boxname]) except subprocess.CalledProcessError as e: logger.debug('tried removing box %s, but is did not exist: %s', boxname, e) - # TODO: remove box files manually - # nesessary when Vagrantfile in ~/.vagrant.d/... is broken. + boxpath = joinpath(expanduser('~'), '.vagrant', + self._vagrant_file_name(boxname)) + if isdir(boxpath): + logger.info("attempting to remove box '%s' by deleting: %s", + boxname, boxpath) + shutil.rmtree(boxpath) class LibvirtBuildVm(FDroidBuildVm): @@ -208,6 +224,22 @@ class LibvirtBuildVm(FDroidBuildVm): except libvirt.libvirtError as e: raise FDroidBuildVmException('could not connect to libvirtd: %s' % (e)) + def check_okay(self): + import libvirt + imagepath = joinpath('var', 'lib', 'libvirt', 'images', + '%s.img' % self._vagrant_file_name(self.srvname)) + image_present = False + if isfile(imagepath): + image_present = True + try: + self.conn.lookupByName(self.srvname) + domain_defined = True + except libvirt.libvirtError: + pass + if image_present and domain_defined: + return True + return False + def destroy(self): super().destroy() @@ -235,9 +267,7 @@ class LibvirtBuildVm(FDroidBuildVm): output = "buildserver.box" logger.debug('no output name set for packaging \'%s\',' + 'defaulting to %s', self.srvname, output) - import libvirt - virConnect = libvirt.open('qemu:///system') - storagePool = virConnect.storagePoolLookupByName('default') + storagePool = self.conn.storagePoolLookupByName('default') if storagePool: if isfile('metadata.json'): @@ -258,7 +288,7 @@ class LibvirtBuildVm(FDroidBuildVm): img_info = json.loads(img_info_raw.decode('utf-8')) metadata = {"provider": "libvirt", "format": img_info['format'], - "virtual_size": math.ceil(img_info['virtual-size'] / (1024. ** 3)) + 1, + "virtual_size": math.ceil(img_info['virtual-size'] / (1024. ** 3)), } if not vagrantfile: @@ -296,6 +326,56 @@ class LibvirtBuildVm(FDroidBuildVm): except subprocess.CalledProcessError as e: logger.info('tired removing \'%s\', file was not present in first place: %s', boxname, e) + def snapshot_create(self, snapshot_name): + try: + _check_call(['virsh', '-c', 'qemu:///system', 'snapshot-create-as', self.srvname, snapshot_name]) + logger.info('...waiting a sec...') + time.sleep(10) + except subprocess.CalledProcessError as e: + raise FDroidBuildVmException("could not cerate snapshot '%s' " + "of libvirt vm '%s'" + % (snapshot_name, self.srvname)) from e + + def snapshot_list(self): + import libvirt + try: + dom = self.conn.lookupByName(self.srvname) + return dom.listAllSnapshots() + except libvirt.libvirtError as e: + raise FDroidBuildVmException('could not list snapshots for domain \'%s\'' % self.srvname) from e + + def snapshot_exists(self, snapshot_name): + import libvirt + try: + dom = self.conn.lookupByName(self.srvname) + return dom.snapshotLookupByName(snapshot_name) is not None + except libvirt.libvirtError: + return False + + def snapshot_revert(self, snapshot_name): + import libvirt + try: + dom = self.conn.lookupByName(self.srvname) + snap = dom.snapshotLookupByName(snapshot_name) + dom.revertToSnapshot(snap) + logger.info('...waiting a sec...') + time.sleep(10) + except libvirt.libvirtError as e: + raise FDroidBuildVmException('could not revert domain \'%s\' to snapshot \'%s\'' + % (self.srvname, snapshot_name)) from e + class VirtualboxBuildVm(FDroidBuildVm): - pass + def snapshot_create(self, snapshot_name): + raise NotImplemented('TODO') + try: + _check_call(['VBoxManage', 'snapshot', self.srvname, 'take', 'fdroidclean'], cwd=self.srvdir) + logger.info('...waiting a sec...') + time.sleep(10) + except subprocess.CalledProcessError as e: + raise FDroidBuildVmException('could not cerate snapshot ' + 'of virtualbox vm %s' + % self.srvname) from e + + def snapshot_available(self, snapshot_name): + raise NotImplemented('TODO') From c749c6848693849442877b5a6d994d00465fd221 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 22 May 2017 17:53:47 +0200 Subject: [PATCH 40/54] test script for vmtools --- tests/extra/manual-vmtools-test.py | 140 +++++++++++++++++++++++++++++ 1 file changed, 140 insertions(+) create mode 100755 tests/extra/manual-vmtools-test.py diff --git a/tests/extra/manual-vmtools-test.py b/tests/extra/manual-vmtools-test.py new file mode 100755 index 00000000..6b6c6964 --- /dev/null +++ b/tests/extra/manual-vmtools-test.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python3 + +import os +import sys +import logging +import textwrap +import tempfile +import inspect +from argparse import ArgumentParser + +localmodule = os.path.realpath( + os.path.join(os.path.dirname(inspect.getfile(inspect.currentframe())), '..', '..')) +print('localmodule: ' + localmodule) +if localmodule not in sys.path: + sys.path.insert(0, localmodule) + +from fdroidserver.vmtools import get_build_vm + + +def main(args): + + if args.provider != None: + if args.provider not in ('libvirt', 'virtualbox'): + logging.critical('provider: %s not supported.', args.provider) + sys.exit(1) + + with tempfile.TemporaryDirectory() as tmpdir: + + # define a simple vagrant vm 'x' + x_dir = os.path.join(tmpdir, 'x') + os.makedirs(x_dir) + with open(os.path.join(x_dir, 'Vagrantfile'), 'w') as f: + f.write(textwrap.dedent("""\ + Vagrant.configure("2") do |config| + config.vm.box = "debian/jessie64" + config.vm.synced_folder ".", "/vagrant", disabled: true + config.ssh.insert_key = false + end + """)) + # define another simple vagrant vm 'y' which uses 'x' as a base box + y_dir = os.path.join(tmpdir, 'y') + os.makedirs(y_dir) + with open(os.path.join(y_dir, 'Vagrantfile'), 'w') as f: + f.write(textwrap.dedent("""\ + Vagrant.configure("2") do |config| + config.vm.box = "x" + config.vm.synced_folder ".", "/vagrant", disabled: true + end + """)) + + # vagrant file for packaging 'x' box + vgrntf=textwrap.dedent("""\ + Vagrant.configure("2") do |config| + + config.vm.synced_folder ".", "/vagrant", type: "nfs", nfs_version: "4", nfs_udp: false + + config.vm.provider :libvirt do |libvirt| + libvirt.driver = "kvm" + libvirt.connect_via_ssh = false + libvirt.username = "root" + libvirt.storage_pool_name = "default" + end + end + """) + + # create a box: x + if not args.skip_create_x: + x = get_build_vm(x_dir, provider=args.provider) + x.destroy() + x.up(provision=True) + x.halt() + x.package(output='x.box', vagrantfile=vgrntf, keep_box_file=False) + x.box_remove('x') + x.box_add('x', 'x.box') + + # use previously created box to spin up a new vm + if not args.skip_create_y: + y = get_build_vm(y_dir, provider=args.provider) + y.destroy() + y.up() + + # create and restore a snapshot + if not args.skip_snapshot_y: + y = get_build_vm(y_dir, provider=args.provider) + + if y.snapshot_exists('clean'): + y.destroy() + y.up() + + y.suspend() + y.snapshot_create('clean') + y.up() + + logging.info('snapshot \'clean\' exsists: %r', y.snapshot_exists('clean')) + + # test if snapshot exists + se = y.snapshot_exists('clean') + logging.info('snapshot \'clean\' available: %r', se) + + # revert snapshot + y.suspend() + logging.info('asdf %s', y.snapshot_revert('clean')) + y.resume() + + # cleanup + if not args.skip_clean: + x = get_build_vm(x_dir, provider=args.provider) + y = get_build_vm(y_dir, provider=args.provider) + y.destroy() + x.destroy() + x.box_remove('x') + +if __name__ == '__main__': + logging.basicConfig(format='%(message)s', level=logging.DEBUG) + + parser = ArgumentParser(description="""\ +This is intended for manually testing vmtools.py + +NOTE: Should this test-run fail it might leave traces of vagrant VMs or boxes + on your system. Those vagrant VMs are named 'x' and 'y'. + """) + parser.add_argument('--provider', help="Force this script use supplied " + "provider instead using our auto provider lookup. " + "Supported values: 'libvirt', 'virtualbox'") + parser.add_argument('--skip-create-x', action="store_true", default=False, + help="Skips: Creating 'x' vm, packaging it into a " + "a box and adding it to vagrant.") + parser.add_argument('--skip-create-y', action="store_true", default=False, + help="Skips: Creating 'y' vm. Depends on having " + "box 'x' added to vagrant.") + parser.add_argument('--skip-snapshot-y', action="store_true", default=False, + help="Skips: Taking a snapshot and restoring a " + "a snapshot of 'y' vm. Requires 'y' mv to be " + "present.") + parser.add_argument('--skip-clean', action="store_true", default=False, + help="Skips: Cleaning up mv images and vagrant " + "metadata on the system.") + args = parser.parse_args() + + main(args) From acf25a399983a578dc45bb2c56b47a902af8ed92 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 22 May 2017 17:53:12 +0200 Subject: [PATCH 41/54] build server use up instead or resume; logging --- fdroidserver/build.py | 4 +++- fdroidserver/vmtools.py | 18 +++++++++++++----- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/fdroidserver/build.py b/fdroidserver/build.py index 4e5be385..24a60344 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -255,10 +255,12 @@ def vm_new_get_clean_builder(serverdir, reset=False): vm.suspend() if reset: + logging.info('buildserver recreated: taking a clean snapshot') vm.snapshot_create('fdroidclean') else: + logging.info('builserver ok: reverting to clean snapshot') vm.snapshot_revert('droidclean') - vm.resume() + vm.up() return get_vagrant_sshinfo() diff --git a/fdroidserver/vmtools.py b/fdroidserver/vmtools.py index c209914c..71e439fd 100644 --- a/fdroidserver/vmtools.py +++ b/fdroidserver/vmtools.py @@ -140,17 +140,22 @@ class FDroidBuildVm(): def up(self, provision=True): try: self.vgrnt.up(provision=provision) + logger.info('...waiting a sec...') + time.sleep(10) except subprocess.CalledProcessError as e: - logger.info('could not bring vm up: %s', e) + raise FDroidBuildVmException("could not bring up vm '%s'" % self.srvname) from e def snapshot_create(self, name): raise NotImplementedError('not implemented, please use a sub-type instance') def suspend(self): - self.vgrnt.suspend() - - def resume(self): - self.vgrnt.resume() + logger.info('suspending buildserver') + try: + self.vgrnt.suspend() + logger.info('...waiting a sec...') + time.sleep(10) + except subprocess.CalledProcessError as e: + raise FDroidBuildVmException("could not suspend vm '%s'" % self.srvname) from e def halt(self): self.vgrnt.halt(force=True) @@ -163,6 +168,7 @@ class FDroidBuildVm(): * vagrant state informations (eg. `.vagrant` folder) * images related to this vm """ + logger.info("destroying vm '%s'", self.srvname) try: self.vgrnt.destroy() logger.debug('vagrant destroy completed') @@ -327,6 +333,7 @@ class LibvirtBuildVm(FDroidBuildVm): logger.info('tired removing \'%s\', file was not present in first place: %s', boxname, e) def snapshot_create(self, snapshot_name): + logger.info("creating snapshot '%s' for vm '%s'", snapshot_name, self.srvname) try: _check_call(['virsh', '-c', 'qemu:///system', 'snapshot-create-as', self.srvname, snapshot_name]) logger.info('...waiting a sec...') @@ -353,6 +360,7 @@ class LibvirtBuildVm(FDroidBuildVm): return False def snapshot_revert(self, snapshot_name): + logger.info("reverting vm '%s' to snapshot '%s'", self.srvname, snapshot_name) import libvirt try: dom = self.conn.lookupByName(self.srvname) From b01d48a4fda49d1893998be9cab904275a9e16b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Sun, 9 Apr 2017 00:57:59 +0200 Subject: [PATCH 42/54] makebuildserver deal with apt cache lock --- makebuildserver | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/makebuildserver b/makebuildserver index c88d665a..02837239 100755 --- a/makebuildserver +++ b/makebuildserver @@ -6,6 +6,7 @@ import re import requests import stat import sys +import shutil import subprocess import vagrant import hashlib @@ -114,6 +115,14 @@ if config['vm_provider'] == 'libvirt': if config['apt_package_cache']: config['aptcachedir'] = cachedir + '/apt/archives' logger.debug('aptcachedir is set to %s', config['aptcachedir']) + aptcachelock = os.path.join(config['aptcachedir'], 'lock') + if os.path.isfile(aptcachelock): + logger.info('apt cache dir is locked, removing lock') + os.remove(aptcachelock) + aptcachepartial = os.path.join(config['aptcachedir'], 'partial') + if os.path.isdir(aptcachepartial): + logger.info('removing partial downloads from apt cache dir') + shutil.rmtree(aptcachepartial) cachefiles = [ ('https://dl.google.com/android/repository/tools_r25.2.3-linux.zip', @@ -486,9 +495,9 @@ def main(): debug_log_vagrant_vm(serverdir, config['domain']) try: v.up(provision=True) - except subprocess.CalledProcessError as e: + except fdroidserver.vmtools.FDroidBuildVmException as e: debug_log_vagrant_vm(serverdir, config['domain']) - logger.critical('could not bring buildserver vm up. %s', e) + logger.exception('could not bring buildserver vm up. %s', e) sys.exit(1) if config['copy_caches_from_host']: From 1b1d6b7d9642cddca6b6d074429bc847957e8975 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Sun, 9 Apr 2017 15:36:31 +0200 Subject: [PATCH 43/54] deal with outdated box images in libvirt storage pool --- fdroidserver/vmtools.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/fdroidserver/vmtools.py b/fdroidserver/vmtools.py index 71e439fd..4c1e5a30 100644 --- a/fdroidserver/vmtools.py +++ b/fdroidserver/vmtools.py @@ -173,7 +173,7 @@ class FDroidBuildVm(): self.vgrnt.destroy() logger.debug('vagrant destroy completed') except subprocess.CalledProcessError as e: - logger.debug('vagrant destroy failed: %s', e) + logger.exception('vagrant destroy failed: %s', e) vgrntdir = joinpath(self.srvdir, '.vagrant') try: shutil.rmtree(vgrntdir) @@ -325,12 +325,22 @@ class LibvirtBuildVm(FDroidBuildVm): logger.warn('could not connect to storage-pool \'default\',' + 'skipping packaging buildserver box') + def box_add(self, boxname, boxfile, force=True): + boximg = '%s_vagrant_box_image_0.img' % (boxname) + if force: + try: + _check_call(['virsh', '-c', 'qemu:///system', 'vol-delete', '--pool', 'default', boximg]) + logger.debug("removed old box image '%s' from libvirt storeage pool", boximg) + except subprocess.CalledProcessError as e: + logger.debug("tired removing old box image '%s', file was not present in first place", boximg, exc_info=e) + super().box_add(boxname, boxfile, force) + def box_remove(self, boxname): super().box_remove(boxname) try: _check_call(['virsh', '-c', 'qemu:///system', 'vol-delete', '--pool', 'default', '%s_vagrant_box_image_0.img' % (boxname)]) except subprocess.CalledProcessError as e: - logger.info('tired removing \'%s\', file was not present in first place: %s', boxname, e) + logger.debug("tired removing '%s', file was not present in first place", boxname, exc_info=e) def snapshot_create(self, snapshot_name): logger.info("creating snapshot '%s' for vm '%s'", snapshot_name, self.srvname) From 358b00d7aaea9633b1dad6d75be9eb4ce585df73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Wed, 12 Apr 2017 17:07:50 +0200 Subject: [PATCH 44/54] removed useless vm validity check; attempted to fix vbox support --- fdroidserver/build.py | 5 +-- fdroidserver/vmtools.py | 75 ++++++++++++++++++++++++----------------- 2 files changed, 46 insertions(+), 34 deletions(-) diff --git a/fdroidserver/build.py b/fdroidserver/build.py index 24a60344..06b93529 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -242,9 +242,6 @@ def vm_new_get_clean_builder(serverdir, reset=False): vm = vmtools.get_build_vm(serverdir) if reset: logging.info('resetting buildserver by request') - elif not vm.check_okay(): - logging.info('resetting buildserver because it appears to be absent or broken') - reset = True elif not vm.snapshot_exists('fdroidclean'): logging.info("resetting buildserver, because snapshot 'fdroidclean' is not present") reset = True @@ -259,7 +256,7 @@ def vm_new_get_clean_builder(serverdir, reset=False): vm.snapshot_create('fdroidclean') else: logging.info('builserver ok: reverting to clean snapshot') - vm.snapshot_revert('droidclean') + vm.snapshot_revert('fdroidclean') vm.up() return get_vagrant_sshinfo() diff --git a/fdroidserver/vmtools.py b/fdroidserver/vmtools.py index 4c1e5a30..578f888c 100644 --- a/fdroidserver/vmtools.py +++ b/fdroidserver/vmtools.py @@ -17,7 +17,7 @@ # along with this program. If not, see . from os import remove as rmfile -from os.path import isdir, isfile, join as joinpath, basename, abspath, expanduser +from os.path import isdir, isfile, join as joinpath, basename, abspath, expanduser, exists as pathexists import math import json import tarfile @@ -134,9 +134,6 @@ class FDroidBuildVm(): import vagrant self.vgrnt = vagrant.Vagrant(root=srvdir, out_cm=vagrant.stdout_cm, err_cm=vagrant.stdout_cm) - def check_okay(self): - return True - def up(self, provision=True): try: self.vgrnt.up(provision=provision) @@ -145,9 +142,6 @@ class FDroidBuildVm(): except subprocess.CalledProcessError as e: raise FDroidBuildVmException("could not bring up vm '%s'" % self.srvname) from e - def snapshot_create(self, name): - raise NotImplementedError('not implemented, please use a sub-type instance') - def suspend(self): logger.info('suspending buildserver') try: @@ -186,10 +180,6 @@ class FDroidBuildVm(): logger.debug('pruning global vagrant status failed: %s', e) def package(self, output=None, vagrantfile=None, keep_box_file=None): - previous_tmp_dir = joinpath(self.srvdir, '_tmp_package') - if isdir(previous_tmp_dir): - logger.info('found previous vagrant package temp dir \'%s\', deleting it', previous_tmp_dir) - shutil.rmtree(previous_tmp_dir) self.vgrnt.package(output=output, vagrantfile=vagrantfile) def _vagrant_file_name(self, name): @@ -219,10 +209,23 @@ class FDroidBuildVm(): boxname, boxpath) shutil.rmtree(boxpath) + def snapshot_create(self, snapshot_name): + raise NotImplementedError('not implemented, please use a sub-type instance') + + def snapshot_list(self): + raise NotImplementedError('not implemented, please use a sub-type instance') + + def snapshot_exists(self, snapshot_name): + raise NotImplementedError('not implemented, please use a sub-type instance') + + def snapshot_revert(self, snapshot_name): + raise NotImplementedError('not implemented, please use a sub-type instance') + class LibvirtBuildVm(FDroidBuildVm): def __init__(self, srvdir): super().__init__(srvdir) + self.provider = 'libvirt' import libvirt try: @@ -230,22 +233,6 @@ class LibvirtBuildVm(FDroidBuildVm): except libvirt.libvirtError as e: raise FDroidBuildVmException('could not connect to libvirtd: %s' % (e)) - def check_okay(self): - import libvirt - imagepath = joinpath('var', 'lib', 'libvirt', 'images', - '%s.img' % self._vagrant_file_name(self.srvname)) - image_present = False - if isfile(imagepath): - image_present = True - try: - self.conn.lookupByName(self.srvname) - domain_defined = True - except libvirt.libvirtError: - pass - if image_present and domain_defined: - return True - return False - def destroy(self): super().destroy() @@ -384,8 +371,13 @@ class LibvirtBuildVm(FDroidBuildVm): class VirtualboxBuildVm(FDroidBuildVm): + + def __init__(self, srvdir): + super().__init__(srvdir) + self.provider = 'virtualbox' + def snapshot_create(self, snapshot_name): - raise NotImplemented('TODO') + logger.info("creating snapshot '%s' for vm '%s'", snapshot_name, self.srvname) try: _check_call(['VBoxManage', 'snapshot', self.srvname, 'take', 'fdroidclean'], cwd=self.srvdir) logger.info('...waiting a sec...') @@ -395,5 +387,28 @@ class VirtualboxBuildVm(FDroidBuildVm): 'of virtualbox vm %s' % self.srvname) from e - def snapshot_available(self, snapshot_name): - raise NotImplemented('TODO') + def snapshot_list(self): + try: + o = _check_output(['VBoxManage', 'snapshot', + self.srvname, 'list', + '--details'], cwd=self.srvdir) + return o + except subprocess.CalledProcessError as e: + raise FDroidBuildVmException("could not list snapshots " + "of virtualbox vm '%s'" + % (self.srvname)) from e + + def snapshot_exists(self, snapshot_name): + return 'fdroidclean' in self.snapshot_list() + + def snapshot_revert(self, snapshot_name): + logger.info("reverting vm '%s' to snapshot '%s'", + self.srvname, snapshot_name) + try: + p = _check_call(['VBoxManage', 'snapshot', self.srvname, + 'restore', 'fdroidclean'], cwd=self.srvdir) + except subprocess.CalledProcessError as e: + raise FDroidBuildVmException("could not load snapshot " + "'fdroidclean' for vm '%s'" + % (self.srvname)) from e + From 0ec542295282047c15d83f23dbb3c210a9f9b810 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Wed, 12 Apr 2017 22:06:21 +0200 Subject: [PATCH 45/54] use uuid for vbox snapshots again --- fdroidserver/vmtools.py | 51 ++++++++++++++++++++++++++++++----------- 1 file changed, 37 insertions(+), 14 deletions(-) diff --git a/fdroidserver/vmtools.py b/fdroidserver/vmtools.py index 578f888c..a47b4c9e 100644 --- a/fdroidserver/vmtools.py +++ b/fdroidserver/vmtools.py @@ -17,7 +17,7 @@ # along with this program. If not, see . from os import remove as rmfile -from os.path import isdir, isfile, join as joinpath, basename, abspath, expanduser, exists as pathexists +from os.path import isdir, isfile, join as joinpath, basename, abspath, expanduser import math import json import tarfile @@ -30,14 +30,14 @@ from logging import getLogger logger = getLogger('fdroidserver-vmtools') -def _check_call(cmd, shell=False): +def _check_call(cmd, shell=False, cwd=None): logger.debug(' '.join(cmd)) - return subprocess.check_call(cmd, shell=shell) + return subprocess.check_call(cmd, shell=shell, cwd=cwd) -def _check_output(cmd, shell=False): +def _check_output(cmd, shell=False, cwd=None): logger.debug(' '.join(cmd)) - return subprocess.check_output(cmd, shell=shell) + return subprocess.check_output(cmd, shell=shell, cwd=cwd) def get_build_vm(srvdir, provider=None): @@ -74,7 +74,7 @@ def get_build_vm(srvdir, provider=None): except subprocess.CalledProcessError: pass try: - vbox_installed = 0 == _check_call(['which', 'VBoxHeadless'], shell=True) + vbox_installed = 0 == _check_call(['which', 'VBoxHeadless']) except subprocess.CalledProcessError: vbox_installed = False if kvm_installed and vbox_installed: @@ -127,6 +127,7 @@ class FDroidBuildVm(): self.srvdir = srvdir self.srvname = basename(srvdir) + '_default' self.vgrntfile = joinpath(srvdir, 'Vagrantfile') + self.srvuuid = self._vagrant_fetch_uuid() if not isdir(srvdir): raise FDroidBuildVmException("Can not init vagrant, directory %s not present" % (srvdir)) if not isfile(self.vgrntfile): @@ -139,6 +140,7 @@ class FDroidBuildVm(): self.vgrnt.up(provision=provision) logger.info('...waiting a sec...') time.sleep(10) + self.srvuuid = self._vagrant_fetch_uuid() except subprocess.CalledProcessError as e: raise FDroidBuildVmException("could not bring up vm '%s'" % self.srvname) from e @@ -185,6 +187,25 @@ class FDroidBuildVm(): def _vagrant_file_name(self, name): return name.replace('/', '-VAGRANTSLASH-') + def _vagrant_fetch_uuid(self): + if isfile(joinpath(self.srvdir, '.vagrant')): + # Vagrant 1.0 - it's a json file... + with open(joinpath(self.srvdir, '.vagrant')) as f: + id = json.load(f)['active']['default'] + logger.debug('vm uuid: %s', id) + return id + elif isfile(joinpath(self.srvdir, '.vagrant', 'machines', + 'default', self.provider, 'id')): + # Vagrant 1.2 (and maybe 1.1?) it's a directory tree... + with open(joinpath(self.srvdir, '.vagrant', 'machines', + 'default', self.provider, 'id')) as f: + id = f.read() + logger.debug('vm uuid: %s', id) + return id + else: + logger.debug('vm uuid is None') + return None + def box_add(self, boxname, boxfile, force=True): """Add vagrant box to vagrant. @@ -224,8 +245,8 @@ class FDroidBuildVm(): class LibvirtBuildVm(FDroidBuildVm): def __init__(self, srvdir): - super().__init__(srvdir) self.provider = 'libvirt' + super().__init__(srvdir) import libvirt try: @@ -373,13 +394,13 @@ class LibvirtBuildVm(FDroidBuildVm): class VirtualboxBuildVm(FDroidBuildVm): def __init__(self, srvdir): - super().__init__(srvdir) self.provider = 'virtualbox' + super().__init__(srvdir) def snapshot_create(self, snapshot_name): logger.info("creating snapshot '%s' for vm '%s'", snapshot_name, self.srvname) try: - _check_call(['VBoxManage', 'snapshot', self.srvname, 'take', 'fdroidclean'], cwd=self.srvdir) + _check_call(['VBoxManage', 'snapshot', self.srvuuid, 'take', 'fdroidclean'], cwd=self.srvdir) logger.info('...waiting a sec...') time.sleep(10) except subprocess.CalledProcessError as e: @@ -390,7 +411,7 @@ class VirtualboxBuildVm(FDroidBuildVm): def snapshot_list(self): try: o = _check_output(['VBoxManage', 'snapshot', - self.srvname, 'list', + self.srvuuid, 'list', '--details'], cwd=self.srvdir) return o except subprocess.CalledProcessError as e: @@ -399,16 +420,18 @@ class VirtualboxBuildVm(FDroidBuildVm): % (self.srvname)) from e def snapshot_exists(self, snapshot_name): - return 'fdroidclean' in self.snapshot_list() + try: + return str(snapshot_name) in str(self.snapshot_list()) + except FDroidBuildVmException: + return False def snapshot_revert(self, snapshot_name): logger.info("reverting vm '%s' to snapshot '%s'", self.srvname, snapshot_name) try: - p = _check_call(['VBoxManage', 'snapshot', self.srvname, - 'restore', 'fdroidclean'], cwd=self.srvdir) + _check_call(['VBoxManage', 'snapshot', self.srvuuid, + 'restore', 'fdroidclean'], cwd=self.srvdir) except subprocess.CalledProcessError as e: raise FDroidBuildVmException("could not load snapshot " "'fdroidclean' for vm '%s'" % (self.srvname)) from e - From 510efaa02498d44f02c6e2b9f4f6064bf950a8a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Thu, 13 Apr 2017 12:49:14 +0200 Subject: [PATCH 46/54] makebuildserver vbox logmessage fix --- makebuildserver | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/makebuildserver b/makebuildserver index 02837239..2e2b1a7b 100755 --- a/makebuildserver +++ b/makebuildserver @@ -83,7 +83,7 @@ if os.path.isfile('/usr/bin/systemd-detect-virt'): config['domain'] = 'buildserver_default' elif virt != 'none': logger.info('Running in an unsupported VM guest (%s)!', virt) -logger.debug('deceted virt: %s', virt) + logger.debug('detected virt: %s', virt) # load config file, if present if os.path.exists('makebuildserver.config.py'): @@ -391,7 +391,7 @@ def update_cache(cachedir, cachefiles): sys.exit(1) -def debug_log_vagrant_vm(vm_dir, vm_name): +def debug_log_vagrant_vm(vm_dir, config): if options.verbosity >= 3: _vagrant_dir = os.path.join(vm_dir, '.vagrant') logger.debug('check %s dir exists? -> %r', _vagrant_dir, os.path.isdir(_vagrant_dir)) @@ -399,10 +399,11 @@ def debug_log_vagrant_vm(vm_dir, vm_name): subprocess.call(['vagrant', 'status'], cwd=vm_dir) logger.debug('> vagrant box list') subprocess.call(['vagrant', 'box', 'list']) - logger.debug('> virsh -c qmeu:///system list --all') - subprocess.call(['virsh', '-c', 'qemu:///system', 'list', '--all']) - logger.debug('> virsh -c qemu:///system snapshot-list %s', vm_name) - subprocess.call(['virsh', '-c', 'qemu:///system', 'snapshot-list', vm_name]) + if config['vm_provider'] == 'libvirt': + logger.debug('> virsh -c qmeu:///system list --all') + subprocess.call(['virsh', '-c', 'qemu:///system', 'list', '--all']) + logger.debug('> virsh -c qemu:///system snapshot-list %s', config['domain']) + subprocess.call(['virsh', '-c', 'qemu:///system', 'snapshot-list', config['domain']]) def main(): @@ -492,11 +493,11 @@ def main(): v.box_remove(config['basebox'], 'virtualbox') logger.info("Configuring build server VM") - debug_log_vagrant_vm(serverdir, config['domain']) + debug_log_vagrant_vm(serverdir, config) try: v.up(provision=True) except fdroidserver.vmtools.FDroidBuildVmException as e: - debug_log_vagrant_vm(serverdir, config['domain']) + debug_log_vagrant_vm(serverdir, config) logger.exception('could not bring buildserver vm up. %s', e) sys.exit(1) From 48159f005a0ad990d933c673759734a92149670c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Thu, 13 Apr 2017 18:41:50 +0200 Subject: [PATCH 47/54] reset buildserver vm if vagrant uuid not present --- fdroidserver/build.py | 5 ++++- fdroidserver/vmtools.py | 6 ++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/fdroidserver/build.py b/fdroidserver/build.py index 06b93529..80af320c 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -242,8 +242,11 @@ def vm_new_get_clean_builder(serverdir, reset=False): vm = vmtools.get_build_vm(serverdir) if reset: logging.info('resetting buildserver by request') + elif not vm.vagrant_uuid_okay(): + logging.info('resetting buildserver, bceause vagrant vm is not okay.') + reset = True elif not vm.snapshot_exists('fdroidclean'): - logging.info("resetting buildserver, because snapshot 'fdroidclean' is not present") + logging.info("resetting buildserver, because snapshot 'fdroidclean' is not present.") reset = True if reset: diff --git a/fdroidserver/vmtools.py b/fdroidserver/vmtools.py index a47b4c9e..ec28ed48 100644 --- a/fdroidserver/vmtools.py +++ b/fdroidserver/vmtools.py @@ -184,6 +184,12 @@ class FDroidBuildVm(): def package(self, output=None, vagrantfile=None, keep_box_file=None): self.vgrnt.package(output=output, vagrantfile=vagrantfile) + def vagrant_uuid_okay(self): + '''Having an uuid means that vagrant up has run successfully.''' + if self.srvuuid is None: + return False + return True + def _vagrant_file_name(self, name): return name.replace('/', '-VAGRANTSLASH-') From 4546929d7f2c2a08f49bd48ef4bc664da58ba6ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Fri, 14 Apr 2017 15:58:16 +0200 Subject: [PATCH 48/54] wait a sec after suspending --- fdroidserver/build.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fdroidserver/build.py b/fdroidserver/build.py index 80af320c..ffc2700b 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -604,9 +604,9 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force): ftp.close() finally: - # Suspend the build server. - vm_suspend_builder() + vm = vmtools.get_build_vm('builder') + vm.suspend() def force_gradle_build_tools(build_dir, build_tools): From 6106b962a2be1d401e2e476cd2260d7baf0ec177 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 25 Apr 2017 10:27:09 +0200 Subject: [PATCH 49/54] build: delete bad builder/ symlinks If builder/ is a symlink but is not detected as a directory by os.path.isdir(), then it is a broken symlink. --- fdroidserver/build.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fdroidserver/build.py b/fdroidserver/build.py index ffc2700b..c146b95f 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -226,6 +226,8 @@ def vm_test_ssh_into_builder(): def vm_new_get_clean_builder(serverdir, reset=False): if not os.path.isdir(serverdir): + if os.path.islink(serverdir): + os.unlink(serverdir) logging.info("buildserver path does not exists, creating %s", serverdir) os.makedirs(serverdir) vagrantfile = os.path.join(serverdir, 'Vagrantfile') From 8f1fabfed618cf564ab844cffc459f0f9682b002 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Tue, 25 Apr 2017 14:45:41 +0200 Subject: [PATCH 50/54] restart builder vm when ssh connection fails --- fdroidserver/build.py | 12 +++++++++++- fdroidserver/vmtools.py | 27 +++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/fdroidserver/build.py b/fdroidserver/build.py index c146b95f..8f844a06 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -41,6 +41,7 @@ from . import scanner from . import vmtools from .common import FDroidPopen, SdkToolsPopen from .exception import FDroidException, BuildException, VCSException +from .vmtools import FDroidBuildVmException try: import paramiko @@ -264,7 +265,16 @@ def vm_new_get_clean_builder(serverdir, reset=False): vm.snapshot_revert('fdroidclean') vm.up() - return get_vagrant_sshinfo() + try: + sshinfo = vm.sshinfo() + except FDroidBuildVmException: + # workaround because libvirt sometimes likes to forget + # about ssh connection info even thou the vm is running + vm.halt() + vm.up() + sshinfo = vm.sshinfo() + + return sshinfo def vm_get_clean_builder(reset=False): diff --git a/fdroidserver/vmtools.py b/fdroidserver/vmtools.py index ec28ed48..f48ea2d5 100644 --- a/fdroidserver/vmtools.py +++ b/fdroidserver/vmtools.py @@ -236,6 +236,33 @@ class FDroidBuildVm(): boxname, boxpath) shutil.rmtree(boxpath) + def sshinfo(self): + """Get ssh connection info for a vagrant VM + + :returns: A dictionary containing 'hostname', 'port', 'user' + and 'idfile' + """ + import paramiko + try: + _check_call(['vagrant ssh-config > sshconfig'], + cwd=self.srvdir, shell=True) + vagranthost = 'default' # Host in ssh config file + sshconfig = paramiko.SSHConfig() + with open(joinpath(self.srvdir, 'sshconfig'), 'r') as f: + sshconfig.parse(f) + sshconfig = sshconfig.lookup(vagranthost) + idfile = sshconfig['identityfile'] + if isinstance(idfile, list): + idfile = idfile[0] + elif idfile.startswith('"') and idfile.endswith('"'): + idfile = idfile[1:-1] + return {'hostname': sshconfig['hostname'], + 'port': int(sshconfig['port']), + 'user': sshconfig['user'], + 'idfile': idfile} + except subprocess.CalledProcessError as e: + raise FDroidBuildVmException("Error getting ssh config") from e + def snapshot_create(self, snapshot_name): raise NotImplementedError('not implemented, please use a sub-type instance') From 9ef936c21a361cbc5c2235a441187b68e453cb38 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Mon, 22 May 2017 15:50:32 +0200 Subject: [PATCH 51/54] leave VirtualBox `vagrant package` as it was originally We only need Vagrantfile hacks for KVM. --- fdroidserver/vmtools.py | 24 +++++++++++++++++------- makebuildserver | 20 +++----------------- 2 files changed, 20 insertions(+), 24 deletions(-) diff --git a/fdroidserver/vmtools.py b/fdroidserver/vmtools.py index f48ea2d5..974ce52d 100644 --- a/fdroidserver/vmtools.py +++ b/fdroidserver/vmtools.py @@ -181,8 +181,8 @@ class FDroidBuildVm(): except subprocess.CalledProcessError as e: logger.debug('pruning global vagrant status failed: %s', e) - def package(self, output=None, vagrantfile=None, keep_box_file=None): - self.vgrnt.package(output=output, vagrantfile=vagrantfile) + def package(self, output=None): + self.vgrnt.package(output=output) def vagrant_uuid_okay(self): '''Having an uuid means that vagrant up has run successfully.''' @@ -309,7 +309,7 @@ class LibvirtBuildVm(FDroidBuildVm): except subprocess.CalledProcessError as e: logger.info("could not undefine libvirt domain '%s': %s", self.srvname, e) - def package(self, output=None, vagrantfile=None, keep_box_file=False): + def package(self, output=None, keep_box_file=False): if not output: output = "buildserver.box" logger.debug('no output name set for packaging \'%s\',' + @@ -338,14 +338,24 @@ class LibvirtBuildVm(FDroidBuildVm): "virtual_size": math.ceil(img_info['virtual-size'] / (1024. ** 3)), } - if not vagrantfile: - logger.debug('no Vagrantfile supplied for box, generating a minimal one...') - vagrantfile = 'Vagrant.configure("2") do |config|\nend' - logger.debug('preparing metadata.json for box %s', output) with open('metadata.json', 'w') as fp: fp.write(json.dumps(metadata)) logger.debug('preparing Vagrantfile for box %s', output) + vagrantfile = textwrap.dedent("""\ + Vagrant.configure("2") do |config| + config.ssh.username = "vagrant" + config.ssh.password = "vagrant" + + config.vm.provider :libvirt do |libvirt| + + libvirt.driver = "kvm" + libvirt.host = "" + libvirt.connect_via_ssh = false + libvirt.storage_pool_name = "default" + + end + end""") with open('Vagrantfile', 'w') as fp: fp.write(vagrantfile) with tarfile.open(output, 'w:gz') as tar: diff --git a/makebuildserver b/makebuildserver index 2e2b1a7b..7f759e68 100755 --- a/makebuildserver +++ b/makebuildserver @@ -30,7 +30,8 @@ parser.add_option('--skip-cache-update', action="store_true", default=False, help="""Skip downloading and checking cache.""" """This assumes that the cache is already downloaded completely.""") parser.add_option('--keep-box-file', action="store_true", default=False, - help="""Box file will not be deleted after adding it to box storage.""") + help="""Box file will not be deleted after adding it to box storage""" + """ (KVM-only).""") options, args = parser.parse_args() logger = logging.getLogger('fdroidserver-makebuildserver') @@ -536,22 +537,7 @@ def main(): if os.path.exists(boxfile): os.remove(boxfile) - vagrantfile = textwrap.dedent("""\ - Vagrant.configure("2") do |config| - config.ssh.username = "vagrant" - config.ssh.password = "vagrant" - - config.vm.provider :libvirt do |libvirt| - - libvirt.driver = "kvm" - libvirt.host = "" - libvirt.connect_via_ssh = false - libvirt.storage_pool_name = "default" - - end - end""") - - vm.package(output=boxfile, vagrantfile=vagrantfile, keep_box_file=options.keep_box_file) + vm.package(output=boxfile) logger.info("Adding box") vm.box_add('buildserver', boxfile, force=True) From fc660048eb104984fd48bb1e93350d64dffa2715 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Tue, 23 May 2017 11:37:09 +0200 Subject: [PATCH 52/54] removed unused buildserver code --- fdroidserver/build.py | 305 ---------------------------------------- fdroidserver/vmtools.py | 1 + makebuildserver | 1 - 3 files changed, 1 insertion(+), 306 deletions(-) diff --git a/fdroidserver/build.py b/fdroidserver/build.py index 8f844a06..c04db9d3 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -26,7 +26,6 @@ import re import tarfile import traceback import time -import json import requests import tempfile import textwrap @@ -49,182 +48,6 @@ except ImportError: pass -def get_vm_provider(): - """Determine vm provider based on .vagrant directory content - """ - if os.path.exists(os.path.join('builder', '.vagrant', 'machines', - 'default', 'libvirt')): - return 'libvirt' - return 'virtualbox' - - -def vm_get_builder_id(provider): - vd = os.path.join('builder', '.vagrant') - if os.path.isdir(vd): - # Vagrant 1.2 (and maybe 1.1?) it's a directory tree... - with open(os.path.join(vd, 'machines', 'default', - provider, 'id')) as vf: - id = vf.read() - return id - else: - # Vagrant 1.0 - it's a json file... - with open(os.path.join('builder', '.vagrant')) as vf: - v = json.load(vf) - return v['active']['default'] - - -def vm_get_builder_status(): - """Get the current status of builder vm. - - :returns: one of: 'running', 'paused', 'shutoff', 'not created' - If something is wrong with vagrant or the vm 'unknown' is returned. - """ - (ret, out) = vagrant(['status'], cwd='builder') - - allowed_providers = 'virtualbox|libvirt' - allowed_states = 'running|paused|shutoff|not created' - - r = re.compile('^\s*(?P\w+)\s+' + - '(?P' + allowed_states + ')' + - '\s+\((?P' + allowed_providers + ')\)\s*$') - - for line in out.split('\n'): - m = r.match(line) - if m: - s = m.group('vm_state') - if options.verbose: - logging.debug('current builder vm status: ' + s) - return s - if options.verbose: - logging.debug('current builder vm status: unknown') - return 'unknown' - - -def vm_is_builder_valid(provider): - """Returns True if we have a valid-looking builder vm - """ - if not os.path.exists(os.path.join('builder', 'Vagrantfile')): - return False - vd = os.path.join('builder', '.vagrant') - if not os.path.exists(vd): - return False - if not os.path.isdir(vd): - # Vagrant 1.0 - if the directory is there, it's valid... - return True - # Vagrant 1.2 - the directory can exist, but the id can be missing... - if not os.path.exists(os.path.join(vd, 'machines', 'default', - provider, 'id')): - return False - return True - - -def vagrant(params, cwd=None, printout=False): - """Run a vagrant command. - - :param: list of parameters to pass to vagrant - :cwd: directory to run in, or None for current directory - :printout: has not effect - :returns: (ret, out) where ret is the return code, and out - is the stdout (and stderr) from vagrant - """ - p = FDroidPopen(['vagrant'] + params, cwd=cwd, output=printout, stderr_to_stdout=printout) - return (p.returncode, p.output) - - -def get_vagrant_sshinfo(): - """Get ssh connection info for a vagrant VM - - :returns: A dictionary containing 'hostname', 'port', 'user' - and 'idfile' - """ - if subprocess.call('vagrant ssh-config >sshconfig', - cwd='builder', shell=True) != 0: - raise BuildException("Error getting ssh config") - vagranthost = 'default' # Host in ssh config file - sshconfig = paramiko.SSHConfig() - sshf = open(os.path.join('builder', 'sshconfig'), 'r') - sshconfig.parse(sshf) - sshf.close() - sshconfig = sshconfig.lookup(vagranthost) - idfile = sshconfig['identityfile'] - if isinstance(idfile, list): - idfile = idfile[0] - elif idfile.startswith('"') and idfile.endswith('"'): - idfile = idfile[1:-1] - return {'hostname': sshconfig['hostname'], - 'port': int(sshconfig['port']), - 'user': sshconfig['user'], - 'idfile': idfile} - - -def vm_shutdown_builder(): - """Turn off builder vm. - """ - if options.server: - if os.path.exists(os.path.join('builder', 'Vagrantfile')): - vagrant(['halt'], cwd='builder') - - -def vm_snapshot_list(provider): - output = options.verbose - if provider is 'virtualbox': - p = FDroidPopen(['VBoxManage', 'snapshot', - vm_get_builder_id(provider), 'list', - '--details'], cwd='builder', - output=output, stderr_to_stdout=output) - elif provider is 'libvirt': - p = FDroidPopen(['virsh', '-c', 'qemu:///system', 'snapshot-list', - vm_get_builder_id(provider)], - output=output, stderr_to_stdout=output) - return p.output - - -def vm_snapshot_clean_available(provider): - return 'fdroidclean' in vm_snapshot_list(provider) - - -def vm_snapshot_restore(provider): - """Does a rollback of the build vm. - """ - output = options.verbose - if provider is 'virtualbox': - p = FDroidPopen(['VBoxManage', 'snapshot', - vm_get_builder_id(provider), 'restore', - 'fdroidclean'], cwd='builder', - output=output, stderr_to_stdout=output) - elif provider is 'libvirt': - p = FDroidPopen(['virsh', '-c', 'qemu:///system', 'snapshot-revert', - vm_get_builder_id(provider), 'fdroidclean'], - output=output, stderr_to_stdout=output) - return p.returncode == 0 - - -def vm_snapshot_create(provider): - output = options.verbose - if provider is 'virtualbox': - p = FDroidPopen(['VBoxManage', 'snapshot', - vm_get_builder_id(provider), - 'take', 'fdroidclean'], cwd='builder', - output=output, stderr_to_stdout=output) - elif provider is 'libvirt': - p = FDroidPopen(['virsh', '-c', 'qemu:///system', 'snapshot-create-as', - vm_get_builder_id(provider), 'fdroidclean'], - output=output, stderr_to_stdout=output) - return p.returncode != 0 - - -def vm_test_ssh_into_builder(): - logging.info("Connecting to virtual machine...") - sshinfo = get_vagrant_sshinfo() - sshs = paramiko.SSHClient() - sshs.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - sshs.connect(sshinfo['hostname'], username=sshinfo['user'], - port=sshinfo['port'], timeout=300, - look_for_keys=False, - key_filename=sshinfo['idfile']) - sshs.close() - - def vm_new_get_clean_builder(serverdir, reset=False): if not os.path.isdir(serverdir): if os.path.islink(serverdir): @@ -277,134 +100,6 @@ def vm_new_get_clean_builder(serverdir, reset=False): return sshinfo -def vm_get_clean_builder(reset=False): - """Get a clean VM ready to do a buildserver build. - - This might involve creating and starting a new virtual machine from - scratch, or it might be as simple (unless overridden by the reset - parameter) as re-using a snapshot created previously. - - A BuildException will be raised if anything goes wrong. - - :reset: True to force creating from scratch. - :returns: A dictionary containing 'hostname', 'port', 'user' - and 'idfile' - """ - provider = get_vm_provider() - - # Reset existing builder machine to a clean state if possible. - vm_ok = False - if not reset: - logging.info("Checking for valid existing build server") - if vm_is_builder_valid(provider): - logging.info("...VM is present (%s)" % provider) - if vm_snapshot_clean_available(provider): - logging.info("...snapshot exists - resetting build server to " + - "clean state") - status = vm_get_builder_status() - if status == 'running': - vm_test_ssh_into_builder() - logging.info("...suspending builder vm") - vagrant(['suspend'], cwd='builder') - logging.info("...waiting a sec...") - time.sleep(10) - elif status == 'shutoff': - logging.info('...starting builder vm') - vagrant(['up'], cwd='builder') - logging.info('...waiting a sec...') - time.sleep(10) - vm_test_ssh_into_builder() - logging.info('...suspending builder vm') - vagrant(['suspend'], cwd='builder') - logging.info("...waiting a sec...") - time.sleep(10) - if options.verbose: - vm_get_builder_status() - - if vm_snapshot_restore(provider): - if options.verbose: - vm_get_builder_status() - logging.info("...reset to snapshot - server is valid") - retcode, output = vagrant(['up'], cwd='builder') - if retcode != 0: - raise BuildException("Failed to start build server") - logging.info("...waiting a sec...") - time.sleep(10) - sshinfo = get_vagrant_sshinfo() - vm_ok = True - else: - logging.info("...failed to reset to snapshot") - else: - logging.info("...snapshot doesn't exist - " - "VBoxManage snapshot list:\n" + - vm_snapshot_list(provider)) - else: - logging.info('...VM not present') - - # If we can't use the existing machine for any reason, make a - # new one from scratch. - if not vm_ok: - if os.path.isdir('builder'): - vm = vmtools.get_build_vm('builder') - vm.destroy() - else: - os.mkdir('builder') - - p = subprocess.Popen(['vagrant', '--version'], - universal_newlines=True, - stdout=subprocess.PIPE) - vver = p.communicate()[0].strip().split(' ')[1] - if vver.split('.')[0] != '1' or int(vver.split('.')[1]) < 4: - raise BuildException("Unsupported vagrant version {0}".format(vver)) - - with open(os.path.join('builder', 'Vagrantfile'), 'w') as vf: - vf.write('Vagrant.configure("2") do |config|\n') - vf.write(' config.vm.box = "buildserver"\n') - vf.write(' config.vm.synced_folder ".", "/vagrant", disabled: true\n') - vf.write('end\n') - - logging.info("Starting new build server") - retcode, _ = vagrant(['up'], cwd='builder') - if retcode != 0: - raise BuildException("Failed to start build server") - provider = get_vm_provider() - sshinfo = get_vagrant_sshinfo() - - # Open SSH connection to make sure it's working and ready... - vm_test_ssh_into_builder() - - logging.info("Saving clean state of new build server") - retcode, _ = vagrant(['suspend'], cwd='builder') - if retcode != 0: - raise BuildException("Failed to suspend build server") - logging.info("...waiting a sec...") - time.sleep(10) - if vm_snapshot_create(provider): - raise BuildException("Failed to take snapshot") - logging.info("...waiting a sec...") - time.sleep(10) - logging.info("Restarting new build server") - retcode, _ = vagrant(['up'], cwd='builder') - if retcode != 0: - raise BuildException("Failed to start build server") - logging.info("...waiting a sec...") - time.sleep(10) - # Make sure it worked... - if not vm_snapshot_clean_available(provider): - raise BuildException("Failed to take snapshot.") - - return sshinfo - - -def vm_suspend_builder(): - """Release the VM previously started with vm_get_clean_builder(). - - This should always be called after each individual app build attempt. - """ - logging.info("Suspending build server") - subprocess.call(['vagrant', 'suspend'], cwd='builder') - - # Note that 'force' here also implies test mode. def build_server(app, build, vcs, build_dir, output_dir, log_dir, force): """Do a build on the builder vm. diff --git a/fdroidserver/vmtools.py b/fdroidserver/vmtools.py index 974ce52d..e502f3d3 100644 --- a/fdroidserver/vmtools.py +++ b/fdroidserver/vmtools.py @@ -24,6 +24,7 @@ import tarfile import time import shutil import subprocess +import textwrap from .common import FDroidException from logging import getLogger diff --git a/makebuildserver b/makebuildserver index 7f759e68..0b376334 100755 --- a/makebuildserver +++ b/makebuildserver @@ -13,7 +13,6 @@ import hashlib import yaml import json import logging -import textwrap from clint.textui import progress from optparse import OptionParser import fdroidserver.tail From c8234919df4b3b58d8eeb19785bab640f957033c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Tue, 23 May 2017 12:53:07 +0200 Subject: [PATCH 53/54] refactored vm related code from build.py to vmtools.py --- fdroidserver/build.py | 57 +---------------------------------------- fdroidserver/vmtools.py | 53 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 56 deletions(-) diff --git a/fdroidserver/build.py b/fdroidserver/build.py index c04db9d3..89131360 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -28,7 +28,6 @@ import traceback import time import requests import tempfile -import textwrap from configparser import ConfigParser from argparse import ArgumentParser import logging @@ -40,7 +39,6 @@ from . import scanner from . import vmtools from .common import FDroidPopen, SdkToolsPopen from .exception import FDroidException, BuildException, VCSException -from .vmtools import FDroidBuildVmException try: import paramiko @@ -48,58 +46,6 @@ except ImportError: pass -def vm_new_get_clean_builder(serverdir, reset=False): - if not os.path.isdir(serverdir): - if os.path.islink(serverdir): - os.unlink(serverdir) - logging.info("buildserver path does not exists, creating %s", serverdir) - os.makedirs(serverdir) - vagrantfile = os.path.join(serverdir, 'Vagrantfile') - if not os.path.isfile(vagrantfile): - with open(os.path.join('builder', 'Vagrantfile'), 'w') as f: - f.write(textwrap.dedent("""\ - # generated file, do not change. - - Vagrant.configure("2") do |config| - config.vm.box = "buildserver" - config.vm.synced_folder ".", "/vagrant", disabled: true - end - """)) - vm = vmtools.get_build_vm(serverdir) - if reset: - logging.info('resetting buildserver by request') - elif not vm.vagrant_uuid_okay(): - logging.info('resetting buildserver, bceause vagrant vm is not okay.') - reset = True - elif not vm.snapshot_exists('fdroidclean'): - logging.info("resetting buildserver, because snapshot 'fdroidclean' is not present.") - reset = True - - if reset: - vm.destroy() - vm.up() - vm.suspend() - - if reset: - logging.info('buildserver recreated: taking a clean snapshot') - vm.snapshot_create('fdroidclean') - else: - logging.info('builserver ok: reverting to clean snapshot') - vm.snapshot_revert('fdroidclean') - vm.up() - - try: - sshinfo = vm.sshinfo() - except FDroidBuildVmException: - # workaround because libvirt sometimes likes to forget - # about ssh connection info even thou the vm is running - vm.halt() - vm.up() - sshinfo = vm.sshinfo() - - return sshinfo - - # Note that 'force' here also implies test mode. def build_server(app, build, vcs, build_dir, output_dir, log_dir, force): """Do a build on the builder vm. @@ -123,8 +69,7 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force): else: logging.getLogger("paramiko").setLevel(logging.WARN) - # sshinfo = vm_get_clean_builder() - sshinfo = vm_new_get_clean_builder('builder') + sshinfo = vmtools.get_clean_builder('builder') try: if not buildserverid: diff --git a/fdroidserver/vmtools.py b/fdroidserver/vmtools.py index e502f3d3..2b198ed1 100644 --- a/fdroidserver/vmtools.py +++ b/fdroidserver/vmtools.py @@ -18,6 +18,7 @@ from os import remove as rmfile from os.path import isdir, isfile, join as joinpath, basename, abspath, expanduser +import os import math import json import tarfile @@ -31,6 +32,58 @@ from logging import getLogger logger = getLogger('fdroidserver-vmtools') +def get_clean_builder(serverdir, reset=False): + if not os.path.isdir(serverdir): + if os.path.islink(serverdir): + os.unlink(serverdir) + logger.info("buildserver path does not exists, creating %s", serverdir) + os.makedirs(serverdir) + vagrantfile = os.path.join(serverdir, 'Vagrantfile') + if not os.path.isfile(vagrantfile): + with open(os.path.join('builder', 'Vagrantfile'), 'w') as f: + f.write(textwrap.dedent("""\ + # generated file, do not change. + + Vagrant.configure("2") do |config| + config.vm.box = "buildserver" + config.vm.synced_folder ".", "/vagrant", disabled: true + end + """)) + vm = get_build_vm(serverdir) + if reset: + logger.info('resetting buildserver by request') + elif not vm.vagrant_uuid_okay(): + logger.info('resetting buildserver, bceause vagrant vm is not okay.') + reset = True + elif not vm.snapshot_exists('fdroidclean'): + logger.info("resetting buildserver, because snapshot 'fdroidclean' is not present.") + reset = True + + if reset: + vm.destroy() + vm.up() + vm.suspend() + + if reset: + logger.info('buildserver recreated: taking a clean snapshot') + vm.snapshot_create('fdroidclean') + else: + logger.info('builserver ok: reverting to clean snapshot') + vm.snapshot_revert('fdroidclean') + vm.up() + + try: + sshinfo = vm.sshinfo() + except FDroidBuildVmException: + # workaround because libvirt sometimes likes to forget + # about ssh connection info even thou the vm is running + vm.halt() + vm.up() + sshinfo = vm.sshinfo() + + return sshinfo + + def _check_call(cmd, shell=False, cwd=None): logger.debug(' '.join(cmd)) return subprocess.check_call(cmd, shell=shell, cwd=cwd) From a01e302cdeebe3837adccbe8fded26ef4bbc7be0 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Tue, 23 May 2017 11:59:53 +0200 Subject: [PATCH 54/54] makebuildserver: make config['domain'] local var where its needed --- makebuildserver | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/makebuildserver b/makebuildserver index 0b376334..48e12f9f 100755 --- a/makebuildserver +++ b/makebuildserver @@ -80,7 +80,6 @@ if os.path.isfile('/usr/bin/systemd-detect-virt'): if virt == 'qemu' or virt == 'kvm' or virt == 'bochs': logger.info('Running in a VM guest, defaulting to QEMU/KVM via libvirt') config['vm_provider'] = 'libvirt' - config['domain'] = 'buildserver_default' elif virt != 'none': logger.info('Running in an unsupported VM guest (%s)!', virt) logger.debug('detected virt: %s', virt) @@ -402,8 +401,9 @@ def debug_log_vagrant_vm(vm_dir, config): if config['vm_provider'] == 'libvirt': logger.debug('> virsh -c qmeu:///system list --all') subprocess.call(['virsh', '-c', 'qemu:///system', 'list', '--all']) - logger.debug('> virsh -c qemu:///system snapshot-list %s', config['domain']) - subprocess.call(['virsh', '-c', 'qemu:///system', 'snapshot-list', config['domain']]) + domain = 'buildserver_default' + logger.debug('> virsh -c qemu:///system snapshot-list %s', domain) + subprocess.call(['virsh', '-c', 'qemu:///system', 'snapshot-list', domain]) def main():