diff --git a/fdroidserver/build.py b/fdroidserver/build.py index 63923584..4e5be385 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -29,6 +29,7 @@ import time import json import requests import tempfile +import textwrap from configparser import ConfigParser from argparse import ArgumentParser import logging @@ -223,6 +224,45 @@ def vm_test_ssh_into_builder(): sshs.close() +def vm_new_get_clean_builder(serverdir, reset=False): + if not os.path.isdir(serverdir): + logging.info("buildserver path does not exists, creating %s", serverdir) + os.makedirs(serverdir) + vagrantfile = os.path.join(serverdir, 'Vagrantfile') + if not os.path.isfile(vagrantfile): + with open(os.path.join('builder', 'Vagrantfile'), 'w') as f: + f.write(textwrap.dedent("""\ + # generated file, do not change. + + Vagrant.configure("2") do |config| + config.vm.box = "buildserver" + config.vm.synced_folder ".", "/vagrant", disabled: true + end + """)) + vm = vmtools.get_build_vm(serverdir) + if reset: + logging.info('resetting buildserver by request') + elif not vm.check_okay(): + logging.info('resetting buildserver because it appears to be absent or broken') + reset = True + elif not vm.snapshot_exists('fdroidclean'): + logging.info("resetting buildserver, because snapshot 'fdroidclean' is not present") + reset = True + + if reset: + vm.destroy() + vm.up() + vm.suspend() + + if reset: + vm.snapshot_create('fdroidclean') + else: + vm.snapshot_revert('droidclean') + vm.resume() + + return get_vagrant_sshinfo() + + def vm_get_clean_builder(reset=False): """Get a clean VM ready to do a buildserver build. @@ -374,7 +414,8 @@ def build_server(app, build, vcs, build_dir, output_dir, log_dir, force): else: logging.getLogger("paramiko").setLevel(logging.WARN) - sshinfo = vm_get_clean_builder() + # sshinfo = vm_get_clean_builder() + sshinfo = vm_new_get_clean_builder('builder') try: if not buildserverid: diff --git a/fdroidserver/vmtools.py b/fdroidserver/vmtools.py index ba8008c4..c209914c 100644 --- a/fdroidserver/vmtools.py +++ b/fdroidserver/vmtools.py @@ -17,13 +17,12 @@ # along with this program. If not, see . from os import remove as rmfile -from os.path import isdir, isfile, join as joinpath, basename, abspath +from os.path import isdir, isfile, join as joinpath, basename, abspath, expanduser import math import json import tarfile import time import shutil -import vagrant import subprocess from .common import FDroidException from logging import getLogger @@ -132,10 +131,11 @@ class FDroidBuildVm(): raise FDroidBuildVmException("Can not init vagrant, directory %s not present" % (srvdir)) if not isfile(self.vgrntfile): raise FDroidBuildVmException("Can not init vagrant, '%s' not present" % (self.vgrntfile)) + import vagrant self.vgrnt = vagrant.Vagrant(root=srvdir, out_cm=vagrant.stdout_cm, err_cm=vagrant.stdout_cm) - def isUpAndRunning(self): - raise NotImplementedError('TODO implement this') + def check_okay(self): + return True def up(self, provision=True): try: @@ -143,6 +143,15 @@ class FDroidBuildVm(): except subprocess.CalledProcessError as e: logger.info('could not bring vm up: %s', e) + def snapshot_create(self, name): + raise NotImplementedError('not implemented, please use a sub-type instance') + + def suspend(self): + self.vgrnt.suspend() + + def resume(self): + self.vgrnt.resume() + def halt(self): self.vgrnt.halt(force=True) @@ -177,6 +186,9 @@ class FDroidBuildVm(): shutil.rmtree(previous_tmp_dir) self.vgrnt.package(output=output, vagrantfile=vagrantfile) + def _vagrant_file_name(self, name): + return name.replace('/', '-VAGRANTSLASH-') + def box_add(self, boxname, boxfile, force=True): """Add vagrant box to vagrant. @@ -194,8 +206,12 @@ class FDroidBuildVm(): _check_call(['vagrant', 'box', 'remove', '--all', '--force', boxname]) except subprocess.CalledProcessError as e: logger.debug('tried removing box %s, but is did not exist: %s', boxname, e) - # TODO: remove box files manually - # nesessary when Vagrantfile in ~/.vagrant.d/... is broken. + boxpath = joinpath(expanduser('~'), '.vagrant', + self._vagrant_file_name(boxname)) + if isdir(boxpath): + logger.info("attempting to remove box '%s' by deleting: %s", + boxname, boxpath) + shutil.rmtree(boxpath) class LibvirtBuildVm(FDroidBuildVm): @@ -208,6 +224,22 @@ class LibvirtBuildVm(FDroidBuildVm): except libvirt.libvirtError as e: raise FDroidBuildVmException('could not connect to libvirtd: %s' % (e)) + def check_okay(self): + import libvirt + imagepath = joinpath('var', 'lib', 'libvirt', 'images', + '%s.img' % self._vagrant_file_name(self.srvname)) + image_present = False + if isfile(imagepath): + image_present = True + try: + self.conn.lookupByName(self.srvname) + domain_defined = True + except libvirt.libvirtError: + pass + if image_present and domain_defined: + return True + return False + def destroy(self): super().destroy() @@ -235,9 +267,7 @@ class LibvirtBuildVm(FDroidBuildVm): output = "buildserver.box" logger.debug('no output name set for packaging \'%s\',' + 'defaulting to %s', self.srvname, output) - import libvirt - virConnect = libvirt.open('qemu:///system') - storagePool = virConnect.storagePoolLookupByName('default') + storagePool = self.conn.storagePoolLookupByName('default') if storagePool: if isfile('metadata.json'): @@ -258,7 +288,7 @@ class LibvirtBuildVm(FDroidBuildVm): img_info = json.loads(img_info_raw.decode('utf-8')) metadata = {"provider": "libvirt", "format": img_info['format'], - "virtual_size": math.ceil(img_info['virtual-size'] / (1024. ** 3)) + 1, + "virtual_size": math.ceil(img_info['virtual-size'] / (1024. ** 3)), } if not vagrantfile: @@ -296,6 +326,56 @@ class LibvirtBuildVm(FDroidBuildVm): except subprocess.CalledProcessError as e: logger.info('tired removing \'%s\', file was not present in first place: %s', boxname, e) + def snapshot_create(self, snapshot_name): + try: + _check_call(['virsh', '-c', 'qemu:///system', 'snapshot-create-as', self.srvname, snapshot_name]) + logger.info('...waiting a sec...') + time.sleep(10) + except subprocess.CalledProcessError as e: + raise FDroidBuildVmException("could not cerate snapshot '%s' " + "of libvirt vm '%s'" + % (snapshot_name, self.srvname)) from e + + def snapshot_list(self): + import libvirt + try: + dom = self.conn.lookupByName(self.srvname) + return dom.listAllSnapshots() + except libvirt.libvirtError as e: + raise FDroidBuildVmException('could not list snapshots for domain \'%s\'' % self.srvname) from e + + def snapshot_exists(self, snapshot_name): + import libvirt + try: + dom = self.conn.lookupByName(self.srvname) + return dom.snapshotLookupByName(snapshot_name) is not None + except libvirt.libvirtError: + return False + + def snapshot_revert(self, snapshot_name): + import libvirt + try: + dom = self.conn.lookupByName(self.srvname) + snap = dom.snapshotLookupByName(snapshot_name) + dom.revertToSnapshot(snap) + logger.info('...waiting a sec...') + time.sleep(10) + except libvirt.libvirtError as e: + raise FDroidBuildVmException('could not revert domain \'%s\' to snapshot \'%s\'' + % (self.srvname, snapshot_name)) from e + class VirtualboxBuildVm(FDroidBuildVm): - pass + def snapshot_create(self, snapshot_name): + raise NotImplemented('TODO') + try: + _check_call(['VBoxManage', 'snapshot', self.srvname, 'take', 'fdroidclean'], cwd=self.srvdir) + logger.info('...waiting a sec...') + time.sleep(10) + except subprocess.CalledProcessError as e: + raise FDroidBuildVmException('could not cerate snapshot ' + 'of virtualbox vm %s' + % self.srvname) from e + + def snapshot_available(self, snapshot_name): + raise NotImplemented('TODO')