From fc660048eb104984fd48bb1e93350d64dffa2715 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20P=C3=B6hn?= Date: Tue, 23 May 2017 11:37:09 +0200 Subject: [PATCH] removed unused buildserver code --- fdroidserver/build.py | 305 ---------------------------------------- fdroidserver/vmtools.py | 1 + makebuildserver | 1 - 3 files changed, 1 insertion(+), 306 deletions(-) diff --git a/fdroidserver/build.py b/fdroidserver/build.py index 8f844a06..c04db9d3 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -26,7 +26,6 @@ import re import tarfile import traceback import time -import json import requests import tempfile import textwrap @@ -49,182 +48,6 @@ except ImportError: pass -def get_vm_provider(): - """Determine vm provider based on .vagrant directory content - """ - if os.path.exists(os.path.join('builder', '.vagrant', 'machines', - 'default', 'libvirt')): - return 'libvirt' - return 'virtualbox' - - -def vm_get_builder_id(provider): - vd = os.path.join('builder', '.vagrant') - if os.path.isdir(vd): - # Vagrant 1.2 (and maybe 1.1?) it's a directory tree... - with open(os.path.join(vd, 'machines', 'default', - provider, 'id')) as vf: - id = vf.read() - return id - else: - # Vagrant 1.0 - it's a json file... - with open(os.path.join('builder', '.vagrant')) as vf: - v = json.load(vf) - return v['active']['default'] - - -def vm_get_builder_status(): - """Get the current status of builder vm. - - :returns: one of: 'running', 'paused', 'shutoff', 'not created' - If something is wrong with vagrant or the vm 'unknown' is returned. - """ - (ret, out) = vagrant(['status'], cwd='builder') - - allowed_providers = 'virtualbox|libvirt' - allowed_states = 'running|paused|shutoff|not created' - - r = re.compile('^\s*(?P\w+)\s+' + - '(?P' + allowed_states + ')' + - '\s+\((?P' + allowed_providers + ')\)\s*$') - - for line in out.split('\n'): - m = r.match(line) - if m: - s = m.group('vm_state') - if options.verbose: - logging.debug('current builder vm status: ' + s) - return s - if options.verbose: - logging.debug('current builder vm status: unknown') - return 'unknown' - - -def vm_is_builder_valid(provider): - """Returns True if we have a valid-looking builder vm - """ - if not os.path.exists(os.path.join('builder', 'Vagrantfile')): - return False - vd = os.path.join('builder', '.vagrant') - if not os.path.exists(vd): - return False - if not os.path.isdir(vd): - # Vagrant 1.0 - if the directory is there, it's valid... - return True - # Vagrant 1.2 - the directory can exist, but the id can be missing... - if not os.path.exists(os.path.join(vd, 'machines', 'default', - provider, 'id')): - return False - return True - - -def vagrant(params, cwd=None, printout=False): - """Run a vagrant command. - - :param: list of parameters to pass to vagrant - :cwd: directory to run in, or None for current directory - :printout: has not effect - :returns: (ret, out) where ret is the return code, and out - is the stdout (and stderr) from vagrant - """ - p = FDroidPopen(['vagrant'] + params, cwd=cwd, output=printout, stderr_to_stdout=printout) - return (p.returncode, p.output) - - -def get_vagrant_sshinfo(): - """Get ssh connection info for a vagrant VM - - :returns: A dictionary containing 'hostname', 'port', 'user' - and 'idfile' - """ - if subprocess.call('vagrant ssh-config >sshconfig', - cwd='builder', shell=True) != 0: - raise BuildException("Error getting ssh config") - vagranthost = 'default' # Host in ssh config file - sshconfig = paramiko.SSHConfig() - sshf = open(os.path.join('builder', 'sshconfig'), 'r') - sshconfig.parse(sshf) - sshf.close() - sshconfig = sshconfig.lookup(vagranthost) - idfile = sshconfig['identityfile'] - if isinstance(idfile, list): - idfile = idfile[0] - elif idfile.startswith('"') and idfile.endswith('"'): - idfile = idfile[1:-1] - return {'hostname': sshconfig['hostname'], - 'port': int(sshconfig['port']), - 'user': sshconfig['user'], - 'idfile': idfile} - - -def vm_shutdown_builder(): - """Turn off builder vm. - """ - if options.server: - if os.path.exists(os.path.join('builder', 'Vagrantfile')): - vagrant(['halt'], cwd='builder') - - -def vm_snapshot_list(provider): - output = options.verbose - if provider is 'virtualbox': - p = FDroidPopen(['VBoxManage', 'snapshot', - vm_get_builder_id(provider), 'list', - '--details'], cwd='builder', - output=output, stderr_to_stdout=output) - elif provider is 'libvirt': - p = FDroidPopen(['virsh', '-c', 'qemu:///system', 'snapshot-list', - vm_get_builder_id(provider)], - output=output, stderr_to_stdout=output) - return p.output - - -def vm_snapshot_clean_available(provider): - return 'fdroidclean' in vm_snapshot_list(provider) - - -def vm_snapshot_restore(provider): - """Does a rollback of the build vm. - """ - output = options.verbose - if provider is 'virtualbox': - p = FDroidPopen(['VBoxManage', 'snapshot', - vm_get_builder_id(provider), 'restore', - 'fdroidclean'], cwd='builder', - output=output, stderr_to_stdout=output) - elif provider is 'libvirt': - p = FDroidPopen(['virsh', '-c', 'qemu:///system', 'snapshot-revert', - vm_get_builder_id(provider), 'fdroidclean'], - output=output, stderr_to_stdout=output) - return p.returncode == 0 - - -def vm_snapshot_create(provider): - output = options.verbose - if provider is 'virtualbox': - p = FDroidPopen(['VBoxManage', 'snapshot', - vm_get_builder_id(provider), - 'take', 'fdroidclean'], cwd='builder', - output=output, stderr_to_stdout=output) - elif provider is 'libvirt': - p = FDroidPopen(['virsh', '-c', 'qemu:///system', 'snapshot-create-as', - vm_get_builder_id(provider), 'fdroidclean'], - output=output, stderr_to_stdout=output) - return p.returncode != 0 - - -def vm_test_ssh_into_builder(): - logging.info("Connecting to virtual machine...") - sshinfo = get_vagrant_sshinfo() - sshs = paramiko.SSHClient() - sshs.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - sshs.connect(sshinfo['hostname'], username=sshinfo['user'], - port=sshinfo['port'], timeout=300, - look_for_keys=False, - key_filename=sshinfo['idfile']) - sshs.close() - - def vm_new_get_clean_builder(serverdir, reset=False): if not os.path.isdir(serverdir): if os.path.islink(serverdir): @@ -277,134 +100,6 @@ def vm_new_get_clean_builder(serverdir, reset=False): return sshinfo -def vm_get_clean_builder(reset=False): - """Get a clean VM ready to do a buildserver build. - - This might involve creating and starting a new virtual machine from - scratch, or it might be as simple (unless overridden by the reset - parameter) as re-using a snapshot created previously. - - A BuildException will be raised if anything goes wrong. - - :reset: True to force creating from scratch. - :returns: A dictionary containing 'hostname', 'port', 'user' - and 'idfile' - """ - provider = get_vm_provider() - - # Reset existing builder machine to a clean state if possible. - vm_ok = False - if not reset: - logging.info("Checking for valid existing build server") - if vm_is_builder_valid(provider): - logging.info("...VM is present (%s)" % provider) - if vm_snapshot_clean_available(provider): - logging.info("...snapshot exists - resetting build server to " + - "clean state") - status = vm_get_builder_status() - if status == 'running': - vm_test_ssh_into_builder() - logging.info("...suspending builder vm") - vagrant(['suspend'], cwd='builder') - logging.info("...waiting a sec...") - time.sleep(10) - elif status == 'shutoff': - logging.info('...starting builder vm') - vagrant(['up'], cwd='builder') - logging.info('...waiting a sec...') - time.sleep(10) - vm_test_ssh_into_builder() - logging.info('...suspending builder vm') - vagrant(['suspend'], cwd='builder') - logging.info("...waiting a sec...") - time.sleep(10) - if options.verbose: - vm_get_builder_status() - - if vm_snapshot_restore(provider): - if options.verbose: - vm_get_builder_status() - logging.info("...reset to snapshot - server is valid") - retcode, output = vagrant(['up'], cwd='builder') - if retcode != 0: - raise BuildException("Failed to start build server") - logging.info("...waiting a sec...") - time.sleep(10) - sshinfo = get_vagrant_sshinfo() - vm_ok = True - else: - logging.info("...failed to reset to snapshot") - else: - logging.info("...snapshot doesn't exist - " - "VBoxManage snapshot list:\n" + - vm_snapshot_list(provider)) - else: - logging.info('...VM not present') - - # If we can't use the existing machine for any reason, make a - # new one from scratch. - if not vm_ok: - if os.path.isdir('builder'): - vm = vmtools.get_build_vm('builder') - vm.destroy() - else: - os.mkdir('builder') - - p = subprocess.Popen(['vagrant', '--version'], - universal_newlines=True, - stdout=subprocess.PIPE) - vver = p.communicate()[0].strip().split(' ')[1] - if vver.split('.')[0] != '1' or int(vver.split('.')[1]) < 4: - raise BuildException("Unsupported vagrant version {0}".format(vver)) - - with open(os.path.join('builder', 'Vagrantfile'), 'w') as vf: - vf.write('Vagrant.configure("2") do |config|\n') - vf.write(' config.vm.box = "buildserver"\n') - vf.write(' config.vm.synced_folder ".", "/vagrant", disabled: true\n') - vf.write('end\n') - - logging.info("Starting new build server") - retcode, _ = vagrant(['up'], cwd='builder') - if retcode != 0: - raise BuildException("Failed to start build server") - provider = get_vm_provider() - sshinfo = get_vagrant_sshinfo() - - # Open SSH connection to make sure it's working and ready... - vm_test_ssh_into_builder() - - logging.info("Saving clean state of new build server") - retcode, _ = vagrant(['suspend'], cwd='builder') - if retcode != 0: - raise BuildException("Failed to suspend build server") - logging.info("...waiting a sec...") - time.sleep(10) - if vm_snapshot_create(provider): - raise BuildException("Failed to take snapshot") - logging.info("...waiting a sec...") - time.sleep(10) - logging.info("Restarting new build server") - retcode, _ = vagrant(['up'], cwd='builder') - if retcode != 0: - raise BuildException("Failed to start build server") - logging.info("...waiting a sec...") - time.sleep(10) - # Make sure it worked... - if not vm_snapshot_clean_available(provider): - raise BuildException("Failed to take snapshot.") - - return sshinfo - - -def vm_suspend_builder(): - """Release the VM previously started with vm_get_clean_builder(). - - This should always be called after each individual app build attempt. - """ - logging.info("Suspending build server") - subprocess.call(['vagrant', 'suspend'], cwd='builder') - - # Note that 'force' here also implies test mode. def build_server(app, build, vcs, build_dir, output_dir, log_dir, force): """Do a build on the builder vm. diff --git a/fdroidserver/vmtools.py b/fdroidserver/vmtools.py index 974ce52d..e502f3d3 100644 --- a/fdroidserver/vmtools.py +++ b/fdroidserver/vmtools.py @@ -24,6 +24,7 @@ import tarfile import time import shutil import subprocess +import textwrap from .common import FDroidException from logging import getLogger diff --git a/makebuildserver b/makebuildserver index 7f759e68..0b376334 100755 --- a/makebuildserver +++ b/makebuildserver @@ -13,7 +13,6 @@ import hashlib import yaml import json import logging -import textwrap from clint.textui import progress from optparse import OptionParser import fdroidserver.tail