mirror of
https://gitlab.com/fdroid/fdroidserver.git
synced 2024-07-04 16:30:12 +02:00
removed unused buildserver code
This commit is contained in:
Michael Pöhn
Hans-Christoph Steiner
parent
9ef936c21a
commit
fc660048eb
|
|
@ -26,7 +26,6 @@ import re
|
|||
import tarfile
|
||||
import traceback
|
||||
import time
|
||||
import json
|
||||
import requests
|
||||
import tempfile
|
||||
import textwrap
|
||||
|
|
@ -49,182 +48,6 @@ except ImportError:
|
|||
pass
|
||||
|
||||
|
||||
def get_vm_provider():
|
||||
"""Determine vm provider based on .vagrant directory content
|
||||
"""
|
||||
if os.path.exists(os.path.join('builder', '.vagrant', 'machines',
|
||||
'default', 'libvirt')):
|
||||
return 'libvirt'
|
||||
return 'virtualbox'
|
||||
|
||||
|
||||
def vm_get_builder_id(provider):
|
||||
vd = os.path.join('builder', '.vagrant')
|
||||
if os.path.isdir(vd):
|
||||
# Vagrant 1.2 (and maybe 1.1?) it's a directory tree...
|
||||
with open(os.path.join(vd, 'machines', 'default',
|
||||
provider, 'id')) as vf:
|
||||
id = vf.read()
|
||||
return id
|
||||
else:
|
||||
# Vagrant 1.0 - it's a json file...
|
||||
with open(os.path.join('builder', '.vagrant')) as vf:
|
||||
v = json.load(vf)
|
||||
return v['active']['default']
|
||||
|
||||
|
||||
def vm_get_builder_status():
|
||||
"""Get the current status of builder vm.
|
||||
|
||||
:returns: one of: 'running', 'paused', 'shutoff', 'not created'
|
||||
If something is wrong with vagrant or the vm 'unknown' is returned.
|
||||
"""
|
||||
(ret, out) = vagrant(['status'], cwd='builder')
|
||||
|
||||
allowed_providers = 'virtualbox|libvirt'
|
||||
allowed_states = 'running|paused|shutoff|not created'
|
||||
|
||||
r = re.compile('^\s*(?P<vagrant_name>\w+)\s+' +
|
||||
'(?P<vm_state>' + allowed_states + ')' +
|
||||
'\s+\((?P<provider>' + allowed_providers + ')\)\s*$')
|
||||
|
||||
for line in out.split('\n'):
|
||||
m = r.match(line)
|
||||
if m:
|
||||
s = m.group('vm_state')
|
||||
if options.verbose:
|
||||
logging.debug('current builder vm status: ' + s)
|
||||
return s
|
||||
if options.verbose:
|
||||
logging.debug('current builder vm status: unknown')
|
||||
return 'unknown'
|
||||
|
||||
|
||||
def vm_is_builder_valid(provider):
|
||||
"""Returns True if we have a valid-looking builder vm
|
||||
"""
|
||||
if not os.path.exists(os.path.join('builder', 'Vagrantfile')):
|
||||
return False
|
||||
vd = os.path.join('builder', '.vagrant')
|
||||
if not os.path.exists(vd):
|
||||
return False
|
||||
if not os.path.isdir(vd):
|
||||
# Vagrant 1.0 - if the directory is there, it's valid...
|
||||
return True
|
||||
# Vagrant 1.2 - the directory can exist, but the id can be missing...
|
||||
if not os.path.exists(os.path.join(vd, 'machines', 'default',
|
||||
provider, 'id')):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def vagrant(params, cwd=None, printout=False):
|
||||
"""Run a vagrant command.
|
||||
|
||||
:param: list of parameters to pass to vagrant
|
||||
:cwd: directory to run in, or None for current directory
|
||||
:printout: has not effect
|
||||
:returns: (ret, out) where ret is the return code, and out
|
||||
is the stdout (and stderr) from vagrant
|
||||
"""
|
||||
p = FDroidPopen(['vagrant'] + params, cwd=cwd, output=printout, stderr_to_stdout=printout)
|
||||
return (p.returncode, p.output)
|
||||
|
||||
|
||||
def get_vagrant_sshinfo():
|
||||
"""Get ssh connection info for a vagrant VM
|
||||
|
||||
:returns: A dictionary containing 'hostname', 'port', 'user'
|
||||
and 'idfile'
|
||||
"""
|
||||
if subprocess.call('vagrant ssh-config >sshconfig',
|
||||
cwd='builder', shell=True) != 0:
|
||||
raise BuildException("Error getting ssh config")
|
||||
vagranthost = 'default' # Host in ssh config file
|
||||
sshconfig = paramiko.SSHConfig()
|
||||
sshf = open(os.path.join('builder', 'sshconfig'), 'r')
|
||||
sshconfig.parse(sshf)
|
||||
sshf.close()
|
||||
sshconfig = sshconfig.lookup(vagranthost)
|
||||
idfile = sshconfig['identityfile']
|
||||
if isinstance(idfile, list):
|
||||
idfile = idfile[0]
|
||||
elif idfile.startswith('"') and idfile.endswith('"'):
|
||||
idfile = idfile[1:-1]
|
||||
return {'hostname': sshconfig['hostname'],
|
||||
'port': int(sshconfig['port']),
|
||||
'user': sshconfig['user'],
|
||||
'idfile': idfile}
|
||||
|
||||
|
||||
def vm_shutdown_builder():
|
||||
"""Turn off builder vm.
|
||||
"""
|
||||
if options.server:
|
||||
if os.path.exists(os.path.join('builder', 'Vagrantfile')):
|
||||
vagrant(['halt'], cwd='builder')
|
||||
|
||||
|
||||
def vm_snapshot_list(provider):
|
||||
output = options.verbose
|
||||
if provider is 'virtualbox':
|
||||
p = FDroidPopen(['VBoxManage', 'snapshot',
|
||||
vm_get_builder_id(provider), 'list',
|
||||
'--details'], cwd='builder',
|
||||
output=output, stderr_to_stdout=output)
|
||||
elif provider is 'libvirt':
|
||||
p = FDroidPopen(['virsh', '-c', 'qemu:///system', 'snapshot-list',
|
||||
vm_get_builder_id(provider)],
|
||||
output=output, stderr_to_stdout=output)
|
||||
return p.output
|
||||
|
||||
|
||||
def vm_snapshot_clean_available(provider):
|
||||
return 'fdroidclean' in vm_snapshot_list(provider)
|
||||
|
||||
|
||||
def vm_snapshot_restore(provider):
|
||||
"""Does a rollback of the build vm.
|
||||
"""
|
||||
output = options.verbose
|
||||
if provider is 'virtualbox':
|
||||
p = FDroidPopen(['VBoxManage', 'snapshot',
|
||||
vm_get_builder_id(provider), 'restore',
|
||||
'fdroidclean'], cwd='builder',
|
||||
output=output, stderr_to_stdout=output)
|
||||
elif provider is 'libvirt':
|
||||
p = FDroidPopen(['virsh', '-c', 'qemu:///system', 'snapshot-revert',
|
||||
vm_get_builder_id(provider), 'fdroidclean'],
|
||||
output=output, stderr_to_stdout=output)
|
||||
return p.returncode == 0
|
||||
|
||||
|
||||
def vm_snapshot_create(provider):
|
||||
output = options.verbose
|
||||
if provider is 'virtualbox':
|
||||
p = FDroidPopen(['VBoxManage', 'snapshot',
|
||||
vm_get_builder_id(provider),
|
||||
'take', 'fdroidclean'], cwd='builder',
|
||||
output=output, stderr_to_stdout=output)
|
||||
elif provider is 'libvirt':
|
||||
p = FDroidPopen(['virsh', '-c', 'qemu:///system', 'snapshot-create-as',
|
||||
vm_get_builder_id(provider), 'fdroidclean'],
|
||||
output=output, stderr_to_stdout=output)
|
||||
return p.returncode != 0
|
||||
|
||||
|
||||
def vm_test_ssh_into_builder():
|
||||
logging.info("Connecting to virtual machine...")
|
||||
sshinfo = get_vagrant_sshinfo()
|
||||
sshs = paramiko.SSHClient()
|
||||
sshs.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
sshs.connect(sshinfo['hostname'], username=sshinfo['user'],
|
||||
port=sshinfo['port'], timeout=300,
|
||||
look_for_keys=False,
|
||||
key_filename=sshinfo['idfile'])
|
||||
sshs.close()
|
||||
|
||||
|
||||
def vm_new_get_clean_builder(serverdir, reset=False):
|
||||
if not os.path.isdir(serverdir):
|
||||
if os.path.islink(serverdir):
|
||||
|
|
@ -277,134 +100,6 @@ def vm_new_get_clean_builder(serverdir, reset=False):
|
|||
return sshinfo
|
||||
|
||||
|
||||
def vm_get_clean_builder(reset=False):
|
||||
"""Get a clean VM ready to do a buildserver build.
|
||||
|
||||
This might involve creating and starting a new virtual machine from
|
||||
scratch, or it might be as simple (unless overridden by the reset
|
||||
parameter) as re-using a snapshot created previously.
|
||||
|
||||
A BuildException will be raised if anything goes wrong.
|
||||
|
||||
:reset: True to force creating from scratch.
|
||||
:returns: A dictionary containing 'hostname', 'port', 'user'
|
||||
and 'idfile'
|
||||
"""
|
||||
provider = get_vm_provider()
|
||||
|
||||
# Reset existing builder machine to a clean state if possible.
|
||||
vm_ok = False
|
||||
if not reset:
|
||||
logging.info("Checking for valid existing build server")
|
||||
if vm_is_builder_valid(provider):
|
||||
logging.info("...VM is present (%s)" % provider)
|
||||
if vm_snapshot_clean_available(provider):
|
||||
logging.info("...snapshot exists - resetting build server to " +
|
||||
"clean state")
|
||||
status = vm_get_builder_status()
|
||||
if status == 'running':
|
||||
vm_test_ssh_into_builder()
|
||||
logging.info("...suspending builder vm")
|
||||
vagrant(['suspend'], cwd='builder')
|
||||
logging.info("...waiting a sec...")
|
||||
time.sleep(10)
|
||||
elif status == 'shutoff':
|
||||
logging.info('...starting builder vm')
|
||||
vagrant(['up'], cwd='builder')
|
||||
logging.info('...waiting a sec...')
|
||||
time.sleep(10)
|
||||
vm_test_ssh_into_builder()
|
||||
logging.info('...suspending builder vm')
|
||||
vagrant(['suspend'], cwd='builder')
|
||||
logging.info("...waiting a sec...")
|
||||
time.sleep(10)
|
||||
if options.verbose:
|
||||
vm_get_builder_status()
|
||||
|
||||
if vm_snapshot_restore(provider):
|
||||
if options.verbose:
|
||||
vm_get_builder_status()
|
||||
logging.info("...reset to snapshot - server is valid")
|
||||
retcode, output = vagrant(['up'], cwd='builder')
|
||||
if retcode != 0:
|
||||
raise BuildException("Failed to start build server")
|
||||
logging.info("...waiting a sec...")
|
||||
time.sleep(10)
|
||||
sshinfo = get_vagrant_sshinfo()
|
||||
vm_ok = True
|
||||
else:
|
||||
logging.info("...failed to reset to snapshot")
|
||||
else:
|
||||
logging.info("...snapshot doesn't exist - "
|
||||
"VBoxManage snapshot list:\n" +
|
||||
vm_snapshot_list(provider))
|
||||
else:
|
||||
logging.info('...VM not present')
|
||||
|
||||
# If we can't use the existing machine for any reason, make a
|
||||
# new one from scratch.
|
||||
if not vm_ok:
|
||||
if os.path.isdir('builder'):
|
||||
vm = vmtools.get_build_vm('builder')
|
||||
vm.destroy()
|
||||
else:
|
||||
os.mkdir('builder')
|
||||
|
||||
p = subprocess.Popen(['vagrant', '--version'],
|
||||
universal_newlines=True,
|
||||
stdout=subprocess.PIPE)
|
||||
vver = p.communicate()[0].strip().split(' ')[1]
|
||||
if vver.split('.')[0] != '1' or int(vver.split('.')[1]) < 4:
|
||||
raise BuildException("Unsupported vagrant version {0}".format(vver))
|
||||
|
||||
with open(os.path.join('builder', 'Vagrantfile'), 'w') as vf:
|
||||
vf.write('Vagrant.configure("2") do |config|\n')
|
||||
vf.write(' config.vm.box = "buildserver"\n')
|
||||
vf.write(' config.vm.synced_folder ".", "/vagrant", disabled: true\n')
|
||||
vf.write('end\n')
|
||||
|
||||
logging.info("Starting new build server")
|
||||
retcode, _ = vagrant(['up'], cwd='builder')
|
||||
if retcode != 0:
|
||||
raise BuildException("Failed to start build server")
|
||||
provider = get_vm_provider()
|
||||
sshinfo = get_vagrant_sshinfo()
|
||||
|
||||
# Open SSH connection to make sure it's working and ready...
|
||||
vm_test_ssh_into_builder()
|
||||
|
||||
logging.info("Saving clean state of new build server")
|
||||
retcode, _ = vagrant(['suspend'], cwd='builder')
|
||||
if retcode != 0:
|
||||
raise BuildException("Failed to suspend build server")
|
||||
logging.info("...waiting a sec...")
|
||||
time.sleep(10)
|
||||
if vm_snapshot_create(provider):
|
||||
raise BuildException("Failed to take snapshot")
|
||||
logging.info("...waiting a sec...")
|
||||
time.sleep(10)
|
||||
logging.info("Restarting new build server")
|
||||
retcode, _ = vagrant(['up'], cwd='builder')
|
||||
if retcode != 0:
|
||||
raise BuildException("Failed to start build server")
|
||||
logging.info("...waiting a sec...")
|
||||
time.sleep(10)
|
||||
# Make sure it worked...
|
||||
if not vm_snapshot_clean_available(provider):
|
||||
raise BuildException("Failed to take snapshot.")
|
||||
|
||||
return sshinfo
|
||||
|
||||
|
||||
def vm_suspend_builder():
|
||||
"""Release the VM previously started with vm_get_clean_builder().
|
||||
|
||||
This should always be called after each individual app build attempt.
|
||||
"""
|
||||
logging.info("Suspending build server")
|
||||
subprocess.call(['vagrant', 'suspend'], cwd='builder')
|
||||
|
||||
|
||||
# Note that 'force' here also implies test mode.
|
||||
def build_server(app, build, vcs, build_dir, output_dir, log_dir, force):
|
||||
"""Do a build on the builder vm.
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ import tarfile
|
|||
import time
|
||||
import shutil
|
||||
import subprocess
|
||||
import textwrap
|
||||
from .common import FDroidException
|
||||
from logging import getLogger
|
||||
|
||||
|
|
|
|||
|
|
@ -13,7 +13,6 @@ import hashlib
|
|||
import yaml
|
||||
import json
|
||||
import logging
|
||||
import textwrap
|
||||
from clint.textui import progress
|
||||
from optparse import OptionParser
|
||||
import fdroidserver.tail
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user