mirror of
https://gitlab.com/fdroid/fdroidserver.git
synced 2024-10-03 17:50:11 +02:00
removed unused buildserver code
This commit is contained in:
Michael Pöhn
Hans-Christoph Steiner
parent
9ef936c21a
commit
fc660048eb
@ -26,7 +26,6 @@ import re
|
|||||||
import tarfile
|
import tarfile
|
||||||
import traceback
|
import traceback
|
||||||
import time
|
import time
|
||||||
import json
|
|
||||||
import requests
|
import requests
|
||||||
import tempfile
|
import tempfile
|
||||||
import textwrap
|
import textwrap
|
||||||
@ -49,182 +48,6 @@ except ImportError:
|
|||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
def get_vm_provider():
|
|
||||||
"""Determine vm provider based on .vagrant directory content
|
|
||||||
"""
|
|
||||||
if os.path.exists(os.path.join('builder', '.vagrant', 'machines',
|
|
||||||
'default', 'libvirt')):
|
|
||||||
return 'libvirt'
|
|
||||||
return 'virtualbox'
|
|
||||||
|
|
||||||
|
|
||||||
def vm_get_builder_id(provider):
|
|
||||||
vd = os.path.join('builder', '.vagrant')
|
|
||||||
if os.path.isdir(vd):
|
|
||||||
# Vagrant 1.2 (and maybe 1.1?) it's a directory tree...
|
|
||||||
with open(os.path.join(vd, 'machines', 'default',
|
|
||||||
provider, 'id')) as vf:
|
|
||||||
id = vf.read()
|
|
||||||
return id
|
|
||||||
else:
|
|
||||||
# Vagrant 1.0 - it's a json file...
|
|
||||||
with open(os.path.join('builder', '.vagrant')) as vf:
|
|
||||||
v = json.load(vf)
|
|
||||||
return v['active']['default']
|
|
||||||
|
|
||||||
|
|
||||||
def vm_get_builder_status():
|
|
||||||
"""Get the current status of builder vm.
|
|
||||||
|
|
||||||
:returns: one of: 'running', 'paused', 'shutoff', 'not created'
|
|
||||||
If something is wrong with vagrant or the vm 'unknown' is returned.
|
|
||||||
"""
|
|
||||||
(ret, out) = vagrant(['status'], cwd='builder')
|
|
||||||
|
|
||||||
allowed_providers = 'virtualbox|libvirt'
|
|
||||||
allowed_states = 'running|paused|shutoff|not created'
|
|
||||||
|
|
||||||
r = re.compile('^\s*(?P<vagrant_name>\w+)\s+' +
|
|
||||||
'(?P<vm_state>' + allowed_states + ')' +
|
|
||||||
'\s+\((?P<provider>' + allowed_providers + ')\)\s*$')
|
|
||||||
|
|
||||||
for line in out.split('\n'):
|
|
||||||
m = r.match(line)
|
|
||||||
if m:
|
|
||||||
s = m.group('vm_state')
|
|
||||||
if options.verbose:
|
|
||||||
logging.debug('current builder vm status: ' + s)
|
|
||||||
return s
|
|
||||||
if options.verbose:
|
|
||||||
logging.debug('current builder vm status: unknown')
|
|
||||||
return 'unknown'
|
|
||||||
|
|
||||||
|
|
||||||
def vm_is_builder_valid(provider):
|
|
||||||
"""Returns True if we have a valid-looking builder vm
|
|
||||||
"""
|
|
||||||
if not os.path.exists(os.path.join('builder', 'Vagrantfile')):
|
|
||||||
return False
|
|
||||||
vd = os.path.join('builder', '.vagrant')
|
|
||||||
if not os.path.exists(vd):
|
|
||||||
return False
|
|
||||||
if not os.path.isdir(vd):
|
|
||||||
# Vagrant 1.0 - if the directory is there, it's valid...
|
|
||||||
return True
|
|
||||||
# Vagrant 1.2 - the directory can exist, but the id can be missing...
|
|
||||||
if not os.path.exists(os.path.join(vd, 'machines', 'default',
|
|
||||||
provider, 'id')):
|
|
||||||
return False
|
|
||||||
return True
|
|
||||||
|
|
||||||
|
|
||||||
def vagrant(params, cwd=None, printout=False):
|
|
||||||
"""Run a vagrant command.
|
|
||||||
|
|
||||||
:param: list of parameters to pass to vagrant
|
|
||||||
:cwd: directory to run in, or None for current directory
|
|
||||||
:printout: has not effect
|
|
||||||
:returns: (ret, out) where ret is the return code, and out
|
|
||||||
is the stdout (and stderr) from vagrant
|
|
||||||
"""
|
|
||||||
p = FDroidPopen(['vagrant'] + params, cwd=cwd, output=printout, stderr_to_stdout=printout)
|
|
||||||
return (p.returncode, p.output)
|
|
||||||
|
|
||||||
|
|
||||||
def get_vagrant_sshinfo():
|
|
||||||
"""Get ssh connection info for a vagrant VM
|
|
||||||
|
|
||||||
:returns: A dictionary containing 'hostname', 'port', 'user'
|
|
||||||
and 'idfile'
|
|
||||||
"""
|
|
||||||
if subprocess.call('vagrant ssh-config >sshconfig',
|
|
||||||
cwd='builder', shell=True) != 0:
|
|
||||||
raise BuildException("Error getting ssh config")
|
|
||||||
vagranthost = 'default' # Host in ssh config file
|
|
||||||
sshconfig = paramiko.SSHConfig()
|
|
||||||
sshf = open(os.path.join('builder', 'sshconfig'), 'r')
|
|
||||||
sshconfig.parse(sshf)
|
|
||||||
sshf.close()
|
|
||||||
sshconfig = sshconfig.lookup(vagranthost)
|
|
||||||
idfile = sshconfig['identityfile']
|
|
||||||
if isinstance(idfile, list):
|
|
||||||
idfile = idfile[0]
|
|
||||||
elif idfile.startswith('"') and idfile.endswith('"'):
|
|
||||||
idfile = idfile[1:-1]
|
|
||||||
return {'hostname': sshconfig['hostname'],
|
|
||||||
'port': int(sshconfig['port']),
|
|
||||||
'user': sshconfig['user'],
|
|
||||||
'idfile': idfile}
|
|
||||||
|
|
||||||
|
|
||||||
def vm_shutdown_builder():
|
|
||||||
"""Turn off builder vm.
|
|
||||||
"""
|
|
||||||
if options.server:
|
|
||||||
if os.path.exists(os.path.join('builder', 'Vagrantfile')):
|
|
||||||
vagrant(['halt'], cwd='builder')
|
|
||||||
|
|
||||||
|
|
||||||
def vm_snapshot_list(provider):
|
|
||||||
output = options.verbose
|
|
||||||
if provider is 'virtualbox':
|
|
||||||
p = FDroidPopen(['VBoxManage', 'snapshot',
|
|
||||||
vm_get_builder_id(provider), 'list',
|
|
||||||
'--details'], cwd='builder',
|
|
||||||
output=output, stderr_to_stdout=output)
|
|
||||||
elif provider is 'libvirt':
|
|
||||||
p = FDroidPopen(['virsh', '-c', 'qemu:///system', 'snapshot-list',
|
|
||||||
vm_get_builder_id(provider)],
|
|
||||||
output=output, stderr_to_stdout=output)
|
|
||||||
return p.output
|
|
||||||
|
|
||||||
|
|
||||||
def vm_snapshot_clean_available(provider):
|
|
||||||
return 'fdroidclean' in vm_snapshot_list(provider)
|
|
||||||
|
|
||||||
|
|
||||||
def vm_snapshot_restore(provider):
|
|
||||||
"""Does a rollback of the build vm.
|
|
||||||
"""
|
|
||||||
output = options.verbose
|
|
||||||
if provider is 'virtualbox':
|
|
||||||
p = FDroidPopen(['VBoxManage', 'snapshot',
|
|
||||||
vm_get_builder_id(provider), 'restore',
|
|
||||||
'fdroidclean'], cwd='builder',
|
|
||||||
output=output, stderr_to_stdout=output)
|
|
||||||
elif provider is 'libvirt':
|
|
||||||
p = FDroidPopen(['virsh', '-c', 'qemu:///system', 'snapshot-revert',
|
|
||||||
vm_get_builder_id(provider), 'fdroidclean'],
|
|
||||||
output=output, stderr_to_stdout=output)
|
|
||||||
return p.returncode == 0
|
|
||||||
|
|
||||||
|
|
||||||
def vm_snapshot_create(provider):
|
|
||||||
output = options.verbose
|
|
||||||
if provider is 'virtualbox':
|
|
||||||
p = FDroidPopen(['VBoxManage', 'snapshot',
|
|
||||||
vm_get_builder_id(provider),
|
|
||||||
'take', 'fdroidclean'], cwd='builder',
|
|
||||||
output=output, stderr_to_stdout=output)
|
|
||||||
elif provider is 'libvirt':
|
|
||||||
p = FDroidPopen(['virsh', '-c', 'qemu:///system', 'snapshot-create-as',
|
|
||||||
vm_get_builder_id(provider), 'fdroidclean'],
|
|
||||||
output=output, stderr_to_stdout=output)
|
|
||||||
return p.returncode != 0
|
|
||||||
|
|
||||||
|
|
||||||
def vm_test_ssh_into_builder():
|
|
||||||
logging.info("Connecting to virtual machine...")
|
|
||||||
sshinfo = get_vagrant_sshinfo()
|
|
||||||
sshs = paramiko.SSHClient()
|
|
||||||
sshs.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
|
||||||
sshs.connect(sshinfo['hostname'], username=sshinfo['user'],
|
|
||||||
port=sshinfo['port'], timeout=300,
|
|
||||||
look_for_keys=False,
|
|
||||||
key_filename=sshinfo['idfile'])
|
|
||||||
sshs.close()
|
|
||||||
|
|
||||||
|
|
||||||
def vm_new_get_clean_builder(serverdir, reset=False):
|
def vm_new_get_clean_builder(serverdir, reset=False):
|
||||||
if not os.path.isdir(serverdir):
|
if not os.path.isdir(serverdir):
|
||||||
if os.path.islink(serverdir):
|
if os.path.islink(serverdir):
|
||||||
@ -277,134 +100,6 @@ def vm_new_get_clean_builder(serverdir, reset=False):
|
|||||||
return sshinfo
|
return sshinfo
|
||||||
|
|
||||||
|
|
||||||
def vm_get_clean_builder(reset=False):
|
|
||||||
"""Get a clean VM ready to do a buildserver build.
|
|
||||||
|
|
||||||
This might involve creating and starting a new virtual machine from
|
|
||||||
scratch, or it might be as simple (unless overridden by the reset
|
|
||||||
parameter) as re-using a snapshot created previously.
|
|
||||||
|
|
||||||
A BuildException will be raised if anything goes wrong.
|
|
||||||
|
|
||||||
:reset: True to force creating from scratch.
|
|
||||||
:returns: A dictionary containing 'hostname', 'port', 'user'
|
|
||||||
and 'idfile'
|
|
||||||
"""
|
|
||||||
provider = get_vm_provider()
|
|
||||||
|
|
||||||
# Reset existing builder machine to a clean state if possible.
|
|
||||||
vm_ok = False
|
|
||||||
if not reset:
|
|
||||||
logging.info("Checking for valid existing build server")
|
|
||||||
if vm_is_builder_valid(provider):
|
|
||||||
logging.info("...VM is present (%s)" % provider)
|
|
||||||
if vm_snapshot_clean_available(provider):
|
|
||||||
logging.info("...snapshot exists - resetting build server to " +
|
|
||||||
"clean state")
|
|
||||||
status = vm_get_builder_status()
|
|
||||||
if status == 'running':
|
|
||||||
vm_test_ssh_into_builder()
|
|
||||||
logging.info("...suspending builder vm")
|
|
||||||
vagrant(['suspend'], cwd='builder')
|
|
||||||
logging.info("...waiting a sec...")
|
|
||||||
time.sleep(10)
|
|
||||||
elif status == 'shutoff':
|
|
||||||
logging.info('...starting builder vm')
|
|
||||||
vagrant(['up'], cwd='builder')
|
|
||||||
logging.info('...waiting a sec...')
|
|
||||||
time.sleep(10)
|
|
||||||
vm_test_ssh_into_builder()
|
|
||||||
logging.info('...suspending builder vm')
|
|
||||||
vagrant(['suspend'], cwd='builder')
|
|
||||||
logging.info("...waiting a sec...")
|
|
||||||
time.sleep(10)
|
|
||||||
if options.verbose:
|
|
||||||
vm_get_builder_status()
|
|
||||||
|
|
||||||
if vm_snapshot_restore(provider):
|
|
||||||
if options.verbose:
|
|
||||||
vm_get_builder_status()
|
|
||||||
logging.info("...reset to snapshot - server is valid")
|
|
||||||
retcode, output = vagrant(['up'], cwd='builder')
|
|
||||||
if retcode != 0:
|
|
||||||
raise BuildException("Failed to start build server")
|
|
||||||
logging.info("...waiting a sec...")
|
|
||||||
time.sleep(10)
|
|
||||||
sshinfo = get_vagrant_sshinfo()
|
|
||||||
vm_ok = True
|
|
||||||
else:
|
|
||||||
logging.info("...failed to reset to snapshot")
|
|
||||||
else:
|
|
||||||
logging.info("...snapshot doesn't exist - "
|
|
||||||
"VBoxManage snapshot list:\n" +
|
|
||||||
vm_snapshot_list(provider))
|
|
||||||
else:
|
|
||||||
logging.info('...VM not present')
|
|
||||||
|
|
||||||
# If we can't use the existing machine for any reason, make a
|
|
||||||
# new one from scratch.
|
|
||||||
if not vm_ok:
|
|
||||||
if os.path.isdir('builder'):
|
|
||||||
vm = vmtools.get_build_vm('builder')
|
|
||||||
vm.destroy()
|
|
||||||
else:
|
|
||||||
os.mkdir('builder')
|
|
||||||
|
|
||||||
p = subprocess.Popen(['vagrant', '--version'],
|
|
||||||
universal_newlines=True,
|
|
||||||
stdout=subprocess.PIPE)
|
|
||||||
vver = p.communicate()[0].strip().split(' ')[1]
|
|
||||||
if vver.split('.')[0] != '1' or int(vver.split('.')[1]) < 4:
|
|
||||||
raise BuildException("Unsupported vagrant version {0}".format(vver))
|
|
||||||
|
|
||||||
with open(os.path.join('builder', 'Vagrantfile'), 'w') as vf:
|
|
||||||
vf.write('Vagrant.configure("2") do |config|\n')
|
|
||||||
vf.write(' config.vm.box = "buildserver"\n')
|
|
||||||
vf.write(' config.vm.synced_folder ".", "/vagrant", disabled: true\n')
|
|
||||||
vf.write('end\n')
|
|
||||||
|
|
||||||
logging.info("Starting new build server")
|
|
||||||
retcode, _ = vagrant(['up'], cwd='builder')
|
|
||||||
if retcode != 0:
|
|
||||||
raise BuildException("Failed to start build server")
|
|
||||||
provider = get_vm_provider()
|
|
||||||
sshinfo = get_vagrant_sshinfo()
|
|
||||||
|
|
||||||
# Open SSH connection to make sure it's working and ready...
|
|
||||||
vm_test_ssh_into_builder()
|
|
||||||
|
|
||||||
logging.info("Saving clean state of new build server")
|
|
||||||
retcode, _ = vagrant(['suspend'], cwd='builder')
|
|
||||||
if retcode != 0:
|
|
||||||
raise BuildException("Failed to suspend build server")
|
|
||||||
logging.info("...waiting a sec...")
|
|
||||||
time.sleep(10)
|
|
||||||
if vm_snapshot_create(provider):
|
|
||||||
raise BuildException("Failed to take snapshot")
|
|
||||||
logging.info("...waiting a sec...")
|
|
||||||
time.sleep(10)
|
|
||||||
logging.info("Restarting new build server")
|
|
||||||
retcode, _ = vagrant(['up'], cwd='builder')
|
|
||||||
if retcode != 0:
|
|
||||||
raise BuildException("Failed to start build server")
|
|
||||||
logging.info("...waiting a sec...")
|
|
||||||
time.sleep(10)
|
|
||||||
# Make sure it worked...
|
|
||||||
if not vm_snapshot_clean_available(provider):
|
|
||||||
raise BuildException("Failed to take snapshot.")
|
|
||||||
|
|
||||||
return sshinfo
|
|
||||||
|
|
||||||
|
|
||||||
def vm_suspend_builder():
|
|
||||||
"""Release the VM previously started with vm_get_clean_builder().
|
|
||||||
|
|
||||||
This should always be called after each individual app build attempt.
|
|
||||||
"""
|
|
||||||
logging.info("Suspending build server")
|
|
||||||
subprocess.call(['vagrant', 'suspend'], cwd='builder')
|
|
||||||
|
|
||||||
|
|
||||||
# Note that 'force' here also implies test mode.
|
# Note that 'force' here also implies test mode.
|
||||||
def build_server(app, build, vcs, build_dir, output_dir, log_dir, force):
|
def build_server(app, build, vcs, build_dir, output_dir, log_dir, force):
|
||||||
"""Do a build on the builder vm.
|
"""Do a build on the builder vm.
|
||||||
|
|||||||
@ -24,6 +24,7 @@ import tarfile
|
|||||||
import time
|
import time
|
||||||
import shutil
|
import shutil
|
||||||
import subprocess
|
import subprocess
|
||||||
|
import textwrap
|
||||||
from .common import FDroidException
|
from .common import FDroidException
|
||||||
from logging import getLogger
|
from logging import getLogger
|
||||||
|
|
||||||
|
|||||||
@ -13,7 +13,6 @@ import hashlib
|
|||||||
import yaml
|
import yaml
|
||||||
import json
|
import json
|
||||||
import logging
|
import logging
|
||||||
import textwrap
|
|
||||||
from clint.textui import progress
|
from clint.textui import progress
|
||||||
from optparse import OptionParser
|
from optparse import OptionParser
|
||||||
import fdroidserver.tail
|
import fdroidserver.tail
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user