From cdfa020ba715a214f132dcc621d93ad493ccca46 Mon Sep 17 00:00:00 2001 From: Ciaran Gultnieks Date: Thu, 6 Sep 2012 18:36:34 +0100 Subject: [PATCH] Buildserver snapshot refinements --- fdroidserver/build.py | 242 ++++++++++++++++++++++-------------------- makebuildserver.sh | 7 +- 2 files changed, 129 insertions(+), 120 deletions(-) diff --git a/fdroidserver/build.py b/fdroidserver/build.py index 239b920e..4efb4aa3 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -49,7 +49,12 @@ def build_server(app, thisbuild, build_dir, output_dir): output = p.communicate()[0] if output.find('fdroidclean') != -1: print "...snapshot exists - resetting build server to clean state" - subprocess.call(['vagrant', 'suspend'], cwd='builder') + p = subprocess.Popen(['vagrant', 'status'], + cwd='builder', stdout=subprocess.PIPE) + output = p.communicate()[0] + if output.find('running') != -1: + print "...suspending" + subprocess.call(['vagrant', 'suspend'], cwd='builder') if subprocess.call(['vagrant', 'snap', 'go', 'fdroidclean'], cwd='builder') == 0: if subprocess.call(['vagrant', 'up'], cwd='builder') != 0: @@ -89,122 +94,129 @@ def build_server(app, thisbuild, build_dir, output_dir): if output.find('fdroidclean') == -1: raise BuildException("Failed to take snapshot.") - # Get SSH configuration settings for us to connect... - subprocess.call('vagrant ssh-config >sshconfig', - cwd='builder', shell=True) - vagranthost = 'default' # Host in ssh config file - - # Load and parse the SSH config... - sshconfig = ssh.SSHConfig() - sshf = open('builder/sshconfig', 'r') - sshconfig.parse(sshf) - sshf.close() - sshconfig = sshconfig.lookup(vagranthost) - - # Open SSH connection... - sshs = ssh.SSHClient() - sshs.set_missing_host_key_policy(ssh.AutoAddPolicy()) - sshs.connect(sshconfig['hostname'], username=sshconfig['user'], - port=int(sshconfig['port']), timeout=10, look_for_keys=False, - key_filename=sshconfig['identityfile']) - - # Get an SFTP connection... - ftp = sshs.open_sftp() - ftp.get_channel().settimeout(15) - - # Put all the necessary files in place... - ftp.chdir('/home/vagrant') - - # Helper to copy the contents of a directory to the server... - def send_dir(path): - root = os.path.dirname(path) - main = os.path.basename(path) - ftp.mkdir(main) - for r, d, f in os.walk(path): - rr = os.path.relpath(r, root) - ftp.chdir(rr) - for dd in d: - ftp.mkdir(dd) - for ff in f: - ftp.put(os.path.join(root, rr, ff), ff) - for i in range(len(rr.split('/'))): - ftp.chdir('..') - ftp.chdir('..') - - print "Preparing server for build..." - serverpath = os.path.abspath(os.path.dirname(__file__)) - ftp.put(os.path.join(serverpath, 'build.py'), 'build.py') - ftp.put(os.path.join(serverpath, 'common.py'), 'common.py') - ftp.put(os.path.join(serverpath, '..', 'config.buildserver.py'), 'config.py') - - # Copy the metadata - just the file for this app... - ftp.mkdir('metadata') - ftp.chdir('metadata') - ftp.put(os.path.join('metadata', app['id'] + '.txt'), - app['id'] + '.txt') - ftp.chdir('..') - # Create the build directory... - ftp.mkdir('build') - ftp.chdir('build') - ftp.mkdir('extlib') - # Copy the main app source code - if os.path.exists(build_dir): - send_dir(build_dir) - # Copy any extlibs that are required... - if thisbuild.has_key('extlibs'): - ftp.chdir('build') - ftp.chdir('extlib') - for lib in thisbuild['extlibs'].split(';'): - lp = lib.split('/') - for d in lp[:-1]: - ftp.mkdir(d) - ftp.chdir(d) - ftp.put(os.path.join('build/extlib', lib), lp[-1]) - for _ in lp[:-1]: - ftp.chdir('..') - ftp.chdir('..') - ftp.chdir('..') - # Copy any srclibs that are required... - if thisbuild.has_key('srclibs'): - ftp.chdir('build') - ftp.chdir('extlib') - for lib in thisbuild['srclibs'].split(';'): - lp = lib.split('@').split('/') - for d in lp[:-1]: - ftp.mkdir(d) - ftp.chdir(d) - ftp.put(os.path.join('build/extlib', lib), lp[-1]) - for _ in lp[:-1]: - ftp.chdir('..') - ftp.chdir('..') - ftp.chdir('..') - - # Execute the build script... - print "Starting build..." - chan = sshs.get_transport().open_session() - stdoutf = chan.makefile('rb') - stderrf = chan.makefile_stderr('rb') - chan.exec_command('python build.py --on-server -p ' + - app['id'] + ' --vercode ' + thisbuild['vercode']) - returncode = chan.recv_exit_status() - output = stdoutf.read() - error = stderrf.read() - if returncode != 0: - raise BuildException("Build.py failed on server for %s:%s" % (app['id'], thisbuild['version']), output.strip(), error.strip()) - - # Retrieve the built files... - ftp.chdir('/home/vagrant/unsigned') - apkfile = app['id'] + '_' + thisbuild['vercode'] + '.apk' - tarball = app['id'] + '_' + thisbuild['vercode'] + '_src' + '.tar.gz' try: - ftp.get(apkfile, os.path.join(output_dir, apkfile)) - ftp.get(tarball, os.path.join(output_dir, tarball)) - except: - raise BuildException("Build failed for %s:%s" % (app['id'], thisbuild['version']), output.strip(), error.strip()) - ftp.close() - # Suspend the buildserver... - subprocess.call(['vagrant', 'suspend'], cwd='builder') + # Get SSH configuration settings for us to connect... + print "Getting ssh configuration..." + subprocess.call('vagrant ssh-config >sshconfig', + cwd='builder', shell=True) + vagranthost = 'default' # Host in ssh config file + + # Load and parse the SSH config... + sshconfig = ssh.SSHConfig() + sshf = open('builder/sshconfig', 'r') + sshconfig.parse(sshf) + sshf.close() + sshconfig = sshconfig.lookup(vagranthost) + + # Open SSH connection... + print "Connecting to virtual machine..." + sshs = ssh.SSHClient() + sshs.set_missing_host_key_policy(ssh.AutoAddPolicy()) + sshs.connect(sshconfig['hostname'], username=sshconfig['user'], + port=int(sshconfig['port']), timeout=10, look_for_keys=False, + key_filename=sshconfig['identityfile']) + + # Get an SFTP connection... + ftp = sshs.open_sftp() + ftp.get_channel().settimeout(15) + + # Put all the necessary files in place... + ftp.chdir('/home/vagrant') + + # Helper to copy the contents of a directory to the server... + def send_dir(path): + root = os.path.dirname(path) + main = os.path.basename(path) + ftp.mkdir(main) + for r, d, f in os.walk(path): + rr = os.path.relpath(r, root) + ftp.chdir(rr) + for dd in d: + ftp.mkdir(dd) + for ff in f: + if not os.path.islink(os.path.join(root, rr, ff)): + ftp.put(os.path.join(root, rr, ff), ff) + for i in range(len(rr.split('/'))): + ftp.chdir('..') + ftp.chdir('..') + + print "Preparing server for build..." + serverpath = os.path.abspath(os.path.dirname(__file__)) + ftp.put(os.path.join(serverpath, 'build.py'), 'build.py') + ftp.put(os.path.join(serverpath, 'common.py'), 'common.py') + ftp.put(os.path.join(serverpath, '..', 'config.buildserver.py'), 'config.py') + + # Copy the metadata - just the file for this app... + ftp.mkdir('metadata') + ftp.chdir('metadata') + ftp.put(os.path.join('metadata', app['id'] + '.txt'), + app['id'] + '.txt') + ftp.chdir('..') + # Create the build directory... + ftp.mkdir('build') + ftp.chdir('build') + ftp.mkdir('extlib') + # Copy the main app source code + if os.path.exists(build_dir): + send_dir(build_dir) + # Copy any extlibs that are required... + if thisbuild.has_key('extlibs'): + ftp.chdir('build') + ftp.chdir('extlib') + for lib in thisbuild['extlibs'].split(';'): + lp = lib.split('/') + for d in lp[:-1]: + ftp.mkdir(d) + ftp.chdir(d) + ftp.put(os.path.join('build/extlib', lib), lp[-1]) + for _ in lp[:-1]: + ftp.chdir('..') + ftp.chdir('..') + ftp.chdir('..') + # Copy any srclibs that are required... + if thisbuild.has_key('srclibs'): + ftp.chdir('build') + ftp.chdir('extlib') + for lib in thisbuild['srclibs'].split(';'): + lp = lib.split('@').split('/') + for d in lp[:-1]: + ftp.mkdir(d) + ftp.chdir(d) + ftp.put(os.path.join('build/extlib', lib), lp[-1]) + for _ in lp[:-1]: + ftp.chdir('..') + ftp.chdir('..') + ftp.chdir('..') + + # Execute the build script... + print "Starting build..." + chan = sshs.get_transport().open_session() + stdoutf = chan.makefile('rb') + stderrf = chan.makefile_stderr('rb') + chan.exec_command('python build.py --on-server -p ' + + app['id'] + ' --vercode ' + thisbuild['vercode']) + returncode = chan.recv_exit_status() + output = stdoutf.read() + error = stderrf.read() + if returncode != 0: + raise BuildException("Build.py failed on server for %s:%s" % (app['id'], thisbuild['version']), output.strip(), error.strip()) + + # Retrieve the built files... + ftp.chdir('/home/vagrant/unsigned') + apkfile = app['id'] + '_' + thisbuild['vercode'] + '.apk' + tarball = app['id'] + '_' + thisbuild['vercode'] + '_src' + '.tar.gz' + try: + ftp.get(apkfile, os.path.join(output_dir, apkfile)) + ftp.get(tarball, os.path.join(output_dir, tarball)) + except: + raise BuildException("Build failed for %s:%s" % (app['id'], thisbuild['version']), output.strip(), error.strip()) + ftp.close() + + finally: + + # Suspend the build server. + subprocess.call(['vagrant', 'suspend'], cwd='builder') def build_local(app, thisbuild, vcs, build_dir, output_dir, extlib_dir, tmp_dir, install, force, verbose=False): diff --git a/makebuildserver.sh b/makebuildserver.sh index 5e2a7c00..b6865cb2 100755 --- a/makebuildserver.sh +++ b/makebuildserver.sh @@ -2,12 +2,9 @@ set -e rm -f buildserver.box cd buildserver +vagrant halt vagrant up -sleep 5 -vagrant ssh -c "sudo shutdown -h now" -cd .. -# Just to wait until it's shut down! -sleep 20 +vagrant halt vagrant package --base `VBoxManage list vms | grep buildserver | sed 's/"\(.*\)".*/\1/'` --output buildserver.box vagrant box add buildserver buildserver.box -f && rm buildserver.box