From 498e7d3c5f6e4a666d7b18f814f2d811aa0915ee Mon Sep 17 00:00:00 2001 From: Ciaran Gultnieks Date: Fri, 3 Feb 2012 16:01:35 +0000 Subject: [PATCH] Work in progress on integrating build server --- .gitignore | 1 + README.buildserver | 17 + build.py | 307 +++++++++++------- builder/.gitignore | 1 + builder/Vagrantfile | 8 + buildserver/.gitignore | 1 + buildserver/Vagrantfile | 26 ++ .../cookbooks/android-ndk/recipes/default.rb | 17 + .../cookbooks/android-sdk/recipes/default.rb | 34 ++ .../fdroidbuild-general/recipes/default.rb | 7 + buildserver/fixpaths.sh | 23 ++ common.py | 2 +- config.buildserver.py | 4 + scanner.py | 2 +- 14 files changed, 334 insertions(+), 116 deletions(-) create mode 100644 README.buildserver create mode 100644 builder/.gitignore create mode 100644 builder/Vagrantfile create mode 100644 buildserver/.gitignore create mode 100644 buildserver/Vagrantfile create mode 100644 buildserver/cookbooks/android-ndk/recipes/default.rb create mode 100644 buildserver/cookbooks/android-sdk/recipes/default.rb create mode 100644 buildserver/cookbooks/fdroidbuild-general/recipes/default.rb create mode 100644 buildserver/fixpaths.sh create mode 100644 config.buildserver.py diff --git a/.gitignore b/.gitignore index 0fb17377..24ea4368 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ tmp/ build_*/ *~ *.pyc +buildserver.box diff --git a/README.buildserver b/README.buildserver new file mode 100644 index 00000000..a28bab97 --- /dev/null +++ b/README.buildserver @@ -0,0 +1,17 @@ + +Setting up a build server: + +1. Install VirtualBox, vagrant and vagrant-snap +2. In the buildserver directory, run 'vagrant up'. Will take a long time. +3. Log in with 'vagrant ssh' +4. Check it all looks ok, then 'sudo shutdown -h now' +5. Back in the main directory, run 'VBoxManage listvms' look for + buildserver_xxxx +6. Run 'vagrant package --base buildserver_xxxx --output buildserver.box'. + Will take a while. +7. You should now have a new 'buildserver.box' + +You should now be able to use the --server option on build.py and builds will +take place in the clean, secure, isolated environment of a fresh virtual +machine for each app built. + diff --git a/build.py b/build.py index ef02ca36..5356aac6 100755 --- a/build.py +++ b/build.py @@ -24,6 +24,7 @@ import subprocess import re import zipfile import tarfile +import traceback from xml.dom.minidom import Document from optparse import OptionParser @@ -44,8 +45,18 @@ parser.add_option("-s", "--stop", action="store_true", default=False, help="Make the build stop on exceptions") parser.add_option("-t", "--test", action="store_true", default=False, help="Test mode - put output in the tmp directory only.") +parser.add_option("--server", action="store_true", default=False, + help="Use build server") +parser.add_option("--on-server", action="store_true", default=False, + help="Specify that we're running on the build server") +parser.add_option("-f", "--force", action="store_true", default=False, + help="Force build of disabled app. Only allowed in test mode.") (options, args) = parser.parse_args() +if options.force and not options.test: + print "Force is only allowed in test mode" + sys.exit(1) + # Get all apps... apps = common.read_metadata(options.verbose) @@ -84,7 +95,7 @@ for app in apps: if options.package and options.package != app['id']: # Silent skip... pass - elif app['Disabled']: + elif app['Disabled'] and not options.force: if options.verbose: print "Skipping %s: disabled" % app['id'] elif (not app['builds']) or app['Repo Type'] =='' or len(app['builds']) == 0: @@ -107,7 +118,7 @@ for app in apps: dest_repo = os.path.join(repo_dir, app['id'] + '_' + thisbuild['vercode'] + '.apk') - if os.path.exists(dest) or os.path.exists(dest_repo): + if os.path.exists(dest) or (not options.test and os.path.exists(dest_repo)): if options.verbose: print "..version " + thisbuild['version'] + " already exists" elif thisbuild['commit'].startswith('!'): @@ -121,131 +132,199 @@ for app in apps: mstart = 'Building version ' print mstart + thisbuild['version'] + ' of ' + app['id'] - # Prepare the source code... - root_dir = common.prepare_source(vcs, app, thisbuild, - build_dir, extlib_dir, sdk_path, ndk_path, - javacc_path, not refreshed_source) - refreshed_source = True + if options.server: - # Scan before building... - buildprobs = common.scan_source(build_dir) - if len(buildprobs) > 0: - print 'Scanner found ' + str(len(buildprobs)) + ' problems:' - for problem in buildprobs: - print '...' + problem - raise BuildException("Can't build due to " + - str(len(buildprobs)) + " scanned problems") + import paramiko - # Build the source tarball right before we build the release... - tarname = app['id'] + '_' + thisbuild['vercode'] + '_src' - tarball = tarfile.open(os.path.join(tmp_dir, - tarname + '.tar.gz'), "w:gz") - def tarexc(f): - if f in ['.svn', '.git', '.hg', '.bzr']: - return True - return False - tarball.add(build_dir, tarname, exclude=tarexc) - tarball.close() + # Start up the virtual maachine... + if subprocess.call(['vagrant', 'up'], cwd='builder') != 0: + # Not a very helpful message yet! + raise BuildException("Failed to set up build server") + # Get SSH configuration settings for us to connect... + subprocess.call('vagrant ssh-config >sshconfig', + cwd='builder', shell=True) + vagranthost = 'default' # Host in ssh config file - # Build native stuff if required... - if thisbuild.get('buildjni', 'no') == 'yes': - ndkbuild = os.path.join(ndk_path, "ndk-build") - p = subprocess.Popen([ndkbuild], cwd=root_dir, - stdout=subprocess.PIPE) - output = p.communicate()[0] + # Load and parse the SSH config... + sshconfig = paramiko.SSHConfig() + sshconfig.parse('builder/sshconfig') + sshconfig = sshconfig.lookup(vagranthost) + + # Open SSH connection... + ssh = paramiko.SSHClient() + ssh.set_missing_host_key_policy(paramiko.AcceptPolicy) + ssh.connect(sshconfig['HostName'], username=sshconfig['Username'], + port=sshconfig['Port'], timeout=10, look_for_keys=False, + key_filename=sshconfig['IdentityFile']) + + # Get an SFTP connection... + ftp = ssh.open_sftp() + ftp.get_channel().settimeout(15) + + # Put all the necessary files in place... + ftp.chdir('/home/vagrant') + ftp.put('build.py', 'build.py') + ftp.put('common.py', 'common.py') + ftp.put('config.buildserver.py', 'config.py') + ftp.mkdir('build') + ftp.chdir('build') + ftp.mkdir('extlib') + def send_dir(path): + lastdir = path + for r, d, f in os.walk(base): + while lastdir != os.path.commonprefix([lastdir, root]): + ftp.chdir('..') + lastdir = os.path.split(lastdir)[0] + lastdir = r + for ff in f: + ftp.put(os.path.join(r, ff), ff) + ftp.send_dir(app['id']) + # TODO: send relevant extlib directories too + ftp.chdir('/home/vagrant') + + # Execute the build script... + ssh.exec_command('python build.py --on-server -p ' + + app['id']) + + # Retrieve the built files... + apkfile = app['id'] + '_' + thisbuild['vercode'] + '.apk' + tarball = app['id'] + '_' + thisbuild['vercode'] + '_src' + '.tar.gz' + ftp.chdir('unsigned') + ftp.get(apkfile, os.path.join(output_dir, apkfile)) + ftp.get(tarball, os.path.join(output_dir, tarball)) + + # Get rid of the virtual machine... + if subprocess.call(['vagrant', 'destroy'], cwd='builder') != 0: + # Not a very helpful message yet! + raise BuildException("Failed to destroy") + + else: + + # Prepare the source code... + root_dir = common.prepare_source(vcs, app, thisbuild, + build_dir, extlib_dir, sdk_path, ndk_path, + javacc_path, not refreshed_source) + refreshed_source = True + + # Scan before building... + buildprobs = common.scan_source(build_dir, root_dir, thisbuild) + if len(buildprobs) > 0: + print 'Scanner found ' + str(len(buildprobs)) + ' problems:' + for problem in buildprobs: + print '...' + problem + raise BuildException("Can't build due to " + + str(len(buildprobs)) + " scanned problems") + + # Build the source tarball right before we build the release... + tarname = app['id'] + '_' + thisbuild['vercode'] + '_src' + tarball = tarfile.open(os.path.join(tmp_dir, + tarname + '.tar.gz'), "w:gz") + def tarexc(f): + if f in ['.svn', '.git', '.hg', '.bzr']: + return True + return False + tarball.add(build_dir, tarname, exclude=tarexc) + tarball.close() + + # Build native stuff if required... + if thisbuild.get('buildjni', 'no') == 'yes': + ndkbuild = os.path.join(ndk_path, "ndk-build") + p = subprocess.Popen([ndkbuild], cwd=root_dir, + stdout=subprocess.PIPE) + output = p.communicate()[0] + if p.returncode != 0: + print output + raise BuildException("NDK build failed for %s:%s" % (app['id'], thisbuild['version'])) + elif options.verbose: + print output + + # Build the release... + if thisbuild.has_key('maven'): + p = subprocess.Popen(['mvn', 'clean', 'install', + '-Dandroid.sdk.path=' + sdk_path], + cwd=root_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + else: + if thisbuild.has_key('antcommand'): + antcommand = thisbuild['antcommand'] + else: + antcommand = 'release' + p = subprocess.Popen(['ant', antcommand], cwd=root_dir, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + output, error = p.communicate() if p.returncode != 0: - print output - raise BuildException("NDK build failed for %s:%s" % (app['id'], thisbuild['version'])) + raise BuildException("Build failed for %s:%s" % (app['id'], thisbuild['version']), output.strip(), error.strip()) elif options.verbose: print output + print "Build successful" - # Build the release... - if thisbuild.has_key('maven'): - p = subprocess.Popen(['mvn', 'clean', 'install', - '-Dandroid.sdk.path=' + sdk_path], - cwd=root_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - else: - if thisbuild.has_key('antcommand'): - antcommand = thisbuild['antcommand'] + # Find the apk name in the output... + if thisbuild.has_key('bindir'): + bindir = os.path.join(build_dir, thisbuild['bindir']) else: - antcommand = 'release' - p = subprocess.Popen(['ant', antcommand], cwd=root_dir, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - output, error = p.communicate() - if p.returncode != 0: - raise BuildException("Build failed for %s:%s" % (app['id'], thisbuild['version']), output.strip(), error.strip()) - elif options.verbose: - print output - print "Build successful" - - # Find the apk name in the output... - if thisbuild.has_key('bindir'): - bindir = os.path.join(build_dir, thisbuild['bindir']) - else: - bindir = os.path.join(root_dir, 'bin') - if thisbuild.get('initfun', 'no') == "yes": - # Special case (again!) for funambol... - src = ("funambol-android-sync-client-" + - thisbuild['version'] + "-unsigned.apk") - src = os.path.join(bindir, src) - elif thisbuild.has_key('maven'): - src = re.match(r".*^\[INFO\] Installing /.*/([^/]*)\.apk", - output, re.S|re.M).group(1) - src = os.path.join(bindir, src) + '.apk' + bindir = os.path.join(root_dir, 'bin') + if thisbuild.get('initfun', 'no') == "yes": + # Special case (again!) for funambol... + src = ("funambol-android-sync-client-" + + thisbuild['version'] + "-unsigned.apk") + src = os.path.join(bindir, src) + elif thisbuild.has_key('maven'): + src = re.match(r".*^\[INFO\] Installing /.*/([^/]*)\.apk", + output, re.S|re.M).group(1) + src = os.path.join(bindir, src) + '.apk' #[INFO] Installing /home/ciaran/fdroidserver/tmp/mainline/application/target/callerid-1.0-SNAPSHOT.apk - else: - src = re.match(r".*^.*Creating (\S+) for release.*$.*", output, - re.S|re.M).group(1) - src = os.path.join(bindir, src) + else: + src = re.match(r".*^.*Creating (\S+) for release.*$.*", output, + re.S|re.M).group(1) + src = os.path.join(bindir, src) - # By way of a sanity check, make sure the version and version - # code in our new apk match what we expect... - print "Checking " + src - p = subprocess.Popen([os.path.join(sdk_path, 'platform-tools', - 'aapt'), - 'dump', 'badging', src], - stdout=subprocess.PIPE) - output = p.communicate()[0] - if thisbuild.get('novcheck', 'no') == "yes": - vercode = thisbuild['vercode'] - version = thisbuild['version'] - else: - vercode = None - version = None - for line in output.splitlines(): - if line.startswith("package:"): - pat = re.compile(".*versionCode='([0-9]*)'.*") - vercode = re.match(pat, line).group(1) - pat = re.compile(".*versionName='([^']*)'.*") - version = re.match(pat, line).group(1) - if version == None or vercode == None: - raise BuildException("Could not find version information in build in output") + # By way of a sanity check, make sure the version and version + # code in our new apk match what we expect... + print "Checking " + src + p = subprocess.Popen([os.path.join(sdk_path, 'platform-tools', + 'aapt'), + 'dump', 'badging', src], + stdout=subprocess.PIPE) + output = p.communicate()[0] + if thisbuild.get('novcheck', 'no') == "yes": + vercode = thisbuild['vercode'] + version = thisbuild['version'] + else: + vercode = None + version = None + for line in output.splitlines(): + if line.startswith("package:"): + pat = re.compile(".*versionCode='([0-9]*)'.*") + vercode = re.match(pat, line).group(1) + pat = re.compile(".*versionName='([^']*)'.*") + version = re.match(pat, line).group(1) + if version == None or vercode == None: + raise BuildException("Could not find version information in build in output") - # Some apps (e.g. Timeriffic) have had the bonkers idea of - # including the entire changelog in the version number. Remove - # it so we can compare. (TODO: might be better to remove it - # before we compile, in fact) - index = version.find(" //") - if index != -1: - version = version[:index] + # Some apps (e.g. Timeriffic) have had the bonkers idea of + # including the entire changelog in the version number. Remove + # it so we can compare. (TODO: might be better to remove it + # before we compile, in fact) + index = version.find(" //") + if index != -1: + version = version[:index] - if (version != thisbuild['version'] or - vercode != thisbuild['vercode']): - raise BuildException(("Unexpected version/version code in output" - "APK: %s / %s" - "Expected: %s / %s") - % (version, str(vercode), thisbuild['version'], str(thisbuild['vercode'])) - ) + if (version != thisbuild['version'] or + vercode != thisbuild['vercode']): + raise BuildException(("Unexpected version/version code in output" + "APK: %s / %s" + "Expected: %s / %s") + % (version, str(vercode), thisbuild['version'], str(thisbuild['vercode'])) + ) - # Copy the unsigned apk to our destination directory for further - # processing (by publish.py)... - shutil.copyfile(src, dest) + # Copy the unsigned apk to our destination directory for further + # processing (by publish.py)... + shutil.copyfile(src, dest) - # Move the source tarball into the output directory... - if output_dir != tmp_dir: - tarfilename = tarname + '.tar.gz' - shutil.move(os.path.join(tmp_dir, tarfilename), - os.path.join(output_dir, tarfilename)) + # Move the source tarball into the output directory... + if output_dir != tmp_dir: + tarfilename = tarname + '.tar.gz' + shutil.move(os.path.join(tmp_dir, tarfilename), + os.path.join(output_dir, tarfilename)) build_succeeded.append(app) except BuildException as be: @@ -264,7 +343,7 @@ for app in apps: except Exception as e: if options.stop: raise - print "Could not build app %s due to unknown error: %s" % (app['id'], e) + print "Could not build app %s due to unknown error: %s" % (app['id'], traceback.format_exc()) failed_apps[app['id']] = e for app in build_succeeded: diff --git a/builder/.gitignore b/builder/.gitignore new file mode 100644 index 00000000..7fc03613 --- /dev/null +++ b/builder/.gitignore @@ -0,0 +1 @@ +sshconfig diff --git a/builder/Vagrantfile b/builder/Vagrantfile new file mode 100644 index 00000000..865a496a --- /dev/null +++ b/builder/Vagrantfile @@ -0,0 +1,8 @@ +Vagrant::Config.run do |config| + + config.vm.box = "buildserver" + config.vm.box_url = "../buildserver.box" + + config.vm.customize ["modifyvm", :id, "--memory", "768"] + +end diff --git a/buildserver/.gitignore b/buildserver/.gitignore new file mode 100644 index 00000000..8000dd9d --- /dev/null +++ b/buildserver/.gitignore @@ -0,0 +1 @@ +.vagrant diff --git a/buildserver/Vagrantfile b/buildserver/Vagrantfile new file mode 100644 index 00000000..d4f8d533 --- /dev/null +++ b/buildserver/Vagrantfile @@ -0,0 +1,26 @@ +Vagrant::Config.run do |config| + + config.vm.box = "debian6-32" + config.vm.box_url = "/shares/software/OS and Boot/debian6-32.box" + + config.vm.customize ["modifyvm", :id, "--memory", "1024"] + + config.vm.provision :shell, :path => "fixpaths.sh" + # Set apt proxy - remove, or adjust this, accordingly! + config.vm.provision :shell, :inline => 'sudo echo "Acquire::http { Proxy \"http://thurlow:3142\"; };" > /etc/apt/apt.conf.d/02proxy && sudo apt-get update' + + config.vm.provision :chef_solo do |chef| + chef.cookbooks_path = "cookbooks" + chef.log_level = :debug + chef.json = { + :settings => { + :sdk_loc => "/home/vagrant/android-sdk", + :ndk_loc => "/home/vagrant/android-ndk", + :user => "vagrant" + } + } + chef.add_recipe "android-sdk" + chef.add_recipe "android-ndk" + chef.add_recipe "fdroidbuild-general" + end +end diff --git a/buildserver/cookbooks/android-ndk/recipes/default.rb b/buildserver/cookbooks/android-ndk/recipes/default.rb new file mode 100644 index 00000000..80e06346 --- /dev/null +++ b/buildserver/cookbooks/android-ndk/recipes/default.rb @@ -0,0 +1,17 @@ + +ndk_loc = node[:settings][:ndk_loc] + +script "setup-android-ndk" do + interpreter "bash" + user node[:settings][:user] + cwd "/tmp" + code " + wget http://dl.google.com/android/ndk/android-ndk-r7-linux-x86.tar.bz2 + tar jxvf android-ndk-r7-linux-x86.tar.bz2 + mv android-ndk-r7 #{ndk_loc} + " + not_if do + File.exists?("#{ndk_loc}") + end +end + diff --git a/buildserver/cookbooks/android-sdk/recipes/default.rb b/buildserver/cookbooks/android-sdk/recipes/default.rb new file mode 100644 index 00000000..91b5a084 --- /dev/null +++ b/buildserver/cookbooks/android-sdk/recipes/default.rb @@ -0,0 +1,34 @@ +%w{openjdk-6-jdk}.each do |pkg| + package pkg do + action :install + end +end + +sdk_loc = node[:settings][:sdk_loc] +user = node[:settings][:user] + +script "setup-android-sdk" do + interpreter "bash" + user user + cwd "/tmp" + code " + wget http://dl.google.com/android/android-sdk_r16-linux.tgz + tar zxvf android-sdk_r16-linux.tgz + mv android-sdk-linux #{sdk_loc} + rm android-sdk_r16-linux.tgz + #{sdk_loc}/tools/android update sdk --no-ui -t platform-tool + #{sdk_loc}/tools/android update sdk --no-ui -t platform + #{sdk_loc}/tools/android update sdk --no-ui -t tool,platform-tool + " + not_if do + File.exists?("#{sdk_loc}") + end +end + +execute "add-android-sdk-path" do + user user + path = "#{sdk_loc}/tools:#{sdk_loc}/platform-tools" + command "echo \"export PATH=\\$PATH:#{path}\" >> /home/#{user}/.bashrc" + not_if "grep #{sdk_loc} /home/#{user}/.bashrc" +end + diff --git a/buildserver/cookbooks/fdroidbuild-general/recipes/default.rb b/buildserver/cookbooks/fdroidbuild-general/recipes/default.rb new file mode 100644 index 00000000..81e4c949 --- /dev/null +++ b/buildserver/cookbooks/fdroidbuild-general/recipes/default.rb @@ -0,0 +1,7 @@ + +%w{ant ant-contrib maven2 javacc python}.each do |pkg| + package pkg do + action :install + end +end + diff --git a/buildserver/fixpaths.sh b/buildserver/fixpaths.sh new file mode 100644 index 00000000..eb8a81fb --- /dev/null +++ b/buildserver/fixpaths.sh @@ -0,0 +1,23 @@ +#!/bin/sh + +fixit() +{ + #Fix sudoers so the PATH gets passed through, otherwise chef + #provisioning doesn't work. + if [ -z "$1" ]; then + export EDITOR=$0 && sudo -E visudo + else + echo "Fix sudoers" + echo "Defaults exempt_group=admin" >> $1 + fi + #Stick the gems bin onto root's path as well. + sudo echo "PATH=$PATH:/var/lib/gems/1.8/bin" >>/root/.bashrc + # Restart sudo so it gets the changes straight away + sudo /etc/init.d/sudo restart +} + +sudo grep "exempt_group" /etc/sudoers -q +if [ "$?" -eq "1" ]; then + fixit +fi + diff --git a/common.py b/common.py index 96086c6a..57f41ec6 100644 --- a/common.py +++ b/common.py @@ -892,7 +892,7 @@ def prepare_source(vcs, app, build, build_dir, extlib_dir, sdk_path, ndk_path, j # Scan the source code in the given directory (and all subdirectories) # and return a list of potential problems. -def scan_source(source_dir): +def scan_source(build_dir, root_dir, thisbuild): problems = [] diff --git a/config.buildserver.py b/config.buildserver.py new file mode 100644 index 00000000..44e5eaac --- /dev/null +++ b/config.buildserver.py @@ -0,0 +1,4 @@ +aapt_path = "/home/vagrant/android-sdk/platform-tools/aapt" +sdk_path = "/home/vagrant/android-sdk" +ndk_path = "/home/vagrant/android-ndk" +javacc_path = "/usr/share/java" diff --git a/scanner.py b/scanner.py index 045d57b4..53c38a99 100755 --- a/scanner.py +++ b/scanner.py @@ -93,7 +93,7 @@ for app in apps: refreshed_source = True # Do the scan... - buildprobs = common.scan_source(build_dir) + buildprobs = common.scan_source(build_dir, root_dir, thisbuild) for problem in buildprobs: problems.append(problem + ' in ' + app['id'] + ' ' + thisbuild['version'])