diff --git a/README.buildserver b/README.buildserver index a28bab97..2c2d83b4 100644 --- a/README.buildserver +++ b/README.buildserver @@ -1,15 +1,14 @@ +Integrating the build server setup into the main scripts is a work in progress. Some things may +not work properly yet. Talk to CiaranG if you're trying to use this and have problems. + Setting up a build server: 1. Install VirtualBox, vagrant and vagrant-snap -2. In the buildserver directory, run 'vagrant up'. Will take a long time. -3. Log in with 'vagrant ssh' -4. Check it all looks ok, then 'sudo shutdown -h now' -5. Back in the main directory, run 'VBoxManage listvms' look for - buildserver_xxxx -6. Run 'vagrant package --base buildserver_xxxx --output buildserver.box'. - Will take a while. -7. You should now have a new 'buildserver.box' +2. Create (or get - ask CiaranG, or wait until I replace this with a download link!) a standard + Debian Squeeze vagrant-compatible base box called 'debian6-32' +3. Run makebuildserver.sh. This will take a long time. The end result is a new base box called + 'buildserver'. You should now be able to use the --server option on build.py and builds will take place in the clean, secure, isolated environment of a fresh virtual diff --git a/build.py b/build.py index fc97f058..f851da1c 100755 --- a/build.py +++ b/build.py @@ -32,6 +32,220 @@ import common from common import BuildException from common import VCSException + +# Do a build on the build server. +def build_server(app, thisbuild, build_dir, output_dir): + import paramiko + + # Destroy the builder vm if it already exists... + # TODO: need to integrate the snapshot stuff so it doesn't have to + # keep wasting time doing this unnecessarily. + if os.path.exists(os.path.join('builder', '.vagrant')): + if subprocess.call(['vagrant', 'destroy'], cwd='builder') != 0: + raise BuildException("Failed to destroy build server") + + # Start up the virtual maachine... + if subprocess.call(['vagrant', 'up'], cwd='builder') != 0: + # Not a very helpful message yet! + raise BuildException("Failed to set up build server") + # Get SSH configuration settings for us to connect... + subprocess.call('vagrant ssh-config >sshconfig', + cwd='builder', shell=True) + vagranthost = 'default' # Host in ssh config file + + # Load and parse the SSH config... + sshconfig = paramiko.SSHConfig() + sshf = open('builder/sshconfig', 'r') + sshconfig.parse(sshf) + sshf.close() + sshconfig = sshconfig.lookup(vagranthost) + + # Open SSH connection... + ssh = paramiko.SSHClient() + ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + print sshconfig + ssh.connect(sshconfig['hostname'], username=sshconfig['user'], + port=int(sshconfig['port']), timeout=10, look_for_keys=False, + key_filename=sshconfig['identityfile']) + + # Get an SFTP connection... + ftp = ssh.open_sftp() + ftp.get_channel().settimeout(15) + + # Put all the necessary files in place... + ftp.chdir('/home/vagrant') + ftp.put('build.py', 'build.py') + ftp.put('common.py', 'common.py') + ftp.put('config.buildserver.py', 'config.py') + ftp.mkdir('metadata') + ftp.chdir('metadata') + ftp.put(os.path.join('metadata', app['id'] + '.txt'), + app['id'] + '.txt') + ftp.chdir('..') + ftp.mkdir('build') + ftp.chdir('build') + ftp.mkdir('extlib') + ftp.mkdir(app['id']) + ftp.chdir('..') + def send_dir(path): + lastdir = path + for r, d, f in os.walk(path): + ftp.chdir(r) + for dd in d: + ftp.mkdir(dd) + for ff in f: + ftp.put(os.path.join(r, ff), ff) + for i in range(len(r.split('/'))): + ftp.chdir('..') + send_dir(build_dir) + # TODO: send relevant extlib and srclib directories too + + # Execute the build script... + ssh.exec_command('python build.py --on-server -p ' + + app['id'] + ' --vercode ' + thisbuild['vercode']) + + # Retrieve the built files... + apkfile = app['id'] + '_' + thisbuild['vercode'] + '.apk' + tarball = app['id'] + '_' + thisbuild['vercode'] + '_src' + '.tar.gz' + ftp.chdir('/home/vagrant/unsigned') + ftp.get(apkfile, os.path.join(output_dir, apkfile)) + ftp.get(tarball, os.path.join(output_dir, tarball)) + + # Get rid of the virtual machine... + if subprocess.call(['vagrant', 'destroy'], cwd='builder') != 0: + # Not a very helpful message yet! + raise BuildException("Failed to destroy") + + +# Do a build locally. +def build_local(app, thisbuild, build_dir, output_dir): + + # Prepare the source code... + root_dir = common.prepare_source(vcs, app, thisbuild, + build_dir, extlib_dir, sdk_path, ndk_path, + javacc_path) + + # Scan before building... + buildprobs = common.scan_source(build_dir, root_dir, thisbuild) + if len(buildprobs) > 0: + print 'Scanner found ' + str(len(buildprobs)) + ' problems:' + for problem in buildprobs: + print '...' + problem + raise BuildException("Can't build due to " + + str(len(buildprobs)) + " scanned problems") + + # Build the source tarball right before we build the release... + tarname = app['id'] + '_' + thisbuild['vercode'] + '_src' + tarball = tarfile.open(os.path.join(tmp_dir, + tarname + '.tar.gz'), "w:gz") + def tarexc(f): + if f in ['.svn', '.git', '.hg', '.bzr']: + return True + return False + tarball.add(build_dir, tarname, exclude=tarexc) + tarball.close() + + # Build native stuff if required... + if thisbuild.get('buildjni', 'no') == 'yes': + ndkbuild = os.path.join(ndk_path, "ndk-build") + p = subprocess.Popen([ndkbuild], cwd=root_dir, + stdout=subprocess.PIPE) + output = p.communicate()[0] + if p.returncode != 0: + print output + raise BuildException("NDK build failed for %s:%s" % (app['id'], thisbuild['version'])) + elif options.verbose: + print output + + # Build the release... + if thisbuild.has_key('maven'): + p = subprocess.Popen(['mvn', 'clean', 'install', + '-Dandroid.sdk.path=' + sdk_path], + cwd=root_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + else: + if thisbuild.has_key('antcommand'): + antcommand = thisbuild['antcommand'] + else: + antcommand = 'release' + p = subprocess.Popen(['ant', antcommand], cwd=root_dir, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + output, error = p.communicate() + if p.returncode != 0: + raise BuildException("Build failed for %s:%s" % (app['id'], thisbuild['version']), output.strip(), error.strip()) + elif options.verbose: + print output + print "Build successful" + + # Find the apk name in the output... + if thisbuild.has_key('bindir'): + bindir = os.path.join(build_dir, thisbuild['bindir']) + else: + bindir = os.path.join(root_dir, 'bin') + if thisbuild.get('initfun', 'no') == "yes": + # Special case (again!) for funambol... + src = ("funambol-android-sync-client-" + + thisbuild['version'] + "-unsigned.apk") + src = os.path.join(bindir, src) + elif thisbuild.has_key('maven'): + src = re.match(r".*^\[INFO\] Installing /.*/([^/]*)\.apk", + output, re.S|re.M).group(1) + src = os.path.join(bindir, src) + '.apk' +#[INFO] Installing /home/ciaran/fdroidserver/tmp/mainline/application/target/callerid-1.0-SNAPSHOT.apk + else: + src = re.match(r".*^.*Creating (\S+) for release.*$.*", output, + re.S|re.M).group(1) + src = os.path.join(bindir, src) + + # By way of a sanity check, make sure the version and version + # code in our new apk match what we expect... + print "Checking " + src + p = subprocess.Popen([os.path.join(sdk_path, 'platform-tools', + 'aapt'), + 'dump', 'badging', src], + stdout=subprocess.PIPE) + output = p.communicate()[0] + if thisbuild.get('novcheck', 'no') == "yes": + vercode = thisbuild['vercode'] + version = thisbuild['version'] + else: + vercode = None + version = None + for line in output.splitlines(): + if line.startswith("package:"): + pat = re.compile(".*versionCode='([0-9]*)'.*") + vercode = re.match(pat, line).group(1) + pat = re.compile(".*versionName='([^']*)'.*") + version = re.match(pat, line).group(1) + if version == None or vercode == None: + raise BuildException("Could not find version information in build in output") + + # Some apps (e.g. Timeriffic) have had the bonkers idea of + # including the entire changelog in the version number. Remove + # it so we can compare. (TODO: might be better to remove it + # before we compile, in fact) + index = version.find(" //") + if index != -1: + version = version[:index] + + if (version != thisbuild['version'] or + vercode != thisbuild['vercode']): + raise BuildException(("Unexpected version/version code in output" + "APK: %s / %s" + "Expected: %s / %s") + % (version, str(vercode), thisbuild['version'], str(thisbuild['vercode'])) + ) + + # Copy the unsigned apk to our destination directory for further + # processing (by publish.py)... + shutil.copyfile(src, dest) + + # Move the source tarball into the output directory... + if output_dir != tmp_dir: + tarfilename = tarname + '.tar.gz' + shutil.move(os.path.join(tmp_dir, tarfilename), + os.path.join(output_dir, tarfilename)) + + #Read configuration... execfile('config.py') @@ -41,6 +255,8 @@ parser.add_option("-v", "--verbose", action="store_true", default=False, help="Spew out even more information than normal") parser.add_option("-p", "--package", default=None, help="Build only the specified package") +parser.add_option("-c", "--vercode", default=None, + help="Build only the specified version code") parser.add_option("-s", "--stop", action="store_true", default=False, help="Make the build stop on exceptions") parser.add_option("-t", "--test", action="store_true", default=False, @@ -89,13 +305,18 @@ if not os.path.isdir(build_dir): os.makedirs(build_dir) extlib_dir = os.path.join(build_dir, 'extlib') +# Filter apps and build versions according to command-line options... +if options.package: + apps = [app for app in apps if app['id'] == options.package] +if options.vercode: + for app in apps: + app['builds'] = [b for b in app['builds'] + if str(b['vercode']) == options.vercode] + # Build applications... for app in apps: - if options.package and options.package != app['id']: - # Silent skip... - pass - elif app['Disabled'] and not options.force: + if app['Disabled'] and not options.force: if options.verbose: print "Skipping %s: disabled" % app['id'] elif (not app['builds']) or app['Repo Type'] =='' or len(app['builds']) == 0: @@ -108,9 +329,6 @@ for app in apps: # Set up vcs interface and make sure we have the latest code... vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir) - refreshed_source = False - - for thisbuild in app['builds']: try: dest = os.path.join(output_dir, app['id'] + '_' + @@ -133,201 +351,9 @@ for app in apps: print mstart + thisbuild['version'] + ' of ' + app['id'] if options.server: - - import paramiko - - # Start up the virtual maachine... - if subprocess.call(['vagrant', 'up'], cwd='builder') != 0: - # Not a very helpful message yet! - raise BuildException("Failed to set up build server") - # Get SSH configuration settings for us to connect... - subprocess.call('vagrant ssh-config >sshconfig', - cwd='builder', shell=True) - vagranthost = 'default' # Host in ssh config file - - # Load and parse the SSH config... - sshconfig = paramiko.SSHConfig() - sshf = open('builder/sshconfig', 'r') - sshconfig.parse(sshf) - sshf.close() - sshconfig = sshconfig.lookup(vagranthost) - - # Open SSH connection... - ssh = paramiko.SSHClient() - ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - print sshconfig - ssh.connect(sshconfig['hostname'], username=sshconfig['user'], - port=int(sshconfig['port']), timeout=10, look_for_keys=False, - key_filename=sshconfig['identityfile']) - - # Get an SFTP connection... - ftp = ssh.open_sftp() - ftp.get_channel().settimeout(15) - - # Put all the necessary files in place... - ftp.chdir('/home/vagrant') - ftp.put('build.py', 'build.py') - ftp.put('common.py', 'common.py') - ftp.put('config.buildserver.py', 'config.py') - ftp.mkdir('build') - ftp.chdir('build') - ftp.mkdir('extlib') - def send_dir(path): - lastdir = path - for r, d, f in os.walk(base): - while lastdir != os.path.commonprefix([lastdir, root]): - ftp.chdir('..') - lastdir = os.path.split(lastdir)[0] - lastdir = r - for ff in f: - ftp.put(os.path.join(r, ff), ff) - send_dir(app['id']) - # TODO: send relevant extlib directories too - ftp.chdir('/home/vagrant') - - # Execute the build script... - ssh.exec_command('python build.py --on-server -p ' + - app['id']) - - # Retrieve the built files... - apkfile = app['id'] + '_' + thisbuild['vercode'] + '.apk' - tarball = app['id'] + '_' + thisbuild['vercode'] + '_src' + '.tar.gz' - ftp.chdir('unsigned') - ftp.get(apkfile, os.path.join(output_dir, apkfile)) - ftp.get(tarball, os.path.join(output_dir, tarball)) - - # Get rid of the virtual machine... - if subprocess.call(['vagrant', 'destroy'], cwd='builder') != 0: - # Not a very helpful message yet! - raise BuildException("Failed to destroy") - + build_server(app, thisbuild, build_dir, output_dir) else: - - # Prepare the source code... - root_dir = common.prepare_source(vcs, app, thisbuild, - build_dir, extlib_dir, sdk_path, ndk_path, - javacc_path, not refreshed_source) - refreshed_source = True - - # Scan before building... - buildprobs = common.scan_source(build_dir, root_dir, thisbuild) - if len(buildprobs) > 0: - print 'Scanner found ' + str(len(buildprobs)) + ' problems:' - for problem in buildprobs: - print '...' + problem - raise BuildException("Can't build due to " + - str(len(buildprobs)) + " scanned problems") - - # Build the source tarball right before we build the release... - tarname = app['id'] + '_' + thisbuild['vercode'] + '_src' - tarball = tarfile.open(os.path.join(tmp_dir, - tarname + '.tar.gz'), "w:gz") - def tarexc(f): - if f in ['.svn', '.git', '.hg', '.bzr']: - return True - return False - tarball.add(build_dir, tarname, exclude=tarexc) - tarball.close() - - # Build native stuff if required... - if thisbuild.get('buildjni', 'no') == 'yes': - ndkbuild = os.path.join(ndk_path, "ndk-build") - p = subprocess.Popen([ndkbuild], cwd=root_dir, - stdout=subprocess.PIPE) - output = p.communicate()[0] - if p.returncode != 0: - print output - raise BuildException("NDK build failed for %s:%s" % (app['id'], thisbuild['version'])) - elif options.verbose: - print output - - # Build the release... - if thisbuild.has_key('maven'): - p = subprocess.Popen(['mvn', 'clean', 'install', - '-Dandroid.sdk.path=' + sdk_path], - cwd=root_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - else: - if thisbuild.has_key('antcommand'): - antcommand = thisbuild['antcommand'] - else: - antcommand = 'release' - p = subprocess.Popen(['ant', antcommand], cwd=root_dir, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - output, error = p.communicate() - if p.returncode != 0: - raise BuildException("Build failed for %s:%s" % (app['id'], thisbuild['version']), output.strip(), error.strip()) - elif options.verbose: - print output - print "Build successful" - - # Find the apk name in the output... - if thisbuild.has_key('bindir'): - bindir = os.path.join(build_dir, thisbuild['bindir']) - else: - bindir = os.path.join(root_dir, 'bin') - if thisbuild.get('initfun', 'no') == "yes": - # Special case (again!) for funambol... - src = ("funambol-android-sync-client-" + - thisbuild['version'] + "-unsigned.apk") - src = os.path.join(bindir, src) - elif thisbuild.has_key('maven'): - src = re.match(r".*^\[INFO\] Installing /.*/([^/]*)\.apk", - output, re.S|re.M).group(1) - src = os.path.join(bindir, src) + '.apk' -#[INFO] Installing /home/ciaran/fdroidserver/tmp/mainline/application/target/callerid-1.0-SNAPSHOT.apk - else: - src = re.match(r".*^.*Creating (\S+) for release.*$.*", output, - re.S|re.M).group(1) - src = os.path.join(bindir, src) - - # By way of a sanity check, make sure the version and version - # code in our new apk match what we expect... - print "Checking " + src - p = subprocess.Popen([os.path.join(sdk_path, 'platform-tools', - 'aapt'), - 'dump', 'badging', src], - stdout=subprocess.PIPE) - output = p.communicate()[0] - if thisbuild.get('novcheck', 'no') == "yes": - vercode = thisbuild['vercode'] - version = thisbuild['version'] - else: - vercode = None - version = None - for line in output.splitlines(): - if line.startswith("package:"): - pat = re.compile(".*versionCode='([0-9]*)'.*") - vercode = re.match(pat, line).group(1) - pat = re.compile(".*versionName='([^']*)'.*") - version = re.match(pat, line).group(1) - if version == None or vercode == None: - raise BuildException("Could not find version information in build in output") - - # Some apps (e.g. Timeriffic) have had the bonkers idea of - # including the entire changelog in the version number. Remove - # it so we can compare. (TODO: might be better to remove it - # before we compile, in fact) - index = version.find(" //") - if index != -1: - version = version[:index] - - if (version != thisbuild['version'] or - vercode != thisbuild['vercode']): - raise BuildException(("Unexpected version/version code in output" - "APK: %s / %s" - "Expected: %s / %s") - % (version, str(vercode), thisbuild['version'], str(thisbuild['vercode'])) - ) - - # Copy the unsigned apk to our destination directory for further - # processing (by publish.py)... - shutil.copyfile(src, dest) - - # Move the source tarball into the output directory... - if output_dir != tmp_dir: - tarfilename = tarname + '.tar.gz' - shutil.move(os.path.join(tmp_dir, tarfilename), - os.path.join(output_dir, tarfilename)) + build_local(app, thisbuild, build_dir, output_dir) build_succeeded.append(app) except BuildException as be: diff --git a/builder/.gitignore b/builder/.gitignore index b8dee1d4..80b72f60 100644 --- a/builder/.gitignore +++ b/builder/.gitignore @@ -1,2 +1,3 @@ sshconfig .vagrant +*.log diff --git a/builder/Vagrantfile b/builder/Vagrantfile index 865a496a..1487a8df 100644 --- a/builder/Vagrantfile +++ b/builder/Vagrantfile @@ -1,7 +1,6 @@ Vagrant::Config.run do |config| config.vm.box = "buildserver" - config.vm.box_url = "../buildserver.box" config.vm.customize ["modifyvm", :id, "--memory", "768"] diff --git a/buildserver/cookbooks/fdroidbuild-general/recipes/default.rb b/buildserver/cookbooks/fdroidbuild-general/recipes/default.rb index 81e4c949..756d6bcc 100644 --- a/buildserver/cookbooks/fdroidbuild-general/recipes/default.rb +++ b/buildserver/cookbooks/fdroidbuild-general/recipes/default.rb @@ -1,5 +1,5 @@ -%w{ant ant-contrib maven2 javacc python}.each do |pkg| +%w{ant ant-contrib maven2 javacc python git-core mercurial subversion bzr}.each do |pkg| package pkg do action :install end diff --git a/common.py b/common.py index 83836500..48be88eb 100644 --- a/common.py +++ b/common.py @@ -657,10 +657,9 @@ def getsrclib(spec, extlib_dir): # 'sdk_path' - the path to the Android SDK # 'ndk_path' - the path to the Android NDK # 'javacc_path' - the path to javacc -# 'refresh' - True to refresh from the remote repo # Returns the root directory, which may be the same as 'build_dir' or may # be a subdirectory of it. -def prepare_source(vcs, app, build, build_dir, extlib_dir, sdk_path, ndk_path, javacc_path, refresh): +def prepare_source(vcs, app, build, build_dir, extlib_dir, sdk_path, ndk_path, javacc_path): # Optionally, the actual app source can be in a subdirectory... if build.has_key('subdir'): diff --git a/makebuildserver.sh b/makebuildserver.sh new file mode 100755 index 00000000..7c982655 --- /dev/null +++ b/makebuildserver.sh @@ -0,0 +1,11 @@ +#!/bin/bash +rm -f buildserver.box +cd buildserver +vagrant up +vagrant ssh -c "sudo shutdown -h now" +cd .. +# Just to wait until it's shut down! +sleep 20 +vagrant package --base `VBoxManage list vms | grep buildserver | sed 's/"\(.*\)".*/\1/'` --output buildserver.box +vagrant box add buildserver buildserver.box -f && rm buildserver.box + diff --git a/scanner.py b/scanner.py index 53c38a99..ee00a3ca 100755 --- a/scanner.py +++ b/scanner.py @@ -75,9 +75,6 @@ for app in apps: # Set up vcs interface and make sure we have the latest code... vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir) - refreshed_source = False - - for thisbuild in app['builds']: if thisbuild['commit'].startswith('!'): @@ -88,9 +85,7 @@ for app in apps: # Prepare the source code... root_dir = common.prepare_source(vcs, app, thisbuild, - build_dir, extlib_dir, sdk_path, ndk_path, javacc_path, - not refreshed_source) - refreshed_source = True + build_dir, extlib_dir, sdk_path, ndk_path, javacc_path) # Do the scan... buildprobs = common.scan_source(build_dir, root_dir, thisbuild)